diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index 661b5ac9..343a195f 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -51,6 +51,11 @@ jobs: run: | make test + - name: Verify Generated Test Cases + run: | + make test_update + git diff --exit-code + - name: Parse Coverage Value From Feature Branch run: | echo "current_coverage=$(tail -n 1 .coverage/coverage.txt | awk '{print $3}' | awk 'sub("%", "")')" >> $GITHUB_ENV @@ -119,6 +124,8 @@ jobs: - name: Run integration tests env: CONNECTION_STRING: postgres://dawgs:weneedbetterpasswords@localhost:5432/dawgs?sslmode=disable + DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE: "1" + DAWGS_INTEGRATION_DISPOSABLE_TARGETS: postgresql://localhost:5432/dawgs run: | make test_integration @@ -156,6 +163,8 @@ jobs: - name: Run integration tests env: CONNECTION_STRING: neo4j://neo4j:weneedbetterpasswords@localhost:7687 + DAWGS_INTEGRATION_ALLOW_DESTRUCTIVE: "1" + DAWGS_INTEGRATION_DISPOSABLE_TARGETS: neo4j://localhost:7687/ run: | make test_integration diff --git a/.gitignore b/.gitignore index 5ba8be64..53866651 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,11 @@ integration/testdata/local/ # Local benchmark comparison output .bench/ +.cache/ # Augment .augment/ + +# Local performance captures and live-run artifacts +/artifacts/perf/ +/artifacts/live/ diff --git a/GOLANG_CODING_STANDARD.md b/GOLANG_CODING_STANDARD.md index c7958ca4..e613a4ab 100644 --- a/GOLANG_CODING_STANDARD.md +++ b/GOLANG_CODING_STANDARD.md @@ -10,11 +10,13 @@ type. The goal is to remove per-type receiver-name churn and reduce cognitive load while reading method bodies. ```go +// Start begins serving requests. func (s *Server) Start() error { go s.loop() return nil } +// Validate reports whether the configuration is supported. func (s Config) Validate() error { if s.Firewall.Backend != "nftables" { return fmt.Errorf("unsupported firewall backend %q", s.Firewall.Backend) @@ -243,6 +245,29 @@ case <-s.joiner.StopC: Avoid splitting tightly coupled statements when the second line is the immediate effect of the first. +## Documentation Comments + +Every function and method declaration and every struct or interface definition +must have a semantically relevant Go doc comment, whether it is exported or +unexported. Document every struct field and interface member individually, +including embedded fields and embedded interface elements. Follow Go doc form +by starting each comment with the declared identifier when applicable. + +Comments must explain the declaration's purpose, meaning, behavior, or contract. +Merely restating the identifier without adding useful information does not +satisfy this requirement. + +```go +// RecordStore retrieves and persists records. +type RecordStore interface { + // Find returns the record identified by key. + Find(ctx context.Context, key string) (Record, error) + + // Save persists record and returns any write failure. + Save(ctx context.Context, record Record) error +} +``` + ## Function Ordering Prefer ordering functions in the same file so dependencies appear before the @@ -256,12 +281,22 @@ Write struct type definitions across multiple lines, with one field per line. Align naturally with `gofmt`; do not compress structs onto one line. ```go +// FirewallConfig controls how the firewall backend manages bans. type FirewallConfig struct { - Backend string `toml:"backend"` - Table string `toml:"table"` - BanSet string `toml:"ban_set"` - Family string `toml:"family"` - DryRunSummaryOnly bool `toml:"dry_run_summary_only"` + // Backend selects the firewall implementation. + Backend string `toml:"backend"` + + // Table identifies the firewall table managed by the backend. + Table string `toml:"table"` + + // BanSet identifies the set containing banned addresses. + BanSet string `toml:"ban_set"` + + // Family selects the address family managed by the backend. + Family string `toml:"family"` + + // DryRunSummaryOnly limits dry-run output to a summary. + DryRunSummaryOnly bool `toml:"dry_run_summary_only"` } ``` @@ -320,13 +355,41 @@ defer fin.Close() ``` Use package-level grouped `const` and `var` declarations for related values. +Every package-level (global) `var` and `const` entity must have a semantically +relevant Go doc comment, whether it is exported or unexported. Document each +member of a grouped declaration individually. Comments on function-local `var` +declarations are optional and left to the author's discretion. ```go +// ErrNotFound indicates that the requested record does not exist. +var ErrNotFound = errors.New("not found") + const ( - ErrNotFound = errors.New("not found") + // fileWatchKey formats a file watch key. + fileWatchKey KeyFormat = "file_watch.%s" + + // hostRecordKey formats a host record key. + hostRecordKey KeyFormat = "hosts.%s" - fileWatchKey KeyFormat = "file_watch.%s" - hostRecordKey KeyFormat = "hosts.%s" + // hostRecordKeyPrefix identifies the host record key namespace. hostRecordKeyPrefix KeyFormat = "hosts." ) ``` + +In grouped `var` and `const` declarations, treat each leading comment and the +member definition it documents as one unit. Separate that unit from the next +comment and member definition with exactly one blank line. The final member +definition may instead be followed directly by the closing `)`. + +```go +var ( + // expansionRootFilter identifies the recursive traversal root filter. + expansionRootFilter = pgsql.Identifier("traversal_root_filter") + + // expansionTerminalFilter identifies the recursive traversal terminal filter. + expansionTerminalFilter = pgsql.Identifier("traversal_terminal_filter") + + // expansionPairFilter identifies the recursive traversal pair filter. + expansionPairFilter = pgsql.Identifier("traversal_pair_filter") +) +``` diff --git a/Makefile b/Makefile index af713148..0008128a 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ THIS_FILE := $(lastword $(MAKEFILE_LIST)) # Go configuration GO_CMD ?= go +GOIMPORTS_CMD ?= goimports CGO_ENABLED ?= 0 BENCH ?= . BENCH_COUNT ?= 10 @@ -33,6 +34,37 @@ METRICS_ENFORCE ?= 0 BENCHMARK_REPORT ?= BENCHMARK_BASELINE ?= BENCHMARK_REGRESSION ?= 0.20 +PERF_BASELINE ?= +PERF_CANDIDATE ?= +PERF_GATE_OUTPUT ?= $(METRICS_DIR)/perf-gate.json +PERF_GATE_SEED ?= 1 +PERF_CONFIDENCE ?= 0.975 +PERF_REGRESSION ?= 0.05 +# Promotion-grade gates must override this with one or more workload names +# whose improvement is required to clear the host A/A-aware materiality floor. +PERF_TARGETS ?= +PERF_MATERIALITY_RATIO ?= 0.95 +PERF_MATERIALITY_ABSOLUTE ?= 100us +PERF_AA_ARTIFACT ?= +PERF_AA_OUTPUT ?= $(METRICS_DIR)/perf-aa-resolution.json +PERF_GATE_AA ?= $(PERF_AA_OUTPUT) +PERF_LEFT ?= +PERF_RIGHT ?= +PERF_CONFIRM_AA ?= +PERF_CONFIRM_OUTPUT ?= $(METRICS_DIR)/perf-confirmation.json +PERF_CASES ?= +PERF_FILTER_CASES ?= +PERF_DIAGNOSTIC_GATE ?= 0 +PERF_BUNDLE_VERIFY_DIR ?= +PERF_BUNDLE_VERIFY_OUTPUT ?= $(METRICS_DIR)/capture-bundle-verification.json +PERF_BUNDLE_REQUIRE_CLEAN ?= 0 +PERF_EXPAND_INTO_ARTIFACT ?= +PERF_EXPAND_INTO_OUTPUT ?= $(METRICS_DIR)/expand-into-study.json +PERF_EXPAND_INTO_PROTOCOL ?= discovery +PERF_TOURNAMENT_ARTIFACT ?= +PERF_TOURNAMENT_OUTPUT ?= $(METRICS_DIR)/reference-tournament.json +PERF_TOURNAMENT_ARMS ?= +PERF_TOURNAMENT_PROTOCOL ?= confirmation FUZZ_REPORT ?= MUTATION_REPORT ?= BACKEND_RESULT_ARGS ?= @@ -56,7 +88,7 @@ QUALITY_INPUTS += -mutation-report $(MUTATION_REPORT) endif QUALITY_INPUTS += -benchmark-regression $(BENCHMARK_REGRESSION) -.PHONY: default all build deps tidy lint format test test_all test_integration test_bdd_integration test_neo4j test_pg test_update plan_corpus complexity complexity_check crap crap_check quality quality_check quality_backend quality_bench metrics metrics_check generate clean help +.PHONY: default all build deps tidy lint format test test_all test_integration test_bdd_integration test_neo4j test_pg test_update plan_corpus perf_gate perf_aa perf_confirm perf_bundle_verify perf_expand_into perf_tournament complexity complexity_check crap crap_check quality quality_check quality_backend quality_bench metrics metrics_check generate clean help # Default target default: help @@ -79,11 +111,12 @@ tidy: # Code quality lint: @echo "Running linter..." - @$(GO_CMD) vet ./... + @$(GO_CMD) vet -unreachable=false ./... + @$(GO_CMD) list ./... | grep -v '/cypher/parser$$' | xargs $(GO_CMD) vet -unreachable format: @echo "Formatting code..." - @find ./ -name '*.go' -print0 | xargs -P 12 -0 -I '{}' goimports -w '{}' + @find ./ \( -path './.git' -o -path './.coverage' \) -prune -o -name '*.go' -print0 | xargs -P 12 -0 -I '{}' $(GOIMPORTS_CMD) -w '{}' # Test targets test: $(METRICS_DIR) @@ -129,6 +162,93 @@ plan_corpus: $(METRICS_DIR) @echo "Capturing Cypher plan corpus..." @$(GO_CMD) run ./cmd/plancorpus +perf_gate: $(METRICS_DIR) + @if [ -z "$(PERF_BASELINE)" ] || [ -z "$(PERF_CANDIDATE)" ]; then \ + echo "PERF_BASELINE and PERF_CANDIDATE are required."; \ + exit 1; \ + fi + @if [ "$(PERF_DIAGNOSTIC_GATE)" != "1" ] && [ -z "$(strip $(PERF_TARGETS))" ]; then \ + echo "PERF_TARGETS is required for a promotion-grade performance gate."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -gate-baseline "$(PERF_BASELINE)" \ + -gate-candidate "$(PERF_CANDIDATE)" \ + -gate-output "$(PERF_GATE_OUTPUT)" \ + -gate-aa "$(PERF_GATE_AA)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" \ + -regression-threshold "$(PERF_REGRESSION)" \ + -gate-targets "$(PERF_TARGETS)" \ + -materiality-ratio "$(PERF_MATERIALITY_RATIO)" \ + -materiality-absolute "$(PERF_MATERIALITY_ABSOLUTE)" \ + -cases "$(PERF_FILTER_CASES)" \ + -diagnostic-gate="$(PERF_DIAGNOSTIC_GATE)" + +perf_aa: $(METRICS_DIR) + @if [ -z "$(PERF_AA_ARTIFACT)" ]; then \ + echo "PERF_AA_ARTIFACT is required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -aa-artifact "$(PERF_AA_ARTIFACT)" \ + -aa-output "$(PERF_AA_OUTPUT)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" + +perf_confirm: $(METRICS_DIR) + @if [ -z "$(PERF_LEFT)" ] || [ -z "$(PERF_RIGHT)" ]; then \ + echo "PERF_LEFT and PERF_RIGHT are required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -confirm-left "$(PERF_LEFT)" \ + -confirm-right "$(PERF_RIGHT)" \ + -confirm-aa "$(PERF_CONFIRM_AA)" \ + -confirm-output "$(PERF_CONFIRM_OUTPUT)" \ + -confirm-cases "$(PERF_CASES)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" + +perf_bundle_verify: $(METRICS_DIR) + @if [ -z "$(PERF_BUNDLE_VERIFY_DIR)" ]; then \ + echo "PERF_BUNDLE_VERIFY_DIR is required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -bundle-verify "$(PERF_BUNDLE_VERIFY_DIR)" \ + -bundle-verify-output "$(PERF_BUNDLE_VERIFY_OUTPUT)" \ + -bundle-require-clean="$(PERF_BUNDLE_REQUIRE_CLEAN)" + +perf_expand_into: $(METRICS_DIR) + @if [ -z "$(PERF_EXPAND_INTO_ARTIFACT)" ]; then \ + echo "PERF_EXPAND_INTO_ARTIFACT is required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -expand-into-artifact "$(PERF_EXPAND_INTO_ARTIFACT)" \ + -expand-into-output "$(PERF_EXPAND_INTO_OUTPUT)" \ + -expand-into-protocol "$(PERF_EXPAND_INTO_PROTOCOL)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" \ + -materiality-ratio "$(PERF_MATERIALITY_RATIO)" \ + -materiality-absolute "$(PERF_MATERIALITY_ABSOLUTE)" + +perf_tournament: $(METRICS_DIR) + @if [ -z "$(PERF_TOURNAMENT_ARTIFACT)" ] || [ -z "$(PERF_TOURNAMENT_ARMS)" ]; then \ + echo "PERF_TOURNAMENT_ARTIFACT and PERF_TOURNAMENT_ARMS are required."; \ + exit 1; \ + fi + @$(GO_CMD) run ./cmd/graphbench \ + -reference-tournament-artifact "$(PERF_TOURNAMENT_ARTIFACT)" \ + -reference-tournament-output "$(PERF_TOURNAMENT_OUTPUT)" \ + -reference-tournament-arms "$(PERF_TOURNAMENT_ARMS)" \ + -reference-tournament-protocol "$(PERF_TOURNAMENT_PROTOCOL)" \ + -seed "$(PERF_GATE_SEED)" \ + -confidence-level "$(PERF_CONFIDENCE)" \ + -materiality-ratio "$(PERF_MATERIALITY_RATIO)" \ + -materiality-absolute "$(PERF_MATERIALITY_ABSOLUTE)" + # Metric targets $(METRICS_DIR): @mkdir -p $(METRICS_DIR) @@ -240,6 +360,12 @@ help: @echo " test_neo4j - Run Neo4j integration tests" @echo " test_pg - Run PostgreSQL integration tests" @echo " plan_corpus - Capture shared corpus query plans for configured backends" + @echo " perf_gate - Compare complete declared GraphBench artifacts" + @echo " perf_aa - Calculate A/A measurement resolution for GraphBench" + @echo " perf_confirm - Build a paired GraphBench confirmation report" + @echo " perf_bundle_verify - Verify a portable GraphBench capture bundle" + @echo " perf_expand_into - Build the fixed-one-hop three-arm study report" + @echo " perf_tournament - Qualify a predeclared three- or five-arm reference tournament" @echo " test_update - Update test cases" @echo " complexity - Report cyclomatic complexity" @echo " crap - Report CRAP scores from unit test coverage" diff --git a/README.md b/README.md index c774fa66..f4b098cd 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,66 @@ plugins. It exposes a backend abstraction for graph queries, with current backen The query interface is built around openCypher, including a PostgreSQL SQL translator for environments that do not support Cypher natively. +The PostgreSQL driver bounds repeated work with immutable 256-entry Cypher AST and SQL translation caches. The shared AST +cache is sharded to avoid serializing unrelated query shapes while preserving same-query AST reuse. Translation +entries are keyed by normalized query text, graph ID, a collision-safe parameter-name/type shape, and the effective +versioned traversal-policy identity; they retain SQL +and parameter-source mappings, never request values or defaults, and fail closed when a required source value is absent. +Cached query text is released by LRU eviction or driver close, and diagnostics expose aggregate counters without query +text. + +### PostgreSQL connection-local translation cache + +The PostgreSQL driver uses connection-local SQL translation caches. Pool construction owns the required connection +lifecycle hooks; callers continue to hand the resulting `*pgxpool.Pool` to `dawgs.Open`: + +```go +poolConfig, err := pgxpool.ParseConfig(connectionString) +if err != nil { + return err +} + +pool, err := pg.NewPool(poolConfig) +if err != nil { + return err +} +database, err := dawgs.Open(ctx, pg.DriverName, dawgs.Config{Pool: pool}) +defer database.Close(ctx) +``` + +`pg.DefaultRuntimeConfig()` uses 64 retained translation entries per live physical PostgreSQL connection and a 5-50 +connection pool. Pass `pg.RuntimeConfig{TranslationCacheEntries: 0}` to `pg.NewPoolWithRuntimeConfig` to disable +connection-local retention, or configure exact bounds with `pg.RuntimeConfig{TranslationCacheEntries: 64, +SharedShortestPathTemplateEntries: 128, Pool: &pg.PoolConfig{MinConnections: 0, MaxConnections: 4}}`. The shared +shortest-path tier retains only immutable SQL templates and fresh bindings are always negotiated per execution; set its +capacity to zero to disable it. Negative cache capacity, negative minimums, zero maximums, and inverted limits are +rejected. The theoretical aggregate entry bound is live physical connections multiplied by this capacity, not a global +bound. + +A cache remains with its physical `*pgx.Conn` across pool lease release and reacquisition, and is removed when that +connection closes or the driver closes. `TranslationCacheStats` reports opaque diagnostic connection IDs and aggregate, +query-text-free counters only. Cached entries retain immutable SQL and parameter-source metadata; every hit binds the +current caller values. The driver keeps the parse cache driver-wide, caches neither results nor routing decisions, and advances +its generation after successful schema assertion and kind refresh. Stable-snapshot traversal workspaces are marked ready +only for the current physical connection and schema generation, then reinitialized after reset, closure, or generation +change. For out-of-band schema changes that affect types or generated SQL, reset or recreate the pool before continuing. + +After schema assertion, applications may opt in to pre-prepare selected hot PostgreSQL statements without executing them: + +```go +if err := database.WarmStatements(ctx, "select 1"); err != nil { + return err +} +``` + +Warm-up touches each currently idle physical connection and uses pgx's normal `CacheStatement` identity, so the first +regular execution adopts the prepared server statement rather than creating a second one. The driver records only SHA-256 +statement identities in its lifecycle state; `TranslationCacheStats` includes aggregate workspace and warm-up counters. +For a persistent, explicitly selected warm set that also applies to new physical connections, use +`database.SetStatementWarmupPolicy(ctx, statements...)`; passing no statements clears the future warm set. Pools supplied +to `pg.NewDriver` must have been created by `pg.NewPool` or `pg.NewPoolWithRuntimeConfig` so those lifecycle hooks are in +place. + ## Quick Start Build the repository: @@ -37,6 +97,12 @@ export CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:654 make test_bdd_integration ``` +Integration suites and fixture-loading GraphBench runs delete graph data. +Existing-graph GraphBench runs reject mutating cases and do not require +fixture loading. +acknowledgement. PostgreSQL sessions remain read-write so temporary traversal +workspaces use the same reset strategy as production. + Use this module from another Go project: ```bash @@ -60,6 +126,16 @@ queries. Its default capacity is 256 entries; callers that need a different capa newly started compilations across every PostgreSQL driver in the process. See [PostgreSQL translation](docs/postgresql_translation.md#translation-cache) for the cache contract and rollback controls. +The direct-write regression benchmark is integration-scoped because it +measures real driver batch APIs. It reloads or clears its fixture outside the +timed region and validates post-state after every iteration: + +```bash +CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs" \ + go test -tags manual_integration ./integration -run '^$' \ + -bench BenchmarkMutationSafeDirectWrites -benchtime=1x +``` + Use `cmd/benchdiff` to compare benchmarks between two committed refs without changing the active worktree: ```bash @@ -77,18 +153,237 @@ The harness writes raw outputs and a Markdown report under `.bench/runs/` by def findings, includes the raw `benchstat` output for each benchmark suite, and ends with a table of all captured benchmark numbers. -The integration benchmark runner includes committed `base`, `adcs_fanout`, and `traversal_shapes` datasets by default. +The integration benchmark runner includes committed `base`, +`fixed_suffix_expansion_fanout`, and `traversal_shapes` datasets by default. The traversal shape suite checks expected result counts for chain, fanout, bounded cycle, disconnected, edge-kind-selective, and multi-path shortest-path scenarios before recording timings. `make plan_corpus` captures plan diagnostics for the shared Cypher integration corpus. It accepts either `CONNECTION_STRING` for one backend or `PG_CONNECTION_STRING` and `NEO4J_CONNECTION_STRING` for both backends, then -writes JSONL captures and markdown/JSON summaries under `.coverage/`. +writes JSONL captures and markdown/JSON summaries under `.coverage/`. Captures record the DAWGS source version, which +can be overridden with a command flag when needed. It reloads fixtures for each selected backend. `go run ./cmd/graphbench` captures runtime diagnostics for the scale corpus under `benchmark/testdata/scale`. The -current modes are `postgres_sql`, `local_traversal`, and `neo4j`; AGE is reference-design input only and is not a direct -comparison mode yet. The command can emit JSONL records plus Markdown and JSON summaries, and can compare current timings -against a previous JSONL baseline. +implemented execution modes are `postgres_sql` and `neo4j`; `local_traversal` is an explicit, non-gating +`not_implemented` diagnostic placeholder, and AGE is reference-design input only. The command can emit JSONL records +plus Markdown and JSON summaries, and can compare current timings against a previous JSONL baseline. Mutating scale +cases must declare a `write_scenario`; each warm-up and timed iteration runs in a rollback transaction and verifies +matched, affected, and post-state cardinality. +Read timings retain every raw warm sample and are bracketed by untimed exact-row +multiset checks. PostgreSQL datasets are vacuumed and analyzed after loading and +before measured reads. Fixture reloads truncate the active relationship and node +partitions together, and +PostgreSQL captures fail before timing unless the active partitions' physical +node and edge counts exactly match the declared fixture; active child-partition +sizes are retained with each fixture. Node-ID expectations and recorded paths use stable +fixture identities rather than backend-assigned IDs, while preserving duplicate +rows and path order. +For sanitized production-like data, `-existing-graph` uses a versioned +logical-key anchor manifest and bypasses every schema/load/clear/vacuum path. +It rejects mutation cases, verifies before/after cardinalities, redacts anchor +values, and supports atomic checkpoints, resume, progress JSONL, and explicitly +labeled adaptive discovery. See `cmd/graphbench/README.md` for the fixed +confirmation and timeout-class workflows. +The executable gate uses the complete corpus/backend declaration instead of the +intersection of successful records, treats Neo4j only as an exact-result and +informational latency oracle, and supports predeclared materiality thresholds. +`make perf_aa` derives host-fingerprinted p50/p95 measurement resolution from +order-balanced repeated A/A captures. Complete normal/envelope performance +gates require that checksummed per-case evidence and use minimum 5%/100us +floors; stress timing remains diagnostic. Exact case/dataset/category/tag selectors create diagnostic-only +artifacts that the complete gate refuses; configured warmups and matched +arm/block/run metadata support isolated confirmation. The GraphBench CLI accepts +repeated `-aa-artifact` inputs so two immutable append-series arms can be +validated without an external merge. `make perf_confirm` +reports paired absolute and relative p50/p95 changes with optional block/reload +A/A floors. Capture bundles can retain the source patch, untracked sources, +module state, binary, manifest, raw records, and checksums. Opt-in pool +concurrency blocks and PostgreSQL component/full-query +references are documented in `cmd/graphbench/README.md`. Path-observed +singleton captures include exact benchmark-only M0/M1 materializer arms with a +shared search boundary; they do not enable an experimental production executor. +Generated fixed-suffix expansion captures provide selectable exact root-reuse, +late-hydration, factored-suffix forward, suffix-seeded reverse, and +backward-viability forward arms plus versioned fixtures with independent +suffix-density and reverse-fan-in controls. V3 fixtures additionally encode +matching-root multiplicity and independent relationship-distinct cycle and +self-loop controls at the productive boundary. The optimizer reports a typed +expansion-search decision. Repository-native +`EXPANSION-SUFFIX-SEEDED-REVERSE` is an exact qualification-only implementation. +When a complete path is observed, that arm carries ordered node IDs alongside +ordered edge IDs and hydrates both arrays directly in the translated statement; +endpoint-only forms keep the smaller edge-only reverse state. The guarded +candidate uses the same materializer while its exact forward fallback retains +the generic path helper. +Production selection remains on the stepwise incumbent because query shape and +available metadata do not provide hard suffix-density or reverse-state bounds. +The now-terminal, tool-only `orientation-probe-v2` experiment uses +`F2 = root_rows + maximum_depth * forward_degree_rows` and +`R2 = suffix_rows + boundary_rows + reverse_degree_rows`, selecting reverse +only when every bounded probe is complete and `4 * R2 < 3 * F2`. Its frozen v3 +corpus contains eight selector-training cases and four evaluation holdouts whose +timings remain unopened. Its frozen historical protocol required matched +`shadow`, `incumbent`, `reverse`, and `guarded` artifacts captured under +Repeatable Read with traversal telemetry. Degree evidence is represented as a +scalar count over the same cap+1-limited adjacency stream, avoiding tuple +materialization without changing the immutable score or fail-closed overflow +rule. A promotable discovery would have needed a clean-tree report and freeze +from the exact eight training cases before confirmation could open the complete +eight-training/four-holdout cohort. That path is now archival only: v2 must not +be recaptured or advanced, and its holdout timings remain unopened. Per-case A/A +evidence also binds the PostgreSQL timing environment, including transaction +isolation, and the exact validated fixture. See +[GraphBench](cmd/graphbench/README.md) for the frozen protocol. +No v2 qualification benchmark has passed. The driver/runtime seam recognizes +exact-query manifests that name either the v1 or v2 selector identity so their +diagnostic and guarded statements remain reproducible. The schema-v2 final +verifier rejects legacy v1 evidence because it cannot bind the required source, +corpus, and frozen cohort; it also terminally rejects v2 because that immutable +policy generation failed its training overhead gate. A future attempt requires +a new policy generation. Neither selector is enabled, and default production +behavior remains unchanged. +The latest five-round, eight-case v2 training prequalification produced exact, +receipt-complete evidence but failed selected-arm overhead on every case: the +guarded statement remained roughly 156-396 microseconds above its selected +exact arm against the immutable 100-microsecond gate. A reduced-gating +prototype also failed a fresh matched capture and was reverted. V2 is therefore +a rejected selector generation rather than a pending clean-tree promotion; any +next attempt must use a new policy identity and qualification freeze. +The subsequent bounded experiment was the independent, tool-only +`suffix-reverse-guard-v1` policy. It enrolls only complete-path fixed-suffix +queries, performs no topology or degree probes, caps suffix payload and reverse +state at 512 rows each, and dispatches either exact suffix-seeded reverse or the +unchanged exact forward traversal in one Repeatable Read statement. Its +six-round training feasibility gate is deliberately not a qualification or +production seam; endpoint-only queries, mutations, protected holdouts, and the +zero-value production translator remain outside the experiment. The gate also +requires process timestamps to prove the declared physical doubled-Williams +order and an artifact-bound schema-v4 A/A chronology proof. The first capture +predates that enforcement and is invalid. A chronology-compliant recapture then +failed the immutable guard-overhead bound on both training cases, so this +generation is terminally stopped before holdout, manifest, driver-policy, or +automatic-selector work. +For the distinct one-fixed-prefix plus selective-terminal-expansion shape, +production uses guarded `EXPANSION-ENDPOINT-SEEDED-REVERSE`: 32 endpoint and +4096 reverse-state caps select either the reverse candidate or an exact +same-statement forward fallback without exposing partial candidate rows. + +The fixed-suffix development generation remains available only to GraphBench +through `-postgres-expansion-suffix-reverse-retry`. It executes a reverse-only +bounded statement, buffers all candidate rows, and retries the exact forward +incumbent after a savepoint rollback in the same Repeatable Read transaction. +Its statement contains no topology probes or inactive forward body. The frozen +development contract and stop gate are documented in +[Suffix reverse transaction retry v1](docs/experiments/suffix_reverse_retry_v1.md). + +Separately, the PostgreSQL V2 driver implements the default-off manifest-v4 +topology route boundary. An installed v4 policy requires a compatible, +current topology synopsis and a read-only Repeatable Read or Serializable +transaction. The first matching query in that transaction records an +incumbent-only decision; only a later matching query with the same snapshot, +parameters, policy identity, and synopsis generation may execute the bounded +reverse candidate with exact forward retry. No v4 promotion manifest ships +with Dawgs, so this implementation does not change default production SQL. + +PostgreSQL recursive shortest-path execution includes contained S3/S4 +singleton selection, a guarded canonical inline witness canary, and an +all-shortest predecessor-DAG executor, with exact same-statement fallback, +reusable session-local workspace-v2 state, late hydration, event-chain runtime +receipts, and +a parameter-shape-aware translation cache. The implementation and its +qualification boundaries are documented in +[Recursive-descent cost controls](docs/recursive_descent_cost_controls.md). + +New inline SP and ordinary-orientation lowerings remain default-off. Canonical +SP-I1 authorization now uses selector `sp-static-v6` and accepts only the +qualified inbound, typed, single-kind, one-path `min=1`/`max=64` bucket; the +automatic `sp-static-v5-contained` S3/S4 choices are unchanged. The +guarded distance identity `SP-I2-C-D` uses selector `sp-static-v8-hidden-fanin` +for exact inbound typed single-kind distance buckets, reverse-physical ID-only +search, the preregistered production-form `state_limit=100000` and +`frontier_limit=100000` guards, and exact S4 fallback. Those caps are immutable +protocol inputs, not qualified values: the dirty-tree rehearsal stopped before +creating a discovery report or freeze, its cycle-control point estimates missed +the frozen bounds, and no protected holdout was opened. Syntax-open +singleton shortest paths now use the documented effective maximum depth 15 and +report `policy_default` depth provenance under `sp-static-v7-contained`. +The PostgreSQL driver's `SetTraversalPolicy` API can expose one eligible +candidate to an explicit normalized-query SHA-256 allowlist under a nonzero +generation. +The benchmark command can exercise that real V2 policy path only with a +GraphBench-verified manifest; its usage and guardrails are documented in +[`cmd/benchmark/README.md`](cmd/benchmark/README.md). +Activation requires the exact promotion manifest, including its measured +execution boundary, independently frozen operational candidate SQL SHA-256, +and evidence digests. The SQL anchor is derived in a non-promotional preflight, +then added to the provisional manifest before formal evidence is recaptured; +the benchmark preflight renders but does not execute a candidate or authorize +it for production policy selection. +Manifest schema v2 also requires every evidence report to repeat the exact +candidate, selector, source, binary, +corpus, cap, bucket, and query-cohort identity; a digest-shaped string alone is +not authorization. The verifier strictly decodes candidate-specific evidence, +recomputes the bound native A/A digest and performance decisions, requires the +reference workload digest to match the exact PostgreSQL A/A workload per cohort +case, and closes every performance receipt against the complete set of resource +case-round receipts. Confirmation and performance expose typed evidence rather +than raw benchmark samples, so final verification can recompute their declared +decisions but cannot independently replay every bootstrap draw; closing that +reproducibility gap requires a producer-schema revision. The operational gate +likewise validates an assembled 32-record native input but does not yet provide +a standalone capture producer. Evidence roles, +bucket names, query identities, and the canonical training/holdout split are +closed sets; duplicate JSON keys, duplicate allowlist entries, extra roles, and +filesystem-symlink escapes fail closed. The legacy +`orientation-probe-v1` report lacks the source/corpus/cohort identity needed by +this closure and therefore cannot authorize promotion. The structurally richer +v2 report remains readable, but final authorization rejects that terminal +generation because its immutable training overhead gate failed. B1/B2 and legacy +unguarded `SP-I1-C-D` remain tooling-only. Endpoint-seeded reverse, +inline ASP, inline canonical witness, and guarded inline distance each have an +evidence-free emergency disable switch. If a manifest-backed candidate carries +a rollback switch, it may carry exactly one and it must match: orientation with +`DisableExpansionOrientation`, ASP-I1 with `DisableInlineASPDAG`, SP-I1 witness +with `DisableInlineSPWitness`, or SP-I2 with `DisableInlineSPDistance`. An +unrelated or second switch is rejected. `DisableEndpointSeededReverse` is +standalone-only; every standalone rollback must omit a candidate and leave +`promotion_manifest_sha256`, `promotion_manifest_json`, and +`query_sha256_allowlist` empty. A matching rollback preserves the installed +manifest and candidate anchor but derives an incumbent-only effective policy +under a new cache generation. Resetting the policy to its zero value immediately +returns all queries to incumbent cache identities. This is a reversible canary +seam, not evidence that a candidate is qualified for broad production use. + +The PostgreSQL scale-plan gate runs as part of `make test_all` when +`CONNECTION_STRING` selects PostgreSQL. It executes every required Cypher scale +representative with `EXPLAIN ANALYZE`, enforces declared result or mutation +cardinality, and checks stable mutation-target and anchored edge-index +invariants. Run it directly with: + +```bash +CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs" \ + go test -tags manual_integration ./cmd/graphbench \ + -run 'Test(PostgreSQLScalePlanInvariants|ScaleCorpusRequiredRepresentativesDeclareCardinality)' \ + -count=1 +``` + +Runtime and plan captures are intentionally generated under the ignored +`.coverage/` directory. Keep them as reviewed environment-specific artifacts; +use the stable `GFSE-*`, `REC-*`, `TRUST-*`, `PRUNE-*`, `HOP-*`, `SCAN-*`, and +`LOOKUP-*` IDs to compare captures with their semantic fixtures and manifest +entries. + +The SP-I2 distance V2 study is terminally stopped before formal timing. Its +frozen 20,000-run prospective calibration, reconstructed from the clean V1 +trace, could not support the fixed A/A and qualification design at the required +power. V2 therefore remains production-off and must not proceed to A/A, +holdout, confirmation, or promotion; see +[`cmd/graphbench/README.md`](cmd/graphbench/README.md) for the reproducible +verification command. In plain terms, comparing the same implementation with +itself still produced uncertainty of roughly `-5.4%` to `+6.3%` and +`-116us` to `+133us`, wider than the allowed plus-or-minus 5% and 100us. The +fixed study recognized the target and control outcomes only about 48% and 51% +of the time, respectively, instead of the required 90% reliability. `go run ./cmd/retriever` dumps and loads live Dawgs graph databases as manifest-based collections of compressed JSONL fragments. It supports @@ -128,8 +423,21 @@ replace github.com/specterops/dawgs => /path/to/dawgs - [Development workflow](docs/development.md): build, test, integration, metrics, quality, and corpus-capture commands. - [Cypher library](cypher/README.md): parser generation and Cypher package overview. - [PostgreSQL translation](docs/postgresql_translation.md): PostgreSQL translator behavior, optimizer lowerings, indexing notes, and validation expectations. +- [CySQL traversal performance priorities](docs/cysql_traversal_priorities.md): source-grounded roadmap for orientation, SP/ASP, probes, statistics, telemetry, and qualification. +- [Traversal priority implementation status](docs/experiments/traversal_priority_implementation_status_v1.md): implemented candidate identities, fail-closed gates, and current no-promotion disposition. +- [Remaining traversal outlier delivery](docs/experiments/remaining_outlier_delivery_v1.md): SP-I2, fixed-suffix, qualification, promotion-closure, and rollback handoff. +- [Production-wide SQL selection](docs/experiments/production_wide_sql_selection_v1.md): staged structural and topology-aware PostgreSQL routing contract. +- [Topology fixed-suffix v4 status](docs/experiments/topology_fixed_suffix_v4_status_v1.md): implemented default-off routing boundary and promotion non-activation record. +- [Topology fixed-suffix v4 capture procedure](docs/experiments/topology_fixed_suffix_v4_capture_v1.md): clean-source evidence sequence required before a v4 policy may be installed. +- [Topology fixed-suffix first-use routing](docs/experiments/topology_fixed_suffix_first_use_v1.md): separately versioned v5 selector and its promotion contract. +- [Composite fixed-suffix lowering](docs/experiments/fixed_suffix_composite_lowering_v1.md): query-atomic multi-region retry substrate. +- [ASP A1 inline hydration disposition](docs/experiments/asp_a1_inline_hydration_v1.md): parameterized attribution, A1 hydration result, and B2 negative disposition. +- [ASP N1 negative-exhaustion disposition](docs/experiments/asp_n1_negative_exhaustion_v1.md): bounded no-path proof with exact A1 fallback and its production-selection boundary. - [Plan corpus capture](cmd/plancorpus/README.md): shared integration corpus plan diagnostics. - [Graph benchmark capture](cmd/graphbench/README.md): runtime diagnostics for scale scenarios. +- [Integration corpus](integration/testdata/README.md): fixture, mutation post-state, and typed-parameter schema. +- [BloodHound regression coverage manifest](regression_coverage_manifest.md): per-query-form layer status and existing primitive links. +- [BloodHound source-parity workflow](docs/regression_source_parity.md): dormant-form activation rules and repeatable BHE/BHCE source audits. - [Cypher syntax support](cypher/Cypher%20Syntax%20Support.md): supported Cypher behavior and semantic notes. ## Repository Map diff --git a/benchmark/testdata/scale/README.md b/benchmark/testdata/scale/README.md index 85c2f788..4971cd49 100644 --- a/benchmark/testdata/scale/README.md +++ b/benchmark/testdata/scale/README.md @@ -12,17 +12,176 @@ Apache AGE is intentionally not a benchmark mode here; it may appear only in Each JSON file contains a list of scale cases with: -- `source`: the source corpus or workload family. - `dataset`: the fixture dataset to load from `integration/testdata`. - `name` and `category`: stable identifiers used in reports. - `cypher`: the Cypher query under test. -- `parameters`: named parameter values. -- `expected_rows`: the expected result cardinality. +- `params`: named parameter values. A typed temporal parameter uses + `{"$type":"datetime","value":"2026-01-02T03:04:05Z"}`. A deterministic + large string list uses + `{"$type":"string_list","prefix":"missing","count":1000,"include":["target"]}`. +- `node_params`: scalar parameters resolved from fixture node names. +- `node_list_params`: list parameters resolved from fixture node names. +- `generated_node_list_params`: high-cardinality fixture-ID lists made from + optional included names plus a prefix/count sequence, for example + `{"ids":{"prefix":"target","count":2000,"include":["matched-target"]}}`. +- `expected.row_count`: the expected result cardinality for a read case. - `observes`: whether the query observes paths, nodes, relationships, properties, or only IDs internally. - `candidate_modes`: the execution modes that should attempt the case. +- `unsupported_modes`: explicit backend-to-reason declarations for matrix + points retained as correctness oracles but not supported by that backend. - `reference_design`: optional design notes, including AGE observations when useful. +Mutations are rejected as ordinary read cases. A mutation must add a +`write_scenario` with: + +- a selection query and `expected_matched` count; +- an `affected_entity` (`node` or `relationship`) and `expected_affected` + count; +- one or more `post_state` queries with expected row counts or integer scalar + values. + +The runner drains the mutation result and validates those expectations inside +one rollback transaction. Warm-up, every timed iteration, and PostgreSQL +`EXPLAIN ANALYZE` therefore start from the same committed fixture state. + +The `generated_reconciliation`, `generated_trust_pruning`, `generated_hops`, +and `generated_scan_lookups` datasets are constructed by +`testutil.NewReconciliationScaleFixture`, +`testutil.NewTrustPruningScaleFixture`, `testutil.NewHopScaleFixture`, and +`testutil.NewScanLookupScaleFixture`; they are intentionally not large +handwritten OpenGraph JSON files. + +The corpus also executes parameterized `generated_shortest_paths_d*_f*` and +`generated_fixed_suffix_expansion_d*_f*_v*_p*` variants. Cases in +`cases/generated_fixed_suffix_expansion.json` use stable `GFSE-*` identifiers. +The normal pairwise subset covers shortest depth 1/2/4/8/16/32/64, fanout +1/16/128/512/1000, +outbound/inbound/directionless, distance/path/all-shortest output, and +disconnected, diamond, cycle, parallel-edge, and self-loop shapes. The +fixed-suffix expansion subset covers depth 0/1/2/4/8/16, fanout +1/10/100/1000, none/sparse/half/all valid branch suffix density, endpoint/path +output, decoys, and a 4 KiB payload. +Each result records the exact configuration name, deterministic graph checksum, +and node/edge cardinality. + +Version-two shortest fixtures use +`generated_shortest_paths_v2_d_o_r_fo_fi_l_k_t_w_x_p_c_s`. +Names are strict and round-trippable: negative values, partial scans, unknown +suffixes, non-canonical numbers, impossible intermediate levels, and partial +parallel configurations are rejected. The fixture has independent outbound +and physical-inbound paths, so hidden downstream fan-in and its mirrored +fan-out control coexist without changing legacy fixture identities. Every edge +has a stable `logical_key`. Metadata records root and per-level degrees, +physical edges by kind, distinct reachable nodes by level, minimum distance, +path cardinalities, predecessor edges, disconnected state, parallel physical +edges and distinct targets, checksum, and loaded physical cardinality. +The ASP qualification subset includes separate training and frozen holdout +cases for outbound and inbound searches, early and maximum-depth targets, +disconnected pairs, parallel relationship kinds, diamond multiplicity, and +stress enumeration. These shapes distinguish stored-helper `ASP-A1-DAG` from +inline `ASP-I1-U-DAG+MAT-M0` at the same full path-multiset boundary. + +`cases/generated_sp_i1_inbound_v1.json` is a separate canonical-witness cohort +for comparing exact S4 with guarded `SP-I1-C-WE+MAT-M0`. Its four training +cases use generated depths 4 and 16 and cover full-depth, early-target, and +disconnected inbound searches. Its three blind holdouts use fresh depths 8 and +32 and cover full-depth and disconnected searches. Every case uses the same +typed one-kind `shortestPath` query with maximum depth 64 and an exact path-set +observation. The disjoint `sp-i1-inbound-v1-training` and +`sp-i1-inbound-v1-holdout` tags are protocol identities; holdout execution is +authorized only after GraphBench validates the training freeze. Ordinary +default, category, dataset, and generic-tag selection omit these protected +holdouts; only the exact holdout protocol tag or an exact holdout case name +enters the frozen authorization path. Partial selections remain forbidden: an +authorized confirmation executes the exact four-training/three-holdout cohort +on PostgreSQL. Neo4j remains in the declaration for cross-backend semantic +coverage, not as a holdout timing arm in this study. + +`cases/generated_sp_i2_distance_v2.json` is the fresh formal scalar-distance +cohort for SP-I2 V2. Its eight training cases freeze three adverse controls +(direct acyclic, direct cyclic, and post-target-cycle shapes) and five efficacy +targets spanning early, full-depth, mixed-fan-in, high-fan-in, and disconnected +execution. Its six protected holdouts use previously unused case and fixture +identities and freeze two adverse controls plus four efficacy targets. Each +declaration carries an immutable `qualification_role`; report code may not +infer that role from timings. Ordinary selectors omit the union of V1 and V2 +protected cases. Exact V1/V2 protocol selectors cannot be mixed, and V2 +holdout resolution is rejected before database setup without V2 authorization. +The protocol declaration binds the training, holdout, and full corpus, +declaration, and resolved-selection digests. + +This V2 cohort is retired and remains checked in only for audit and semantic +regression coverage. The frozen prospective study found that ordinary timing +variation was wider than its A/A limits and that target/control decisions were +reliable only about half the time instead of the required 90%. No formal A/A +or candidate timing began, and no holdout was opened. GraphBench rejects the +formal V2 executor and any explicit formal-cohort execution before database +setup. A future attempt must use a newly named successor protocol rather than +changing V2's sample count or limits. + +`shape.fixture_tier` is one of `normal`, `envelope`, or `stress`. +`shape.qualification_split` is independently one of `training`, `holdout`, or +`diagnostic`; selector thresholds may use training records but must be frozen +before holdout records are opened. Direction, +relationship-kind count, expected state class, and result-cardinality class +are stored alongside it. Stress cases remain exact diagnostics and are not +silently promoted to release p95 evidence. + +Version-two fixed-suffix expansion fixtures use +`generated_fixed_suffix_expansion_v2_d_f_r_x_i_m_z_p`. +Unlike the legacy modulus form, every integer is exact: `r0` represents zero +reachable branch suffixes, `x` varies false boundaries independently, `i` +controls reverse fan-in, `m` controls physical suffix multiplicity, and `z` is +either zero or one. Fixture records include declared root rows, forward +expansion states, suffix rows/boundaries, expected reverse states, output +trails, physical cardinality, and checksum. Semantic relationships carry +deterministic `logical_key` properties so relationship-distinct paths can be +compared across backends whose physical IDs differ. + +Version-three fixed-suffix fixtures extend that exact grammar as +`generated_fixed_suffix_expansion_v3_d_f_r_x_i_m_q_z_c_s_p`. +`q` independently controls how many distinct `ExpansionRoot` nodes match the +root predicate; only the primary root owns the declared fanout and suffixes. +`c1` adds two distinctly keyed `Expand` relationships from the deterministic +productive boundary through a dedicated node and back, while `s1` adds one +distinctly keyed `Expand` self-loop at that boundary. The primary root is the +productive boundary when only `z1` supplies a reachable suffix; otherwise the +first reachable branch boundary is used. Both controls require a productive +boundary, may be enabled independently, and preserve Cypher's +relationship-distinct trail semantics. V3 names reject implicit populations, +invalid booleans, unproductive fan-in/topology controls, and noncanonical +numbers. Metadata derives exact forward/reverse relationship-distinct states +and complete output trails from the generated graph. +The orientation-v2 declaration freezes eight training cases spanning every +encoded dimension and four holdouts at previously unused depths 7, 11, 13, +and 15. `orientation-v2-training` and `orientation-v2-holdout` are disjoint +cohort tags; the legacy v2 declarations retain their original v1 evidence +splits. + +`cases/fixed_suffix_expansion_limits.json` is an optimization-neutral cardinality +holdout suite. It covers 511, 512, 513, and 600 physical suffix rows, productive +endpoint and full-path observations, and exactly 512 physical rows with two +suffix paths per boundary to prove bag multiplicity. These `GFSE-BOUNDARY-*` +cases are not owned by any one optimization design; archived experiment reports +retain their historical case names. +The file-backed `fixed_suffix_expansion_adversarial` fixture adds 17 distinct +root lanes converging on one boundary, a reusable-node cycle, two physical suffix +paths, and noncanonical logical IDs. Its 68-row endpoint bag proves +relationship-trail rejection and multiplicity independently of the generated +limit fixtures. + +The file-backed `expand_into` fixture and `cases/expand_into.json` form the +fixed-one-hop, bound-pair plan study. They cover typed, wildcard, and multi-kind +matches; cross-kind relationship multiplicity; duplicate and missing outer +pairs; self-loops; and both asymmetric degree orientations. The +`source_lower_degree` and `target_lower_degree` cases deliberately reverse which +endpoint has the cheaper typed adjacency so the pair join, lower-degree scan, +and statement-local pair-cache references are compared at the same complete +relationship observation boundary. + Use `cmd/graphbench` to run this corpus and produce JSONL, Markdown, and JSON -summaries. +summaries. Exact case/dataset/category/tag selectors are intended for targeted +diagnosis and mark their outputs diagnostic-only; they never replace a complete +corpus capture. diff --git a/benchmark/testdata/scale/cases/expand_into.json b/benchmark/testdata/scale/cases/expand_into.json new file mode 100644 index 00000000..b24de17a --- /dev/null +++ b/benchmark/testdata/scale/cases/expand_into.json @@ -0,0 +1,147 @@ +{ + "cases": [ + { + "name": "EXPAND-INTO-01-typed-singleton-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "typed", "singleton-pair"] + }, + { + "name": "EXPAND-INTO-02-wildcard-cross-kind-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "direction": "outbound", "relationship_kind_count": 0, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "wildcard", "cross-kind"] + }, + { + "name": "EXPAND-INTO-03-multi-kind-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "multi-kind", "cross-kind"] + }, + { + "name": "EXPAND-INTO-04-duplicate-pair-multiplicity", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-source", "pair-missing", "pair-source"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 4}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "pair-cache", "duplicate-outer-rows", "hit-miss", "holdout"] + }, + { + "name": "EXPAND-INTO-05-self-loop", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-loop"]}, + "node_params": {"end_id": "pair-loop"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "self-loop", "holdout"] + }, + { + "name": "EXPAND-INTO-06-missing-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r]->(e) RETURN r", + "node_list_params": {"start_ids": ["pair-missing"]}, + "node_params": {"end_id": "pair-target"}, + "expected": {"row_count": 0}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "relationship_kind_count": 0, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "missing-pair", "holdout"] + }, + { + "name": "EXPAND-INTO-07-source-lower-degree", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["low-source"]}, + "node_params": {"end_id": "high-target"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false, "expected_state_class": "source_lower_degree"}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "lower-degree", "source-lower-degree", "holdout"] + }, + { + "name": "EXPAND-INTO-08-target-lower-degree", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r", + "node_list_params": {"start_ids": ["high-source"]}, + "node_params": {"end_id": "low-target"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "outbound", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false, "expected_state_class": "target_lower_degree"}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "lower-degree", "target-lower-degree", "holdout"] + }, + { + "name": "EXPAND-INTO-09-directionless-reversed-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN r", + "node_list_params": {"start_ids": ["pair-target"]}, + "node_params": {"end_id": "pair-source"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "directionless", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "directionless", "cross-kind", "holdout"] + }, + { + "name": "EXPAND-INTO-10-inbound-pair", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)<-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN r", + "node_list_params": {"start_ids": ["pair-target"]}, + "node_params": {"end_id": "pair-source"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "inbound", "edge_kinds": ["ExpandIntoKindA", "ExpandIntoKindB"], "relationship_kind_count": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "inbound", "cross-kind", "holdout"] + }, + { + "name": "EXPAND-INTO-11-directionless-self-loop", + "dataset": "expand_into", + "category": "expand_into_one_hop", + "cypher": "UNWIND $start_ids AS start_id MATCH (s), (e) WHERE id(s) = start_id AND id(e) = $end_id MATCH (s)-[r:ExpandIntoKindA]-(e) RETURN r", + "node_list_params": {"start_ids": ["pair-loop"]}, + "node_params": {"end_id": "pair-loop"}, + "expected": {"row_count": 1}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "envelope", "direction": "directionless", "edge_kinds": ["ExpandIntoKindA"], "relationship_kind_count": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["expand-into", "directionless", "self-loop", "holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/fixed_suffix_expansion_limits.json b/benchmark/testdata/scale/cases/fixed_suffix_expansion_limits.json new file mode 100644 index 00000000..c8537773 --- /dev/null +++ b/benchmark/testdata/scale/cases/fixed_suffix_expansion_limits.json @@ -0,0 +1,311 @@ +{ + "cases": [ + { + "name": "GFSE-BOUNDARY-S511-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x510_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-511", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S512-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x511_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-512", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S513-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x512_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-513", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S600-productive-endpoint", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x599_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-600", + "nonempty-remainder", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S513-productive-path", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x512_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 1, + "result_kind": "path_set" + }, + "observes": { + "paths": true, + "nodes": true, + "relationships": true, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": true + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-513", + "path", + "holdout" + ] + }, + { + "name": "GFSE-BOUNDARY-S512-physical-bag-multiplicity", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f129_r0_x255_i0_m2_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": { + "root_key": "generated-fse-root" + }, + "expected": { + "row_count": 2, + "result_kind": "id_rows", + "id_rows": [ + [ + "fse-head-root-00", + "fse-terminal" + ], + [ + "fse-head-root-01", + "fse-terminal" + ] + ] + }, + "observes": { + "paths": false, + "nodes": false, + "relationships": false, + "properties": true + }, + "shape": { + "qualification_split": "holdout", + "fixture_tier": "normal", + "root_predicate": "selective_property", + "terminal_predicate": "fixed_suffix", + "edge_kinds": [ + "Expand", + "EnterSuffix", + "ContinueSuffix", + "CompleteSuffix" + ], + "min_depth": 0, + "max_depth": 16, + "path_materialization_required": false + }, + "candidate_modes": [ + "postgres_sql", + "neo4j" + ], + "tags": [ + "generated", + "normal-tier", + "fixed-suffix-expansion-boundary", + "suffix-cardinality-512", + "physical-bag-multiplicity", + "holdout" + ] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_endpoint_seeded_expansion_v1.json b/benchmark/testdata/scale/cases/generated_endpoint_seeded_expansion_v1.json new file mode 100644 index 00000000..72f6f92e --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_endpoint_seeded_expansion_v1.json @@ -0,0 +1,37 @@ +{ + "cases": [ + { + "name": "GESE-01-guard-admitted", + "dataset": "generated_endpoint_seeded_expansion_v1_d3_e2_q1_w2_o1_x1_m1_c0_p8", + "category": "generated_endpoint_seeded_expansion", + "cypher": "MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..64]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN count(*)", + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "root_predicate": "unbound", "terminal_predicate": "selective_property", "edge_kinds": ["HasSession", "MemberOf"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["endpoint-seeded-expansion", "guard-admitted", "scalar"] + }, + { + "name": "GESE-02-endpoint-guard-fallback", + "dataset": "generated_endpoint_seeded_expansion_v1_d2_e33_q0_w33_o0_x0_m1_c0_p0", + "category": "generated_endpoint_seeded_expansion", + "cypher": "MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..64]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN count(*)", + "expected": {"row_count": 1, "scalar_int": 33, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "unbound", "terminal_predicate": "selective_property", "edge_kinds": ["HasSession", "MemberOf"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["endpoint-seeded-expansion", "endpoint-guard-overflow", "fallback", "scalar", "holdout"] + }, + { + "name": "GESE-03-state-guard-fallback", + "dataset": "generated_endpoint_seeded_expansion_v1_d1_e1_q0_w1_o0_x4097_m1_c0_p0", + "category": "generated_endpoint_seeded_expansion", + "cypher": "MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..64]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN count(*)", + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "unbound", "terminal_predicate": "selective_property", "edge_kinds": ["HasSession", "MemberOf"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["endpoint-seeded-expansion", "state-guard-overflow", "fallback", "scalar", "holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_fixed_suffix_expansion.json b/benchmark/testdata/scale/cases/generated_fixed_suffix_expansion.json new file mode 100644 index 00000000..2e895520 --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_fixed_suffix_expansion.json @@ -0,0 +1,377 @@ +{ + "cases": [ + { + "name": "GFSE-V2-D16-F1000-R1-X1-M1-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"], ["fse-head-branch-0000-depth-16-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "endpoint-ids", "depth-16", "fanout-1000", "reachable-1", "disconnected-1", "discovery"] + }, + { + "name": "GFSE-V2-D16-F1000-R1-X1-M1-sparse_path", + "dataset": "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-branch-0000-level-08","fse-branch-0000-level-09","fse-branch-0000-level-10","fse-branch-0000-level-11","fse-branch-0000-level-12","fse-branch-0000-level-13","fse-branch-0000-level-14","fse-branch-0000-level-15","fse-branch-0000-level-16","fse-head-branch-0000-depth-16-00","fse-middle-branch-0000-depth-16-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-level-08","branch-0000-level-09","branch-0000-level-10","branch-0000-level-11","branch-0000-level-12","branch-0000-level-13","branch-0000-level-14","branch-0000-level-15","branch-0000-level-16","branch-0000-depth-16:enter","branch-0000-depth-16:continue","branch-0000-depth-16:complete"]},{"nodes":["fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "path", "depth-16", "fanout-1000", "reachable-1", "disconnected-1", "discovery"] + }, + { + "name": "GFSE-V2-D08-F512-R0-X512-zero_reachable", + "dataset": "generated_fixed_suffix_expansion_v2_d8_f512_r0_x512_i0_m1_z0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 0, "result_kind": "id_rows", "id_rows": []}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "endpoint-ids", "zero-result", "reachable-0", "disconnected-512", "adversarial", "holdout"] + }, + { + "name": "GFSE-V2-D08-F016-R1-I1000-high_reverse_fanin", + "dataset": "generated_fixed_suffix_expansion_v2_d8_f16_r1_x0_i1000_m1_z0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-08-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "endpoint-ids", "reverse-fanin-1000", "adversarial", "holdout"] + }, + { + "name": "GFSE-P1-TRAIN-D09-F017-R0-X2-I1024-M1-Q1-high_reverse_fanin_path", + "dataset": "generated_fixed_suffix_expansion_v2_d9_f17_r0_x2_i1024_m1_z1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..9]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 9, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "suffix-reverse-retry-v1-training", "path", "reverse-fanin-1024", "p1-open", "training"] + }, + { + "name": "GFSE-P1-TRAIN-D09-F513-R0-X512-no_path_exhaustion", + "dataset": "generated_fixed_suffix_expansion_v2_d9_f513_r0_x512_i0_m1_z0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..9]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 0, "result_kind": "path_set", "path_rows": []}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 9, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "suffix-reverse-retry-v1-training", "path", "zero-result", "suffix-cardinality-512", "p1-open", "training"] + }, + { + "name": "GFSE-P1-TRAIN-D00-F001-R0-X0-M4-P2100000-output_byte_retry_path", + "dataset": "generated_fixed_suffix_expansion_v2_d0_f1_r0_x0_i0_m4_z1_p2100000", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..0]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 4, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-head-root-01","fse-middle-root-01","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-head-root-02","fse-middle-root-02","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-head-root-03","fse-middle-root-03","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 0, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "P1 retry-buffer output-byte control is PostgreSQL-specific; the hydrated payload does not complete within the 90-second Neo4j diagnostic deadline"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v2", "suffix-reverse-retry-v1-training", "path", "output-byte-retry", "payload-2100000", "p1-open", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q1-C0-S0-root_baseline", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q1_z1_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-1", "productive-boundary-controls-none", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C0-S0-root_multiplicity", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-4", "productive-boundary-controls-none", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C1-S0-productive_cycle", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"], ["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-4", "productive-boundary-cycle", "relationship-distinct", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C0-S1-productive_self_loop", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-root-00", "fse-terminal"], ["fse-head-root-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "root-rows-4", "productive-boundary-self-loop", "relationship-distinct", "training"] + }, + { + "name": "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 3, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-productive-boundary-cycle","fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["productive-boundary-cycle-enter","productive-boundary-cycle-return","root:enter","root:continue","root:complete"]},{"nodes":["fse-root","fse-root","fse-head-root-00","fse-middle-root-00","fse-terminal"],"relationship_kinds":["Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["productive-boundary-self-loop","root:enter","root:continue","root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "path", "root-rows-4", "productive-boundary-cycle", "productive-boundary-self-loop", "relationship-distinct", "training"] + }, + { + "name": "GFSE-V3-TRAIN-D03-F006-R1-X0-I4-M2-Q2-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d3_f6_r1_x0_i4_m2_q2_z0_c0_s0_p32", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-03-00", "fse-terminal"], ["fse-head-branch-0000-depth-03-01", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "reachable-sparse", "reverse-fanin", "suffix-multiplicity-2", "root-rows-2", "payload", "training"] + }, + { + "name": "GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path", + "dataset": "generated_fixed_suffix_expansion_v3_d5_f8_r4_x3_i0_m1_q3_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..5]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 4, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-head-branch-0000-depth-05-00","fse-middle-branch-0000-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-depth-05:enter","branch-0000-depth-05:continue","branch-0000-depth-05:complete"]},{"nodes":["fse-root","fse-branch-0001-level-01","fse-branch-0001-level-02","fse-branch-0001-level-03","fse-branch-0001-level-04","fse-branch-0001-level-05","fse-head-branch-0001-depth-05-00","fse-middle-branch-0001-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0001-level-01","branch-0001-level-02","branch-0001-level-03","branch-0001-level-04","branch-0001-level-05","branch-0001-depth-05:enter","branch-0001-depth-05:continue","branch-0001-depth-05:complete"]},{"nodes":["fse-root","fse-branch-0002-level-01","fse-branch-0002-level-02","fse-branch-0002-level-03","fse-branch-0002-level-04","fse-branch-0002-level-05","fse-head-branch-0002-depth-05-00","fse-middle-branch-0002-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0002-level-01","branch-0002-level-02","branch-0002-level-03","branch-0002-level-04","branch-0002-level-05","branch-0002-depth-05:enter","branch-0002-depth-05:continue","branch-0002-depth-05:complete"]},{"nodes":["fse-root","fse-branch-0003-level-01","fse-branch-0003-level-02","fse-branch-0003-level-03","fse-branch-0003-level-04","fse-branch-0003-level-05","fse-head-branch-0003-depth-05-00","fse-middle-branch-0003-depth-05-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0003-level-01","branch-0003-level-02","branch-0003-level-03","branch-0003-level-04","branch-0003-level-05","branch-0003-depth-05:enter","branch-0003-depth-05:continue","branch-0003-depth-05:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 5, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "path", "reachable-half", "disconnected-3", "root-rows-3", "training"] + }, + { + "name": "GFSE-V3-TRAIN-D06-F010-R10-X1-I7-M3-Q1-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d6_f10_r10_x1_i7_m3_q1_z0_c0_s0_p64", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..6]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 30, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 6, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-training", "endpoint-ids", "reachable-all", "disconnected-1", "reverse-fanin", "suffix-multiplicity-3", "payload", "training"] + }, + { + "name": "GFSE-V3-HOLDOUT-D07-F005-R1-X3-I6-M2-Q6-C1-S1-path", + "dataset": "generated_fixed_suffix_expansion_v3_d7_f5_r1_x3_i6_m2_q6_z0_c1_s1_p24", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..7]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-head-branch-0000-depth-07-00","fse-middle-branch-0000-depth-07-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-depth-07:enter","branch-0000-depth-07:continue","branch-0000-depth-07:complete"]},{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-head-branch-0000-depth-07-01","fse-middle-branch-0000-depth-07-01","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-depth-07:enter","branch-0000-depth-07:continue","branch-0000-depth-07:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 7, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "path", "productive-boundary-cycle", "productive-boundary-self-loop", "relationship-distinct", "holdout"] + }, + { + "name": "GFSE-V3-HOLDOUT-D11-F007-R0-X4-I0-M3-Q2-C1-S0-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d11_f7_r0_x4_i0_m3_q2_z1_c1_s0_p96", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..11]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 6, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 11, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "endpoint-ids", "zero-depth-suffix", "productive-boundary-cycle", "relationship-distinct", "holdout"] + }, + { + "name": "GFSE-V3-HOLDOUT-D13-F009-R4-X1-I2-M1-Q7-C0-S1-path", + "dataset": "generated_fixed_suffix_expansion_v3_d13_f9_r4_x1_i2_m1_q7_z0_c0_s1_p8", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..13]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 4, "result_kind": "path_set", "path_rows": [{"nodes":["fse-root","fse-branch-0000-level-01","fse-branch-0000-level-02","fse-branch-0000-level-03","fse-branch-0000-level-04","fse-branch-0000-level-05","fse-branch-0000-level-06","fse-branch-0000-level-07","fse-branch-0000-level-08","fse-branch-0000-level-09","fse-branch-0000-level-10","fse-branch-0000-level-11","fse-branch-0000-level-12","fse-branch-0000-level-13","fse-head-branch-0000-depth-13-00","fse-middle-branch-0000-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0000-level-01","branch-0000-level-02","branch-0000-level-03","branch-0000-level-04","branch-0000-level-05","branch-0000-level-06","branch-0000-level-07","branch-0000-level-08","branch-0000-level-09","branch-0000-level-10","branch-0000-level-11","branch-0000-level-12","branch-0000-level-13","branch-0000-depth-13:enter","branch-0000-depth-13:continue","branch-0000-depth-13:complete"]},{"nodes":["fse-root","fse-branch-0001-level-01","fse-branch-0001-level-02","fse-branch-0001-level-03","fse-branch-0001-level-04","fse-branch-0001-level-05","fse-branch-0001-level-06","fse-branch-0001-level-07","fse-branch-0001-level-08","fse-branch-0001-level-09","fse-branch-0001-level-10","fse-branch-0001-level-11","fse-branch-0001-level-12","fse-branch-0001-level-13","fse-head-branch-0001-depth-13-00","fse-middle-branch-0001-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0001-level-01","branch-0001-level-02","branch-0001-level-03","branch-0001-level-04","branch-0001-level-05","branch-0001-level-06","branch-0001-level-07","branch-0001-level-08","branch-0001-level-09","branch-0001-level-10","branch-0001-level-11","branch-0001-level-12","branch-0001-level-13","branch-0001-depth-13:enter","branch-0001-depth-13:continue","branch-0001-depth-13:complete"]},{"nodes":["fse-root","fse-branch-0002-level-01","fse-branch-0002-level-02","fse-branch-0002-level-03","fse-branch-0002-level-04","fse-branch-0002-level-05","fse-branch-0002-level-06","fse-branch-0002-level-07","fse-branch-0002-level-08","fse-branch-0002-level-09","fse-branch-0002-level-10","fse-branch-0002-level-11","fse-branch-0002-level-12","fse-branch-0002-level-13","fse-head-branch-0002-depth-13-00","fse-middle-branch-0002-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0002-level-01","branch-0002-level-02","branch-0002-level-03","branch-0002-level-04","branch-0002-level-05","branch-0002-level-06","branch-0002-level-07","branch-0002-level-08","branch-0002-level-09","branch-0002-level-10","branch-0002-level-11","branch-0002-level-12","branch-0002-level-13","branch-0002-depth-13:enter","branch-0002-depth-13:continue","branch-0002-depth-13:complete"]},{"nodes":["fse-root","fse-branch-0003-level-01","fse-branch-0003-level-02","fse-branch-0003-level-03","fse-branch-0003-level-04","fse-branch-0003-level-05","fse-branch-0003-level-06","fse-branch-0003-level-07","fse-branch-0003-level-08","fse-branch-0003-level-09","fse-branch-0003-level-10","fse-branch-0003-level-11","fse-branch-0003-level-12","fse-branch-0003-level-13","fse-head-branch-0003-depth-13-00","fse-middle-branch-0003-depth-13-00","fse-terminal"],"relationship_kinds":["Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","Expand","EnterSuffix","ContinueSuffix","CompleteSuffix"],"relationship_keys":["branch-0003-level-01","branch-0003-level-02","branch-0003-level-03","branch-0003-level-04","branch-0003-level-05","branch-0003-level-06","branch-0003-level-07","branch-0003-level-08","branch-0003-level-09","branch-0003-level-10","branch-0003-level-11","branch-0003-level-12","branch-0003-level-13","branch-0003-depth-13:enter","branch-0003-depth-13:continue","branch-0003-depth-13:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 13, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "path", "productive-boundary-self-loop", "relationship-distinct", "holdout"] + }, + { + "name": "GFSE-V3-HOLDOUT-D15-F012-R6-X6-I9-M2-Q3-Z1-endpoint", + "dataset": "generated_fixed_suffix_expansion_v3_d15_f12_r6_x6_i9_m2_q3_z1_c0_s0_p128", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..15]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 14, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 15, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "orientation-v2-holdout", "endpoint-ids", "zero-depth-suffix", "reverse-fanin", "suffix-multiplicity-2", "payload", "holdout"] + }, + { + "name": "GFSE-D00-F001-none_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d0_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..0]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 0, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-0", "fanout-1", "density-none"] + }, + { + "name": "GFSE-D00-F001-none_path", + "dataset": "generated_fixed_suffix_expansion_d0_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..0]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 0, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-0", "fanout-1", "density-none"] + }, + { + "name": "GFSE-D01-F010-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d1_f10_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..1]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-1", "fanout-10", "density-sparse"] + }, + { + "name": "GFSE-D01-F010-sparse_path", + "dataset": "generated_fixed_suffix_expansion_d1_f10_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..1]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 1, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-1", "fanout-10", "density-sparse"] + }, + { + "name": "GFSE-D02-F100-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d2_f100_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 11}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-2", "fanout-100", "density-sparse"] + }, + { + "name": "GFSE-D02-F100-sparse_path", + "dataset": "generated_fixed_suffix_expansion_d2_f100_v10_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 11, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-2", "fanout-100", "density-sparse"] + }, + { + "name": "GFSE-D04-F010-half_payload_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d4_f10_v2_p4096", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..4]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 6}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-4", "fanout-10", "density-half", "payload-4k"] + }, + { + "name": "GFSE-D04-F010-half_payload_path", + "dataset": "generated_fixed_suffix_expansion_d4_f10_v2_p4096", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..4]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 6, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-4", "fanout-10", "density-half", "payload-4k"] + }, + { + "name": "GFSE-D08-F001-all_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d8_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-8", "fanout-1", "density-all"] + }, + { + "name": "GFSE-D08-F001-all_path", + "dataset": "generated_fixed_suffix_expansion_d8_f1_v1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-8", "fanout-1", "density-all"] + }, + { + "name": "GFSE-D16-F1000-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_d16_f1000_v1000_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "endpoint-ids", "depth-16", "fanout-1000", "density-sparse"] + }, + { + "name": "GFSE-D16-F1000-sparse_path", + "dataset": "generated_fixed_suffix_expansion_d16_f1000_v1000_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "fixed-suffix-expansion", "path", "depth-16", "fanout-1000", "density-sparse"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_shortest_paths.json b/benchmark/testdata/scale/cases/generated_shortest_paths.json new file mode 100644 index 00000000..1c07682f --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_shortest_paths.json @@ -0,0 +1,320 @@ +{ + "cases": [ + { + "name": "GSP-D01-F001_distance", + "dataset": "generated_shortest_paths_d1_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 1, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-1", "fanout-1"] + }, + { + "name": "GSP-D01-F001_path", + "dataset": "generated_shortest_paths_d1_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 1, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-1", "fanout-1"] + }, + { + "name": "GSP-D00-F001_path_zero", + "dataset": "generated_shortest_paths_d1_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*0..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-start"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-start"], "relationship_kinds": []}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 0, "max_depth": 1, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "zero-depth", "depth-0", "fanout-1"] + }, + { + "name": "GSP-D02-F016_distance", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D02-F016_path", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D04-F128_distance", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 4, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D04-F128_path", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D08-F001_distance_inbound", + "dataset": "generated_shortest_paths_d8_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((e)<-[:Traverse*1..8]-(s)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 8, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "inbound", "depth-8", "fanout-1"] + }, + { + "name": "GSP-D08-F001_path_inbound", + "dataset": "generated_shortest_paths_d8_f1", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((e)<-[:Traverse*1..8]-(s)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "inbound", "depth-8", "fanout-1"] + }, + { + "name": "GSP-D08-F128_path_directionless", + "dataset": "generated_shortest_paths_d8_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..8]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["neo4j"], + "unsupported_modes": {"postgres_sql": "the PostgreSQL translator does not support directionless variable-length expansion"}, + "tags": ["generated", "normal-tier", "path", "directionless", "depth-8", "fanout-128"] + }, + { + "name": "GSP-D16-F016_distance", + "dataset": "generated_shortest_paths_d16_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 16, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "depth-16", "fanout-16"] + }, + { + "name": "GSP-D16-F016_path", + "dataset": "generated_shortest_paths_d16_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "depth-16", "fanout-16"] + }, + { + "name": "GSP-D04-F128_disconnected", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-disconnected"}, + "expected": {"row_count": 0}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "disconnected", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D04-F128_path_disconnected", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-disconnected"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "disconnected", "depth-4", "fanout-128"] + }, + { + "name": "GSP-D02-F016_distance_cycle", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-cycle-b"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "cycle", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D02-F016_path_cycle", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-cycle-b"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "cycle", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D01-F016_distance_parallel", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse|TypedTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-parallel-end"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse", "TypedTraverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "parallel-edges", "depth-1", "fanout-16"] + }, + { + "name": "GSP-D01-F016_path_parallel", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse|TypedTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-parallel-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse", "TypedTraverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "parallel-edges", "depth-1", "fanout-16"] + }, + { + "name": "GSP-D02-F016_distance_self_loop", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-self-loop-exit"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "distance", "self-loop", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D02-F016_path_self_loop", + "dataset": "generated_shortest_paths_d2_f16", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-self-loop-exit"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 4, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "path", "self-loop", "depth-2", "fanout-16"] + }, + { + "name": "GSP-D32-F512_distance", + "dataset": "generated_shortest_paths_d32_f512", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..32]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 32, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 32, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "distance", "depth-32", "fanout-512"] + }, + { + "name": "GSP-D32-F512_path", + "dataset": "generated_shortest_paths_d32_f512", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..32]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 32, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "path", "depth-32", "fanout-512"] + }, + { + "name": "GSP-D64-F1000_distance", + "dataset": "generated_shortest_paths_d64_f1000", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "scalar_int": 64, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "distance", "depth-64", "fanout-1000"] + }, + { + "name": "GSP-D64-F1000_path", + "dataset": "generated_shortest_paths_d64_f1000", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "path", "depth-64", "fanout-1000"] + }, + { + "name": "GSP-D64-F1000_disconnected", + "dataset": "generated_shortest_paths_d64_f1000", + "category": "generated_shortest_path", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-start", "end_id": "sp-disconnected"}, + "expected": {"row_count": 0}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "envelope-tier", "distance", "disconnected", "depth-64", "fanout-1000"] + }, + { + "name": "GSP-D04-F128_all_shortest_diamond", + "dataset": "generated_shortest_paths_d4_f128", + "category": "generated_all_shortest_paths", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse|TypedTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-start", "end_id": "sp-diamond-end"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [ + {"nodes": ["sp-start", "sp-diamond-left", "sp-diamond-end"], "relationship_kinds": ["Traverse", "TypedTraverse"]}, + {"nodes": ["sp-start", "sp-diamond-right", "sp-diamond-end"], "relationship_kinds": ["Traverse", "TypedTraverse"]} + ]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse", "TypedTraverse"], "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "normal-tier", "all-shortest", "diamond", "ties"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_shortest_paths_v2.json b/benchmark/testdata/scale/cases/generated_shortest_paths_v2.json new file mode 100644 index 00000000..c11380eb --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_shortest_paths_v2.json @@ -0,0 +1,352 @@ +{ + "cases": [ + { + "name": "GSPV2-NORMAL-outbound-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..3]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "mirrored_fanout", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "outbound"] + }, + { + "name": "GSPV2-NORMAL-hidden-fanin-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..3]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "hidden_intermediate_fan_in", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in"] + }, + { + "name": "GSPV2-NORMAL-outbound-all-shortest-depth3", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..3]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02", "sp-v2-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02", "primary-03"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "two_sided_predecessor_dag", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "predecessor-dag", "training"] + }, + { + "name": "GSPV2-TRAINING-early-depth1-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-01"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01"], "relationship_kinds": ["Traverse"], "relationship_keys": ["primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "early-target", "early-depth-1", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-early-depth2-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-02"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02"], "relationship_kinds": ["Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "early-target", "early-depth-2", "max-64", "training"] + }, + { + "name": "GSPV2-TRAINING-early-depth3-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-03"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02", "sp-v2-linear-03"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02", "primary-03"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "early-target", "early-depth-3", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-inbound-early-depth1-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((r)<-[:Traverse*1..16]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-01"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01"], "relationship_kinds": ["Traverse"], "relationship_keys": ["inbound-primary-08"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "inbound", "early-target", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-inbound-early-depth3-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-03"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_early_target_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "inbound", "early-target", "max-64", "training"] + }, + { + "name": "GSPV2-TRAINING-cycle-dead-tail-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_cycle_dead_tail", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "cycle-dead-tail", "max-64", "training"] + }, + { + "name": "GSPV2-TRAINING-reconvergent-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:DiamondTraverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-diamond-start", "end_id": "sp-v2-diamond-end"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000000", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000000-a", "diamond-000000-b"]}, {"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000001", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000001-a", "diamond-000001-b"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["DiamondTraverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_reconvergence", "result_cardinality_class": "small_multi", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "reconvergence", "max-16", "training"] + }, + { + "name": "GSPV2-TRAINING-disconnected-all-shortest-max64", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-disconnected-start", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 64, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "disconnected", "max-miss", "max-64", "training"] + }, + { + "name": "GSPV2-NORMAL-hidden-fanin-path", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..3]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-03", "inbound-primary-02", "inbound-primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "hidden_intermediate_fan_in", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 3, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in"] + }, + { + "name": "GSPV2-NORMAL-parallel-kind-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"], "direction": "outbound", "relationship_kind_count": 7, "fixture_tier": "normal", "expected_state_class": "parallel_kind_high_cardinality", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 2, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "parallel-kinds", "holdout"] + }, + { + "name": "GSPV2-NORMAL-parallel-kind-path", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"], "direction": "outbound", "relationship_kind_count": 7, "fixture_tier": "normal", "expected_state_class": "parallel_kind_high_cardinality", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "parallel-kinds", "holdout"] + }, + { + "name": "GSPV2-NORMAL-diamond-all-shortest", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:DiamondTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-diamond-start", "end_id": "sp-v2-diamond-end"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000000", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000000-a", "diamond-000000-b"]}, {"nodes": ["sp-v2-diamond-start", "sp-v2-diamond-000001", "sp-v2-diamond-end"], "relationship_kinds": ["DiamondTraverse", "DiamondTraverse"], "relationship_keys": ["diamond-000001-a", "diamond-000001-b"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["DiamondTraverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag", "result_cardinality_class": "small_multi", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "diamond", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-outbound-distance", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "scalar_int": 8, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "outbound", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-inbound-path", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..8]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_hidden_fanin_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-all-shortest", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01", "sp-v2-linear-02", "sp-v2-linear-03", "sp-v2-linear-04", "sp-v2-linear-05", "sp-v2-linear-06", "sp-v2-linear-07", "sp-v2-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], "relationship_keys": ["primary-01", "primary-02", "primary-03", "primary-04", "primary-05", "primary-06", "primary-07", "primary-08"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_predecessor_dag_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-disconnected-depth8", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-disconnected-start", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "recursive_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "disconnected", "max-miss", "recursive-kernel", "holdout"] + }, + { + "name": "GSPV2-DIAGNOSTIC-inbound-disconnected-depth8", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..8]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-start"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "disconnected", "max-miss", "sp-i2", "diagnostic"] + }, + { + "name": "GSPV2-HOLDOUT-depth8-inbound-all-shortest", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((r)<-[:Traverse*1..8]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-end"], "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "independent_recursive_predecessor_dag_hidden_fanin_depth8", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "inbound", "hidden-fan-in", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-disconnected-all-shortest-depth8", + "dataset": "generated_shortest_paths_v2_d8_o4_r2_fo8_fi64_l4_k3_t16_w4_x32_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-disconnected-start", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_disconnected_max_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 8, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "disconnected", "max-miss", "holdout"] + }, + { + "name": "GSPV2-HOLDOUT-parallel-kind-all-shortest", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + "expected": {"row_count": 7, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "holdout", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"], "direction": "outbound", "relationship_kind_count": 7, "fixture_tier": "normal", "expected_state_class": "predecessor_dag_parallel_kind_multiplicity", "result_cardinality_class": "small_multi", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "all-shortest", "parallel-kinds", "holdout"] + }, + { + "name": "GSPV2-DIAGNOSTIC-early-target-all-shortest-max16", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-linear-01"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["sp-v2-start", "sp-v2-linear-01"], "relationship_kinds": ["Traverse"], "relationship_keys": ["primary-01"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "predecessor_dag_early_target_max_slack", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "all-shortest", "early-target", "max-slack", "diagnostic"] + }, + { + "name": "GSPV2-NORMAL-implicit-max15-distance", + "dataset": "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "implicit_policy_depth15", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 15, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "implicit-maximum", "diagnostic"] + }, + { + "name": "GSPV2-STRESS-explicit-max15-depth16-control", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((s)-[:Traverse*1..15]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "explicit_depth15_boundary_miss", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 15, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "distance", "explicit-maximum", "depth-boundary", "diagnostic"] + }, + { + "name": "GSPV2-STRESS-hidden-fanin-distance", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..16]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 16, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "hidden_intermediate_fan_in", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "distance", "inbound", "hidden-fan-in"] + }, + { + "name": "GSPV2-STRESS-outbound-all-shortest-depth16", + "dataset": "generated_shortest_paths_v2_d16_o16_r1_fo16_fi16384_l2_k30_t1024_w100_x1024_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:Traverse*1..16]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-start", "end_id": "sp-v2-end"}, + "expected": {"row_count": 1, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "two_sided_predecessor_dag_hidden_fanout", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 16, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "all-shortest", "predecessor-dag"] + }, + { + "name": "GSPV2-STRESS-diamond-all-shortest-128", + "dataset": "generated_shortest_paths_v2_d3_o0_r0_fo0_fi0_l0_k0_t0_w128_x0_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = allShortestPaths((s)-[:DiamondTraverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + "node_params": {"start_id": "sp-v2-diamond-start", "end_id": "sp-v2-diamond-end"}, + "expected": {"row_count": 128, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "diagnostic", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["DiamondTraverse"], "direction": "outbound", "relationship_kind_count": 1, "fixture_tier": "stress", "expected_state_class": "predecessor_output_multiplicity", "result_cardinality_class": "large_multi", "min_depth": 1, "max_depth": 2, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "stress-tier", "all-shortest", "diamond", "output-multiplicity"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_sp_i1_inbound_v1.json b/benchmark/testdata/scale/cases/generated_sp_i1_inbound_v1.json new file mode 100644 index 00000000..3c901654 --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_sp_i1_inbound_v1.json @@ -0,0 +1,226 @@ +{ + "cases": [ + { + "name": "GSP-I1-V1-TRAIN-D04-FI016-full", + "dataset": "generated_shortest_paths_v2_d4_o0_r4_fo0_fi16_l2_k0_t0_w0_x4_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_16", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-TRAIN-D16-FI256-early-d04", + "dataset": "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-04"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-16", "inbound-primary-15", "inbound-primary-14", "inbound-primary-13"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_early_target_fanin_256", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "early-target", "early-depth-4", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-TRAIN-D16-FI256-full", + "dataset": "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-linear-08", "sp-v2-inbound-linear-09", "sp-v2-inbound-linear-10", "sp-v2-inbound-linear-11", "sp-v2-inbound-linear-12", "sp-v2-inbound-linear-13", "sp-v2-inbound-linear-14", "sp-v2-inbound-linear-15", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-16", "inbound-primary-15", "inbound-primary-14", "inbound-primary-13", "inbound-primary-12", "inbound-primary-11", "inbound-primary-10", "inbound-primary-09", "inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_256", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-TRAIN-D16-FI256-disconnected", + "dataset": "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "training", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_disconnected_fanin_256", + "result_cardinality_class": "empty", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "disconnected", "max-miss", "sp-i1-inbound-v1-training"] + }, + { + "name": "GSP-I1-V1-HOLDOUT-D08-FI031-full", + "dataset": "generated_shortest_paths_v2_d8_o0_r3_fo0_fi31_l3_k0_t0_w0_x7_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "holdout", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_31", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "holdout", "sp-i1-inbound-v1-holdout"] + }, + { + "name": "GSP-I1-V1-HOLDOUT-D32-FI191-full", + "dataset": "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": { + "row_count": 1, + "result_kind": "path_set", + "path_rows": [{ + "nodes": ["sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-linear-03", "sp-v2-inbound-linear-04", "sp-v2-inbound-linear-05", "sp-v2-inbound-linear-06", "sp-v2-inbound-linear-07", "sp-v2-inbound-linear-08", "sp-v2-inbound-linear-09", "sp-v2-inbound-linear-10", "sp-v2-inbound-linear-11", "sp-v2-inbound-linear-12", "sp-v2-inbound-linear-13", "sp-v2-inbound-linear-14", "sp-v2-inbound-linear-15", "sp-v2-inbound-linear-16", "sp-v2-inbound-linear-17", "sp-v2-inbound-linear-18", "sp-v2-inbound-linear-19", "sp-v2-inbound-linear-20", "sp-v2-inbound-linear-21", "sp-v2-inbound-linear-22", "sp-v2-inbound-linear-23", "sp-v2-inbound-linear-24", "sp-v2-inbound-linear-25", "sp-v2-inbound-linear-26", "sp-v2-inbound-linear-27", "sp-v2-inbound-linear-28", "sp-v2-inbound-linear-29", "sp-v2-inbound-linear-30", "sp-v2-inbound-linear-31", "sp-v2-inbound-end"], + "relationship_kinds": ["Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse", "Traverse"], + "relationship_keys": ["inbound-primary-32", "inbound-primary-31", "inbound-primary-30", "inbound-primary-29", "inbound-primary-28", "inbound-primary-27", "inbound-primary-26", "inbound-primary-25", "inbound-primary-24", "inbound-primary-23", "inbound-primary-22", "inbound-primary-21", "inbound-primary-20", "inbound-primary-19", "inbound-primary-18", "inbound-primary-17", "inbound-primary-16", "inbound-primary-15", "inbound-primary-14", "inbound-primary-13", "inbound-primary-12", "inbound-primary-11", "inbound-primary-10", "inbound-primary-09", "inbound-primary-08", "inbound-primary-07", "inbound-primary-06", "inbound-primary-05", "inbound-primary-04", "inbound-primary-03", "inbound-primary-02", "inbound-primary-01"] + }] + }, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "holdout", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_full_depth_fanin_191", + "result_cardinality_class": "singleton", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "holdout", "sp-i1-inbound-v1-holdout"] + }, + { + "name": "GSP-I1-V1-HOLDOUT-D32-FI191-disconnected", + "dataset": "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "path_set"}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": { + "qualification_split": "holdout", + "fallback_expectation": "forbidden", + "root_predicate": "bound_id", + "terminal_predicate": "bound_id", + "edge_kinds": ["Traverse"], + "direction": "inbound", + "relationship_kind_count": 1, + "fixture_tier": "normal", + "expected_state_class": "inbound_predecessor_disconnected_fanin_191", + "result_cardinality_class": "empty", + "min_depth": 1, + "max_depth": 64, + "path_materialization_required": true + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in", "disconnected", "max-miss", "holdout", "sp-i1-inbound-v1-holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_sp_i2_distance_v1.json b/benchmark/testdata/scale/cases/generated_sp_i2_distance_v1.json new file mode 100644 index 00000000..6b77dcb7 --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_sp_i2_distance_v1.json @@ -0,0 +1,124 @@ +{ + "cases": [ + { + "name": "GSP-I2-V1-TRAIN-D03-RI064-FI032-full", + "dataset": "generated_shortest_paths_v2_d3_o0_r64_fo0_fi32_l2_k0_t0_w0_x3_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_full_depth_root_fanin_64_intermediate_fanin_32", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "sp-i2-distance-v1-training"] + }, + { + "name": "GSP-I2-V1-TRAIN-D08-RI128-FI064-early-d02", + "dataset": "generated_shortest_paths_v2_d8_o0_r128_fo0_fi64_l4_k0_t0_w0_x8_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-02"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_early_target_root_fanin_128", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "early-target", "early-depth-2", "sp-i2-distance-v1-training"] + }, + { + "name": "GSP-I2-V1-TRAIN-D08-RI128-FI064-full", + "dataset": "generated_shortest_paths_v2_d8_o0_r128_fo0_fi64_l4_k0_t0_w0_x8_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 8, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_full_depth_root_fanin_128_intermediate_fanin_64", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "sp-i2-distance-v1-training"] + }, + { + "name": "GSP-I2-V1-TRAIN-D16-RI256-FI512-full", + "dataset": "generated_shortest_paths_v2_d16_o0_r256_fo0_fi512_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 16, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_full_depth_root_fanin_256_intermediate_fanin_512", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "sp-i2-distance-v1-training"] + }, + { + "name": "GSP-I2-V1-TRAIN-D16-RI256-FI512-disconnected", + "dataset": "generated_shortest_paths_v2_d16_o0_r256_fo0_fi512_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_disconnected_root_fanin_256", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "disconnected", "max-miss", "sp-i2-distance-v1-training"] + }, + { + "name": "GSP-I2-V1-TRAIN-cycle-control", + "dataset": "generated_shortest_paths_v2_d6_o0_r0_fo0_fi0_l0_k0_t0_w0_x6_p0_c1_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-cycle-a", "end_id": "sp-v2-cycle-b"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "training", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_cycle_control", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "cycle", "adverse-control", "sp-i2-distance-v1-training"] + }, + { + "name": "GSP-I2-V1-HOLDOUT-D05-RI047-FI023-full", + "dataset": "generated_shortest_paths_v2_d5_o0_r47_fo0_fi23_l2_k0_t0_w0_x5_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 5, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_full_depth_root_fanin_47_intermediate_fanin_23", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "holdout", "sp-i2-distance-v1-holdout"] + }, + { + "name": "GSP-I2-V1-HOLDOUT-D13-RI191-FI383-full", + "dataset": "generated_shortest_paths_v2_d13_o0_r191_fo0_fi383_l7_k0_t0_w0_x13_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 13, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_full_depth_root_fanin_191_intermediate_fanin_383", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "holdout", "sp-i2-distance-v1-holdout"] + }, + { + "name": "GSP-I2-V1-HOLDOUT-D13-RI191-FI383-early-d03", + "dataset": "generated_shortest_paths_v2_d13_o0_r191_fo0_fi383_l7_k0_t0_w0_x13_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-03"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_early_target_root_fanin_191", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "early-target", "early-depth-3", "holdout", "sp-i2-distance-v1-holdout"] + }, + { + "name": "GSP-I2-V1-HOLDOUT-D21-RI127-FI255-disconnected", + "dataset": "generated_shortest_paths_v2_d21_o0_r127_fo0_fi255_l11_k0_t0_w0_x21_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"qualification_split": "holdout", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "reverse_physical_disconnected_root_fanin_127", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "normal-tier", "distance", "inbound", "hidden-fan-in", "disconnected", "max-miss", "holdout", "sp-i2-distance-v1-holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_sp_i2_distance_v2.json b/benchmark/testdata/scale/cases/generated_sp_i2_distance_v2.json new file mode 100644 index 00000000..e1c12d8e --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_sp_i2_distance_v2.json @@ -0,0 +1,173 @@ +{ + "cases": [ + { + "name": "GSP-I2-V2-TRAIN-direct-acyclic-shallow", + "dataset": "generated_shortest_paths_v2_d1_o0_r4_fo0_fi0_l0_k0_t0_w0_x1_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_direct_acyclic_shallow_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "direct", "adverse-control", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-direct-cycle-control", + "dataset": "generated_shortest_paths_v2_d7_o0_r0_fo0_fi0_l0_k0_t0_w0_x7_p0_c1_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-cycle-a", "end_id": "sp-v2-cycle-b"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_direct_two_node_cycle", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "direct", "cycle", "adverse-control", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-D02-post-target-cycle", + "dataset": "generated_shortest_paths_v2_d5_o0_r19_fo0_fi11_l3_k0_t0_w0_x5_p0_c1_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-02"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_depth_two_post_target_cycle", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "cycle", "early-target", "adverse-control", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-D02-hidden-intermediate-fanin", + "dataset": "generated_shortest_paths_v2_d4_o0_r37_fo0_fi73_l2_k0_t0_w0_x4_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-02"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_early_target_hidden_intermediate_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "early-target", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-D03-hidden-root-fanin", + "dataset": "generated_shortest_paths_v2_d3_o0_r83_fo0_fi41_l2_k0_t0_w0_x3_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_full_depth_hidden_root_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-D08-mixed-fanin", + "dataset": "generated_shortest_paths_v2_d8_o0_r149_fo0_fi79_l4_k0_t0_w0_x8_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 8, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_full_depth_mixed_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-D16-high-fanin", + "dataset": "generated_shortest_paths_v2_d16_o0_r263_fo0_fi521_l8_k0_t0_w0_x16_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 16, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_full_depth_high_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "sp-i2-distance-v2-training"] + }, + { + "name": "GSP-I2-V2-TRAIN-D16-disconnected-cyclic-exhaustion", + "dataset": "generated_shortest_paths_v2_d16_o0_r271_fo0_fi527_l8_k0_t0_w0_x17_p0_c1_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_disconnected_cyclic_exhaustion", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "disconnected", "cycle", "sp-i2-distance-v2-training"] + }, + + { + "name": "GSP-I2-V2-HOLDOUT-direct-parallel-asymmetric-cycle", + "dataset": "generated_shortest_paths_v2_d1_o0_r17_fo0_fi0_l0_k3_t2_w0_x1_p0_c1_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 1, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "holdout", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_direct_parallel_asymmetric_cycle", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "direct", "parallel", "cycle", "holdout", "adverse-control", "sp-i2-distance-v2-holdout"] + }, + { + "name": "GSP-I2-V2-HOLDOUT-D02-longer-competing-cycle", + "dataset": "generated_shortest_paths_v2_d6_o0_r43_fo0_fi29_l3_k0_t0_w0_x6_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-02"}, + "expected": {"row_count": 1, "scalar_int": 2, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "holdout", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_depth_two_longer_competing_cycle", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "cycle", "holdout", "adverse-control", "sp-i2-distance-v2-holdout"] + }, + { + "name": "GSP-I2-V2-HOLDOUT-D03-irrelevant-high-fanout", + "dataset": "generated_shortest_paths_v2_d9_o0_r173_fo0_fi97_l4_k0_t0_w0_x9_p0_c0_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-03"}, + "expected": {"row_count": 1, "scalar_int": 3, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "holdout", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_early_target_irrelevant_high_fanout", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "early-target", "hidden-fan-in", "holdout", "sp-i2-distance-v2-holdout"] + }, + { + "name": "GSP-I2-V2-HOLDOUT-D11-medium-fanin", + "dataset": "generated_shortest_paths_v2_d11_o0_r197_fo0_fi211_l6_k0_t0_w0_x11_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 11, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "holdout", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_medium_full_depth_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "holdout", "sp-i2-distance-v2-holdout"] + }, + { + "name": "GSP-I2-V2-HOLDOUT-D23-deep-fanin", + "dataset": "generated_shortest_paths_v2_d23_o0_r307_fo0_fi601_l12_k0_t0_w0_x23_p0_c0_s0", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + "expected": {"row_count": 1, "scalar_int": 23, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "holdout", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_deep_full_depth_hidden_fanin", "result_cardinality_class": "singleton", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "holdout", "sp-i2-distance-v2-holdout"] + }, + { + "name": "GSP-I2-V2-HOLDOUT-D27-disconnected-cycles", + "dataset": "generated_shortest_paths_v2_d27_o0_r313_fo0_fi607_l14_k0_t0_w0_x29_p0_c1_s1", + "category": "generated_shortest_path_v2", + "cypher": "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p) AS distance", + "node_params": {"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-disconnected-end"}, + "expected": {"row_count": 0, "result_kind": "scalar"}, + "observes": {}, + "shape": {"qualification_split": "holdout", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "root_predicate": "bound_id", "terminal_predicate": "bound_id", "edge_kinds": ["Traverse"], "direction": "inbound", "relationship_kind_count": 1, "fixture_tier": "normal", "expected_state_class": "v2_deep_disconnected_graph_with_cycles", "result_cardinality_class": "empty", "min_depth": 1, "max_depth": 64, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["generated", "v2", "distance", "inbound", "hidden-fan-in", "disconnected", "cycle", "holdout", "sp-i2-distance-v2-holdout"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/generated_suffix_route_component_v1.json b/benchmark/testdata/scale/cases/generated_suffix_route_component_v1.json new file mode 100644 index 00000000..82ca71cb --- /dev/null +++ b/benchmark/testdata/scale/cases/generated_suffix_route_component_v1.json @@ -0,0 +1,138 @@ +{ + "cases": [ + { + "name": "GFSE-SRC-V1-TARGET-D16-F1024-sparse_endpoint_ids", + "dataset": "generated_fixed_suffix_expansion_v3_d16_f1024_r1_x1_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-16-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 16, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "endpoint-ids", "sparse-suffix", "efficacy-target", "training"] + }, + { + "name": "GFSE-SRC-V1-TARGET-D17-F1025-sparse_path", + "dataset": "generated_fixed_suffix_expansion_v3_d17_f1025_r1_x2_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..17]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "path_set", "path_rows": [{"nodes": ["fse-root", "fse-branch-0000-level-01", "fse-branch-0000-level-02", "fse-branch-0000-level-03", "fse-branch-0000-level-04", "fse-branch-0000-level-05", "fse-branch-0000-level-06", "fse-branch-0000-level-07", "fse-branch-0000-level-08", "fse-branch-0000-level-09", "fse-branch-0000-level-10", "fse-branch-0000-level-11", "fse-branch-0000-level-12", "fse-branch-0000-level-13", "fse-branch-0000-level-14", "fse-branch-0000-level-15", "fse-branch-0000-level-16", "fse-branch-0000-level-17", "fse-head-branch-0000-depth-17-00", "fse-middle-branch-0000-depth-17-00", "fse-terminal"], "relationship_kinds": ["Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["branch-0000-level-01", "branch-0000-level-02", "branch-0000-level-03", "branch-0000-level-04", "branch-0000-level-05", "branch-0000-level-06", "branch-0000-level-07", "branch-0000-level-08", "branch-0000-level-09", "branch-0000-level-10", "branch-0000-level-11", "branch-0000-level-12", "branch-0000-level-13", "branch-0000-level-14", "branch-0000-level-15", "branch-0000-level-16", "branch-0000-level-17", "branch-0000-depth-17:enter", "branch-0000-depth-17:continue", "branch-0000-depth-17:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "efficacy_target", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 17, "path_materialization_required": true}, + "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "path", "sparse-suffix", "efficacy-target", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-D08-F017-I1024-high_reverse_fanin", + "dataset": "generated_fixed_suffix_expansion_v3_d8_f17_r1_x3_i1024_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..8]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 1, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-08-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 8, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "high-reverse-fanin", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-D05-F016-dense_suffix", + "dataset": "generated_fixed_suffix_expansion_v3_d5_f16_r16_x4_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..5]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 16, "result_kind": "id_rows", "id_rows": [["fse-head-branch-0000-depth-05-00", "fse-terminal"], ["fse-head-branch-0001-depth-05-00", "fse-terminal"], ["fse-head-branch-0002-depth-05-00", "fse-terminal"], ["fse-head-branch-0003-depth-05-00", "fse-terminal"], ["fse-head-branch-0004-depth-05-00", "fse-terminal"], ["fse-head-branch-0005-depth-05-00", "fse-terminal"], ["fse-head-branch-0006-depth-05-00", "fse-terminal"], ["fse-head-branch-0007-depth-05-00", "fse-terminal"], ["fse-head-branch-0008-depth-05-00", "fse-terminal"], ["fse-head-branch-0009-depth-05-00", "fse-terminal"], ["fse-head-branch-0010-depth-05-00", "fse-terminal"], ["fse-head-branch-0011-depth-05-00", "fse-terminal"], ["fse-head-branch-0012-depth-05-00", "fse-terminal"], ["fse-head-branch-0013-depth-05-00", "fse-terminal"], ["fse-head-branch-0014-depth-05-00", "fse-terminal"], ["fse-head-branch-0015-depth-05-00", "fse-terminal"]]}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 5, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "dense-suffix", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-D09-F513-no_path", + "dataset": "generated_fixed_suffix_expansion_v3_d9_f513_r0_x513_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..9]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 0, "result_kind": "id_rows", "id_rows": []}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 9, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "no-path", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-CAP511", + "dataset": "generated_fixed_suffix_expansion_v3_d3_f7_r0_x511_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, "expected": {"row_count": 0, "result_kind": "id_rows", "id_rows": []}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "suffix-cap-511", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-CAP512", + "dataset": "generated_fixed_suffix_expansion_v3_d3_f8_r0_x512_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, "expected": {"row_count": 0, "result_kind": "id_rows", "id_rows": []}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "suffix-cap-512", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-CAP513", + "dataset": "generated_fixed_suffix_expansion_v3_d3_f9_r0_x513_i0_m1_q1_z0_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "generated-fse-root"}, "expected": {"row_count": 0, "result_kind": "id_rows", "id_rows": []}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "suffix-cap-513", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-productive_cycle_path", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f4_r0_x5_i0_m1_q1_z1_c1_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["fse-root", "fse-head-root-00", "fse-middle-root-00", "fse-terminal"], "relationship_kinds": ["EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["root:enter", "root:continue", "root:complete"]}, {"nodes": ["fse-root", "fse-productive-boundary-cycle", "fse-root", "fse-head-root-00", "fse-middle-root-00", "fse-terminal"], "relationship_kinds": ["Expand", "Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["productive-boundary-cycle-enter", "productive-boundary-cycle-return", "root:enter", "root:continue", "root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": true}, "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "productive-cycle", "relationship-distinct", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-productive_self_loop_path", + "dataset": "generated_fixed_suffix_expansion_v3_d2_f5_r0_x6_i0_m1_q1_z1_c0_s1_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["fse-root", "fse-head-root-00", "fse-middle-root-00", "fse-terminal"], "relationship_kinds": ["EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["root:enter", "root:continue", "root:complete"]}, {"nodes": ["fse-root", "fse-root", "fse-head-root-00", "fse-middle-root-00", "fse-terminal"], "relationship_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["productive-boundary-self-loop", "root:enter", "root:continue", "root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 2, "path_materialization_required": true}, "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "productive-self-loop", "relationship-distinct", "adverse-control", "training"] + }, + { + "name": "GFSE-SRC-V1-CONTROL-multiple_relationship_distinct_paths", + "dataset": "generated_fixed_suffix_expansion_v3_d1_f6_r0_x7_i0_m2_q1_z1_c0_s0_p0", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..1]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", + "params": {"root_key": "generated-fse-root"}, + "expected": {"row_count": 2, "result_kind": "path_set", "path_rows": [{"nodes": ["fse-root", "fse-head-root-00", "fse-middle-root-00", "fse-terminal"], "relationship_kinds": ["EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["root:enter", "root:continue", "root:complete"]}, {"nodes": ["fse-root", "fse-head-root-01", "fse-middle-root-01", "fse-terminal"], "relationship_kinds": ["EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "relationship_keys": ["root:enter", "root:continue", "root:complete"]}]}, + "observes": {"paths": true, "nodes": true, "relationships": true, "properties": true}, + "shape": {"qualification_split": "training", "qualification_role": "adverse_control", "fallback_expectation": "forbidden", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 1, "path_materialization_required": true}, "candidate_modes": ["postgres_sql"], + "unsupported_modes": {"neo4j": "suffix-route-component-v1 is a PostgreSQL-only diagnostic roster"}, + "tags": ["generated", "normal-tier", "fixed-suffix-expansion-v3", "suffix-route-component-v1", "multiple-path", "relationship-distinct", "adverse-control", "training"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/hops.json b/benchmark/testdata/scale/cases/hops.json new file mode 100644 index 00000000..2835c2ac --- /dev/null +++ b/benchmark/testdata/scale/cases/hops.json @@ -0,0 +1,92 @@ +{ + "cases": [ + { + "name": "HOP-01_dense_outbound_bound_anchor", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) = $anchor RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "edge_kinds": ["HopKind01"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-01", "outbound", "dense", "full-direction"] + }, + { + "name": "HOP-02_dense_inbound_bound_anchor", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) = $anchor RETURN r, s", + "node_params": {"anchor": "hop-in-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"terminal_predicate": "bound_end_id", "edge_kinds": ["HopKind01"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-02", "inbound", "dense", "full-direction"] + }, + { + "name": "HOP-03_dense_thirty_kind_outbound_anchor", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09|HopKind10|HopKind11|HopKind12|HopKind13|HopKind14|HopKind15|HopKind16|HopKind17|HopKind18|HopKind19|HopKind20|HopKind21|HopKind22|HopKind23|HopKind24|HopKind25|HopKind26|HopKind27|HopKind28|HopKind29|HopKind30]->(e) WHERE id(s) = $anchor RETURN r, e", + "node_params": {"anchor": "hop-kind-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "edge_kinds": ["HopKind01", "HopKind02", "HopKind03", "HopKind04", "HopKind05", "HopKind06", "HopKind07", "HopKind08", "HopKind09", "HopKind10", "HopKind11", "HopKind12", "HopKind13", "HopKind14", "HopKind15", "HopKind16", "HopKind17", "HopKind18", "HopKind19", "HopKind20", "HopKind21", "HopKind22", "HopKind23", "HopKind24", "HopKind25", "HopKind26", "HopKind27", "HopKind28", "HopKind29", "HopKind30"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-03", "outbound", "dense", "30-kinds"] + }, + { + "name": "HOP-04_dense_opposite_endpoint_kind_disjunction", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopTypedEdge]->(e) WHERE id(s) = $anchor AND (e:HopEndA OR e:HopEndB) RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "terminal_predicate": "endpoint_kind_disjunction", "edge_kinds": ["HopTypedEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-04", "dense", "endpoint-kinds", "multi-kind-nodes"] + }, + { + "name": "HOP-05_thousand_endpoint_IDs_with_sparse_matches", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $anchor AND id(e) IN $end_ids RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "generated_node_list_params": {"end_ids": {"prefix": "hop-id-target", "count": 1000}}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "terminal_predicate": "large_end_id_list", "edge_kinds": ["HopIDEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-05", "1000-list", "endpoint-ids", "sparse-match"] + }, + { + "name": "HOP-07_nested_branch_selectivity", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopNestedEdge]->(e:HopTemplate) WHERE id(s) = $anchor AND ((e.requiresmanagerapproval = false AND e.schemaversion > 1 AND e.authorizedsignatures = 0 AND e.authenticationenabled = true) OR (e.requiresmanagerapproval = false AND e.schemaversion = 1 AND e.authenticationenabled = true)) RETURN r, e", + "node_params": {"anchor": "hop-out-root"}, + "expected": {"row_count": 64}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "bound_start_id", "terminal_predicate": "nested_property_disjunction", "edge_kinds": ["HopNestedEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-07", "nested-or", "branch-local", "selectivity"] + }, + { + "name": "HOP-09_dense_two_sided_ID_sets", + "dataset": "generated_hops", + "category": "standalone_one_hop", + "cypher": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r, e", + "generated_node_list_params": { + "start_ids": {"prefix": "hop-set-start", "count": 32}, + "end_ids": {"prefix": "hop-set-end", "count": 32} + }, + "expected": {"row_count": 1024}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "start_id_list", "terminal_predicate": "end_id_list", "edge_kinds": ["HopSetEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["HOP-09", "dense", "two-sided-ids", "32x32"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/reconciliation.json b/benchmark/testdata/scale/cases/reconciliation.json new file mode 100644 index 00000000..c45bafb2 --- /dev/null +++ b/benchmark/testdata/scale/cases/reconciliation.json @@ -0,0 +1,150 @@ +{ + "cases": [ + { + "name": "REC-01_inbound_30_kind_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->(e:ADEntity) WHERE e.objectid = $object_id DELETE r", + "params": {"object_id": "rec-in"}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": { + "terminal_predicate": "typed_endpoint_property", + "edge_kinds": ["RecKind01", "RecKind02", "RecKind03", "RecKind04", "RecKind05", "RecKind06", "RecKind07", "RecKind08", "RecKind09", "RecKind10", "RecKind11", "RecKind12", "RecKind13", "RecKind14", "RecKind15", "RecKind16", "RecKind17", "RecKind18", "RecKind19", "RecKind20", "RecKind21", "RecKind22", "RecKind23", "RecKind24", "RecKind25", "RecKind26", "RecKind27", "RecKind28", "RecKind29", "RecKind30"], + "path_materialization_required": false + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-01", "mutation", "inbound", "30-kinds"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->(e:ADEntity) WHERE e.objectid = $object_id RETURN id(r)", + "params": {"object_id": "rec-in"}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "target relationships deleted", "cypher": "MATCH ()-[r]->(e:ADEntity) WHERE e.objectid = $object_id RETURN count(r)", "params": {"object_id": "rec-in"}, "expected": {"scalar_int": 0}}, + {"name": "wrong endpoint survivor", "cypher": "MATCH ()-[r:RecKind02]->(e:ADEntity) WHERE e.objectid = $object_id RETURN count(r)", "params": {"object_id": "survivor"}, "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-02_outbound_30_kind_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH (s:ADEntity)-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->() WHERE s.objectid = $object_id DELETE r", + "params": {"object_id": "rec-out"}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": { + "root_predicate": "typed_endpoint_property", + "edge_kinds": ["RecKind01", "RecKind02", "RecKind03", "RecKind04", "RecKind05", "RecKind06", "RecKind07", "RecKind08", "RecKind09", "RecKind10", "RecKind11", "RecKind12", "RecKind13", "RecKind14", "RecKind15", "RecKind16", "RecKind17", "RecKind18", "RecKind19", "RecKind20", "RecKind21", "RecKind22", "RecKind23", "RecKind24", "RecKind25", "RecKind26", "RecKind27", "RecKind28", "RecKind29", "RecKind30"], + "path_materialization_required": false + }, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-02", "mutation", "outbound", "30-kinds"], + "write_scenario": { + "selection_cypher": "MATCH (s:ADEntity)-[r:RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30]->() WHERE s.objectid = $object_id RETURN id(r)", + "params": {"object_id": "rec-out"}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "target relationships deleted", "cypher": "MATCH (s:ADEntity)-[r]->() WHERE s.objectid = $object_id RETURN count(r)", "params": {"object_id": "rec-out"}, "expected": {"scalar_int": 0}}, + {"name": "wrong start survivor", "cypher": "MATCH (s:Source)-[r:RecKind02]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-04_large_high_match_object_id_list_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids DELETE r", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-high", "count": 2000, "include": ["rec-list"]}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "large_property_list", "edge_kinds": ["ADReconcile"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-04", "mutation", "large-list", "high-match"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids RETURN id(r)", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-high", "count": 2000, "include": ["rec-list"]}}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "selected list relationships deleted", "cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid = $object_id RETURN count(r)", "params": {"object_id": "rec-list"}, "expected": {"scalar_int": 0}}, + {"name": "unrelated relationship survives", "cypher": "MATCH ()-[r:Survivor]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-04_thousand_item_no_match_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids DELETE r", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-only", "count": 1000}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "large_property_list", "edge_kinds": ["ADReconcile"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-04", "mutation", "1000-list", "no-match"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:ADReconcile]->(e:ADEntity) WHERE e.objectid IN $object_ids RETURN id(r)", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-only", "count": 1000}}, + "affected_entity": "relationship", + "expected_matched": 0, + "expected_affected": 0, + "post_state": [ + {"name": "all list relationships survive", "cypher": "MATCH ()-[r:ADReconcile]->() RETURN count(r)", "expected": {"scalar_int": 2}} + ] + } + }, + { + "name": "REC-06_large_endpoint_id_list_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH ()-[r:DelegatedEnrollmentAgent]->(e:CertTemplate) WHERE id(e) IN $template_ids DELETE r", + "generated_node_list_params": {"template_ids": {"prefix": "scale-template", "count": 2000, "include": ["template"]}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "large_id_list", "edge_kinds": ["DelegatedEnrollmentAgent"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-06", "mutation", "large-id-list"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:DelegatedEnrollmentAgent]->(e:CertTemplate) WHERE id(e) IN $template_ids RETURN id(r)", + "generated_node_list_params": {"template_ids": {"prefix": "scale-template", "count": 2000, "include": ["template"]}}, + "affected_entity": "relationship", + "expected_matched": 2, + "expected_affected": 2, + "post_state": [ + {"name": "delegations deleted", "cypher": "MATCH ()-[r:DelegatedEnrollmentAgent]->() RETURN count(r)", "expected": {"scalar_int": 0}}, + {"name": "unrelated relationship survives", "cypher": "MATCH ()-[r:Survivor]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "REC-08_large_list_high_degree_detach_delete", + "dataset": "generated_reconciliation", + "category": "reconciliation_mutation", + "cypher": "MATCH (n:ADEntity) WHERE n.objectid IN $object_ids DETACH DELETE n", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-node", "count": 2000, "include": ["delete-target"]}}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "large_property_list", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["REC-08", "mutation", "large-list", "high-degree", "detach-delete"], + "write_scenario": { + "selection_cypher": "MATCH (n:ADEntity) WHERE n.objectid IN $object_ids RETURN id(n)", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-node", "count": 2000, "include": ["delete-target"]}}, + "affected_entity": "node", + "expected_matched": 1, + "expected_affected": 1, + "post_state": [ + {"name": "target node deleted", "cypher": "MATCH (n:ADEntity) WHERE n.objectid = $object_id RETURN count(n)", "params": {"object_id": "delete-target"}, "expected": {"scalar_int": 0}}, + {"name": "all incident relationships cascaded", "cypher": "MATCH ()-[r:Incident]->() RETURN count(r)", "expected": {"scalar_int": 0}}, + {"name": "decoy node survives", "cypher": "MATCH (n:ADEntity) WHERE n.objectid = $object_id RETURN count(n)", "params": {"object_id": "survivor"}, "expected": {"scalar_int": 1}} + ] + } + } + ] +} diff --git a/benchmark/testdata/scale/cases/scans_lookups.json b/benchmark/testdata/scale/cases/scans_lookups.json new file mode 100644 index 00000000..b3ae498e --- /dev/null +++ b/benchmark/testdata/scale/cases/scans_lookups.json @@ -0,0 +1,214 @@ +{ + "cases": [ + { + "name": "SCAN-01_dense_base_endpoint_relationship_ID_scan", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:ScanPostProcessed]->(e) WHERE s:ADBase AND e:AZBase RETURN id(r)", + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "start_base_kind", "terminal_predicate": "end_base_kind", "edge_kinds": ["ScanPostProcessed"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-01", "dense", "relationship-id", "projection-id-only"] + }, + { + "name": "SCAN-02_dense_non_Meta_relationship_hydration", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:TrackerA|TrackerB]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND NOT (e:Meta OR e:MetaDetail) RETURN r", + "expected": {"row_count": 256}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"root_predicate": "excluded_start_kinds", "terminal_predicate": "excluded_end_kinds", "edge_kinds": ["TrackerA", "TrackerB"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-02", "dense", "full-relationship", "projection-full-hydration"] + }, + { + "name": "SCAN-03_present_lastseen_selectivity", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:MigratedEdge]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND r.lastseen IS NOT NULL AND NOT (e:Meta OR e:MetaDetail) RETURN id(r)", + "expected": {"row_count": 64, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "excluded_start_kinds", "terminal_predicate": "relationship_property_presence", "edge_kinds": ["MigratedEdge"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-03", "null-missing", "selective-property"] + }, + { + "name": "SCAN-04_dense_raw_ownership_hydration", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s:Entity)-[r:OwnsRaw]->() RETURN r", + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"root_predicate": "start_entity_kind", "edge_kinds": ["OwnsRaw"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-04", "dense", "full-relationship"] + }, + { + "name": "SCAN-05_nine_kind_bound_end_inbound_scan", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s:Entity)-[r:ScanEdge01|ScanEdge02|ScanEdge03|ScanEdge04|ScanEdge05|ScanEdge06|ScanEdge07|ScanEdge08|ScanEdge09]->(e) WHERE id(e) = $target RETURN r, s", + "node_params": {"target": "scan-nine-kind-target"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": true, "properties": true}, + "shape": {"root_predicate": "start_entity_kind", "terminal_predicate": "bound_end_id", "edge_kinds": ["ScanEdge01", "ScanEdge02", "ScanEdge03", "ScanEdge04", "ScanEdge05", "ScanEdge06", "ScanEdge07", "ScanEdge08", "ScanEdge09"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-05", "nine-kinds", "dense-inbound", "full-direction"] + }, + { + "name": "SCAN-06_dense_shallow_IDs_and_kind_projection", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:LocalToComputer]->(e:Computer) RETURN id(s), id(r), type(r), id(e)", + "expected": {"row_count": 256, "result_kind": "shallow_ids_kind"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"terminal_predicate": "typed_end", "edge_kinds": ["LocalToComputer"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-06", "dense", "shallow-projection", "projection-shallow-ids-kind"] + }, + { + "name": "SCAN-07_dense_directed_ID_pairs", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:MemberOf|MemberOfLocalGroup]->(e) RETURN id(s), id(e)", + "expected": {"row_count": 256, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"edge_kinds": ["MemberOf", "MemberOfLocalGroup"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-07", "dense", "directed-pairs", "duplicate-endpoints"] + }, + { + "name": "SCAN-08_thousand_victim_IDs_scenario_A", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)", + "generated_node_list_params": {"victims": {"prefix": "scan-victim", "count": 1000}}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "three_start_kinds", "terminal_predicate": "large_end_id_list", "edge_kinds": ["GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL", "WritePublicInformation"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-08", "1000-list", "scenario-a", "dense"] + }, + { + "name": "SCAN-08_thousand_victim_IDs_scenario_B", + "dataset": "generated_scan_lookups", + "category": "relationship_scans", + "cypher": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL]->(e:Computer) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)", + "generated_node_list_params": {"victims": {"prefix": "scan-victim", "count": 1000}}, + "expected": {"row_count": 64, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "three_start_kinds", "terminal_predicate": "typed_large_end_id_list", "edge_kinds": ["GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["SCAN-08", "1000-list", "scenario-b", "selective-end-kind"] + }, + { + "name": "LOOKUP-02_repeated_exact_objectid_lookup", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Computer) WHERE n.objectid = $objectid RETURN id(n)", + "params": {"objectid": "S-1-5-21-scale"}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "exact_objectid", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-02", "exact-property", "multiple-hit"] + }, + { + "name": "LOOKUP-04_suffix_kind_and_domain_filter", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Group) WHERE n.objectid ENDS WITH $suffix AND n.domainsid = $domain RETURN id(n)", + "params": {"suffix": "-512", "domain": "S-1-5-21"}, + "expected": {"row_count": 64, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "suffix_and_equality", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-04", "suffix", "selectivity"] + }, + { + "name": "LOOKUP-05_repeated_case_insensitive_prefix", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Group) WHERE toLower(n.name) STARTS WITH $prefix RETURN id(n)", + "params": {"prefix": "remote desktop users"}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "case_insensitive_prefix", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-05", "case-insensitive", "repeated"] + }, + { + "name": "LOOKUP-09_thousand_ID_full_node_hydration", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n) WHERE id(n) IN $ids RETURN n", + "generated_node_list_params": {"ids": {"prefix": "lookup-id-target", "count": 1000}}, + "expected": {"row_count": 1000}, + "observes": {"paths": false, "nodes": true, "relationships": false, "properties": true}, + "shape": {"root_predicate": "large_node_id_list", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-09", "1000-list", "dense", "full-node", "projection-full-hydration"] + }, + { + "name": "LOOKUP-11_tenant_adjacency_thousand_property_list", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (s)-[:Contains]->(e:AZRole) WHERE id(s) = $tenant AND e.roletemplateid IN $roles RETURN e", + "node_params": {"tenant": "lookup-tenant"}, + "params": {"roles": {"$type": "string_list", "prefix": "role-template", "count": 1000}}, + "expected": {"row_count": 1000}, + "observes": {"paths": false, "nodes": true, "relationships": false, "properties": true}, + "shape": {"root_predicate": "bound_tenant", "terminal_predicate": "large_endpoint_property_list", "edge_kinds": ["Contains"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-11", "1000-list", "tenant-adjacency", "full-node"] + }, + { + "name": "LOOKUP-13_dense_suffix_bound_endpoint", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (s)-[:LocalToComputer]->(e) WHERE s.objectid ENDS WITH $suffix AND id(e) = $target RETURN s", + "params": {"suffix": "-555"}, + "node_params": {"target": "lookup-local-target"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": true, "relationships": false, "properties": true}, + "shape": {"root_predicate": "start_property_suffix", "terminal_predicate": "bound_end_id", "edge_kinds": ["LocalToComputer"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-13", "dense-inbound", "suffix", "bound-end"] + }, + { + "name": "LOOKUP-15_all_node_count", + "dataset": "generated_scan_lookups", + "category": "counts", + "cypher": "MATCH (n) RETURN count(n)", + "expected": {"row_count": 1, "scalar_int": 3774, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-15", "count", "dense-graph"] + }, + { + "name": "LOOKUP-15_all_relationship_count", + "dataset": "generated_scan_lookups", + "category": "counts", + "cypher": "MATCH ()-[r]->() RETURN count(r)", + "expected": {"row_count": 1, "scalar_int": 2408, "result_kind": "scalar"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-15", "count", "dense-graph"] + }, + { + "name": "LOOKUP-16_typed_four_property_NTLM_filter", + "dataset": "generated_scan_lookups", + "category": "lookups", + "cypher": "MATCH (n:Computer) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapavailable = true AND n.ldapsigning = false RETURN id(n)", + "params": {"domain": "S-1-5-21"}, + "expected": {"row_count": 128, "result_kind": "id_set"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "four_property_equalities", "terminal_predicate": "node_kind", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["LOOKUP-16", "typed", "four-properties", "dense"] + } + ] +} diff --git a/benchmark/testdata/scale/cases/shortest_paths.json b/benchmark/testdata/scale/cases/shortest_paths.json index b9539b36..9902a8c1 100644 --- a/benchmark/testdata/scale/cases/shortest_paths.json +++ b/benchmark/testdata/scale/cases/shortest_paths.json @@ -39,7 +39,13 @@ }, "expected": { "row_count": 1, - "result_kind": "path_set" + "result_kind": "path_set", + "path_rows": [ + { + "nodes": ["n1", "n2", "n3"], + "relationship_kinds": ["EdgeKind1", "EdgeKind2"] + } + ] }, "observes": { "paths": true, @@ -58,4 +64,3 @@ } ] } - diff --git a/benchmark/testdata/scale/cases/traversal.json b/benchmark/testdata/scale/cases/traversal.json index 2bab928d..86cf247c 100644 --- a/benchmark/testdata/scale/cases/traversal.json +++ b/benchmark/testdata/scale/cases/traversal.json @@ -83,16 +83,22 @@ "tags": ["path-materialization"] }, { - "name": "adcs_p1_endpoint_ids", - "dataset": "adcs_fanout", - "category": "bloodhound_search", - "cypher": "MATCH (n:Group) WHERE n.objectid = $objectid MATCH (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) RETURN id(ca), id(d)", + "name": "fixed_suffix_expansion_endpoint_ids", + "dataset": "fixed_suffix_expansion_fanout", + "category": "fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", "params": { - "objectid": "S-1-5-21-2643190041-1319121918-239771340-513" + "root_key": "fixed-suffix-fanout-root" }, "expected": { "row_count": 4, - "result_kind": "id_rows" + "result_kind": "id_rows", + "id_rows": [ + ["fse-head", "fse-terminal"], + ["fse-head", "fse-terminal"], + ["fse-head", "fse-terminal"], + ["fse-head", "fse-terminal"] + ] }, "observes": { "paths": false, @@ -103,24 +109,43 @@ "shape": { "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", - "edge_kinds": ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], + "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, + "max_depth": 16, "path_materialization_required": false }, "candidate_modes": ["postgres_sql", "local_traversal", "neo4j"], - "tags": ["bloodhound", "adcs", "id-only", "local-traversal-candidate"] + "tags": ["fixed-suffix-expansion", "fanout", "id-only", "local-traversal-candidate"] }, { - "name": "adcs_p1_path_observed", - "dataset": "adcs_fanout", - "category": "bloodhound_search", - "cypher": "MATCH (n:Group) WHERE n.objectid = $objectid MATCH p = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) RETURN p", + "name": "GFSE-BOUNDARY-cyclic-relationship-distinct-bag", + "dataset": "fixed_suffix_expansion_adversarial", + "category": "generated_fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH (root)-[:Expand*0..3]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)", + "params": {"root_key": "suffix-overflow-adversarial-root"}, + "expected": {"row_count": 68, "result_kind": "id_rows"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"qualification_split": "holdout", "fixture_tier": "normal", "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, "max_depth": 3, "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["normal-tier", "fixed-suffix-expansion-boundary", "suffix-overflow", "cycle", "relationship-distinct", "physical-bag-multiplicity", "noncanonical-logical-ids", "holdout"] + }, + { + "name": "fixed_suffix_expansion_path_observed", + "dataset": "fixed_suffix_expansion_fanout", + "category": "fixed_suffix_expansion", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = $root_key MATCH p = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p", "params": { - "objectid": "S-1-5-21-2643190041-1319121918-239771340-513" + "root_key": "fixed-suffix-fanout-root" }, "expected": { "row_count": 4, - "result_kind": "path_set" + "result_kind": "path_set", + "path_rows": [ + {"nodes": ["fse-root", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["EnterSuffix", "ContinueSuffix", "CompleteSuffix"]}, + {"nodes": ["fse-root", "fse-expansion-a", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"]}, + {"nodes": ["fse-root", "fse-expansion-b", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"]}, + {"nodes": ["fse-root", "fse-expansion-b", "fse-expansion-c", "fse-head", "fse-middle", "fse-terminal"], "relationship_kinds": ["Expand", "Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"]} + ] }, "observes": { "paths": true, @@ -131,13 +156,13 @@ "shape": { "root_predicate": "selective_property", "terminal_predicate": "fixed_suffix", - "edge_kinds": ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], + "edge_kinds": ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], "min_depth": 0, + "max_depth": 16, "path_materialization_required": true }, "candidate_modes": ["postgres_sql", "neo4j"], - "tags": ["bloodhound", "adcs", "path-materialization"] + "tags": ["fixed-suffix-expansion", "fanout", "path-materialization"] } ] } - diff --git a/benchmark/testdata/scale/cases/trust_pruning.json b/benchmark/testdata/scale/cases/trust_pruning.json new file mode 100644 index 00000000..bcf9cda8 --- /dev/null +++ b/benchmark/testdata/scale/cases/trust_pruning.json @@ -0,0 +1,133 @@ +{ + "cases": [ + { + "name": "TRUST-01_dense_same_forest_relationship_ids", + "dataset": "generated_trust_pruning", + "category": "trust_reconciliation", + "cypher": "MATCH (s:Domain)-[r:SameForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN id(r)", + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "typed_temporal_disjunction", "edge_kinds": ["SameForestTrust"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["TRUST-01", "dense", "relationship-ids", "temporal-or"] + }, + { + "name": "TRUST-02_dense_cross_forest_relationship_hydration", + "dataset": "generated_trust_pruning", + "category": "trust_reconciliation", + "cypher": "MATCH (s:Domain)-[r:CrossForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN r", + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": true, "properties": true}, + "shape": {"root_predicate": "typed_temporal_disjunction", "edge_kinds": ["CrossForestTrust"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["TRUST-02", "dense", "relationship-hydration", "temporal-or"] + }, + { + "name": "TRUST-03_directional_branch_local_kinds", + "dataset": "generated_trust_pruning", + "category": "trust_reconciliation", + "cypher": "MATCH (s:Domain)-[r]->(e:Domain) WHERE (id(s) = $forward_start AND id(e) = $forward_end AND r:AbuseTGTDelegation) OR (id(s) = $forward_end AND id(e) = $forward_start AND r:SpoofSIDHistory) RETURN id(r)", + "node_params": {"forward_start": "trust-late-a", "forward_end": "trust-late-b"}, + "expected": {"row_count": 2}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": false}, + "shape": {"root_predicate": "directional_id_disjunction", "edge_kinds": ["AbuseTGTDelegation", "SpoofSIDHistory"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["TRUST-03", "directional", "branch-local-kind", "relationship-ids"] + }, + { + "name": "PRUNE-01_dense_old_relationship_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH ()-[r]->() WHERE NOT (r:HasSession OR r:MetaIncludes) AND datetime(r.lastseen) < datetime($threshold) RETURN id(r)", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "expected": {"row_count": 128}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "kind_negation_and_temporal", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-01", "dense", "kind-negation", "relationship-ids"] + }, + { + "name": "PRUNE-02_dense_missing_or_old_session_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH ()-[r:HasSession]->() WHERE r.lastseen IS NULL OR datetime(r.lastseen) < datetime($threshold) RETURN id(r)", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "expected": {"row_count": 256}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "missing_or_temporal", "edge_kinds": ["HasSession"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-02", "dense", "missing-property", "relationship-ids"] + }, + { + "name": "PRUNE-03_dense_missing_or_old_node_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH (n:PruneCandidate) WHERE NOT n:Domain AND (n.lastseen IS NULL OR datetime(n.lastseen) < datetime($threshold)) RETURN id(n)", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "expected": {"row_count": 262}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "kind_negation_and_missing_or_temporal", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-03", "dense", "missing-property", "node-ids"] + }, + { + "name": "PRUNE-04_dense_orphan_sid_selection", + "dataset": "generated_trust_pruning", + "category": "pruning_selection", + "cypher": "MATCH (n) WHERE NOT n:Domain AND n.name IS NULL AND n.objectid STARTS WITH $sid_prefix RETURN id(n)", + "params": {"sid_prefix": "S-1-5"}, + "expected": {"row_count": 130}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "kind_negation_missing_name_prefix", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-04", "dense", "missing-property", "prefix", "node-ids"] + }, + { + "name": "PRUNE-05_dense_relationship_batch_delete_equivalent", + "dataset": "generated_trust_pruning", + "category": "pruning_mutation", + "cypher": "MATCH ()-[r:PruneBatch]->() WHERE r.remove = $flag DELETE r", + "params": {"flag": true}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "relationship_property", "edge_kinds": ["PruneBatch"], "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-05", "mutation", "direct-batch-equivalent"], + "write_scenario": { + "selection_cypher": "MATCH ()-[r:PruneBatch]->() WHERE r.remove = $flag RETURN id(r)", + "params": {"flag": true}, + "affected_entity": "relationship", + "expected_matched": 128, + "expected_affected": 128, + "post_state": [ + {"name": "selected relationships deleted", "cypher": "MATCH ()-[r:PruneBatch]->() WHERE r.remove = $flag RETURN count(r)", "params": {"flag": true}, "expected": {"scalar_int": 0}}, + {"name": "survivor relationship remains", "cypher": "MATCH ()-[r:PruneBatchSurvivor]->() RETURN count(r)", "expected": {"scalar_int": 1}} + ] + } + }, + { + "name": "PRUNE-06_high_degree_node_batch_delete_equivalent", + "dataset": "generated_trust_pruning", + "category": "pruning_mutation", + "cypher": "MATCH (n:PruneBatchNode) WHERE n.remove = $flag DETACH DELETE n", + "params": {"flag": true}, + "expected": {"result_kind": "mutation"}, + "observes": {"paths": false, "nodes": false, "relationships": false, "properties": true}, + "shape": {"root_predicate": "node_property_high_degree", "path_materialization_required": false}, + "candidate_modes": ["postgres_sql", "neo4j"], + "tags": ["PRUNE-06", "mutation", "high-degree", "cascade", "direct-batch-equivalent"], + "write_scenario": { + "selection_cypher": "MATCH (n:PruneBatchNode) WHERE n.remove = $flag RETURN id(n)", + "params": {"flag": true}, + "affected_entity": "node", + "expected_matched": 65, + "expected_affected": 65, + "post_state": [ + {"name": "selected nodes deleted", "cypher": "MATCH (n:PruneBatchNode) WHERE n.remove = $flag RETURN count(n)", "params": {"flag": true}, "expected": {"scalar_int": 0}}, + {"name": "high degree incident relationships cascaded", "cypher": "MATCH ()-[r:PruneIncident]->() RETURN count(r)", "expected": {"scalar_int": 0}}, + {"name": "unselected batch nodes survive", "cypher": "MATCH (n:PruneBatchNode) RETURN count(n)", "expected": {"scalar_int": 65}} + ] + } + } + ] +} diff --git a/benchmark/testdata/scale/protocols/asp_p4_i1_disconnected_preflight_v1.json b/benchmark/testdata/scale/protocols/asp_p4_i1_disconnected_preflight_v1.json new file mode 100644 index 00000000..8d1ac4d7 --- /dev/null +++ b/benchmark/testdata/scale/protocols/asp_p4_i1_disconnected_preflight_v1.json @@ -0,0 +1,105 @@ +{ + "schema": "asp-p4-i1-disconnected-preflight-v1", + "generation": "asp-p4-i1-disconnected-preflight-v1", + "status": "terminally_rejected_current_i1_arm", + "production_default": "ASP-A1-DAG", + "purpose": "training-only A1-versus-inline-I1 telemetry preflight for the selected all-shortest disconnected target; not a power study, performance qualification, or selector change", + "predecessor": { + "protocol": "asp-p4-open-baseline-v1", + "source_commit": "bf055b3aaeda1f887e652a399b065290db560236", + "artifact": ".coverage/p4-open-baseline-bf055b3/round-1.jsonl", + "artifact_sha256": "69c29c0a79e2ca566bbd54e533c896138a7770b245d030dc347ebc9413e6b6fe", + "selection": "GSPV2-TRAINING-disconnected-all-shortest-max64; four matched PostgreSQL/Neo4j median ratios were 7.0414939412490645, 5.8250091002483355, 10.559733195363082, and 5.227608712020583" + }, + "identities": { + "incumbent": "ASP-A1-DAG", + "candidate": "ASP-I1-U-DAG+MAT-M0", + "candidate_fallback": "ASP-A1-DAG", + "candidate_telemetry_schema": "asp-i1-guarded-v1", + "incumbent_telemetry_reader": "public.read_all_shortest_paths_a1_diagnostic_v1" + }, + "design": { + "modes": ["postgres_sql"], + "pool_size": 1, + "postgres_isolation": "repeatable_read", + "postgres_traversal_telemetry": "diagnostic", + "warmup_iterations": 1, + "timed_iterations": 5, + "rounds": 4, + "arm_orders": [ + ["A1", "I1"], + ["I1", "A1"], + ["A1", "I1"], + ["I1", "A1"] + ], + "cap_overrides_permitted": false, + "reference_and_concurrency_permitted": false + }, + "corpus": { + "target_cases": [ + "GSPV2-TRAINING-disconnected-all-shortest-max64" + ], + "adverse_control_cases": [ + "GSPV2-TRAINING-early-depth1-all-shortest-max16", + "GSPV2-TRAINING-early-depth2-all-shortest-max64", + "GSPV2-TRAINING-reconvergent-all-shortest-max16" + ], + "excluded": [ + "all generated_shortest_paths_v2 holdout cases", + "all generated_shortest_paths_v2 diagnostic and stress cases", + "all other open baseline cases", + "all ASP-B1 and ASP-B2 candidate executors", + "all historical ASP-I1 artifacts because they mixed protected holdouts" + ] + }, + "acceptance_requirements": { + "exact_public_path_multiset": true, + "incumbent_runtime_identity": "ASP-A1-DAG", + "incumbent_complete_a1_telemetry": true, + "candidate_runtime_identity": "ASP-I1-U-DAG+MAT-M0", + "candidate_runtime_branches": { + "GSPV2-TRAINING-disconnected-all-shortest-max64": "inline_no_path", + "adverse_controls": "inline_predecessor_dag" + }, + "candidate_complete_typed_inline_telemetry": true, + "candidate_fallback_disqualifies": true, + "complete_hydration_and_workspace_telemetry": true, + "missing_hidden_or_contradictory_counter_disqualifies": true + }, + "result": { + "source_commit": "307e62f4d0e102384752e031f9c2850d6a73dbfe", + "binary_sha256": "ce101efdc55d173176aaa221b3ca3a18b4d40e3fb3a970fa346fdc98c125557c", + "artifact_directory": ".coverage/p4-i1-disconnected-307e62f", + "a1_artifact": "a1.jsonl", + "a1_artifact_sha256": "03114fd4b2cfdc50959697eebbfe08e4932ab307c38845b88415d8a287a3778f", + "i1_artifact": "i1.jsonl", + "i1_artifact_sha256": "f147df180b8b351bf14ed83977105beaf48b96a822a143a33229afbb979083b6", + "capture_ledger": "capture-ledger.json", + "capture_ledger_sha256": "9baa5f3d25f47d5efd1a496ac266fee013edd419d9a696eb3f953e65c40a29f5", + "records": 32, + "timed_samples": 160, + "warmup_samples": 32, + "public_observations": "all 32 records exact; A1 and I1 public path multisets match in every case/round pair", + "candidate_telemetry": "all 16 I1 records have complete asp-i1-guarded-v1 telemetry, ASP-I1-U-DAG+MAT-M0 runtime identity, and zero fallback; disconnected records use inline_no_path and every control uses inline_predecessor_dag", + "capture_condition": "CPU governor powersave; diagnostic preflight only", + "pooled_timing_i1_to_a1": { + "GSPV2-TRAINING-disconnected-all-shortest-max64": {"median_ratio": 0.19176668726324123, "p95_ratio": 0.24634721331782422}, + "GSPV2-TRAINING-early-depth1-all-shortest-max16": {"median_ratio": 1.3645783077358358, "p95_ratio": 1.5156226026744484}, + "GSPV2-TRAINING-early-depth2-all-shortest-max64": {"median_ratio": 1.410286298627804, "p95_ratio": 1.0702295806459974}, + "GSPV2-TRAINING-reconvergent-all-shortest-max16": {"median_ratio": 1.200003319543841, "p95_ratio": 0.8959548837461973} + }, + "disposition": "ASP-I1-U-DAG+MAT-M0 is much faster on the selected disconnected no-path target but fails every frozen shallow/reconvergent adverse control at the pooled median. The P4 stop gate therefore terminally rejects the current I1 arm before power, broader timing, holdout, selector, or manifest work." + }, + "next_authorization": { + "requires": [ + "a distinct executor identity and separately frozen roster before reopening P4 candidate timing", + "a new clean source capture; this result may not be retuned, pooled, or repurposed as promotion evidence" + ], + "does_not_authorize": [ + "further ASP-I1-U-DAG+MAT-M0 timing under this P4 generation", + "formal power or performance qualification", + "protected holdout or diagnostic/stress capture", + "production selector, manifest, or default changes" + ] + } +} diff --git a/benchmark/testdata/scale/protocols/asp_p4_open_baseline_v1.json b/benchmark/testdata/scale/protocols/asp_p4_open_baseline_v1.json new file mode 100644 index 00000000..b481d679 --- /dev/null +++ b/benchmark/testdata/scale/protocols/asp_p4_open_baseline_v1.json @@ -0,0 +1,122 @@ +{ + "schema": "asp-p4-open-baseline-v1", + "generation": "asp-p4-open-baseline-v1", + "status": "completed_target_selected_pending_candidate_preflight_roster", + "production_default": "ASP-A1-DAG", + "purpose": "training-only cross-backend baseline and A1 telemetry inventory; not an ASP candidate comparison or performance qualification", + "baseline": { + "predecessor_commit": "7f5d0f9dcc7bd1a86dd2846e7180e06b7795c13c", + "note": "The retained P0 all-shortest ratio is a single protected shallow-diamond observation and cannot select a P4 target; the clean capture commit is recorded with the result." + }, + "identities": { + "postgres_incumbent": "ASP-A1-DAG", + "postgres_selector": "asp-static-v1", + "neo4j_reference": "neo4j-default" + }, + "design": { + "modes": ["postgres_sql", "neo4j"], + "pool_size": 1, + "postgres_isolation": "repeatable_read", + "postgres_traversal_telemetry": "diagnostic", + "warmup_iterations": 1, + "timed_iterations": 5, + "rounds": 4, + "backend_order": "GraphBench alternates the requested backend order on even rounds", + "postgres_force_shortest_executor": "ASP-A1-DAG", + "cap_overrides_permitted": false, + "reference_and_concurrency_permitted": false + }, + "corpus": { + "training_cases": [ + "GSPV2-NORMAL-outbound-all-shortest-depth3", + "GSPV2-TRAINING-early-depth1-all-shortest-max16", + "GSPV2-TRAINING-early-depth2-all-shortest-max64", + "GSPV2-TRAINING-early-depth3-all-shortest-max16", + "GSPV2-TRAINING-inbound-early-depth1-all-shortest-max16", + "GSPV2-TRAINING-inbound-early-depth3-all-shortest-max64", + "GSPV2-TRAINING-cycle-dead-tail-all-shortest-max64", + "GSPV2-TRAINING-reconvergent-all-shortest-max16", + "GSPV2-TRAINING-disconnected-all-shortest-max64" + ], + "excluded": [ + "all generated_shortest_paths_v2 holdout cases", + "all generated_shortest_paths_v2 diagnostic and stress cases", + "all legacy ASP-I1 archived captures because they mixed protected holdouts", + "all ASP-I1, ASP-B1, and ASP-B2 candidate executors" + ] + }, + "preflight_requirements": { + "exact_public_path_multiset": true, + "postgres_expected_runtime_identity": "ASP-A1-DAG", + "postgres_complete_all_shortest_telemetry": true, + "postgres_complete_hydration_and_workspace_telemetry": true, + "no_fallback_or_hidden_counter": true, + "neo4j_records_required": true + }, + "next_authorization": { + "requires": [ + "a separately frozen A1-versus-one-candidate P4 telemetry preflight roster for GSPV2-TRAINING-disconnected-all-shortest-max64", + "complete exact public observations and complete candidate telemetry for every preflight record", + "a new clean source commit before any follow-on candidate capture" + ], + "does_not_authorize": [ + "ASP-I1, ASP-B1, or ASP-B2 timing outside the separately frozen one-candidate P4 preflight", + "formal power or performance qualification", + "protected holdout or diagnostic/stress capture", + "production selector or manifest changes" + ] + }, + "prior_stop_result": { + "source_commit": "3a74d14be83f2c99b1694109d54840501ebbc3f5", + "binary_sha256": "4e47611c0e06d508e81011cc35d348a167c4c9a1d863095e3b72add821780c91", + "artifact": ".coverage/p4-open-baseline-3a74d14/round-1.jsonl", + "artifact_sha256": "b6c09fbd9dd1e46bd262af224469c12a9a69367f5ad7e504e853da5247fb53f6", + "records": 18, + "postgres_records": 9, + "neo4j_records": 9, + "timed_samples": 90, + "public_observations": "all 18 records exact; PostgreSQL reported ASP-A1-DAG with no fallback", + "stop_reason": "Every PostgreSQL diagnostic reported hidden_counters_unavailable because the outer Function Scan does not expose invocation-local traversal work counters. The remaining rounds were not run." + }, + "result": { + "source_commit": "bf055b3aaeda1f887e652a399b065290db560236", + "binary_sha256": "8a8cdd0998ef9391776ee4a6de3689f31f0a2133675bc96d9cffd95d6caceb31", + "artifact": ".coverage/p4-open-baseline-bf055b3/round-1.jsonl", + "artifact_sha256": "69c29c0a79e2ca566bbd54e533c896138a7770b245d030dc347ebc9413e6b6fe", + "records": 72, + "postgres_records": 36, + "neo4j_records": 36, + "timed_samples": 360, + "warmup_samples": 72, + "public_observations": "all 72 records exact; every PostgreSQL record reported ASP-A1-DAG, no fallback, and complete all-shortest, hydration, and workspace telemetry", + "capture_condition": "CPU governor powersave; descriptive target selection only, not performance qualification" + }, + "target_selection": { + "name": "GSPV2-TRAINING-disconnected-all-shortest-max64", + "selection_basis": "four matched backend rounds with PostgreSQL/Neo4j median ratio 5.8250091002483355 and p95 ratio 5.263006089799886", + "all_round_median_ratios": [7.0414939412490645, 5.8250091002483355, 10.559733195363082, 5.227608712020583], + "runtime_branch": "search_no_path", + "a1_receipt": { + "candidate_edges": 32, + "seen_peak": 33, + "predecessor_peak": 32, + "output_paths": 0, + "workspace_bytes": 229376 + }, + "excluded_from_matching": [ + "GSPV2-TRAINING-cycle-dead-tail-all-shortest-max64 has exact per-backend records but nonmatching cross-backend serialized observations" + ] + }, + "a1_diagnostic_prerequisite": { + "status": "implemented_validated_and_used_in_clean_recapture", + "reader": "public.read_all_shortest_paths_a1_diagnostic_v1", + "scope": "untimed Repeatable Read GraphBench replay on the timed physical PostgreSQL connection; separate from B1/B2 telemetry", + "unarmed_executor_behavior": "one local GUC check and no diagnostic workspace or telemetry-table operations", + "validation": [ + "complete shallow, recursive, reconvergent, inbound, and no-path A1 receipts", + "stale and contradictory receipt rejection", + "session isolation, cancellation rollback, and backend reuse" + ], + "next_step": "freeze the selected target's distinct A1-versus-one-candidate preflight roster" + } +} diff --git a/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1.json b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1.json new file mode 100644 index 00000000..6e5b5a57 --- /dev/null +++ b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1.json @@ -0,0 +1,99 @@ +{ + "schema": "p5-adjacency-materialization-feasibility-v1", + "generation": "p5-adjacency-materialization-feasibility-v1", + "status": "implemented_validated_pending_clean_capture", + "purpose": "prospective shadow-only feasibility measurement for a graph-scoped directed adjacency materialization; not a Cypher executor, selector, read-performance qualification, or production schema change", + "predecessor": { + "source_commit": "6a0a252bad44f1a73b3fdace9de8d84d7fed11f5", + "inventory": ".coverage/p5-feasibility-4db030c/inventory.json", + "inventory_sha256": "6913d46209601b5bd8bd955b28738f1040dc6c14a514d95aeae081d2977ffb92", + "selection_basis": "The base edge layout already has outgoing/incoming covering indexes, while a topology synopsis lacks a mutation-epoch/cache contract. The locally testable shadow materialization was selected only for its feasibility inventory." + }, + "architecture": { + "name": "P5-ADJACENCY-MATERIALIZATION-V1", + "relation": "public.p5_adjacency_v1", + "layout": "one graph-partitioned relation with exactly two rows per base edge: outbound anchor=start_id/neighbor=end_id and inbound anchor=end_id/neighbor=start_id", + "columns": ["graph_id", "direction", "anchor_id", "neighbor_id", "edge_id", "kind_id"], + "lookup_index": "(graph_id, direction, anchor_id, kind_id, edge_id) include (neighbor_id)", + "maintenance": "same-transaction edge insert/delete maintenance; edge property-only updates do not rewrite adjacency; node-delete cascade and graph drop must remove corresponding rows", + "read_boundary": "no Cypher translator, executor, policy, cache key, or production selector may read this relation" + }, + "baseline": { + "relation": "base edge partitions", + "outbound_index": "(start_id, kind_id) include (id, end_id)", + "inbound_index": "(end_id, kind_id) include (id, start_id)", + "comparison": "matched clean fixture reloads with and without the shadow materialization; alternate condition order at the block level" + }, + "design": { + "database": "postgresql_disposable_only", + "source_state": "clean committed tree", + "rounds": 4, + "warmup_iterations": 1, + "timed_iterations": 5, + "isolation": "repeatable_read_for_read_probes", + "write_measurement": "each warm-up and timed mutation runs in a transaction that validates state and rolls back", + "wal_measurement": "record pg_current_wal_lsn before and after each committed setup/mutation calibration only while autovacuum is quiescent; record EXPLAIN (ANALYZE, WAL) statement-local mutation bytes as the attributed mutation value and retain the LSN delta as a cross-check outside timed Cypher observations", + "read_measurement": "raw parameterized adjacency lookup only after exact shadow-state validation; no public Cypher result may use the relation", + "cap_overrides_permitted": false, + "protected_case_access_permitted": false + }, + "mutation_roster": { + "fixture": "testutil.NewDirectWriteScaleFixture", + "sizes": [1, 1000, 2000], + "operations": [ + "batch relationship create", + "relationship upsert/conflict property merge", + "relationship property-only update", + "batched relationship delete", + "batched node delete with incident-edge cascade", + "graph clear/reload and graph drop" + ], + "state_oracles": [ + "every base edge has exactly one outbound and one inbound shadow row", + "every shadow row maps to exactly one base edge with matching graph, kind, anchor, neighbor, and direction", + "property-only edge updates preserve the same shadow-row identity", + "rollback, cancellation, pool reuse, reload, and graph drop leave no committed stale rows" + ] + }, + "resource_roster": [ + "per-operation elapsed time and p50/p95 distribution", + "WAL bytes from LSN deltas", + "shadow heap and index bytes", + "base edge heap and index bytes", + "read-probe elapsed time, plan, buffers, and result cardinality", + "trigger/maintenance execution count and rollback outcome" + ], + "required_outcome": { + "exact_shadow_state": true, + "no_committed_stale_rows": true, + "separate_write_wal_storage_and_read_measurements": true, + "same_connection_cancellation_and_pool_reuse": true, + "checksummed_clean_source_artifacts": true, + "no_automatic_budget_or_candidate_pass": true + }, + "implementation": { + "status": "opt_in_shadow_only", + "install": "query.On(tx).InstallP5AdjacencyShadow", + "remove": "query.On(tx).DropP5AdjacencyShadow", + "capture_runner": "graphbench -p5-adjacency-feasibility-output ; fixed four counterbalanced blocks, one warm-up, five rollback-only timed samples, autovacuum-quiescent setup LSN calibration plus statement-local EXPLAIN WAL mutation calibration, and raw SQL read probes only", + "normal_schema_change": "none; schema_up.sql and normal driver startup do not install or read p5_adjacency_v1", + "validation": [ + "unit SQL boundary test", + "PostgreSQL lifecycle test for backfill, insert, endpoint update, property-only update, node cascade, rollback, cancellation, pool reuse, graph deletion, and removal" + ] + }, + "next_authorization": { + "requires": [ + "all frozen shadow-state and lifecycle oracles pass", + "a complete resource report that separately reports base and shadow write, WAL, storage, and raw read values", + "a separately frozen budget decision before any Cypher candidate or selector experiment" + ], + "does_not_authorize": [ + "Cypher translation or executor reads from p5_adjacency_v1", + "production schema deployment", + "automatic policy or cache-key changes", + "protected corpus timing", + "a performance or promotion claim" + ] + } +} diff --git a/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1_rejection.json b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1_rejection.json new file mode 100644 index 00000000..e375419b --- /dev/null +++ b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1_rejection.json @@ -0,0 +1,25 @@ +{ + "schema": "p5-adjacency-materialization-feasibility-v1-rejection", + "generation": "p5-adjacency-materialization-feasibility-v1", + "source_commit": "5b6332184320bea2f1cb2cef24264f8e88d35b78", + "protocol_sha256": "497ec91c9f1ffbd922b9ec5b32ccff4374e95c06cbec9a4f0ff9ee366338eea1", + "report_sha256": "9f9f0e1703cb06522b3b8532a932a6dd4b00a832fde3967bae6a15c83ce3cd7d", + "completed_oracles": { + "conditions": 24, + "committed_calibrations": 42, + "cancellation_and_pool_reuse": true, + "shadow_cleanup": true, + "autovacuum_quiescent": true + }, + "failed_requirement": { + "name": "attributed_trigger_maintenance_wal", + "reason": "EXPLAIN (ANALYZE, WAL) reports plan-node WAL only and omits row-trigger maintenance writes. In an isolated 1,000-edge shadow delete, EXPLAIN reported 54,000 bytes while pg_stat_statements recorded 162,000 bytes for the same top-level statement." + }, + "consequence": { + "resource_report_accepted": false, + "budget_decision_created": false, + "cypher_candidate_authorized": false, + "successor_protocol": "p5-adjacency-materialization-feasibility-v2" + }, + "terminal": true +} diff --git a/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2.json b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2.json new file mode 100644 index 00000000..ecd49e78 --- /dev/null +++ b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2.json @@ -0,0 +1,83 @@ +{ + "schema": "p5-adjacency-materialization-feasibility-v2", + "generation": "p5-adjacency-materialization-feasibility-v2", + "status": "implemented_validated_pending_clean_capture", + "purpose": "prospective shadow-only feasibility measurement for a graph-scoped directed adjacency materialization; not a Cypher executor, selector, read-performance qualification, or production schema change", + "predecessor": { + "generation": "p5-adjacency-materialization-feasibility-v1", + "rejection": "benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1_rejection.json", + "reason": "V1 EXPLAIN plan WAL omitted row-trigger maintenance writes, so its completed capture cannot supply attributable physical WAL evidence." + }, + "architecture": { + "name": "P5-ADJACENCY-MATERIALIZATION-V1", + "relation": "public.p5_adjacency_v1", + "layout": "one graph-partitioned relation with exactly two rows per base edge: outbound anchor=start_id/neighbor=end_id and inbound anchor=end_id/neighbor=start_id", + "columns": ["graph_id", "direction", "anchor_id", "neighbor_id", "edge_id", "kind_id"], + "lookup_index": "(graph_id, direction, anchor_id, kind_id, edge_id) include (neighbor_id)", + "maintenance": "same-transaction edge insert/delete maintenance; edge property-only updates do not rewrite adjacency; node-delete cascade and graph drop must remove corresponding rows", + "read_boundary": "no Cypher translator, executor, policy, cache key, or production selector may read this relation" + }, + "baseline": { + "relation": "base edge partitions", + "outbound_index": "(start_id, kind_id) include (id, end_id)", + "inbound_index": "(end_id, kind_id) include (id, start_id)" + }, + "roster": { + "fixture": "testutil.NewDirectWriteScaleFixture", + "sizes": [1, 1000, 2000], + "operations": [ + "batch_relationship_create", + "relationship_upsert_conflict_merge", + "relationship_property_only_update", + "batched_relationship_delete", + "batched_node_delete_cascade", + "graph_clear_reload", + "graph_drop" + ], + "conditions": ["base", "shadow"], + "rounds": 4, + "warmup_iterations": 1, + "timed_iterations": 5, + "isolation": "repeatable_read_for_read_probes", + "write_measurement": "each warm-up and timed mutation runs in a transaction that validates state and rolls back", + "wal_measurement": "for each committed calibration, an operation-specific no-op CTE marker identifies the ordinary top-level mutation in pg_stat_statements; its before/after one-call delta records WAL records, full-page images, and bytes including trigger maintenance. The capture may create and later remove pg_stat_statements only in the disposable database, and requires PostgreSQL to preload it. Quiescent pg_current_wal_lsn setup and mutation deltas remain diagnostic cross-checks only.", + "read_measurement": "raw parameterized adjacency lookup only after exact shadow-state validation; no public Cypher result may use the relation", + "cap_overrides_permitted": false, + "protected_case_access_permitted": false + }, + "state_oracles": { + "exact_two_rows_per_base_edge": true, + "property_only_update_preserves_shadow_row_identity": true, + "rollback_restores_fixture": true, + "cancellation_and_pool_reuse": true, + "graph_cleanup": true, + "no_cypher_read_path": true, + "checksummed_clean_source_artifacts": true, + "no_automatic_budget_or_candidate_pass": true + }, + "implementation": { + "status": "opt_in_shadow_only", + "install": "query.On(tx).InstallP5AdjacencyShadow", + "remove": "query.On(tx).DropP5AdjacencyShadow", + "capture_runner": "graphbench -p5-adjacency-feasibility-output ; fixed four counterbalanced blocks, one warm-up, five rollback-only timed samples, temporary pg_stat_statements statement-WAL calibration, quiescent LSN diagnostics, and raw SQL read probes only", + "normal_schema_change": "none; schema_up.sql and normal driver startup do not install or read p5_adjacency_v1", + "validation": [ + "unit SQL boundary test", + "PostgreSQL lifecycle test for backfill, insert, endpoint update, property-only update, node cascade, rollback, cancellation, pool reuse, graph deletion, and removal" + ] + }, + "next_authorization": { + "requires": [ + "all frozen shadow-state and lifecycle oracles pass", + "a complete resource report that separately reports base and shadow write, pg_stat_statements WAL, storage, and raw read values", + "a separately frozen budget decision before any Cypher candidate or selector experiment" + ], + "does_not_authorize": [ + "Cypher translation or executor reads from p5_adjacency_v1", + "production schema deployment", + "automatic policy or cache-key changes", + "protected corpus timing", + "a performance or promotion claim" + ] + } +} diff --git a/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2_disposition.json b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2_disposition.json new file mode 100644 index 00000000..dea3b8e3 --- /dev/null +++ b/benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2_disposition.json @@ -0,0 +1,48 @@ +{ + "schema": "p5-adjacency-materialization-feasibility-v2-disposition", + "generation": "p5-adjacency-materialization-feasibility-v2", + "report": { + "sha256": "c8c4d7f88d0904bd00d1b85355f2f5806dcb74cf84a41d80237bf081a7222d04", + "source_commit": "d257cd921134970583acea19c960f279f4d8fd5a", + "protocol_sha256": "f291f062c5099a1648ed6999db91ee4337aee2a3aaffa5f5d90872206446aee4", + "source_archive_sha256": "b6b2e40e1e12039fe9990bd2a84882ab08118fbf84697f8ad59392d843118063", + "captured_at": "2026-08-18T22:50:32.030359211Z" + }, + "capture_validation": { + "passed": true, + "conditions": 24, + "committed_calibrations": 42, + "all_statement_wal_single_call": true, + "all_statement_wal_positive": true, + "all_lsn_cross_checks_quiescent": true, + "cancellation_and_pool_reuse": true, + "cleanup_complete": true + }, + "physical_evidence": { + "storage_combined_over_base": { + "targets_1000": 1.9171974522292994, + "targets_2000": 1.9489795918367347 + }, + "statement_wal_shadow_over_base": { + "batch_relationship_create_targets_2000": 2.2165474653460677, + "batched_relationship_delete_targets_2000": 3, + "batched_node_delete_cascade_targets_2000": 2.4236811990934997, + "graph_clear_reload_targets_2000": 2.4498487823236283, + "graph_drop_targets_2000": 2.4492160409045787 + }, + "timing_shadow_over_base_upper_median": { + "batch_relationship_create_targets_2000": 8.61297789497717, + "batched_relationship_delete_targets_2000": 1324.2618967987025, + "batched_node_delete_cascade_targets_2000": 524.4560213961504, + "graph_clear_reload_targets_2000": 639.7219324653876, + "graph_drop_targets_2000": 558.9572434471479 + }, + "raw_lookup_is_not_a_candidate": true + }, + "decision": { + "candidate_authorized": false, + "budget_decision_created": false, + "reason": "The shadow relation adds nearly one base-relation's worth of storage, doubles or triples attributed WAL for structural writes, and creates two-to-four orders of magnitude destructive-write latency regressions. Raw lookup probes do not offset those physical maintenance costs and are not Cypher results.", + "terminal": true + } +} diff --git a/benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v1.json b/benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v1.json new file mode 100644 index 00000000..1b60f4ef --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v1.json @@ -0,0 +1,101 @@ +{ + "schema": "sp-bidirectional-p3-preflight-v1", + "generation": "sp-bidirectional-p3-preflight-v1", + "status": "superseded_before_direct_floor_capture", + "production_default": "off", + "purpose": "telemetry and component-boundary readiness only; not a performance qualification", + "identities": { + "incumbent_distance": "SP-S4-C-D", + "incumbent_witness": "SP-S4-C-WE+MAT-M0", + "reference_distance": "SP-S3-U-D", + "reference_witness": "SP-S3-U-E+MAT-M0", + "b1_distance": "SP-B1-C-ALT-NODE-D", + "b1_witness": "SP-B1-C-ALT-NODE-WE+MAT-M0", + "b2_distance": "SP-B2-C-MIN-LEVEL-D", + "b2_witness": "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "direct_floor": "SP-S0-DIRECT", + "diagnostic_source": "public.read_bidirectional_shortest_path_diagnostic_v1" + }, + "baseline": { + "source_commit": "57be1681140a2642639df0c06f7167bc17203e9b", + "round1_path": ".coverage/p0-clean-57be168-round1.jsonl", + "round1_sha256": "5cd14dc4b13008f5e307d44a16c56ff608eb79596b2ecaddf59d1eb70c31c6a1", + "round2_path": ".coverage/p0-clean-57be168-round2.jsonl", + "round2_sha256": "3bb71d1951b66559677abd4bba5441d844567269e6c1b1694cc95b67b4bc1f4d" + }, + "design": { + "pool_size": 1, + "isolation": "repeatable_read", + "traversal_telemetry": "diagnostic", + "warmup_iterations": 1, + "timed_iterations": 5, + "primary_rounds": 4, + "primary_arm_order": [ + ["S4", "B1", "S3", "B2"], + ["B1", "B2", "S4", "S3"], + ["B2", "S3", "B1", "S4"], + ["S3", "S4", "B2", "B1"] + ], + "direct_floor_order": ["S4", "S0", "S0", "S4"], + "cap_overrides_permitted": false, + "reference_and_concurrency_permitted": false + }, + "corpus": { + "target_cases": [ + "GSP-D08-F001_distance_inbound", + "GSP-D08-F001_path_inbound", + "GSP-D64-F1000_path" + ], + "control_cases": [ + "GSP-D16-F016_distance", + "GSP-D16-F016_path", + "GSP-D04-F128_disconnected", + "GSP-D04-F128_path_disconnected", + "GSP-D02-F016_distance_cycle", + "GSP-D02-F016_path_cycle", + "GSP-D02-F016_distance_self_loop", + "GSP-D02-F016_path_self_loop", + "GSP-D01-F016_distance_parallel", + "GSP-D01-F016_path_parallel", + "shortest_distance_bound_pair", + "one_shortest_path_bound_pair" + ], + "direct_floor_cases": [ + "GSP-D01-F001_distance", + "GSP-D01-F001_path" + ], + "excluded": [ + "generated_shortest_paths_v2 protected training and holdout declarations", + "generated_sp_i1_inbound_v1 declarations", + "generated_sp_i2_distance_v1 declarations", + "generated_sp_i2_distance_v2 declarations" + ] + }, + "readiness_requirements": { + "exact_public_observations": true, + "expected_runtime_identity": true, + "expected_scheduler": true, + "one_search_call": true, + "complete_bidirectional_counters": true, + "complete_path_hydration_counters_for_witnesses": true, + "measured_workspace_high_water": true, + "measured_plan_buffers_temp_and_wal": true, + "fallback_disqualifies": true, + "missing_or_hidden_counter_disqualifies": true, + "unattributed_component_disqualifies": true + }, + "next_authorization": { + "requires": [ + "all frozen preflight records exact", + "all B1/B2 target and control records have complete telemetry", + "separate component-boundary implementation for workspace reset, temporary-table access, search, witness recovery, hydration, and decoding", + "a separately frozen power simulation calibrated from the preflight trace" + ], + "does_not_authorize": [ + "production selector", + "formal performance tournament", + "protected holdout access" + ] + }, + "disposition": "The primary S3/S4/B1/B2 records were diagnostic-only. The direct-floor schedule named nonexistent SP-S4-C-DIRECT arms, so V1 cannot complete or qualify. V2 has distinct observation-specific direct-floor arms." +} diff --git a/benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v2.json b/benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v2.json new file mode 100644 index 00000000..be6aecc4 --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v2.json @@ -0,0 +1,100 @@ +{ + "schema": "sp-bidirectional-p3-preflight-v2", + "generation": "sp-bidirectional-p3-preflight-v2", + "status": "terminally_rejected_current_b1_b2_arms", + "production_default": "off", + "purpose": "telemetry and component-boundary readiness only; not a performance qualification", + "supersedes": "sp-bidirectional-p3-preflight-v1", + "identities": { + "incumbent_distance": "SP-S4-C-D", + "incumbent_witness": "SP-S4-C-WE+MAT-M0", + "reference_distance": "SP-S3-U-D", + "reference_witness": "SP-S3-U-E+MAT-M0", + "b1_distance": "SP-B1-C-ALT-NODE-D", + "b1_witness": "SP-B1-C-ALT-NODE-WE+MAT-M0", + "b2_distance": "SP-B2-C-MIN-LEVEL-D", + "b2_witness": "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "direct_floor": "SP-S0-DIRECT", + "diagnostic_source": "public.read_bidirectional_shortest_path_diagnostic_v1" + }, + "baseline": { + "source_commit": "57be1681140a2642639df0c06f7167bc17203e9b", + "round1_path": ".coverage/p0-clean-57be168-round1.jsonl", + "round1_sha256": "5cd14dc4b13008f5e307d44a16c56ff608eb79596b2ecaddf59d1eb70c31c6a1", + "round2_path": ".coverage/p0-clean-57be168-round2.jsonl", + "round2_sha256": "3bb71d1951b66559677abd4bba5441d844567269e6c1b1694cc95b67b4bc1f4d" + }, + "design": { + "pool_size": 1, + "isolation": "repeatable_read", + "traversal_telemetry": "diagnostic", + "warmup_iterations": 1, + "timed_iterations": 5, + "primary_rounds": 4, + "primary_arm_order": [ + ["S4", "B1", "S3", "B2"], + ["B1", "B2", "S4", "S3"], + ["B2", "S3", "B1", "S4"], + ["S3", "S4", "B2", "B1"] + ], + "direct_floor_rounds_per_observation": 2, + "direct_floor_arm_order": [["S4", "S0"], ["S0", "S4"]], + "cap_overrides_permitted": false, + "reference_and_concurrency_permitted": false + }, + "corpus": { + "target_cases": ["GSP-D08-F001_distance_inbound", "GSP-D08-F001_path_inbound", "GSP-D64-F1000_path"], + "control_cases": [ + "GSP-D16-F016_distance", "GSP-D16-F016_path", "GSP-D04-F128_disconnected", "GSP-D04-F128_path_disconnected", + "GSP-D02-F016_distance_cycle", "GSP-D02-F016_path_cycle", "GSP-D02-F016_distance_self_loop", "GSP-D02-F016_path_self_loop", + "GSP-D01-F016_distance_parallel", "GSP-D01-F016_path_parallel", "shortest_distance_bound_pair", "one_shortest_path_bound_pair" + ], + "direct_floor_distance_cases": ["GSP-D01-F001_distance"], + "direct_floor_witness_cases": ["GSP-D01-F001_path"], + "excluded": [ + "generated_shortest_paths_v2 protected training and holdout declarations", + "generated_sp_i1_inbound_v1 declarations", + "generated_sp_i2_distance_v1 declarations", + "generated_sp_i2_distance_v2 declarations" + ] + }, + "readiness_requirements": { + "exact_public_observations": true, + "expected_runtime_identity": true, + "expected_scheduler": true, + "one_search_call": true, + "complete_bidirectional_counters": true, + "complete_path_hydration_counters_for_witnesses": true, + "measured_workspace_high_water": true, + "measured_plan_buffers_temp_and_wal": true, + "fallback_disqualifies": true, + "missing_or_hidden_counter_disqualifies": true, + "unattributed_component_disqualifies": true + }, + "next_authorization": { + "requires": [ + "all frozen preflight records exact", + "all B1/B2 target and control records have complete telemetry", + "separate component-boundary implementation for workspace reset, temporary-table access, search, witness recovery, hydration, and decoding", + "a separately frozen power simulation calibrated from the preflight trace" + ], + "does_not_authorize": ["production selector", "formal performance tournament", "protected holdout access"] + }, + "result": { + "source_commit": "d77409674d6da4b00c3e379356955a5678dccbae", + "binary_sha256": "e08dd6d7f95b83421d91e3af19e7462b35d56b3a704b26d11d2e200d44d57ae4", + "artifact_directory": ".coverage/p3-preflight-d774096", + "primary_artifacts": 32, + "direct_floor_artifacts": 8, + "exact_records": 248, + "timed_samples": 1240, + "capture_ledger_sha256": "edd64b293ea0bb8a19fac41ee8e58d7042a511090b8043d5f85736fa4a2b567a", + "b1_b2_telemetry": "120 records exact with complete invocation-local counters, zero fallback, and matching runtime identity", + "target_median_ratio_to_s4": { + "GSP-D08-F001_distance_inbound": {"b1": 4.587563248375879, "b2": 3.265546008201876}, + "GSP-D08-F001_path_inbound": {"b1": 4.652223697591718, "b2": 3.7439601040489365}, + "GSP-D64-F1000_path": {"b1": 8.125883158637967, "b2": 4.190687443553809} + }, + "disposition": "Both existing compact B1/B2 function-workspace arms fail every frozen target before a power study. Retain S4 and S3 references; do not add component work, formal timing, a selector, or a holdout for these identities." + } +} diff --git a/benchmark/testdata/scale/protocols/sp_i2_distance_v1_rejection.json b/benchmark/testdata/scale/protocols/sp_i2_distance_v1_rejection.json new file mode 100644 index 00000000..c2ab3505 --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_i2_distance_v1_rejection.json @@ -0,0 +1,17 @@ +{ + "schema": "sp-i2-terminal-rejection-v1", + "generation": "sp-i2-distance-v1", + "executor": "SP-I2-C-D", + "policy": "sp-i2-distance-guarded-v1", + "selector": "sp-static-v8-hidden-fanin", + "source_commit": "3865cbc57758b7b20b7ffe431f27235873422eed", + "discovery_report_sha256": "f80b0f54624de79e9161673f7c9971662bcd5286bf70829176febc6de2681309", + "failed_gate": { + "metric": "p95_ratio_upper", + "observed": 1.2528773826285173, + "limit": 1.05 + }, + "freeze_created": false, + "holdout_opened": false, + "terminal": true +} diff --git a/benchmark/testdata/scale/protocols/sp_i2_distance_v2.json b/benchmark/testdata/scale/protocols/sp_i2_distance_v2.json new file mode 100644 index 00000000..3fb90e07 --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_i2_distance_v2.json @@ -0,0 +1,143 @@ +{ + "schema": "sp-i2-tail-protocol-v2", + "generation": "sp-i2-distance-v2", + "status": "terminated_inadequate_power", + "production_default": "off", + "identities": { + "executor": "SP-I2-C-D-V2", + "policy": "sp-i2-distance-guarded-v2", + "selector": "sp-static-v9-hidden-fanin-tail", + "fallback_executor": "SP-S4-C-D", + "fallback_internal_executor": "SP-S3-U-E+MAT-M0", + "training_tag": "sp-i2-distance-v2-training", + "holdout_tag": "sp-i2-distance-v2-holdout", + "development_tag": "sp-i2-distance-v2-development", + "qualification_schema": "sp-i2-tail-qualification-v2", + "freeze_schema": "sp-i2-tail-freeze-v2", + "aa_schema": "sp-i2-tail-aa-v2", + "promotion_manifest_schema": 3, + "rollback_switch": "DisableInlineSPDistance", + "statistical_implementation": "sp-i2-hier-bootstrap-v2/chacha8-sha256" + }, + "development_executors": [ + "SP-I2-C-D-V2-E0", + "SP-I2-C-D-V2-E1", + "SP-I2-C-D-V2-E1D", + "SP-I2-C-D-V2-E1P", + "SP-I2-C-D-V2-E1DP" + ], + "selected_architecture": "E1", + "limits": { + "state_rows": 100000, + "frontier_rows": 100000, + "maximum_depth": 64, + "minimum_depth": 1 + }, + "design": { + "seed": 1, + "confidence_level": 0.975, + "bootstrap_replicates": 100000, + "rounds": 40, + "ordinary_warmups": 25, + "attested_stabilizations": 1, + "timed_samples_per_round": 100, + "pool_size": 1, + "isolation": "repeatable_read", + "arm_order": "odd_a_then_b_even_b_then_a" + }, + "gates": { + "target_median_ratio_upper": 0.95, + "target_median_saving_lower_us": 100, + "control_median_ratio_upper": 1.10, + "control_median_overhead_upper_us": 100, + "p95_ratio_upper": 1.05, + "control_p95_overhead_upper_us": 100, + "aa_equivalence_ratio": 1.05, + "aa_first_position_ratio_upper": 1.10, + "aa_first_position_overhead_upper_us": 100, + "session_first_p95_ratio_upper": 1.25, + "session_first_p95_overhead_upper_us": 250 + }, + "operational_design": { + "blocks": 40, + "fresh_sessions_per_arm_case_block": 25, + "samples_per_arm_case": 1000, + "plan_cache_mode": "auto" + }, + "bootstrap": { + "domain": "sp-i2-tail-bootstrap-v2", + "case_order": ["dataset", "case"], + "ratio_scale": "log", + "lower_percentile": 0.0125, + "upper_percentile": 0.9875, + "quantile": "nearest_rank", + "round_resampling": "paired", + "within_round_resampling": "independent_by_arm" + }, + "simulation": { + "implementation": "sp-i2-power-simulation-v2/chacha8-sha256-normal-pivot", + "runs_per_scenario": 20000, + "wilson_confidence": 0.95, + "required_power_lower": 0.90, + "required_coverage": 0.975, + "p95_boundary_false_pass_upper": 0.015, + "decision_false_pass_upper": 0.0275, + "trace_rescaling_transform": "piecewise_log_quantile_anchor_then_paired_empirical_round_drift", + "source_commit": "3865cbc57758b7b20b7ffe431f27235873422eed", + "baseline_trace_sha256": "ac3ceb27ee92e3f4e21e3994ff9ee82d483b8081e9d44ddcef8e695ffdb1b6d0", + "candidate_trace_sha256": "f6d79e81bdaafedaa95568d57140c14e0808fbb6fc261387abc916081137785a", + "p50_round_drift": [1.0549272325683534, 0.9553178433721615, 1.0824521582712268, 0.8532441514494672, 0.99483899482769, 1.1475597655779528, 1.0347648948072776, 1.1314630755587145, 1.1245150395381482, 1.0711601501580643, 1.0339160440070687, 1.0999062206416197, 1.0641741066579358, 1.0163905551993713, 0.8966331506031865, 0.9042675610985601, 0.8323795952194755, 0.911215305974392, 0.9634264387781224, 0.9156526124180061], + "p95_round_drift": [1.0971737648364577, 0.9279078355320133, 0.8946432636451367, 0.9216323659919106, 0.9510067432444674, 1.0664886537522253, 1.103054457976633, 1.011789884513237, 1.0810830535670386, 1.181220892042835, 1.0682390242574165, 1.1260231541796215, 1.0077572383123325, 1.0566105552995742, 0.8842778982677586, 0.9341845524461532, 0.9249072950605768, 0.8916442634975811, 0.9987108508828467, 0.9457081910432685], + "log_standard_errors": {"pooled": 0.025959, "order_stratum": 0.036712, "first_position": 0.036712}, + "absolute_standard_errors_us": {"pooled": 59.338, "order_stratum": 83.917, "first_position": 83.917}, + "scenarios": [ + {"name":"aa_identity","kind":"aa_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"5a1a1fbb2242081a8afd1f06c2e9a23356cd52ac126d1506e5d9c93a53e07c81"}, + {"name":"aa_upper_margin","kind":"aa_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1050,"candidate_p95_us":2100,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"176d7120be1d53de6daf77a5ab5bd7d89bb9267b4eb2400ac76cef5a407f2ccb"}, + {"name":"aa_lower_margin","kind":"aa_boundary","baseline_p50_us":1050,"baseline_p95_us":2100,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"7aac1267f5fe3b3f90c6b200454850e3465806ea5fc871a1f3f50b4610177c94"}, + {"name":"target_power","kind":"target_power","baseline_p50_us":2000,"baseline_p95_us":4000,"candidate_p50_us":1800,"candidate_p95_us":3880,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"8797df36131eb1818f01314cbfb648c4aa889ac476cb20045dfd282106b9d10d"}, + {"name":"target_boundary","kind":"target_boundary","baseline_p50_us":2000,"baseline_p95_us":4000,"candidate_p50_us":1900,"candidate_p95_us":4200,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"0f7875ce519647e8d4705cd5c96712ec0e0f8cb816c4aa144696385312242b79"}, + {"name":"control_power","kind":"control_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":1940,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"69b1fe3e760738992eefcb95b4b12e4e27d68f34e207e3e8096a5cfddc174cc9"}, + {"name":"control_boundary","kind":"control_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1100,"candidate_p95_us":2100,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"388dd072ef7d6ff7044bc73c8e1a31754a1ce6d9efc1689b9968a507442c00cd"}, + {"name":"aa_order_odd_high","kind":"aa_order_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1.025,"even_candidate_multiplier":0.975609756097561,"seed":"710db9dee43c4a38bd8f63d75f9b388eb6e9674dacf97811e6e330a3090f3d2d"}, + {"name":"aa_order_even_high","kind":"aa_order_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":0.975609756097561,"even_candidate_multiplier":1.025,"seed":"950a3c42d102af5db8c2e263d1aee331aaa96b5b5e50dd39bf93bbeb5a4cdc84"}, + {"name":"aa_order_upper_margin","kind":"aa_order_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1.05,"even_candidate_multiplier":0.9523809523809523,"seed":"14441415f9ce978bff00f2a179983e76378e35c52f04a5cc387c903bb3e05c85"}, + {"name":"aa_order_lower_margin","kind":"aa_order_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":0.9523809523809523,"even_candidate_multiplier":1.05,"seed":"5bbc873e5b7069a072a16487562f8e9e7cf8ea7e70ff6779675d55810a1dc967"} + ] + }, + "host_admission": { + "sequence": ["S4/S4", "V2/V2", "S4/V2"], + "maximum_s4_remediations": 1, + "candidate_epoch_locked_on_first_invocation": true, + "machine_thresholds": { + "runner_process_overlap_count": 0, + "thermal_throttle_events": 0, + "maximum_steal_time_percent": 1.0, + "cpu_governor": "performance", + "postgresql_session_settings": "exact_match" + }, + "remediable_causes": [ + "postgresql_or_session_setting_mismatch", + "runner_process_overlap", + "cpu_governor_violation", + "thermal_throttle_violation", + "steal_time_violation" + ] + }, + "corpus": { + "source": "cases/generated_sp_i2_distance_v2.json", + "training_cases": 8, + "holdout_cases": 6, + "training_corpus_sha256": "b57e77369d9686123a847e24fd4037d7ee4bc4c9b3d6f73b67eca7a956b0493b", + "holdout_corpus_sha256": "16a4f8663cdc1c537c99a85571931c3e7d3f9f71500cb1311375aa9930f8c201", + "full_corpus_sha256": "f057a779bd1587ff08596459de42ef51f7befcc0365a9bf86f894a77e0e06d0e", + "training_declaration_sha256": "5d704f62c70fea909565ae0541d8a74a925c6cc14587a49a9a6422d5aa077133", + "holdout_declaration_sha256": "009101538c650a213e807189bd45dede5ab6785dd5ac9e93dc3f1ad328b3fcfa", + "full_declaration_sha256": "1721f48e724b227e0bf4d9a1e03b0471f10fc23ef7402e47536b664af6b96a69", + "training_resolved_sha256": "75802b0d76034fac1b2c144c125069f8b971180997540b6b2bf46b89523fbacc", + "holdout_resolved_sha256": "d08d149fcaf29e91750fde1e1eae1f3b2f2a6819608a073558ee4d6f13d82ce9", + "full_resolved_sha256": "fa1abc601d60d295add2095c0ff343c47d013165fe23b9a09eeca429254318c2" + }, + "multiplicity_rule": "intersection_union_all_cases_must_pass", + "v1_evidence_reuse": false, + "holdout_authorization_required_before_database_setup": true +} diff --git a/benchmark/testdata/scale/protocols/sp_i2_distance_v2_rejection.json b/benchmark/testdata/scale/protocols/sp_i2_distance_v2_rejection.json new file mode 100644 index 00000000..88349b34 --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_i2_distance_v2_rejection.json @@ -0,0 +1,24 @@ +{ + "schema": "sp-i2-terminal-rejection-v2", + "generation": "sp-i2-distance-v2", + "source_commit": "5df040c2992dd92cf0480beed887c4068c3052b2", + "protocol_sha256": "17cddc5100bc4f523122b0664ec63d3b4954ae2c01000f04864f10fdd00e1e89", + "simulation_report_sha256": "cbf4fc593a0adfa72ead23f4f391d530790a474a292a9cc47788a18048b17875", + "simulation_implementation": "sp-i2-power-simulation-v2/chacha8-sha256-normal-pivot", + "runs_per_scenario": 20000, + "failed_gates": [ + {"scenario":"aa_identity","metric":"admission_power_wilson_lower","observed":0,"required":0.9}, + {"scenario":"target_power","metric":"full_decision_power_wilson_lower","observed":0.4724809842358317,"required":0.9}, + {"scenario":"control_power","metric":"full_decision_power_wilson_lower","observed":0.5053708806725798,"required":0.9}, + {"scenario":"aa_order_odd_high","metric":"admission_power_wilson_lower","observed":0,"required":0.9}, + {"scenario":"aa_order_even_high","metric":"admission_power_wilson_lower","observed":0,"required":0.9} + ], + "coverage_calibrated": true, + "formal_aa_started": false, + "capture_plan_created": false, + "sealed_preregistration_created": false, + "holdout_opened": false, + "production_activated": false, + "successor_protocol_required": true, + "terminal": true +} diff --git a/benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3.json b/benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3.json new file mode 100644 index 00000000..cea06f1f --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3.json @@ -0,0 +1,58 @@ +{ + "schema": "sp-i2-successor-power-study-v3", + "generation": "sp-i2-distance-v3-power-study", + "implementation": "sp-i2-power-simulation-v3/chacha8-sha256-normal-pivot", + "status": "prospective", + "archived_trace": { + "source_commit": "3865cbc57758b7b20b7ffe431f27235873422eed", + "baseline_trace_sha256": "ac3ceb27ee92e3f4e21e3994ff9ee82d483b8081e9d44ddcef8e695ffdb1b6d0", + "candidate_trace_sha256": "f6d79e81bdaafedaa95568d57140c14e0808fbb6fc261387abc916081137785a", + "rounds": 20, + "case_records_per_round": 12, + "timed_samples_per_record": 10 + }, + "design": { + "blocks": 800, + "ordinary_warmups": 25, + "timed_samples_per_arm_case_block": 100, + "pool_size": 1, + "isolation": "repeatable_read", + "arm_order": "odd_incumbent_then_candidate_even_candidate_then_incumbent" + }, + "statistics": { + "bootstrap_confidence": 0.975, + "bootstrap_replicates": 100000, + "quantile": "nearest_rank", + "wilson_confidence": 0.95, + "simulation_runs_per_scenario": 20000, + "required_power_lower": 0.9, + "required_coverage": 0.975, + "p95_boundary_false_pass_upper": 0.015, + "decision_false_pass_upper": 0.0275, + "trace_rescaling_transform": "scaled_v2_calibration_then_paired_empirical_round_drift" + }, + "gates": { + "aa_equivalence_ratio": 1.05, + "aa_first_position_ratio_upper": 1.1, + "aa_first_position_overhead_upper_us": 100, + "target_median_ratio_upper": 0.95, + "target_median_saving_lower_us": 100, + "control_median_ratio_upper": 1.1, + "control_median_overhead_upper_us": 100, + "p95_ratio_upper": 1.05, + "control_p95_overhead_upper_us": 100 + }, + "scenarios": [ + {"name":"aa_identity","kind":"aa_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"3970c10649c771c2f49b69623a7c595d60526a06221c8b5c55170183d66a6db8"}, + {"name":"aa_upper_margin","kind":"aa_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1050,"candidate_p95_us":2100,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"c4df99870132870a933d6916b6cbee9bc5edbf660064d4f6482c5ab8e018f7ed"}, + {"name":"aa_lower_margin","kind":"aa_boundary","baseline_p50_us":1050,"baseline_p95_us":2100,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"ce22d3a7048e8a14dd32ea29df8630961e40bf112f317da70b0487759f3e8f94"}, + {"name":"target_power","kind":"target_power","baseline_p50_us":2000,"baseline_p95_us":4000,"candidate_p50_us":1800,"candidate_p95_us":3880,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"e93f43b16a398747395c1e835184071d7a2ffcc5645dc904069b36f9d769dd9b"}, + {"name":"target_boundary","kind":"target_boundary","baseline_p50_us":2000,"baseline_p95_us":4000,"candidate_p50_us":1900,"candidate_p95_us":4200,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"f1629a1a2cbf342252293e539bfa06051c94a2c23a415b1280a348d9bd435efa"}, + {"name":"control_power","kind":"control_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":1940,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"69bee821d4b945680c719a27c4cdd0a6c9e659ef305a90d562253e7624a759ca"}, + {"name":"control_boundary","kind":"control_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1100,"candidate_p95_us":2100,"odd_candidate_multiplier":1,"even_candidate_multiplier":1,"seed":"69f30fc2544e3e3b2818bbffda9a76878bf31be125ce3e3c1afd98db6c53edf7"}, + {"name":"aa_order_odd_high","kind":"aa_order_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1.025,"even_candidate_multiplier":0.975609756097561,"seed":"64e31acbe58eb690d245d593b5a3bf389d3fe93e40b484b563ba68806069dfdd"}, + {"name":"aa_order_even_high","kind":"aa_order_power","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":0.975609756097561,"even_candidate_multiplier":1.025,"seed":"471c1217381f594193578334c61c31a2d010222f90e2bee7ed211db4c1d63080"}, + {"name":"aa_order_upper_margin","kind":"aa_order_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":1.05,"even_candidate_multiplier":0.9523809523809523,"seed":"f5a51d2ae1cf3d8623ee0686a84ecafb2111f226c7909a14f7fb42583c780e32"}, + {"name":"aa_order_lower_margin","kind":"aa_order_boundary","baseline_p50_us":1000,"baseline_p95_us":2000,"candidate_p50_us":1000,"candidate_p95_us":2000,"odd_candidate_multiplier":0.9523809523809523,"even_candidate_multiplier":1.05,"seed":"0cd7214a86eb3a539bff75a43f72722a9f876e3c5c4aa24363a12e36c6724b5f"} + ] +} diff --git a/benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3_rejection.json b/benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3_rejection.json new file mode 100644 index 00000000..5ae44577 --- /dev/null +++ b/benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3_rejection.json @@ -0,0 +1,16 @@ +{ + "schema": "sp-i2-successor-power-study-rejection-v3", + "generation": "sp-i2-distance-v3-power-study", + "protocol_sha256": "e11090bbbe73cc36dfae2af97e26b6e1fc4d42590fc6fd331b2204c7a9e04f31", + "implementation": "sp-i2-power-simulation-v3/chacha8-sha256-normal-pivot", + "runs_per_scenario": 20000, + "failed_gates": [ + {"scenario":"aa_order_odd_high","metric":"admission_power_wilson_lower","observed":0.14201232557116983,"required":0.9}, + {"scenario":"aa_order_even_high","metric":"admission_power_wilson_lower","observed":0.14723913101703448,"required":0.9} + ], + "candidate_implemented": false, + "corpus_created": false, + "database_timing_started": false, + "holdout_opened": false, + "terminal": true +} diff --git a/benchmark/testdata/scale/protocols/sql_strategy_routing_component_closure_v1.json b/benchmark/testdata/scale/protocols/sql_strategy_routing_component_closure_v1.json new file mode 100644 index 00000000..4701546d --- /dev/null +++ b/benchmark/testdata/scale/protocols/sql_strategy_routing_component_closure_v1.json @@ -0,0 +1,135 @@ +{ + "schema": "sql-strategy-routing-component-closure-v1", + "generation": "sql-strategy-routing-component-closure-v1", + "status": "frozen_measurement_closure", + "production_default": "off", + "purpose": "close the client/raw-PGX prepared-state and temporary-workspace observation gaps left by sql-strategy-routing-preflight-v1; this is not a selector, route cache, translation cache change, retry, schema change, or production qualification", + "predecessor": { + "protocol": "sql-strategy-routing-preflight-v1", + "clean_component_source_commit": "aaecb745c328128115273b4da7fa71a8de3351b7", + "artifact_directory": ".coverage/sql-routing-preflight-v1-clean-aaecb74", + "required_findings": [ + "all open target/control observations exact", + "direct reverse component telemetry complete", + "both sparse target classes materially faster", + "cancellation rollback and size-one pool reuse exact" + ] + }, + "identities": { + "incumbent": "EXPANSION-STEPWISE-FORWARD", + "component": "EXPANSION-SUFFIX-SEEDED-REVERSE", + "component_diagnostic_label": "suffix-route-component-v1", + "closure_label": "suffix-route-component-closure-v1", + "selector": "none", + "fallback": "forbidden" + }, + "execution_boundary": { + "statement": "one exact incumbent statement or one exact forced reverse component statement", + "selection": "external diagnostic arm labeling only", + "transaction": "caller-owned repeatable_read with rollback-only raw-PGX closure transactions", + "fresh_session": "one dedicated newly opened PostgreSQL session executes its first prepared-statement miss", + "same_session": "the same dedicated session executes exactly the declared prepared-statement hits", + "pool_reacquisition": "a dedicated newly opened size-one raw-PGX pool executes one prepared miss, releases it, then every declared hit reacquires the same backend PID", + "inactive_forward_body": "forbidden in component SQL", + "same_statement_probe": "forbidden", + "transaction_retry": "forbidden", + "route_cache": "forbidden", + "translation_cache_change": "forbidden", + "persistent_schema_or_synopsis": "forbidden" + }, + "roster": { + "tag": "suffix-route-component-v1", + "status": "open_training_only", + "targets": [ + "GFSE-SRC-V1-TARGET-D16-F1024-sparse_endpoint_ids", + "GFSE-SRC-V1-TARGET-D17-F1025-sparse_path" + ], + "controls": [ + "GFSE-SRC-V1-CONTROL-D08-F017-I1024-high_reverse_fanin", + "GFSE-SRC-V1-CONTROL-D05-F016-dense_suffix", + "GFSE-SRC-V1-CONTROL-D09-F513-no_path", + "GFSE-SRC-V1-CONTROL-CAP511", + "GFSE-SRC-V1-CONTROL-CAP512", + "GFSE-SRC-V1-CONTROL-CAP513", + "GFSE-SRC-V1-CONTROL-productive_cycle_path", + "GFSE-SRC-V1-CONTROL-productive_self_loop_path", + "GFSE-SRC-V1-CONTROL-multiple_relationship_distinct_paths" + ], + "excluded": [ + "protected declarations", + "terminal-generation declarations", + "shortest-path and all-shortest-path declarations", + "endpoint-seeded expansion declarations" + ] + }, + "design": { + "mode": "postgres_sql", + "pool_size": 1, + "traversal_telemetry": "diagnostic", + "require_clean_source": true, + "warmup_iterations": 1, + "timed_iterations": 5, + "rounds": 4, + "arm_orders": [ + ["incumbent", "reverse_component"], + ["reverse_component", "incumbent"], + ["incumbent", "reverse_component"], + ["reverse_component", "incumbent"] + ], + "session_memory_ceiling_bytes": 1048576, + "pool_memory_ceiling_bytes": 1048576, + "raw_pgx_per_case": { + "fresh_session_prepared_misses": 1, + "same_session_prepared_hits": 5, + "pooled_prepared_misses": 1, + "pooled_release_reacquire_prepared_hits": 5 + } + }, + "required_observations": [ + "exact_public_rows_and_paths_for_every_timed_and_raw_pgx_execution_with_per_sample_normalized_observation_sha256", + "component_reverse_runtime_identity_and_zero_inactive_forward_work", + "complete_suffix_boundary_reverse_receipt_ordered_hydration_and_workspace_counters", + "client_parse_optimize_translate_render_waterfall", + "raw_pgx_transaction_bind_prepare_first_row_decode_drain_close_and_total_waterfall", + "explicit_fresh_prepared_miss_same_session_prepared_hit_and_pool_reacquired_prepared_hit_strata", + "same_backend_pid_after_pool_release_reacquisition", + "per_query_fresh_session_pooled_session_and_pool_temporary_workspace_high_water", + "plan_buffers_temp_and_wal_with_no_public_rows_before_complete_status", + "separate_timeout_cancellation_rollback_and_exact_pool_replay" + ], + "workspace_measurement": { + "query": "sum(pg_total_relation_size) over pg_my_temp_schema() after result drain and before rollback", + "exclusions": [ + "traversal_runtime_attestation_v1", + "telemetry-named temporary relations" + ], + "timing_effect": "the workspace observation is outside bind/first-row/decode/drain/total timing intervals", + "pool_interpretation": "the dedicated raw-PGX pool has size one, making pooled-session and pool high water directly comparable without inheriting the runner pool statement cache" + }, + "acceptance": { + "all_records_status_ok": true, + "all_exact_oracles_required": true, + "complete_component_counter_status_required": true, + "component_workspace_required_families": ["suffix_component", "workspace"], + "all_raw_pgx_samples_require_nonnegative_workspace_and_stable_rows": true, + "all_pool_reacquired_samples_require_same_backend_pid": true, + "workspace_ceilings_must_not_be_exceeded": true, + "both_sparse_target_classes_must_remain_materially_improved_in_main_timed_samples": true, + "adverse_controls_must_remain_descriptive_only": true + }, + "next_authorization": { + "requires": [ + "this closure capture passes every acceptance condition", + "a separate frozen suffix-route-cache-feasibility-v1 protocol" + ], + "does_not_authorize": [ + "automatic routing", + "route-decision cache implementation", + "translation cache key changes", + "persistent topology schema", + "transaction retry", + "protected holdout access", + "promotion or Neo4j release-gate claims" + ] + } +} diff --git a/benchmark/testdata/scale/protocols/sql_strategy_routing_preflight_v1.json b/benchmark/testdata/scale/protocols/sql_strategy_routing_preflight_v1.json new file mode 100644 index 00000000..3d20a0d2 --- /dev/null +++ b/benchmark/testdata/scale/protocols/sql_strategy_routing_preflight_v1.json @@ -0,0 +1,113 @@ +{ + "schema": "sql-strategy-routing-preflight-v1", + "generation": "sql-strategy-routing-preflight-v1", + "status": "frozen_preimplementation", + "production_default": "off", + "purpose": "diagnostic-only feasibility for externally selected single-arm fixed-suffix reverse SQL; not a retry, same-statement probe, cache implementation, schema change, production selector, or performance qualification", + "predecessors": { + "p0": { + "source_commit": "a4b29f2", + "binary_sha256": "6745fca9c3b79b48875a0d5555f375f458af34d938e20e545bb6bce27379fca2", + "round_1": ".coverage/p0-sql-routing-20260819/round-1.jsonl", + "round_1_sha256": "a50836f08eaf85fac839f09d15420f060445ffb5312bf3bf609552fc86ad26ca", + "round_2": ".coverage/p0-sql-routing-20260819/round-2.jsonl", + "round_2_sha256": "fb6198257d15c71f2dcb348e1d2c3d7a9579b27ef44d983b3d5fc50f0336e262", + "backend_delta_sha256": "7be78bef932ee10a2c5d7b11237c419ece655e8cedc239adaa13f6997adf0e55" + }, + "terminal_generations": [ + "orientation-probe-v2", + "suffix-reverse-guard-v1", + "suffix-reverse-retry-v1" + ] + }, + "identities": { + "incumbent": "EXPANSION-STEPWISE-FORWARD", + "component": "EXPANSION-SUFFIX-SEEDED-REVERSE", + "diagnostic_label": "suffix-route-component-v1", + "selector": "none", + "fallback": "forbidden" + }, + "execution_boundary": { + "statement": "one exact forced reverse statement or one exact incumbent statement", + "selection": "external diagnostic arm selection only", + "transaction": "caller-owned repeatable_read", + "same_statement_probe": "forbidden", + "inactive_forward_body": "forbidden", + "transaction_retry": "forbidden", + "persistent_cache": "forbidden", + "persistent_synopsis": "forbidden" + }, + "roster": { + "tag": "suffix-route-component-v1", + "status": "open_training_only", + "requires_new_fixture_identities": true, + "targets": [ + "GFSE-SRC-V1-TARGET-D16-F1024-sparse_endpoint_ids", + "GFSE-SRC-V1-TARGET-D17-F1025-sparse_path" + ], + "adverse_controls": [ + "GFSE-SRC-V1-CONTROL-D08-F017-I1024-high_reverse_fanin", + "GFSE-SRC-V1-CONTROL-D05-F016-dense_suffix", + "GFSE-SRC-V1-CONTROL-D09-F513-no_path", + "GFSE-SRC-V1-CONTROL-CAP511", + "GFSE-SRC-V1-CONTROL-CAP512", + "GFSE-SRC-V1-CONTROL-CAP513", + "GFSE-SRC-V1-CONTROL-productive_cycle_path", + "GFSE-SRC-V1-CONTROL-productive_self_loop_path", + "GFSE-SRC-V1-CONTROL-multiple_relationship_distinct_paths" + ], + "excluded": [ + "all protected declarations", + "all terminal-generation declarations", + "all shortest-path and all-shortest-path declarations", + "endpoint-seeded expansion declarations" + ] + }, + "design": { + "modes": ["postgres_sql"], + "pool_size": 1, + "isolation": "repeatable_read", + "traversal_telemetry": "diagnostic", + "require_clean_source": true, + "warmup_iterations": 1, + "timed_iterations": 5, + "rounds": 4, + "comparison": "matched incumbent versus direct reverse component", + "records_per_arm_round": 11, + "timed_samples_per_arm_round": 55, + "arm_orders": [ + ["incumbent", "reverse_component"], + ["reverse_component", "incumbent"], + ["incumbent", "reverse_component"], + ["reverse_component", "incumbent"] + ], + "cap_overrides_permitted": false, + "protected_case_access_permitted": false + }, + "required_observations": [ + "exact_public_rows_and_paths", + "reverse_runtime_identity", + "zero_inactive_forward_executor_work", + "suffix_boundary_reverse_receipt_and_ordered_hydration_counters", + "planning_execution_decode_and_first_session_timings", + "buffers_temp_wal_and_workspace_high_water", + "cancellation_and_pool_reuse", + "no_public_rows_before_complete_status" + ], + "next_authorization": { + "requires": [ + "new fixture declarations and exact oracles", + "all direct component records exact with complete telemetry", + "material improvement on both frozen sparse target classes", + "a separately frozen cache-feasibility protocol before any automatic decision cache" + ], + "does_not_authorize": [ + "production routing", + "translation-cache key changes", + "persistent topology schema", + "transaction retry", + "protected holdout access", + "promotion or Neo4j release-gate claims" + ] + } +} diff --git a/benchmark/testdata/scale/protocols/suffix_route_cache_feasibility_v1.json b/benchmark/testdata/scale/protocols/suffix_route_cache_feasibility_v1.json new file mode 100644 index 00000000..13348216 --- /dev/null +++ b/benchmark/testdata/scale/protocols/suffix_route_cache_feasibility_v1.json @@ -0,0 +1,110 @@ +{ + "schema": "suffix-route-cache-feasibility-v1", + "generation": "suffix-route-cache-feasibility-v1", + "status": "frozen_preimplementation_feasibility", + "production_default": "off", + "purpose": "define the only admissible transaction-scoped routing-decision cache feasibility study after the passed direct-component boundary closure; this is not automatic routing, a translation-cache change, result caching, a schema change, or production qualification", + "predecessor": { + "protocol": "sql-strategy-routing-component-closure-v1", + "source_commit": "c490f3cfcd6bbb54fb42a5c3b466979fb5080ac5", + "artifact_directory": ".coverage/sql-routing-component-closure-v1-c490f3c", + "run_uuid": "e4800916-4eee-4dcf-ae99-d68068dcf4d5", + "required_findings": [ + "88 successful records with exact public rows and paths", + "all twelve raw-PGX prepared-state observations per record agree with the primary observation", + "both sparse targets remain materially improved", + "temporary workspace is zero and pool backend identity is stable" + ] + }, + "scope": { + "backend": "postgres_sql", + "fixtures": "same eleven open suffix-route-component-v1 training declarations only", + "transaction": "caller-owned read-only repeatable_read transaction", + "cache_location": "application memory owned by exactly one active PostgreSQL transaction", + "cache_contents": "immutable routing decision only; never result rows, graph values, translated SQL, plans, or graph metadata", + "selector": "disabled outside explicitly named feasibility arms" + }, + "cache_key": { + "schema_version": "suffix-route-cache-feasibility-v1", + "required_components": [ + "opaque transaction-owner token minted after BEGIN", + "graph_id", + "normalized Cypher shape fingerprint", + "canonical parameter names, types, and values fingerprint", + "frozen routing-policy identity and threshold version", + "transaction-local invalidation generation" + ], + "snapshot_rule": "an entry is valid only while its owning transaction remains active under its original repeatable-read snapshot; no process, pool, connection, graph, query, or snapshot identifier may enable reuse after commit, rollback, release, reacquisition, retry, or a new BEGIN", + "missing_or_unverifiable_component": "bypass cache and execute the exact ordinary incumbent" + }, + "ownership_and_invalidation": { + "creation": "allocate after BEGIN and destroy before the transaction object is returned to its caller", + "allowed_transactions": "read-only transactions with no savepoint lifecycle and no graph mutation", + "write_or_savepoint_boundary": "invalidate every entry, increment the transaction-local generation, and bypass cache for the remainder of that transaction", + "commit": "discard every entry before or during transaction completion; no entry survives commit", + "rollback_or_cancellation": "discard every entry before rollback completes; cancellation must not publish or retain a partially computed decision", + "retry": "forbidden; a replacement transaction receives a new owner token and an empty cache", + "pool_reacquisition": "forbidden as a reuse mechanism; a released connection cannot carry an entry into another transaction" + }, + "miss_and_hit_contract": { + "miss": "execute the exact ordinary EXPANSION-STEPWISE-FORWARD incumbent and record cache-miss provenance; a miss must not execute the reverse component or a hidden qualifying probe", + "entry_publication": "publish only a complete immutable decision produced inside the owner transaction after all required safety checks; failures, cancellation, timeout, or capacity exhaustion publish nothing", + "hit": "may select the already-qualified direct EXPANSION-SUFFIX-SEEDED-REVERSE statement only in the same active owner transaction when every key and ownership condition matches; it remains subject to exact public-result validation and full runtime telemetry", + "negative_or_unknown_decision": "remain incumbent-only and may be cached only for the same owner, key, and generation", + "fallback": "any cache validation, decoding, telemetry, or exactness failure stops the feasibility generation rather than widening routing" + }, + "resource_and_write_boundary": { + "maximum_entries_per_transaction": 64, + "maximum_total_bytes_per_transaction": 65536, + "maximum_entry_bytes": 4096, + "eviction": "none; capacity exhaustion bypasses cache and does not replace a valid entry", + "allocation": "hits must not allocate unboundedly or retain caller-owned mutable buffers", + "persistent_state": "forbidden: no table, index, extension, trigger, function, graph epoch, catalog mutation, temp relation, or translation-cache key change", + "read_path_wal": "zero cache-attributable WAL bytes; every feasibility read arm must prove no data-modifying cache SQL, durable write, or WAL-producing maintenance action", + "rollback_removal": "the cache has no database-side removal work because it is application-memory-only; transaction cleanup is mandatory even when PostgreSQL rollback itself succeeds" + }, + "required_evidence": [ + "exact public rows and paths for disabled, miss, hit, capacity-exhausted, invalidated, cancelled, and rolled-back states", + "transaction owner token, backend PID, snapshot/transaction provenance, key fingerprint, invalidation generation, and cache-state receipt without retaining user values", + "proof that every miss executed the exact incumbent and every hit used only a same-transaction qualified decision", + "memory entry count and byte high-water bounded by the declared limits", + "zero cache-attributable WAL, durable writes, temporary relations, and persistent schema changes", + "cancellation rollback followed by a fresh empty transaction and exact replay", + "pool release/reacquisition proof that no cache entry crossed a transaction boundary" + ], + "acceptance": { + "all_open_training_oracles_exact": true, + "all_misses_incumbent_only": true, + "all_hits_owner_snapshot_key_and_generation_bound": true, + "all_transaction_end_and_invalidation_boundaries_empty_cache": true, + "all_memory_limits_observed": true, + "zero_cache_attributable_wal_and_persistent_state": true, + "cancellation_and_rollback_replay_exact": true, + "no_selector_or_translation_cache_change": true + }, + "stop_conditions": [ + "any cache reuse across a transaction, snapshot, connection release, retry, graph mutation, or savepoint boundary", + "any miss that executes a candidate statement or hidden probe", + "any stale, absent, malformed, or oversized entry that changes public results or routing", + "any cache-attributable WAL, durable write, temporary relation, schema object, or unbounded allocation", + "any failure to remove entries on cancellation, rollback, commit, or capacity exhaustion", + "any protected declaration, automatic routing, production policy, or Neo4j release-gate claim" + ], + "next_authorization": { + "requires": [ + "this frozen protocol", + "a separately reviewed implementation limited to this feasibility scope", + "all acceptance conditions satisfied by new clean-source evidence" + ], + "does_not_authorize": [ + "production routing", + "process- or pool-scoped decision reuse", + "translation-cache key changes", + "result caching", + "persistent graph metadata or schema", + "transaction retry", + "protected holdout access", + "promotion or Neo4j release claims" + ] + } +} diff --git a/benchmark/testdata/scale/protocols/topology_selected_routing_v1.json b/benchmark/testdata/scale/protocols/topology_selected_routing_v1.json new file mode 100644 index 00000000..04675446 --- /dev/null +++ b/benchmark/testdata/scale/protocols/topology_selected_routing_v1.json @@ -0,0 +1,43 @@ +{ + "schema": "topology-selected-routing-v1", + "status": "frozen_implementation_protocol", + "production_default": "off", + "manifest_versions": { + "v2": "exact_query", + "v3": "graph_independent_structural", + "v4": "topology_selected_fixed_suffix" + }, + "transaction": { + "required_isolation": ["repeatable_read", "serializable"], + "read_only": true, + "same_snapshot_synopsis_read": true + }, + "selector": { + "estimator_version": "topology-fixed-suffix-counts-v1", + "maximum_edge_to_node_ratio_per_mille": 1000, + "comparison": "edge_count * 1000 <= node_count * maximum_edge_to_node_ratio_per_mille" + }, + "route_cache": { + "scope": "one_active_transaction", + "maximum_entries": 64, + "maximum_total_bytes": 65536, + "maximum_entry_bytes": 4096, + "miss": "incumbent_only", + "eviction": "forbidden", + "invalidation": ["write", "savepoint", "rollback", "cancellation", "retry", "pool_release", "transaction_end"] + }, + "execution": { + "candidate": "new_fixed_suffix_production_identity", + "single_arm": true, + "candidate_output_buffered": true, + "fallback": "exact_forward_same_snapshot" + }, + "fallback_conditions": ["missing", "building", "failed", "incompatible", "stale", "ambiguous", "resource_limited"], + "activation": { + "semantic_oracles": "exact", + "p50_improvement": "at_least_5_percent_or_100us", + "p95_ratio_maximum": 1.05, + "selector_overhead": "at_most_1.10x_or_100us", + "evidence": "clean_source_training_then_frozen_holdout" + } +} diff --git a/cmd/benchmark/README.md b/cmd/benchmark/README.md index 1cba4e28..c156e12d 100644 --- a/cmd/benchmark/README.md +++ b/cmd/benchmark/README.md @@ -5,14 +5,14 @@ Runs query scenarios against a real database and outputs markdown, JSON, or benc ## Usage ```bash -# Default datasets (base, adcs_fanout, and traversal_shapes) +# Default datasets (base, fixed_suffix_expansion_fanout, and traversal_shapes) go run ./cmd/benchmark -connection "postgresql://dawgs:dawgs@localhost:5432/dawgs" # Traversal shape dataset only go run ./cmd/benchmark -connection "..." -dataset traversal_shapes -# ADCS fanout dataset with PostgreSQL EXPLAIN diagnostics -go run ./cmd/benchmark -connection "..." -dataset adcs_fanout -json-output report.json -explain +# Fixed-suffix expansion fanout dataset with PostgreSQL EXPLAIN diagnostics +go run ./cmd/benchmark -connection "..." -dataset fixed_suffix_expansion_fanout -json-output report.json -explain # Local dataset (not committed to repo) go run ./cmd/benchmark -connection "..." -dataset local/phantom @@ -23,6 +23,22 @@ go run ./cmd/benchmark -connection "..." -local-dataset local/phantom # Neo4j go run ./cmd/benchmark -driver neo4j -connection "neo4j://neo4j:password@localhost:7687" +# PostgreSQL connection-local translation cache +go run ./cmd/benchmark -driver pg -connection "..." -iterations 10 + +# Cold and warm concurrent cache measurements (4 workers × 20 samples) +go run ./cmd/benchmark -driver pg -connection "..." -dataset traversal_shapes -pg-min-conns 0 -pg-max-conns 4 -workers 4 -warmup 0 -iterations 20 +go run ./cmd/benchmark -driver pg -connection "..." -dataset traversal_shapes -pg-min-conns 0 -pg-max-conns 4 -workers 4 -warmup 2 -iterations 20 + +# Benchmark the guarded inline predecessor-DAG executor without enabling it in production +go run ./cmd/benchmark -driver pg -connection "..." -dataset traversal_shapes -pg-shortest-path-executor 'ASP-I1-U-DAG+MAT-M0' -pg-plan-cache-mode auto -iterations 20 + +# Exercise one verified manifest-authorized query through the real V2 policy path +go run ./cmd/benchmark -driver pg -connection "..." -dataset traversal_shapes -pg-traversal-policy-manifest .coverage/promotion.json -pg-traversal-policy-generation 7 -pg-plan-cache-mode auto -iterations 20 + +# Derive a non-promotional exact SQL anchor for one provisional traversal-policy bucket +go run ./cmd/benchmark -driver pg -connection "..." -dataset traversal_shapes -pg-traversal-policy-preflight-manifest .coverage/provisional.json -pg-traversal-policy-preflight-output .coverage/policy-preflight.json + # Save to file go run ./cmd/benchmark -connection "..." -output report.md @@ -40,7 +56,20 @@ go run ./cmd/benchmark -connection "..." -format benchfmt -output report.bench | `-driver` | `pg` | Database driver (`pg`, `neo4j`) | | `-connection` | | Connection string (or `CONNECTION_STRING` env) | | `-iterations` | `10` | Timed iterations per scenario | -| `-explain` | `false` | Capture PostgreSQL `EXPLAIN (ANALYZE, BUFFERS)` and translated SQL for Cypher scenarios in JSON output | +| `-warmup` | `1` | Untimed iterations per worker; use `0` to include cold-query cost | +| `-workers` | `1` | Concurrent workers per scenario; each contributes `-iterations` samples | +| `-pg-cache-entries` | `64` | Translations retained per physical PostgreSQL connection | +| `-pg-shared-shortest-path-template-entries` | `128` | Immutable shortest-path SQL templates shared across physical connections; zero disables the L2 tier | +| `-pg-shortest-path-executor` | | Benchmark-only qualified executor identity; production routing remains manifest-controlled | +| `-pg-traversal-policy-manifest` | | A GraphBench-verified promotion manifest to install through PostgreSQL's real `SetTraversalPolicy` path | +| `-pg-traversal-policy-generation` | `1` | Nonzero traversal-policy generation used for cache identity in manifest policy mode | +| `-pg-traversal-policy-preflight-manifest` | | Provisional one-query manifest used only to render the candidate SQL anchor | +| `-pg-traversal-policy-preflight-output` | | JSON destination for the non-promotional preflight record; required with the preflight manifest | +| `-pg-plan-cache-mode` | `auto` | Plan mode for forced or manifest-policy PostgreSQL shortest-path runs (`auto`, `force_custom_plan`, `force_generic_plan`) | +| `-pg-jit` | `true` | Enable PostgreSQL JIT transaction-locally during forced or manifest-policy shortest-path runs | +| `-pg-min-conns` | `5` | Minimum physical PostgreSQL connections | +| `-pg-max-conns` | `50` | Maximum physical PostgreSQL connections | +| `-explain` | `false` | Capture PostgreSQL JSON `EXPLAIN (ANALYZE, BUFFERS, SETTINGS)` and translated SQL for Cypher scenarios | | `-dataset` | | Run only this dataset | | `-local-dataset` | | Add a local dataset to the default set | | `-dataset-dir` | `integration/testdata` | Path to testdata directory | @@ -50,9 +79,48 @@ go run ./cmd/benchmark -connection "..." -format benchfmt -output report.bench Use `-format benchfmt` when comparing scenario timings with `benchstat`. Each timed scenario iteration is emitted as a separate `ns/op` sample so two benchmark runs can be compared directly. -The committed default datasets are `base`, `adcs_fanout`, and `traversal_shapes`. `traversal_shapes` covers chain, -fanout, bounded cycle, disconnected, edge-kind-selective, and multi-path shortest-path traversal shapes. Scenarios with -declared expected row counts fail before reporting timings if a query returns the wrong result shape. +`pg` constructs the connection-local runtime with the selected pool and cache settings, uses the default 64-entry cache +per physical PostgreSQL connection, and supports PostgreSQL EXPLAIN capture. +Its JSON and Markdown reports also include query-text-free translation-cache, traversal-workspace, and prepared-statement +counters, structured PostgreSQL planning/execution timings, and configured pool limits. For shortest paths it also reports +query-text-free parse, cache/bind, translation, formatting, and dispatch totals plus shared-template L2 activity. Use a cold (`-warmup 0`) and warm (`-warmup 2`) run with the same worker count and +pool configuration to measure cache effectiveness; do not compare their latency distributions without accounting for the +intentionally different warm-up state. + +`-pg-shortest-path-executor` is deliberately a tool-only comparison mode: it +retranslates at the benchmark boundary and bypasses production policy +selection. `-pg-traversal-policy-manifest` instead installs the supplied +document on the newly opened PostgreSQL driver, runs at Repeatable Read, and leaves +translation, SQL-anchor verification, cache lookup, and fallback selection to +the driver. The report identifies this as `production_policy` and records only +the policy generation and manifest SHA-256, never its raw contents. + +Before using manifest policy mode, verify the complete evidence closure with +`go run ./cmd/graphbench -promotion-manifest `. The driver fails closed +on an invalid candidate, manifest, query set, snapshot, or SQL anchor, but the +benchmark command does not treat a digest-shaped document as promotion proof. +The current policy contract authorizes exactly one query digest, so policy mode +requires `-dataset` and runs exactly one matching parameterized Cypher +scenario. `-explain` is intentionally unavailable in this mode because the +standalone explainer would otherwise bypass the live policy gate and report a +different statement. + +When a formal manifest needs its candidate SQL anchor, use +`-pg-traversal-policy-preflight-manifest` with the same selected benchmark +dataset. Its provisional manifest supplies the candidate, selector, caps, and +single query bucket; the command loads the graph and renders that exact +translation, then writes only the query and SQL SHA-256 values plus translation +metadata. It does not install a traversal policy, execute the candidate SQL, +or create verification evidence. Copy the SQL hash into the provisional +manifest and recapture the complete GraphBench evidence closure before using +the resulting document with policy mode. The output path must be new: preflight +refuses to overwrite either the provisional manifest or an earlier record. + +The committed default datasets are `base`, `fixed_suffix_expansion_fanout`, and +`traversal_shapes`. `traversal_shapes` covers chain, fanout, bounded cycle, +disconnected, edge-kind-selective, and multi-path shortest-path traversal +shapes. Scenarios with declared expected row counts fail before reporting +timings if a query returns the wrong result shape. ## Example: Neo4j on local/phantom diff --git a/cmd/benchmark/explain.go b/cmd/benchmark/explain.go index ca1334f7..0bb4731f 100644 --- a/cmd/benchmark/explain.go +++ b/cmd/benchmark/explain.go @@ -18,29 +18,71 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" + "strings" + "time" "github.com/specterops/dawgs/cypher/frontend" "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" "github.com/specterops/dawgs/graph" ) // ExplainResult captures PostgreSQL-specific plan diagnostics for a scenario. type ExplainResult struct { - SQL string `json:"sql"` - Plan []string `json:"plan"` - Optimization translate.OptimizationSummary `json:"optimization"` + SQL string `json:"sql"` + SQLFingerprint string `json:"sql_fingerprint"` + Plan []string `json:"plan"` + Optimization translate.OptimizationSummary `json:"optimization"` + PostgreSQL PostgreSQLExplainMetrics `json:"postgresql"` +} + +// PostgreSQLExplainMetrics contains structured server-side timings and +// configuration emitted by EXPLAIN. It is intentionally independent of the +// human-readable plan text so benchmark comparisons need not parse it. +type PostgreSQLExplainMetrics struct { + PlanningTime time.Duration `json:"planning_time"` + ExecutionTime time.Duration `json:"execution_time"` + Settings map[string]string `json:"settings,omitempty"` + Stages map[string]PostgreSQLExplainStage `json:"stages,omitempty"` +} + +// PostgreSQLExplainStage is an additive node timing derived from the exact +// JSON plan. It carries no SQL, parameters, or result values. +type PostgreSQLExplainStage struct { + NodeType string `json:"node_type"` + Rows int64 `json:"rows"` + Loops int64 `json:"loops"` + Total time.Duration `json:"total"` +} + +type postgresExplainDocument struct { + PlanningTime float64 `json:"Planning Time"` + ExecutionTime float64 `json:"Execution Time"` + Settings map[string]string `json:"Settings"` } func newPostgresExplainer(kindMapper pgsql.KindMapper, graphID int32) ExplainFunc { - return func(ctx context.Context, tx graph.Transaction, cypherQuery string) (*ExplainResult, error) { - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + return newPostgresExplainerWithExecutor(kindMapper, graphID, "") +} + +func newPostgresExplainerWithExecutor(kindMapper pgsql.KindMapper, graphID int32, executor optimize.ShortestPathExecutor) ExplainFunc { + return func(ctx context.Context, tx graph.Transaction, scenario Scenario) (*ExplainResult, error) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), scenario.Cypher) if err != nil { return nil, err } - translation, err := translate.Translate(ctx, regularQuery, kindMapper, nil, graphID) + var translation translate.Result + if executor != "" && strings.Contains(strings.ToLower(scenario.Cypher), "shortestpath") { + translation, err = translate.TranslateForTool(ctx, regularQuery, kindMapper, scenario.Parameters, graphID, translate.ToolOptions{ForceShortestPathExecutor: executor}) + } else { + translation, err = translate.Translate(ctx, regularQuery, kindMapper, scenario.Parameters, graphID) + } if err != nil { return nil, err } @@ -50,17 +92,22 @@ func newPostgresExplainer(kindMapper pgsql.KindMapper, graphID int32) ExplainFun return nil, err } - result := tx.Raw("EXPLAIN (ANALYZE, BUFFERS) "+sqlQuery, translation.Parameters) + result := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON) "+sqlQuery, translation.Parameters) defer result.Close() var plan []string + var metrics PostgreSQLExplainMetrics for result.Next() { values := result.Values() if len(values) == 0 { continue } - plan = append(plan, fmt.Sprint(values[0])) + rawPlan := explainValueString(values[0]) + plan = append(plan, rawPlan) + if parsed, err := parsePostgreSQLExplainMetrics(rawPlan); err == nil { + metrics = parsed + } } if err := result.Error(); err != nil { @@ -68,9 +115,102 @@ func newPostgresExplainer(kindMapper pgsql.KindMapper, graphID int32) ExplainFun } return &ExplainResult{ - SQL: sqlQuery, - Plan: plan, - Optimization: translation.Optimization, + SQL: sqlQuery, + SQLFingerprint: sqlFingerprint(sqlQuery), + Plan: plan, + Optimization: translation.Optimization, + PostgreSQL: metrics, }, nil } } + +func explainValueString(value any) string { + switch typed := value.(type) { + case []byte: + return string(typed) + case string: + return typed + default: + if encoded, err := json.Marshal(value); err == nil { + return string(encoded) + } + return fmt.Sprint(value) + } +} + +func sqlFingerprint(sqlQuery string) string { + digest := sha256.Sum256([]byte(sqlQuery)) + return hex.EncodeToString(digest[:]) +} + +func parsePostgreSQLExplainMetrics(raw string) (PostgreSQLExplainMetrics, error) { + var documents []postgresExplainDocument + if err := json.Unmarshal([]byte(raw), &documents); err != nil { + return PostgreSQLExplainMetrics{}, err + } + if len(documents) == 0 { + return PostgreSQLExplainMetrics{}, fmt.Errorf("PostgreSQL EXPLAIN JSON is empty") + } + metrics := PostgreSQLExplainMetrics{ + PlanningTime: time.Duration(documents[0].PlanningTime * float64(time.Millisecond)), + ExecutionTime: time.Duration(documents[0].ExecutionTime * float64(time.Millisecond)), + Settings: documents[0].Settings, + Stages: map[string]PostgreSQLExplainStage{}, + } + var rawDocuments []map[string]any + if err := json.Unmarshal([]byte(raw), &rawDocuments); err != nil { + return PostgreSQLExplainMetrics{}, err + } + if len(rawDocuments) > 0 { + collectPostgreSQLExplainStages(rawDocuments[0]["Plan"], metrics.Stages) + } + if len(metrics.Stages) == 0 { + metrics.Stages = nil + } + return metrics, nil +} + +func collectPostgreSQLExplainStages(value any, stages map[string]PostgreSQLExplainStage) { + node, ok := value.(map[string]any) + if !ok { + return + } + name := "" + for _, key := range []string{"CTE Name", "Function Name", "Subplan Name"} { + if candidate, ok := node[key].(string); ok && candidate != "" { + name = candidate + break + } + } + if name != "" { + stage := PostgreSQLExplainStage{NodeType: stringExplainValue(node["Node Type"]), Rows: int64ExplainValue(node["Actual Rows"]), Loops: int64ExplainValue(node["Actual Loops"])} + stage.Total = time.Duration(floatExplainValue(node["Actual Total Time"]) * float64(time.Millisecond) * float64(stage.Loops)) + if existing, found := stages[name]; found { + existing.Rows += stage.Rows + existing.Loops += stage.Loops + existing.Total += stage.Total + stages[name] = existing + } else { + stages[name] = stage + } + } + if children, ok := node["Plans"].([]any); ok { + for _, child := range children { + collectPostgreSQLExplainStages(child, stages) + } + } +} + +func stringExplainValue(value any) string { + result, _ := value.(string) + return result +} + +func floatExplainValue(value any) float64 { + result, _ := value.(float64) + return result +} + +func int64ExplainValue(value any) int64 { + return int64(floatExplainValue(value)) +} diff --git a/cmd/benchmark/explain_test.go b/cmd/benchmark/explain_test.go new file mode 100644 index 00000000..0f364d26 --- /dev/null +++ b/cmd/benchmark/explain_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestParsePostgreSQLExplainMetrics(t *testing.T) { + metrics, err := parsePostgreSQLExplainMetrics(`[{"Plan": {}, "Planning Time": 1.25, "Execution Time": 3.5, "Settings": {"plan_cache_mode": "force_custom_plan"}}]`) + require.NoError(t, err) + require.Equal(t, 1250*time.Microsecond, metrics.PlanningTime) + require.Equal(t, 3500*time.Microsecond, metrics.ExecutionTime) + require.Equal(t, "force_custom_plan", metrics.Settings["plan_cache_mode"]) + require.Equal(t, "822ae07d4783158bc1912bb623e5107cc9002d519e1143a9c200ed6ee18b6d0f", sqlFingerprint("select 1")) +} + +func TestParsePostgreSQLExplainMetricsCollectsNamedStages(t *testing.T) { + metrics, err := parsePostgreSQLExplainMetrics(`[{"Plan":{"Node Type":"CTE Scan","CTE Name":"asp_distance","Actual Rows":3,"Actual Loops":2,"Actual Total Time":1.5,"Plans":[{"Node Type":"Function Scan","Function Name":"all_shortest_paths_dag","Actual Rows":3,"Actual Loops":1,"Actual Total Time":2.0}]},"Planning Time":0.1,"Execution Time":3.1}]`) + require.NoError(t, err) + require.Equal(t, 2, len(metrics.Stages)) + require.Equal(t, 3*time.Millisecond, metrics.Stages["asp_distance"].Total) + require.Equal(t, 2*time.Millisecond, metrics.Stages["all_shortest_paths_dag"].Total) +} + +func TestParsePostgreSQLExplainMetricsRejectsNonJSON(t *testing.T) { + _, err := parsePostgreSQLExplainMetrics("Seq Scan on node") + require.Error(t, err) +} + +func TestExplainValueStringPreservesJSONBytes(t *testing.T) { + require.Equal(t, `[{"Plan": {}}]`, explainValueString([]byte(`[{"Plan": {}}]`))) + require.JSONEq(t, `[{"Plan":{"Node Type":"Result"}}]`, explainValueString([]any{map[string]any{"Plan": map[string]any{"Node Type": "Result"}}})) +} diff --git a/cmd/benchmark/main.go b/cmd/benchmark/main.go index 77d7c0e3..95a6877d 100644 --- a/cmd/benchmark/main.go +++ b/cmd/benchmark/main.go @@ -27,7 +27,9 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/specterops/dawgs" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/drivers/pg/model" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" @@ -36,18 +38,36 @@ import ( _ "github.com/specterops/dawgs/drivers/neo4j" ) +type postgresBenchmarkDriver interface { + KindMapper() pg.KindMapper + DefaultGraph() (model.Graph, bool) +} + func main() { var ( - driver = flag.String("driver", "pg", "database driver (pg, neo4j)") - connStr = flag.String("connection", "", "database connection string (or CONNECTION_STRING)") - iterations = flag.Int("iterations", 10, "timed iterations per scenario") - output = flag.String("output", "", "output file (default: stdout)") - format = flag.String("format", reportFormatMarkdown, "output format (markdown, json, benchfmt)") - jsonOutput = flag.String("json-output", "", "JSON output file for baseline comparison") - explain = flag.Bool("explain", false, "capture PostgreSQL EXPLAIN (ANALYZE, BUFFERS) for Cypher scenarios") - datasetDir = flag.String("dataset-dir", "integration/testdata", "path to testdata directory") - localDataset = flag.String("local-dataset", "", "additional local dataset (e.g. local/phantom)") - onlyDataset = flag.String("dataset", "", "run only this dataset (e.g. diamond, local/phantom)") + driver = flag.String("driver", "pg", "database driver (pg, neo4j)") + connStr = flag.String("connection", "", "database connection string (or CONNECTION_STRING)") + iterations = flag.Int("iterations", 10, "timed iterations per scenario") + warmup = flag.Int("warmup", 1, "untimed iterations per worker (zero measures cold queries)") + workers = flag.Int("workers", 1, "concurrent workers per scenario") + pgCache = flag.Int("pg-cache-entries", pg.DefaultRuntimeConfig().TranslationCacheEntries, "translations retained per physical connection") + pgSharedSP = flag.Int("pg-shared-shortest-path-template-entries", pg.DefaultRuntimeConfig().SharedShortestPathTemplateEntries, "immutable shortest-path templates shared across physical connections (zero disables)") + pgSPExecutor = flag.String("pg-shortest-path-executor", "", "benchmark-only qualified shortest-path executor identity (default uses production routing)") + pgPolicy = flag.String("pg-traversal-policy-manifest", "", "verified promotion manifest to install through the PostgreSQL traversal-policy path") + pgPolicyGen = flag.Uint64("pg-traversal-policy-generation", 1, "nonzero generation for -pg-traversal-policy-manifest") + pgPolicyPreflight = flag.String("pg-traversal-policy-preflight-manifest", "", "provisional manifest used only to derive the candidate SQL anchor") + pgPolicyPreflightOutput = flag.String("pg-traversal-policy-preflight-output", "", "JSON destination for the non-promotional traversal-policy preflight") + pgPlanMode = flag.String("pg-plan-cache-mode", "auto", "PostgreSQL plan cache mode for shortest-path benchmark modes (auto, force_custom_plan, force_generic_plan)") + pgJIT = flag.Bool("pg-jit", true, "enable PostgreSQL JIT transaction-locally for shortest-path benchmark modes") + pgMinConns = flag.Int("pg-min-conns", int(pg.DefaultRuntimeConfig().Pool.MinConnections), "minimum physical PostgreSQL connections") + pgMaxConns = flag.Int("pg-max-conns", int(pg.DefaultRuntimeConfig().Pool.MaxConnections), "maximum physical PostgreSQL connections") + output = flag.String("output", "", "output file (default: stdout)") + format = flag.String("format", reportFormatMarkdown, "output format (markdown, json, benchfmt)") + jsonOutput = flag.String("json-output", "", "JSON output file for baseline comparison") + explain = flag.Bool("explain", false, "capture PostgreSQL EXPLAIN (ANALYZE, BUFFERS) for Cypher scenarios") + datasetDir = flag.String("dataset-dir", "integration/testdata", "path to testdata directory") + localDataset = flag.String("local-dataset", "", "additional local dataset (e.g. local/phantom)") + onlyDataset = flag.String("dataset", "", "run only this dataset (e.g. diamond, local/phantom)") ) flag.Parse() @@ -55,9 +75,36 @@ func main() { if err := validateIterations(*iterations); err != nil { fatal("%v", err) } + if err := validateBenchmarkConcurrency(*warmup, *workers); err != nil { + fatal("%v", err) + } if !isReportFormat(*format) { fatal("unsupported output format %q", *format) } + if *pgPolicy != "" || *pgPolicyPreflight != "" { + if *driver != pg.DriverName { + fatal("traversal-policy benchmark modes require -driver pg") + } + if *pgSPExecutor != "" { + fatal("traversal-policy benchmark modes cannot be combined with -pg-shortest-path-executor") + } + if *onlyDataset == "" { + fatal("traversal-policy benchmark modes require -dataset so their exact-query path is unambiguous") + } + if *pgPolicy != "" && *explain { + fatal("-explain cannot be combined with -pg-traversal-policy-manifest because the explainer does not bypass the live policy gate") + } + } + if *pgPolicy != "" && *pgPolicyPreflight != "" { + fatal("-pg-traversal-policy-manifest cannot be combined with -pg-traversal-policy-preflight-manifest") + } + if *pgPolicyPreflight != "" && *pgPolicyPreflightOutput == "" { + fatal("-pg-traversal-policy-preflight-manifest requires -pg-traversal-policy-preflight-output") + } + runtimeConfig, err := benchmarkRuntimeConfig(*pgCache, *pgSharedSP, *pgMinConns, *pgMaxConns) + if err != nil { + fatal("invalid PostgreSQL runtime configuration: %v", err) + } conn := *connStr if conn == "" { @@ -67,31 +114,37 @@ func main() { fatal("no connection string: set -connection flag or CONNECTION_STRING env var") } - var ( - ctx = context.Background() - cfg = dawgs.Config{ - GraphQueryMemoryLimit: size.Gibibyte, - ConnectionString: conn, - } - ) + ctx := context.Background() + db, err := openBenchmarkDatabaseWithRuntimeConfig(ctx, *driver, conn, size.Gibibyte, runtimeConfig) + if err != nil { + fatal("failed to open database: %v", err) + } + defer db.Close(ctx) - if *driver == pg.DriverName { - poolCfg, err := pgxpool.ParseConfig(conn) + var traversalPolicy *pg.TraversalPolicy + var traversalPolicyPreflight *benchmarkTraversalPromotionManifest + if *pgPolicy != "" { + policy, err := loadBenchmarkTraversalPolicy(*pgPolicy, *pgPolicyGen) if err != nil { - fatal("failed to parse pool configuration: %v", err) + fatal("load PostgreSQL traversal policy manifest: %v", err) } - pool, err := pg.NewPool(poolCfg) - if err != nil { - fatal("failed to create pool: %v", err) + policyDriver, ok := db.(traversalPolicyBenchmarkDriver) + if !ok { + fatal("PostgreSQL benchmark driver does not support traversal-policy installation") + } + if err := policyDriver.SetTraversalPolicy(policy); err != nil { + fatal("install PostgreSQL traversal policy: %v", err) } - cfg.Pool = pool + traversalPolicy = &policy + fmt.Fprintf(os.Stderr, "installed PostgreSQL traversal policy generation=%d candidate=%s manifest=%s\n", policy.Generation, policy.ShortestPathExecutor, policy.PromotionManifestSHA256) } - - db, err := dawgs.Open(ctx, *driver, cfg) - if err != nil { - fatal("failed to open database: %v", err) + if *pgPolicyPreflight != "" { + _, manifest, err := loadBenchmarkTraversalPromotionManifest(*pgPolicyPreflight) + if err != nil { + fatal("load PostgreSQL traversal policy preflight manifest: %v", err) + } + traversalPolicyPreflight = &manifest } - defer db.Close(ctx) // Build dataset list var datasets []string @@ -103,6 +156,22 @@ func main() { datasets = append(datasets, *localDataset) } } + if traversalPolicy != nil || traversalPolicyPreflight != nil { + queryAllowlist := []string(nil) + if traversalPolicy != nil { + queryAllowlist = traversalPolicy.QuerySHA256Allowlist + } else { + for _, bucket := range traversalPolicyPreflight.Buckets { + queryAllowlist = append(queryAllowlist, bucket.QuerySHA256...) + } + } + selectionPolicy := pg.TraversalPolicy{QuerySHA256Allowlist: queryAllowlist} + for _, dataset := range datasets { + if _, err := selectTraversalPolicyScenarios(scenariosForDataset(dataset, opengraph.IDMap{}), selectionPolicy); err != nil { + fatal("select manifest-authorized traversal benchmark scenario: %v", err) + } + } + } // Scan all datasets for kinds and assert schema nodeKinds, edgeKinds := scanKinds(*datasetDir, datasets) @@ -122,22 +191,60 @@ func main() { var runOptions RunOptions if *explain { - if *driver != pg.DriverName { + if !isPostgresBenchmarkDriver(*driver) { fmt.Fprintf(os.Stderr, " explain capture is only supported for pg; continuing without plans\n") - } else if pgDB, ok := db.(*pg.Driver); !ok { + } else if pgDB, ok := db.(postgresBenchmarkDriver); !ok { fmt.Fprintf(os.Stderr, " explain capture unavailable for %T; continuing without plans\n", db) } else if defaultGraph, hasDefaultGraph := pgDB.DefaultGraph(); !hasDefaultGraph { fatal("failed to resolve default graph for explain capture") } else { - runOptions.Explain = newPostgresExplainer(pgDB.KindMapper(), defaultGraph.ID) + runOptions.Explain = newPostgresExplainerWithExecutor(pgDB.KindMapper(), defaultGraph.ID, optimize.ShortestPathExecutor(*pgSPExecutor)) + } + } + if *pgSPExecutor != "" { + if *driver != pg.DriverName { + fatal("-pg-shortest-path-executor requires -driver pg") + } + pgDB, ok := db.(postgresBenchmarkDriver) + if !ok { + fatal("PostgreSQL benchmark driver does not expose translation metadata") } + defaultGraph, found := pgDB.DefaultGraph() + if !found { + fatal("failed to resolve default graph for shortest-path executor benchmark") + } + wrapped, err := newShortestExecutorBenchmarkDatabase(db, pgDB.KindMapper(), defaultGraph, optimize.ShortestPathExecutor(*pgSPExecutor), *pgPlanMode, *pgJIT) + if err != nil { + fatal("configure shortest-path executor benchmark: %v", err) + } + db = wrapped + } + if traversalPolicy != nil { + wrapped, err := newTraversalPolicyBenchmarkDatabase(db, *pgPlanMode, *pgJIT) + if err != nil { + fatal("configure traversal-policy benchmark: %v", err) + } + db = wrapped } report := Report{ - Driver: *driver, - GitRef: gitRef(), - Date: time.Now().Format("2006-01-02"), - Iterations: *iterations, + Driver: *driver, + GitRef: gitRef(), + Date: time.Now().Format("2006-01-02"), + Iterations: *iterations, + WarmupIterations: *warmup, + Workers: *workers, + ShortestPathExecutor: *pgSPExecutor, + PostgreSQLPlanCacheMode: *pgPlanMode, + PostgreSQLJIT: *pgJIT, + } + if traversalPolicy != nil { + report.ShortestPathExecutor = string(traversalPolicy.ShortestPathExecutor) + report.ShortestPathMode = shortestPathModeProductionPolicy + report.TraversalPolicyGeneration = traversalPolicy.Generation + report.TraversalPolicyManifestSHA256 = traversalPolicy.PromotionManifestSHA256 + } else if *pgSPExecutor != "" { + report.ShortestPathMode = shortestPathModeForced } for _, ds := range datasets { @@ -162,7 +269,46 @@ func main() { fmt.Fprintf(os.Stderr, " loaded %d nodes\n", len(idMap)) // Run scenarios - for _, s := range scenariosForDataset(ds, idMap) { + scenarios := scenariosForDataset(ds, idMap) + if traversalPolicy != nil || traversalPolicyPreflight != nil { + queryAllowlist := []string(nil) + if traversalPolicy != nil { + queryAllowlist = traversalPolicy.QuerySHA256Allowlist + } else { + for _, bucket := range traversalPolicyPreflight.Buckets { + queryAllowlist = append(queryAllowlist, bucket.QuerySHA256...) + } + } + scenarios, err = selectTraversalPolicyScenarios(scenarios, pg.TraversalPolicy{QuerySHA256Allowlist: queryAllowlist}) + if err != nil { + fatal("select manifest-authorized traversal benchmark scenario: %v", err) + } + if traversalPolicy != nil { + fmt.Fprintf(os.Stderr, " manifest-authorized scenario: %s/%s\n", scenarios[0].Section, scenarios[0].Label) + } + } + if traversalPolicyPreflight != nil { + pgDB, ok := db.(postgresBenchmarkDriver) + if !ok { + fatal("PostgreSQL benchmark driver does not expose translation metadata for policy preflight") + } + defaultGraph, found := pgDB.DefaultGraph() + if !found { + fatal("failed to resolve default graph for traversal policy preflight") + } + preflight, err := renderTraversalPolicyPreflight(ctx, pgDB.KindMapper(), defaultGraph, scenarios[0], *traversalPolicyPreflight) + if err != nil { + fatal("render traversal policy preflight: %v", err) + } + if err := writeTraversalPolicyPreflight(*pgPolicyPreflight, *pgPolicyPreflightOutput, preflight); err != nil { + fatal("write traversal policy preflight: %v", err) + } + fmt.Fprintf(os.Stderr, " wrote non-promotional traversal policy preflight %s (query=%s sql=%s)\n", *pgPolicyPreflightOutput, preflight.QuerySHA256, preflight.SQLSHA256) + return + } + for _, s := range scenarios { + runOptions.WarmupIterations = *warmup + runOptions.Workers = *workers result, err := runScenario(ctx, db, s, *iterations, runOptions) if err != nil { fmt.Fprintf(os.Stderr, " %s/%s failed: %v\n", s.Section, s.Label, err) @@ -182,6 +328,23 @@ func main() { ) } } + if statsProvider, ok := db.(interface{ TranslationCacheStats() pg.Stats }); ok { + stats := statsProvider.TranslationCacheStats() + report.TranslationCache = &stats + fmt.Fprintf(os.Stderr, "PostgreSQL connection state: cache hits=%d misses=%d bypasses=%d evictions=%d; workspaces initialized=%d reused=%d; statements prepared=%d reused=%d; live_connections=%d pool=%d-%d\n", + stats.Aggregate.Hits, + stats.Aggregate.Misses, + stats.Aggregate.Bypasses, + stats.Aggregate.Evictions, + stats.TraversalWorkspace.Initializations, + stats.TraversalWorkspace.Reuses, + stats.PreparedStatements.Prepared, + stats.PreparedStatements.Reuses, + stats.LiveConnections, + stats.MinConnections, + stats.MaxConnections, + ) + } // Write report var mdOut *os.File @@ -218,6 +381,64 @@ func main() { } } +func openBenchmarkDatabase(ctx context.Context, driverName, connection string, graphQueryMemoryLimit size.Size) (graph.Database, error) { + return openBenchmarkDatabaseWithRuntimeConfig(ctx, driverName, connection, graphQueryMemoryLimit, pg.DefaultRuntimeConfig()) +} + +func openBenchmarkDatabaseWithRuntimeConfig(ctx context.Context, driverName, connection string, graphQueryMemoryLimit size.Size, runtimeConfig pg.RuntimeConfig) (graph.Database, error) { + cfg := dawgs.Config{ + GraphQueryMemoryLimit: graphQueryMemoryLimit, + ConnectionString: connection, + } + + switch driverName { + case pg.DriverName: + poolConfig, err := pgxpool.ParseConfig(connection) + if err != nil { + return nil, fmt.Errorf("parse PostgreSQL pool configuration: %w", err) + } + pool, err := pg.NewPoolWithRuntimeConfig(ctx, poolConfig, runtimeConfig) + if err != nil { + return nil, fmt.Errorf("create PostgreSQL pool: %w", err) + } + return pg.NewDriver(graphQueryMemoryLimit, pool), nil + + default: + return dawgs.Open(ctx, driverName, cfg) + } +} + +func benchmarkRuntimeConfig(cacheEntries, sharedShortestPathTemplateEntries, minConnections, maxConnections int) (pg.RuntimeConfig, error) { + const maxInt32 = int(^uint32(0) >> 1) + if cacheEntries < 0 { + return pg.RuntimeConfig{}, fmt.Errorf("translation cache entries must not be negative: %d", cacheEntries) + } + if sharedShortestPathTemplateEntries < 0 { + return pg.RuntimeConfig{}, fmt.Errorf("shared shortest-path template entries must not be negative: %d", sharedShortestPathTemplateEntries) + } + if minConnections < 0 || minConnections > maxInt32 { + return pg.RuntimeConfig{}, fmt.Errorf("minimum connections must be between 0 and %d: %d", maxInt32, minConnections) + } + if maxConnections < 1 || maxConnections > maxInt32 { + return pg.RuntimeConfig{}, fmt.Errorf("maximum connections must be between 1 and %d: %d", maxInt32, maxConnections) + } + if minConnections > maxConnections { + return pg.RuntimeConfig{}, fmt.Errorf("minimum connections %d exceeds maximum connections %d", minConnections, maxConnections) + } + return pg.RuntimeConfig{ + TranslationCacheEntries: cacheEntries, + SharedShortestPathTemplateEntries: sharedShortestPathTemplateEntries, + Pool: &pg.PoolConfig{ + MinConnections: int32(minConnections), + MaxConnections: int32(maxConnections), + }, + }, nil +} + +func isPostgresBenchmarkDriver(driverName string) bool { + return driverName == pg.DriverName +} + func scanKinds(datasetDir string, datasets []string) (graph.Kinds, graph.Kinds) { var nodeKinds, edgeKinds graph.Kinds diff --git a/cmd/benchmark/main_integration_test.go b/cmd/benchmark/main_integration_test.go new file mode 100644 index 00000000..56c749fb --- /dev/null +++ b/cmd/benchmark/main_integration_test.go @@ -0,0 +1,217 @@ +//go:build manual_integration + +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/util/size" + "github.com/stretchr/testify/require" +) + +func postgresBenchmarkIntegrationConnection(t *testing.T) string { + t.Helper() + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + normalized := strings.ToLower(connection) + if !strings.HasPrefix(normalized, "postgres://") && !strings.HasPrefix(normalized, "postgresql://") { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + return connection +} + +// TestPostgresV2BenchmarkMode proves the benchmark's explicit v2 path opens +// a live database, loads a graph fixture, and measures a Cypher scenario. +func TestPostgresV2BenchmarkMode(t *testing.T) { + ctx := context.Background() + database, err := openBenchmarkDatabase(ctx, pg.DriverName, postgresBenchmarkIntegrationConnection(t), size.Gibibyte) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, database.Close(context.Background())) + }) + + nodeKind := graph.StringKind("BenchmarkV2IntegrationNode") + schema := graph.Schema{ + Graphs: []graph.Graph{{ + Name: "benchmark_v2_integration", + Nodes: graph.Kinds{nodeKind}, + }}, + DefaultGraph: graph.Graph{Name: "benchmark_v2_integration"}, + } + require.NoError(t, database.AssertSchema(ctx, schema)) + t.Cleanup(func() { + _ = database.WriteTransaction(context.Background(), func(tx graph.Transaction) error { + return tx.Nodes().Delete() + }) + }) + require.NoError(t, database.WriteTransaction(ctx, func(tx graph.Transaction) error { + if _, err := tx.CreateNode(graph.NewProperties(), nodeKind); err != nil { + return err + } + _, err := tx.CreateNode(graph.NewProperties(), nodeKind) + return err + })) + + scenario := expectScenarioRows(cypherScenario("v2", "live", "nodes", "MATCH (n:BenchmarkV2IntegrationNode) RETURN n"), 2) + result, err := runScenario(ctx, database, scenario, 2, RunOptions{}) + require.NoError(t, err) + require.Equal(t, int64(2), result.RowCount) + + statsProvider, ok := database.(*pg.Driver) + require.True(t, ok) + stats := statsProvider.TranslationCacheStats() + require.NotEmpty(t, stats.Connections) + require.GreaterOrEqual(t, stats.Aggregate.Misses, uint64(1)) +} + +// TestPostgresV2BenchmarkPolicyPath proves a benchmark reaches the real V2 +// manifest gate rather than a tool-only forced translation. The candidate SQL +// anchor is rendered before policy installation, then validated by the driver +// when the parameterized scenario executes at repeatable read. +func TestPostgresV2BenchmarkPolicyPath(t *testing.T) { + ctx := context.Background() + database, err := openBenchmarkDatabase(ctx, pg.DriverName, postgresBenchmarkIntegrationConnection(t), size.Gibibyte) + require.NoError(t, err) + driver, ok := database.(*pg.Driver) + require.True(t, ok) + t.Cleanup(func() { + require.NoError(t, driver.SetTraversalPolicy(pg.TraversalPolicy{})) + require.NoError(t, database.Close(context.Background())) + }) + + nodeKind := graph.StringKind("BenchmarkV2PolicyNode") + edgeKind := graph.StringKind("BenchmarkV2PolicyEdge") + schema := graph.Schema{ + Graphs: []graph.Graph{{ + Name: "benchmark_v2_policy_integration", + Nodes: graph.Kinds{nodeKind}, + Edges: graph.Kinds{edgeKind}, + }}, + DefaultGraph: graph.Graph{Name: "benchmark_v2_policy_integration"}, + } + require.NoError(t, database.AssertSchema(ctx, schema)) + t.Cleanup(func() { + _ = database.WriteTransaction(context.Background(), func(tx graph.Transaction) error { + return tx.Nodes().Delete() + }) + }) + + var startID, endID graph.ID + require.NoError(t, database.WriteTransaction(ctx, func(tx graph.Transaction) error { + start, err := tx.CreateNode(graph.NewProperties(), nodeKind) + if err != nil { + return err + } + end, err := tx.CreateNode(graph.NewProperties(), nodeKind) + if err != nil { + return err + } + startID, endID = start.ID, end.ID + _, err = tx.CreateRelationshipByIDs(startID, endID, edgeKind, graph.NewProperties()) + return err + })) + + const cypher = "MATCH p = allShortestPaths((s)-[:BenchmarkV2PolicyEdge*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + parameters := map[string]any{"start_id": startID, "end_id": endID} + scenario := expectScenarioRows(cypherScenarioWithParameters("Shortest Paths", "policy", "candidate", cypher, parameters), 1) + defaultGraph, found := driver.DefaultGraph() + require.True(t, found) + productionOptions := translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 1000, PredecessorLimit: 1000, EnumerationLimit: 1000, OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "outbound", ObservationMode: "all_paths", MinimumDepth: 1, MaximumDepth: 4, RelationshipKindCount: 1, + }, + SelectorVersion: "benchmark-policy-path-v1", + } + parsed, err := frontend.ParseCypher(frontend.NewContext(), cypher) + require.NoError(t, err) + translation, err := translate.TranslateWithProductionOptions(ctx, parsed, driver.KindMapper(), parameters, defaultGraph.ID, productionOptions) + require.NoError(t, err) + sqlQuery, err := translate.Translated(translation) + require.NoError(t, err) + sqlDigest := sha256.Sum256([]byte(sqlQuery)) + + queryDigest := pg.TraversalPolicyQuerySHA256(cypher) + preflightManifest := benchmarkTraversalPromotionManifest{ + Candidate: string(optimize.ShortestPathExecutorASPI1DAG), + SelectorVersion: "benchmark-policy-path-v1", + Caps: map[string]int64{ + "state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20, + }, + Buckets: []benchmarkPolicyBucket{{ + QuerySHA256: []string{queryDigest}, Direction: "outbound", ObservationMode: "all_paths", + MinimumDepth: 1, MaximumDepth: 4, RelationshipKindCount: 1, + }}, + } + preflight, err := renderTraversalPolicyPreflight(ctx, driver.KindMapper(), defaultGraph, scenario, preflightManifest) + require.NoError(t, err) + require.Equal(t, queryDigest, preflight.QuerySHA256) + require.Equal(t, hex.EncodeToString(sqlDigest[:]), preflight.SQLSHA256) + require.Equal(t, string(optimize.ShortestPathExecutorASPI1DAG), preflight.Candidate) + + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"path": role + ".json", "sha256": strings.Repeat("0", sha256.Size*2)} + } + manifest, err := json.Marshal(map[string]any{ + "version": 2, "candidate": string(optimize.ShortestPathExecutorASPI1DAG), "selector_version": "benchmark-policy-path-v1", + "source_commit": "benchmark-integration", "source_sha256": strings.Repeat("0", sha256.Size*2), + "binary_sha256": hex.EncodeToString(sqlDigest[:]), "corpus_sha256": strings.Repeat("0", sha256.Size*2), + "operational_candidate_sql_sha256": hex.EncodeToString(sqlDigest[:]), + "execution_boundary": "guarded_dual_arm", "fallback_executor": string(optimize.ShortestPathExecutorASPA1DAG), + "caps": map[string]int64{"state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20}, + "buckets": []map[string]any{{ + "name": "benchmark-policy-path", "query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}, + "direction": "outbound", "observation_mode": "all_paths", "minimum_depth": 1, "maximum_depth": 4, + "relationship_kind_count": 1, "untyped_relationship": false, + }}, + "evidence": evidence, + }) + require.NoError(t, err) + manifestPath := t.TempDir() + "/manifest.json" + require.NoError(t, os.WriteFile(manifestPath, manifest, 0o600)) + policy, err := loadBenchmarkTraversalPolicy(manifestPath, 1) + require.NoError(t, err) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + wrapped, err := newTraversalPolicyBenchmarkDatabase(database, "auto", true) + require.NoError(t, err) + result, err := runScenario(ctx, wrapped, scenario, 2, RunOptions{WarmupIterations: 1}) + require.NoError(t, err) + require.Equal(t, int64(1), result.RowCount) + + stats := driver.TranslationCacheStats() + require.GreaterOrEqual(t, stats.Aggregate.Misses, uint64(1)) +} diff --git a/cmd/benchmark/main_test.go b/cmd/benchmark/main_test.go new file mode 100644 index 00000000..dd2909f5 --- /dev/null +++ b/cmd/benchmark/main_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/drivers/pg" + "github.com/stretchr/testify/require" +) + +// TestPostgresBenchmarkDriverModes verifies PostgreSQL-only explain support. +func TestPostgresBenchmarkDriverModes(t *testing.T) { + require.True(t, isPostgresBenchmarkDriver(pg.DriverName)) + require.False(t, isPostgresBenchmarkDriver("neo4j")) +} + +func TestBenchmarkRuntimeConfigValidatesAndConvertsPoolLimits(t *testing.T) { + config, err := benchmarkRuntimeConfig(32, 16, 0, 4) + require.NoError(t, err) + require.Equal(t, 32, config.TranslationCacheEntries) + require.Equal(t, 16, config.SharedShortestPathTemplateEntries) + require.Equal(t, &pg.PoolConfig{MinConnections: 0, MaxConnections: 4}, config.Pool) + + for _, arguments := range [][4]int{{-1, 0, 0, 1}, {1, -1, 0, 1}, {1, 0, -1, 1}, {1, 0, 1, 0}, {1, 0, 2, 1}} { + _, err := benchmarkRuntimeConfig(arguments[0], arguments[1], arguments[2], arguments[3]) + require.Error(t, err) + } +} + +func TestBenchmarkShortestPathExecutorAllowlist(t *testing.T) { + require.True(t, benchmarkShortestPathExecutor(optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance)) + require.True(t, benchmarkShortestPathExecutor(optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness)) + require.True(t, benchmarkShortestPathExecutor(optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG)) + require.True(t, benchmarkShortestPathExecutor(optimize.ShortestPathExecutorASPI1DAG)) + require.True(t, benchmarkShortestPathExecutor(optimize.ShortestPathExecutorASPB1AlternatingNodeDAG)) + require.False(t, benchmarkShortestPathExecutor(optimize.ShortestPathExecutorIncumbentWorkspace)) + require.False(t, benchmarkShortestPathExecutor("unknown")) + require.True(t, benchmarkPlanCacheMode("auto")) + require.True(t, benchmarkPlanCacheMode("force_custom_plan")) + require.True(t, benchmarkPlanCacheMode("force_generic_plan")) + require.False(t, benchmarkPlanCacheMode("invalid")) +} diff --git a/cmd/benchmark/report.go b/cmd/benchmark/report.go index 26e1635b..352aec55 100644 --- a/cmd/benchmark/report.go +++ b/cmd/benchmark/report.go @@ -24,21 +24,35 @@ import ( "strings" "time" "unicode" + + "github.com/specterops/dawgs/drivers/pg" ) const ( reportFormatBenchfmt = "benchfmt" reportFormatJSON = "json" reportFormatMarkdown = "markdown" + + shortestPathModeForced = "forced" + shortestPathModeProductionPolicy = "production_policy" ) // Report holds all benchmark results and metadata. type Report struct { - Driver string `json:"driver"` - GitRef string `json:"git_ref"` - Date string `json:"date"` - Iterations int `json:"iterations"` - Results []Result `json:"results"` + Driver string `json:"driver"` + GitRef string `json:"git_ref"` + Date string `json:"date"` + Iterations int `json:"iterations"` + WarmupIterations int `json:"warmup_iterations"` + Workers int `json:"workers"` + ShortestPathExecutor string `json:"shortest_path_executor,omitempty"` + ShortestPathMode string `json:"shortest_path_mode,omitempty"` + TraversalPolicyGeneration uint64 `json:"traversal_policy_generation,omitempty"` + TraversalPolicyManifestSHA256 string `json:"traversal_policy_manifest_sha256,omitempty"` + PostgreSQLPlanCacheMode string `json:"postgresql_plan_cache_mode,omitempty"` + PostgreSQLJIT bool `json:"postgresql_jit"` + TranslationCache *pg.Stats `json:"translation_cache,omitempty"` + Results []Result `json:"results"` } func writeReport(w io.Writer, r Report, format string) error { @@ -72,7 +86,17 @@ func writeJSON(w io.Writer, r Report) error { } func writeMarkdown(w io.Writer, r Report) error { - fmt.Fprintf(w, "# Benchmarks — %s @ %s (%s, %d iterations)\n\n", r.Driver, r.GitRef, r.Date, r.Iterations) + fmt.Fprintf(w, "# Benchmarks — %s @ %s (%s, %d iterations × %d workers, %d warm-up iterations)\n\n", r.Driver, r.GitRef, r.Date, r.Iterations, r.Workers, r.WarmupIterations) + if r.ShortestPathExecutor != "" { + switch r.ShortestPathMode { + case shortestPathModeProductionPolicy: + fmt.Fprintf(w, "Shortest-path executor: `%s` through production traversal policy generation %d (manifest `%s`); PostgreSQL plan cache: `%s`; JIT: `%t`.\n\n", r.ShortestPathExecutor, r.TraversalPolicyGeneration, r.TraversalPolicyManifestSHA256, r.PostgreSQLPlanCacheMode, r.PostgreSQLJIT) + case shortestPathModeForced: + fmt.Fprintf(w, "Shortest-path executor: `%s` forced at the benchmark boundary; PostgreSQL plan cache: `%s`; JIT: `%t`.\n\n", r.ShortestPathExecutor, r.PostgreSQLPlanCacheMode, r.PostgreSQLJIT) + default: + fmt.Fprintf(w, "Shortest-path executor: `%s`; PostgreSQL plan cache: `%s`; JIT: `%t`.\n\n", r.ShortestPathExecutor, r.PostgreSQLPlanCacheMode, r.PostgreSQLJIT) + } + } fmt.Fprintf(w, "| Query | Dataset | Rows | Distinct Rows | Duplicate Rows | Median | P95 | Max | Explain |\n") fmt.Fprintf(w, "|-------|---------|-----:|--------------:|---------------:|-------:|----:|----:|:--------|\n") @@ -96,6 +120,28 @@ func writeMarkdown(w io.Writer, r Report) error { } fmt.Fprintln(w) + if r.TranslationCache != nil { + cache := r.TranslationCache.Aggregate + workspaces := r.TranslationCache.TraversalWorkspace + statements := r.TranslationCache.PreparedStatements + fmt.Fprintf(w, "PostgreSQL connection state: cache %d hits, %d misses, %d bypasses, %d evictions; workspaces %d initialized/%d reused; statements %d prepared/%d reused across %d live connections.\n\n", cache.Hits, cache.Misses, cache.Bypasses, cache.Evictions, workspaces.Initializations, workspaces.Reuses, statements.Prepared, statements.Reuses, r.TranslationCache.LiveConnections) + shortest := r.TranslationCache.SQLGeneration.ShortestPath + if shortest.Count > 0 { + fmt.Fprintf(w, "PostgreSQL shortest-path generation (%d samples): parse %s, cache/bind %s, translate %s, format %s, dispatch %s total. Shared L2: %d hits, %d misses, %d entries/%d capacity.\n\n", shortest.Count, fmtDuration(shortest.Parse), fmtDuration(shortest.Cache), fmtDuration(shortest.Translate), fmtDuration(shortest.Format), fmtDuration(shortest.Dispatch), r.TranslationCache.SharedShortestPathTemplates.Hits, r.TranslationCache.SharedShortestPathTemplates.Misses, r.TranslationCache.SharedShortestPathTemplates.Entries, r.TranslationCache.SharedShortestPathTemplates.Capacity) + } + selection := r.TranslationCache.StrategySelection + if selection.Incumbent+selection.ExactQueryCanary+selection.StructuralAuthorized > 0 { + fmt.Fprintf(w, "PostgreSQL strategy selection: %d incumbent, %d exact-query canary, %d structurally authorized, %d structural-shadow, %d shape-unavailable observations.\n\n", selection.Incumbent, selection.ExactQueryCanary, selection.StructuralAuthorized, selection.StructuralShadow, selection.ShapeUnavailable) + } + shapeCache := r.TranslationCache.TraversalShapeCache + if shapeCache.Hits+shapeCache.Misses > 0 { + fmt.Fprintf(w, "PostgreSQL traversal shape cache: %d hits, %d misses, %d entries/%d capacity.\n\n", shapeCache.Hits, shapeCache.Misses, shapeCache.Entries, shapeCache.Capacity) + } + route := r.TranslationCache.TraversalRouteDecision + if route.Disabled+route.SynopsisUnavailable+route.ShadowMiss+route.ShadowHit+route.Capacity+route.ParametersInvalid > 0 { + fmt.Fprintf(w, "PostgreSQL topology routing shadow: %d misses, %d hits, %d synopsis-unavailable, %d disabled, %d capacity, %d parameter-invalid.\n\n", route.ShadowMiss, route.ShadowHit, route.SynopsisUnavailable, route.Disabled, route.Capacity, route.ParametersInvalid) + } + } return nil } diff --git a/cmd/benchmark/report_test.go b/cmd/benchmark/report_test.go index 92b460bb..f0be29e0 100644 --- a/cmd/benchmark/report_test.go +++ b/cmd/benchmark/report_test.go @@ -24,9 +24,11 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" "github.com/stretchr/testify/require" ) +// TestWriteJSONEmitsBaselineFriendlyReport verifies that JSON retains row diagnostics, timing values, SQL, and every optimizer decision needed for baseline comparisons. func TestWriteJSONEmitsBaselineFriendlyReport(t *testing.T) { var ( distinctRows = int64(2) @@ -43,10 +45,19 @@ func TestWriteJSONEmitsBaselineFriendlyReport(t *testing.T) { }}, } report = Report{ - Driver: "pg", - GitRef: "abc123", - Date: "2026-05-14", - Iterations: 3, + Driver: "pg", + GitRef: "abc123", + Date: "2026-05-14", + Iterations: 3, + WarmupIterations: 1, + Workers: 2, + TranslationCache: &pg.Stats{ + LiveConnections: 2, + Aggregate: pg.TranslationCacheStats{Hits: 4, Misses: 2}, + TraversalWorkspace: pg.TraversalWorkspaceStats{Initializations: 1, Reuses: 3}, + PreparedStatements: pg.PreparedStatementStats{Prepared: 2, Reuses: 4}, + StrategySelection: pg.StrategySelectionStats{Incumbent: 3, ExactQueryCanary: 1, StructuralShadow: 2, ShapeUnavailable: 2}, + }, Results: []Result{{ Section: "Traversal", Dataset: "base", @@ -85,6 +96,10 @@ func TestWriteJSONEmitsBaselineFriendlyReport(t *testing.T) { for _, expected := range []string{ `"driver": "pg"`, `"git_ref": "abc123"`, + `"warmup_iterations": 1`, + `"workers": 2`, + `"translation_cache": {`, + `"hits": 4`, `"median": 10000000`, `"row_count": 2`, `"distinct_row_count": 2`, @@ -105,18 +120,28 @@ func TestWriteJSONEmitsBaselineFriendlyReport(t *testing.T) { } } +// TestWriteMarkdownIncludesDiagnosticColumns verifies that Markdown exposes distinct and duplicate row counts alongside timing and plan-capture status. func TestWriteMarkdownIncludesDiagnosticColumns(t *testing.T) { var ( distinctRows = int64(2) duplicateRows = int64(0) report = Report{ - Driver: "pg", - GitRef: "abc123", - Date: "2026-05-14", - Iterations: 3, + Driver: "pg", + GitRef: "abc123", + Date: "2026-05-14", + Iterations: 3, + WarmupIterations: 1, + Workers: 2, + TranslationCache: &pg.Stats{ + LiveConnections: 2, + Aggregate: pg.TranslationCacheStats{Hits: 4, Misses: 2}, + TraversalWorkspace: pg.TraversalWorkspaceStats{Initializations: 1, Reuses: 3}, + PreparedStatements: pg.PreparedStatementStats{Prepared: 2, Reuses: 4}, + StrategySelection: pg.StrategySelectionStats{Incumbent: 3, ExactQueryCanary: 1, StructuralShadow: 2, ShapeUnavailable: 2}, + }, Results: []Result{{ - Section: "ADCS Fanout", - Dataset: "adcs_fanout", + Section: "Fixed Suffix Expansion Fanout", + Dataset: "fixed_suffix_expansion_fanout", Label: "combined", RowCount: 2, DistinctRowCount: &distinctRows, @@ -138,22 +163,56 @@ func TestWriteMarkdownIncludesDiagnosticColumns(t *testing.T) { for _, expected := range []string{ "Distinct Rows", "Duplicate Rows", - "| ADCS Fanout / combined | adcs_fanout | 2 | 2 | 0 | 10.0ms | 20.0ms | 30.0ms | captured |", + "| Fixed Suffix Expansion Fanout / combined | fixed_suffix_expansion_fanout | 2 | 2 | 0 | 10.0ms | 20.0ms | 30.0ms | captured |", + "PostgreSQL connection state: cache 4 hits, 2 misses, 0 bypasses, 0 evictions; workspaces 1 initialized/3 reused; statements 2 prepared/4 reused across 2 live connections.", + "PostgreSQL strategy selection: 3 incumbent, 1 exact-query canary, 0 structurally authorized, 2 structural-shadow, 2 shape-unavailable observations.", } { require.Contains(t, text, expected) } } +func TestWriteMarkdownIdentifiesProductionPolicyPath(t *testing.T) { + report := Report{ + Driver: pg.DriverName, + GitRef: "abcdef0", + Date: "2026-08-20", + Iterations: 2, + ShortestPathExecutor: string(optimize.ShortestPathExecutorASPI1DAG), + ShortestPathMode: shortestPathModeProductionPolicy, + TraversalPolicyGeneration: 42, + TraversalPolicyManifestSHA256: "0123456789abcdef", + PostgreSQLPlanCacheMode: "auto", + PostgreSQLJIT: true, + } + var output bytes.Buffer + + require.NoError(t, writeMarkdown(&output, report)) + require.Contains(t, output.String(), "through production traversal policy generation 42") + require.Contains(t, output.String(), "manifest `0123456789abcdef`") +} + +// TestValidateIterationsRejectsZero verifies that benchmark execution requires at least one measured iteration. func TestValidateIterationsRejectsZero(t *testing.T) { require.Error(t, validateIterations(0)) require.NoError(t, validateIterations(1)) } +// TestValidateBenchmarkConcurrencyRejectsInvalidInputs verifies that cold and +// concurrent measurements reject invalid values before database work starts. +func TestValidateBenchmarkConcurrencyRejectsInvalidInputs(t *testing.T) { + require.Error(t, validateBenchmarkConcurrency(-1, 1)) + require.Error(t, validateBenchmarkConcurrency(0, 0)) + require.NoError(t, validateBenchmarkConcurrency(0, 1)) + require.NoError(t, validateBenchmarkConcurrency(2, 4)) +} + +// TestWriteReportRejectsUnknownFormat verifies that report dispatch fails instead of silently choosing a serializer for an unsupported format. func TestWriteReportRejectsUnknownFormat(t *testing.T) { err := writeReport(&bytes.Buffer{}, Report{}, "xml") require.ErrorContains(t, err, "unsupported output format") } +// TestWriteJSON verifies that JSON dispatch preserves the selected driver and emits raw duration samples in nanoseconds. func TestWriteJSON(t *testing.T) { report := testReport() var out bytes.Buffer @@ -165,6 +224,7 @@ func TestWriteJSON(t *testing.T) { require.Contains(t, out.String(), `1000000`) } +// TestWriteBenchfmt verifies that benchfmt output carries platform metadata, a stable benchmark name, and one ns/op observation per sample. func TestWriteBenchfmt(t *testing.T) { report := testReport() var out bytes.Buffer @@ -180,6 +240,7 @@ func TestWriteBenchfmt(t *testing.T) { require.Contains(t, output, "\t1\t2000000 ns/op") } +// TestSanitizeBenchNamePart verifies that benchmark labels normalize whitespace and arrows without destroying hierarchy separators, and that empty labels receive a fallback. func TestSanitizeBenchNamePart(t *testing.T) { require.Equal(t, "Shortest_Paths", sanitizeBenchNamePart("Shortest Paths")) require.Equal(t, "n1_-_n3", sanitizeBenchNamePart("n1 -> n3")) @@ -187,6 +248,7 @@ func TestSanitizeBenchNamePart(t *testing.T) { require.Equal(t, "unknown", sanitizeBenchNamePart("")) } +// TestWriteMarkdownOmitsSamples verifies that Markdown reports aggregate timings without leaking the raw nanosecond sample series. func TestWriteMarkdownOmitsSamples(t *testing.T) { report := testReport() var out bytes.Buffer @@ -198,6 +260,7 @@ func TestWriteMarkdownOmitsSamples(t *testing.T) { require.False(t, strings.Contains(output, "1000000")) } +// testReport returns a representative report used by serializer tests. func testReport() Report { return Report{ Driver: "pg", diff --git a/cmd/benchmark/runner.go b/cmd/benchmark/runner.go index 87faf8a7..d9cbdfda 100644 --- a/cmd/benchmark/runner.go +++ b/cmd/benchmark/runner.go @@ -20,15 +20,21 @@ import ( "context" "fmt" "sort" + "sync" "time" "github.com/specterops/dawgs/graph" ) -type ExplainFunc func(ctx context.Context, tx graph.Transaction, cypher string) (*ExplainResult, error) +// ExplainFunc captures a plan for the exact scenario values that were timed. +// Passing the Scenario prevents parameterized endpoint plans from silently +// explaining an empty query instead of the workload under measurement. +type ExplainFunc func(ctx context.Context, tx graph.Transaction, scenario Scenario) (*ExplainResult, error) type RunOptions struct { - Explain ExplainFunc + Explain ExplainFunc + WarmupIterations int + Workers int } // Stats holds computed timing statistics for a scenario. @@ -56,21 +62,18 @@ func runScenario(ctx context.Context, db graph.Database, s Scenario, iterations if err := validateIterations(iterations); err != nil { return Result{}, err } - - // Warm-up: one untimed run. - measurement, err := runScenarioOnce(ctx, db, s) - if err != nil { + if options.Workers == 0 { + options.Workers = 1 + } + if err := validateRunOptions(options); err != nil { return Result{}, err } - durations := make([]time.Duration, iterations) - - for i := range iterations { - start := time.Now() - if _, err := runScenarioOnce(ctx, db, s); err != nil { - return Result{}, err - } - durations[i] = time.Since(start) + measurement, durations, err := runScenarioSamples(iterations, options.WarmupIterations, options.Workers, func() (Measurement, error) { + return runScenarioOnce(ctx, db, s) + }) + if err != nil { + return Result{}, err } result := Result{ @@ -86,7 +89,7 @@ func runScenario(ctx context.Context, db graph.Database, s Scenario, iterations if options.Explain != nil && s.Cypher != "" { if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { - explain, err := options.Explain(ctx, tx, s.Cypher) + explain, err := options.Explain(ctx, tx, s) result.Explain = explain return err }); err != nil { @@ -97,6 +100,72 @@ func runScenario(ctx context.Context, db graph.Database, s Scenario, iterations return result, nil } +// runScenarioSamples executes one logical benchmark scenario across workers. +// Every worker warms its own leased connection before it contributes timed +// samples, which makes connection-local cache behavior observable without +// mixing warm-up latency into the reported distribution. +func runScenarioSamples(iterations, warmupIterations, workers int, run func() (Measurement, error)) (Measurement, []time.Duration, error) { + if err := validateIterations(iterations); err != nil { + return Measurement{}, nil, err + } + if err := validateBenchmarkConcurrency(warmupIterations, workers); err != nil { + return Measurement{}, nil, err + } + + type workerResult struct { + measurement Measurement + durations []time.Duration + err error + } + + results := make(chan workerResult, workers) + var group sync.WaitGroup + group.Add(workers) + for range workers { + go func() { + defer group.Done() + var measurement Measurement + for range warmupIterations { + next, err := run() + if err != nil { + results <- workerResult{err: err} + return + } + measurement = next + } + + durations := make([]time.Duration, iterations) + for index := range iterations { + started := time.Now() + next, err := run() + if err != nil { + results <- workerResult{err: err} + return + } + if warmupIterations == 0 && index == 0 { + measurement = next + } + durations[index] = time.Since(started) + } + results <- workerResult{measurement: measurement, durations: durations} + }() + } + group.Wait() + close(results) + + durations := make([]time.Duration, 0, iterations*workers) + var measurement Measurement + for result := range results { + if result.err != nil { + return Measurement{}, nil, result.err + } + measurement = result.measurement + durations = append(durations, result.durations...) + } + + return measurement, durations, nil +} + func validateIterations(iterations int) error { if iterations < 1 { return fmt.Errorf("iterations must be at least 1") @@ -105,6 +174,20 @@ func validateIterations(iterations int) error { return nil } +func validateRunOptions(options RunOptions) error { + return validateBenchmarkConcurrency(options.WarmupIterations, options.Workers) +} + +func validateBenchmarkConcurrency(warmupIterations, workers int) error { + if warmupIterations < 0 { + return fmt.Errorf("warm-up iterations must not be negative: %d", warmupIterations) + } + if workers < 1 { + return fmt.Errorf("workers must be at least 1: %d", workers) + } + return nil +} + func runScenarioOnce(ctx context.Context, db graph.Database, s Scenario) (Measurement, error) { var measurement Measurement if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { diff --git a/cmd/benchmark/runner_test.go b/cmd/benchmark/runner_test.go new file mode 100644 index 00000000..dc252055 --- /dev/null +++ b/cmd/benchmark/runner_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestRunScenarioSamplesWarmsEachWorker verifies that every worker receives +// its own untimed warm-up and contributes the configured number of samples. +func TestRunScenarioSamplesWarmsEachWorker(t *testing.T) { + var calls atomic.Int64 + measurement, samples, err := runScenarioSamples(3, 2, 4, func() (Measurement, error) { + calls.Add(1) + return Measurement{RowCount: 7}, nil + }) + require.NoError(t, err) + require.Equal(t, int64(7), measurement.RowCount) + require.Len(t, samples, 12) + require.Equal(t, int64(20), calls.Load()) +} + +// TestRunScenarioSamplesSupportsColdMeasurements verifies that zero warm-ups +// keep the first timed query in the timing distribution. +func TestRunScenarioSamplesSupportsColdMeasurements(t *testing.T) { + var calls atomic.Int64 + measurement, samples, err := runScenarioSamples(2, 0, 3, func() (Measurement, error) { + return Measurement{RowCount: calls.Add(1)}, nil + }) + require.NoError(t, err) + require.NotZero(t, measurement.RowCount) + require.Len(t, samples, 6) + require.Equal(t, int64(6), calls.Load()) +} diff --git a/cmd/benchmark/scenarios.go b/cmd/benchmark/scenarios.go index ef819e62..dc8941f3 100644 --- a/cmd/benchmark/scenarios.go +++ b/cmd/benchmark/scenarios.go @@ -25,35 +25,47 @@ import ( "github.com/specterops/dawgs/opengraph" ) -// Measurement captures the warm-up result shape for a benchmark scenario. +// Measurement pairs a benchmark duration with the number of rows observed. type Measurement struct { - RowCount int64 - DistinctRowCount *int64 + // RowCount records the number of rows produced. + RowCount int64 + // DistinctRowCount records unique rows returned by the benchmark scenario. + DistinctRowCount *int64 + // DuplicateRowCount records repeated rows retained by the benchmark scenario. DuplicateRowCount *int64 } -// Scenario defines a single benchmark query to run against a loaded dataset. +// Scenario defines one query, its parameters, and expected cardinality. type Scenario struct { - Section string // grouping key in the report (e.g. "Match Nodes") - Dataset string - Label string // human-readable row label + // Section groups baseline rows under a Markdown summary section. + Section string // grouping key in the report (e.g. "Match Nodes") + // Dataset identifies the fixture dataset. + Dataset string + // Label provides the benchfmt label for the benchmark scenario. + Label string // human-readable row label + // ExpectedRows sets the row count required for a scenario to succeed. ExpectedRows *int64 - Cypher string - Query func(tx graph.Transaction) (Measurement, error) + // Cypher contains the Cypher statement under test. + Cypher string + // Parameters supplies the immutable parameters bound when Cypher is executed. + Parameters map[string]any + // Query executes the scenario in a transaction and returns its duration and observed row count. + Query func(tx graph.Transaction) (Measurement, error) } +// traversalShapesDataset is the fixture key shared by traversal-shape scenario selection and dataset loading. const traversalShapesDataset = "traversal_shapes" // defaultDatasets is the set of datasets committed to the repo. -var defaultDatasets = []string{"base", "adcs_fanout", traversalShapesDataset} +var defaultDatasets = []string{"base", "fixed_suffix_expansion_fanout", traversalShapesDataset} // scenariosForDataset returns all benchmark scenarios for a given dataset and its loaded ID map. func scenariosForDataset(dataset string, idMap opengraph.IDMap) []Scenario { switch dataset { case "base": return baseScenarios(idMap) - case "adcs_fanout": - return adcsFanoutScenarios() + case "fixed_suffix_expansion_fanout": + return fixedSuffixExpansionFanoutScenarios() case traversalShapesDataset: return traversalShapesScenarios(idMap) case "local/phantom": @@ -63,21 +75,31 @@ func scenariosForDataset(dataset string, idMap opengraph.IDMap) []Scenario { } } +// expectRows returns an addressable row expectation so zero expected rows remains distinguishable from an unspecified expectation. func expectRows(rows int64) *int64 { return &rows } +// countNodes measures the transaction-visible node cardinality for dataset sanity benchmarks. func countNodes(tx graph.Transaction) (int64, error) { return tx.Nodes().Count() } +// countEdges measures the transaction-visible relationship cardinality for dataset sanity benchmarks. func countEdges(tx graph.Transaction) (int64, error) { return tx.Relationships().Count() } +// cypherQuery adapts Cypher text into a benchmark callback that drains the result and records returned row count. func cypherQuery(cypher string) func(tx graph.Transaction) (Measurement, error) { + return cypherQueryWithParameters(cypher, nil) +} + +// cypherQueryWithParameters adapts parameterized Cypher text into a benchmark +// callback that drains the result and records returned row count. +func cypherQueryWithParameters(cypher string, parameters map[string]any) func(tx graph.Transaction) (Measurement, error) { return func(tx graph.Transaction) (Measurement, error) { - result := tx.Query(cypher, nil) + result := tx.Query(cypher, parameters) defer result.Close() var rowCount int64 @@ -89,6 +111,7 @@ func cypherQuery(cypher string) func(tx graph.Transaction) (Measurement, error) } } +// countQuery adapts a cardinality callback into a benchmark Measurement while preserving the callback error. func countQuery(query func(tx graph.Transaction) (int64, error)) func(tx graph.Transaction) (Measurement, error) { return func(tx graph.Transaction) (Measurement, error) { rowCount, err := query(tx) @@ -100,16 +123,26 @@ func countQuery(query func(tx graph.Transaction) (int64, error)) func(tx graph.T } } +// cypherScenario builds a row-counting Scenario from its corpus identity and Cypher text. func cypherScenario(section, dataset, label, cypher string) Scenario { + return cypherScenarioWithParameters(section, dataset, label, cypher, nil) +} + +// cypherScenarioWithParameters builds a row-counting Scenario whose Cypher +// identity remains stable while its endpoint values vary with fixture loading. +// That stability is required by the exact-query traversal-policy allowlist. +func cypherScenarioWithParameters(section, dataset, label, cypher string, parameters map[string]any) Scenario { return Scenario{ - Section: section, - Dataset: dataset, - Label: label, - Cypher: cypher, - Query: cypherQuery(cypher), + Section: section, + Dataset: dataset, + Label: label, + Cypher: cypher, + Parameters: parameters, + Query: cypherQueryWithParameters(cypher, parameters), } } +// cypherPathScenario builds a Scenario that validates and counts path-valued columns while consuming results. func cypherPathScenario(section, dataset, label, cypher string, pathColumns int) Scenario { return Scenario{ Section: section, @@ -120,11 +153,13 @@ func cypherPathScenario(section, dataset, label, cypher string, pathColumns int) } } +// expectScenarioRows returns scenario with an explicit correctness expectation attached. func expectScenarioRows(scenario Scenario, rows int64) Scenario { scenario.ExpectedRows = expectRows(rows) return scenario } +// cypherPathQuery adapts Cypher text into a benchmark callback that validates path columns and hashes their node/edge identities while draining rows. func cypherPathQuery(cypher string, pathColumns int) func(tx graph.Transaction) (Measurement, error) { return func(tx graph.Transaction) (Measurement, error) { result := tx.Query(cypher, nil) @@ -171,6 +206,7 @@ func cypherPathQuery(cypher string, pathColumns int) func(tx graph.Transaction) } } +// pathRowKey serializes path node and edge IDs into an unambiguous key used to prevent result materialization from being optimized away. func pathRowKey(paths []graph.Path) string { var builder strings.Builder @@ -213,15 +249,16 @@ func pathRowKey(paths []graph.Path) string { // --- Base dataset scenarios (n1 -> n2 -> n3) --- +// baseScenarios defines cardinality, lookup, and one-hop checks for the three-node base fixture. func baseScenarios(idMap opengraph.IDMap) []Scenario { ds := "base" return []Scenario{ {Section: "Match Nodes", Dataset: ds, Label: ds, ExpectedRows: expectRows(3), Query: countQuery(countNodes)}, {Section: "Match Edges", Dataset: ds, Label: ds, ExpectedRows: expectRows(2), Query: countQuery(countEdges)}, - expectScenarioRows(cypherScenario("Shortest Paths", ds, "n1 -> n3", fmt.Sprintf( - "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = %d AND id(e) = %d RETURN p", - idMap["n1"], idMap["n3"], - )), 1), + expectScenarioRows(cypherScenarioWithParameters("Shortest Paths", ds, "n1 -> n3", + "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + map[string]any{"start_id": idMap["n1"], "end_id": idMap["n3"]}, + ), 1), expectScenarioRows(cypherScenario("Traversal", ds, "n1", fmt.Sprintf( "MATCH (s)-[*1..]->(e) WHERE id(s) = %d RETURN e", idMap["n1"], @@ -235,46 +272,49 @@ func baseScenarios(idMap opengraph.IDMap) []Scenario { } } -const adcsFanoutObjectID = "S-1-5-21-2643190041-1319121918-239771340-513" +// fixedSuffixFanoutRootKey identifies the fanout fixture root whose generated ID is injected into fixed-suffix scenarios. +const fixedSuffixFanoutRootKey = "fixed-suffix-fanout-root" -func adcsFanoutScenarios() []Scenario { +// fixedSuffixExpansionFanoutScenarios exercises bounded reverse-suffix expansion at increasing depths and with path projection enabled. +func fixedSuffixExpansionFanoutScenarios() []Scenario { var ( - ds = "adcs_fanout" + ds = "fixed_suffix_expansion_fanout" p1 = fmt.Sprintf(` - MATCH (n:Group) WHERE n.objectid = '%s' - MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) + MATCH (root:ExpansionRoot) WHERE root.root_key = '%s' + MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN p1 - `, adcsFanoutObjectID) + `, fixedSuffixFanoutRootKey) p2 = fmt.Sprintf(` - MATCH (n:Group) WHERE n.objectid = '%s' - MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d:Domain) - WHERE ct.authenticationenabled = true - AND ct.requiresmanagerapproval = false - AND ct.enrolleesuppliessubject = true - AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) + MATCH (root:ExpansionRoot) WHERE root.root_key = '%s' + MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head:SuffixHead)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal:SuffixTerminal) + WHERE predicate.eligible = true + AND predicate.requires_review = false + AND predicate.allows_direct = true + AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN p2 - `, adcsFanoutObjectID) + `, fixedSuffixFanoutRootKey) combinedMatch = fmt.Sprintf(` - MATCH (n:Group) WHERE n.objectid = '%s' - MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) - MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) - WHERE ct.authenticationenabled = true - AND ct.requiresmanagerapproval = false - AND ct.enrolleesuppliessubject = true - AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) - `, adcsFanoutObjectID) + MATCH (root:ExpansionRoot) WHERE root.root_key = '%s' + MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) + WHERE predicate.eligible = true + AND predicate.requires_review = false + AND predicate.allows_direct = true + AND (predicate.version = 1 OR predicate.required_approvals = 0) + `, fixedSuffixFanoutRootKey) ) return []Scenario{ - cypherPathScenario("ADCS Fanout", ds, "p1 only", p1, 1), - cypherPathScenario("ADCS Fanout", ds, "p2 only", p2, 1), - cypherPathScenario("ADCS Fanout", ds, "combined", combinedMatch+"RETURN p1,p2", 2), - cypherScenario("ADCS Fanout", ds, "combined endpoints", combinedMatch+"RETURN id(ca), id(d), id(ct)"), + cypherPathScenario("Fixed Suffix Expansion Fanout", ds, "p1 only", p1, 1), + cypherPathScenario("Fixed Suffix Expansion Fanout", ds, "p2 only", p2, 1), + cypherPathScenario("Fixed Suffix Expansion Fanout", ds, "combined", combinedMatch+"RETURN p1,p2", 2), + cypherScenario("Fixed Suffix Expansion Fanout", ds, "combined endpoints", combinedMatch+"RETURN id(head), id(terminal), id(predicate)"), } } // --- Traversal shape scenarios --- +// traversalShapesScenarios covers single-hop, bounded variable-length, shortest-path, and repeated-edge traversal forms over the shared fixture. func traversalShapesScenarios(idMap opengraph.IDMap) []Scenario { ds := traversalShapesDataset return []Scenario{ @@ -320,19 +360,20 @@ func traversalShapesScenarios(idMap opengraph.IDMap) []Scenario { "MATCH (s)-[*1..]->(e) WHERE id(s) = %d RETURN e", idMap["s0"], )), 6), - expectScenarioRows(cypherScenario("Shortest Paths", ds, "diamond many paths", fmt.Sprintf( - "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = %d AND id(e) = %d RETURN p", - idMap["d0"], idMap["d4"], - )), 3), - expectScenarioRows(cypherScenario("Shortest Paths", ds, "disconnected", fmt.Sprintf( - "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = %d AND id(e) = %d RETURN p", - idMap["x0"], idMap["x1"], - )), 0), + expectScenarioRows(cypherScenarioWithParameters("Shortest Paths", ds, "diamond many paths", + "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + map[string]any{"start_id": idMap["d0"], "end_id": idMap["d4"]}, + ), 3), + expectScenarioRows(cypherScenarioWithParameters("Shortest Paths", ds, "disconnected", + "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = $disconnected_start_id AND id(e) = $disconnected_end_id RETURN p", + map[string]any{"disconnected_start_id": idMap["x0"], "disconnected_end_id": idMap["x1"]}, + ), 0), } } // --- Phantom scenarios (hardcoded node IDs from the dataset) --- +// phantomScenarios preserves legacy benchmark cases that intentionally address the phantom fixture by its stable generated IDs. func phantomScenarios(idMap opengraph.IDMap) []Scenario { var ( ds = "local/phantom" diff --git a/cmd/benchmark/scenarios_test.go b/cmd/benchmark/scenarios_test.go index 0206c020..fe39008a 100644 --- a/cmd/benchmark/scenarios_test.go +++ b/cmd/benchmark/scenarios_test.go @@ -25,6 +25,30 @@ import ( "github.com/stretchr/testify/require" ) +// TestShortestPathScenariosUseStableParameterizedCypher verifies that fixture +// reloads vary only bindings, not the exact Cypher identity authorized by a +// traversal policy manifest. +func TestShortestPathScenariosUseStableParameterizedCypher(t *testing.T) { + first := traversalShapesScenarios(opengraph.IDMap{ + "d0": graph.ID(1), "d4": graph.ID(2), "x0": graph.ID(3), "x1": graph.ID(4), + }) + second := traversalShapesScenarios(opengraph.IDMap{ + "d0": graph.ID(101), "d4": graph.ID(102), "x0": graph.ID(103), "x1": graph.ID(104), + }) + + firstShortest := shortestPathScenarios(first) + secondShortest := shortestPathScenarios(second) + require.Len(t, firstShortest, 2) + require.Len(t, secondShortest, 2) + for index := range firstShortest { + require.Equal(t, firstShortest[index].Cypher, secondShortest[index].Cypher) + require.NotEmpty(t, firstShortest[index].Parameters) + require.NotEqual(t, firstShortest[index].Parameters, secondShortest[index].Parameters) + } + require.NotEqual(t, firstShortest[0].Cypher, firstShortest[1].Cypher, "one manifest query digest must select exactly one scenario") +} + +// TestBaseScenariosDeclareExpectedRows verifies the canonical row-count contract for every query family in the base fixture. func TestBaseScenariosDeclareExpectedRows(t *testing.T) { scenarios := baseScenarios(opengraph.IDMap{ "n1": graph.ID(1), @@ -41,6 +65,7 @@ func TestBaseScenariosDeclareExpectedRows(t *testing.T) { requireExpectedRows(t, scenarios, "Filter By Kind", "NodeKind2", 2) } +// TestTraversalShapesDatasetIsValid verifies that the checked-in traversal fixture parses and retains its expected 45-node, 41-edge topology. func TestTraversalShapesDatasetIsValid(t *testing.T) { file, err := os.Open("../../integration/testdata/traversal_shapes.json") require.NoError(t, err) @@ -52,6 +77,7 @@ func TestTraversalShapesDatasetIsValid(t *testing.T) { require.Len(t, doc.Graph.Edges, 41) } +// TestTraversalShapesScenariosDeclareExpectedRows verifies the expected cardinalities for depth, fanout, cycle, dead-end, kind-filtered, and shortest-path fixture cases. func TestTraversalShapesScenariosDeclareExpectedRows(t *testing.T) { scenarios := traversalShapesScenarios(traversalShapesIDMap()) @@ -71,11 +97,13 @@ func TestTraversalShapesScenariosDeclareExpectedRows(t *testing.T) { requireExpectedRows(t, scenarios, "Shortest Paths", "disconnected", 0) } +// TestDefaultDatasetsIncludeTraversalShapes verifies that ordinary benchmark runs include both traversal-shape and fixed-suffix fanout coverage. func TestDefaultDatasetsIncludeTraversalShapes(t *testing.T) { require.Contains(t, defaultDatasets, traversalShapesDataset) - require.Contains(t, defaultDatasets, "adcs_fanout") + require.Contains(t, defaultDatasets, "fixed_suffix_expansion_fanout") } +// TestValidateScenarioRows verifies that observed cardinality must match the scenario contract and that failures identify the scenario and both counts. func TestValidateScenarioRows(t *testing.T) { scenario := Scenario{ Section: "Traversal", @@ -88,6 +116,7 @@ func TestValidateScenarioRows(t *testing.T) { require.ErrorContains(t, validateScenarioRows(scenario, 1), "Traversal/n1 on base expected 2 rows, got 1") } +// traversalShapesIDMap resolves traversal-shape fixture node keys to database identifiers. func traversalShapesIDMap() opengraph.IDMap { ids := []string{ "c0", "c10", @@ -106,6 +135,7 @@ func traversalShapesIDMap() opengraph.IDMap { return idMap } +// requireExpectedRows locates a scenario by section and label and asserts its declared cardinality. func requireExpectedRows(t *testing.T, scenarios []Scenario, section, label string, expectedRows int64) { t.Helper() @@ -119,3 +149,13 @@ func requireExpectedRows(t *testing.T, scenarios []Scenario, section, label stri require.Failf(t, "scenario not found", "%s/%s", section, label) } + +func shortestPathScenarios(scenarios []Scenario) []Scenario { + shortest := make([]Scenario, 0) + for _, scenario := range scenarios { + if scenario.Section == "Shortest Paths" { + shortest = append(shortest, scenario) + } + } + return shortest +} diff --git a/cmd/benchmark/shortest_executor.go b/cmd/benchmark/shortest_executor.go new file mode 100644 index 00000000..e2bf3edc --- /dev/null +++ b/cmd/benchmark/shortest_executor.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/drivers/pg/model" + "github.com/specterops/dawgs/graph" +) + +// shortestExecutorBenchmarkDatabase forces one repository-qualified executor +// only at the benchmark boundary. Production driver routing remains governed +// by the signed traversal-policy manifest. +type shortestExecutorBenchmarkDatabase struct { + graph.Database + mapper pg.KindMapper + graph model.Graph + executor optimize.ShortestPathExecutor + planCacheMode string + jitEnabled bool +} + +func newShortestExecutorBenchmarkDatabase(database graph.Database, mapper pg.KindMapper, target model.Graph, executor optimize.ShortestPathExecutor, planCacheMode string, jitEnabled bool) (graph.Database, error) { + if !benchmarkShortestPathExecutor(executor) { + return nil, fmt.Errorf("unsupported benchmark shortest-path executor %q", executor) + } + if !benchmarkPlanCacheMode(planCacheMode) { + return nil, fmt.Errorf("unsupported PostgreSQL plan cache mode %q", planCacheMode) + } + return &shortestExecutorBenchmarkDatabase{Database: database, mapper: mapper, graph: target, executor: executor, planCacheMode: planCacheMode, jitEnabled: jitEnabled}, nil +} + +func benchmarkPlanCacheMode(mode string) bool { + return mode == "auto" || mode == "force_custom_plan" || mode == "force_generic_plan" +} + +func benchmarkShortestPathExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorASPN1NegativeExhaustion, + optimize.ShortestPathExecutorI2GuardedDistanceV2: + return true + default: + return false + } +} + +func (s *shortestExecutorBenchmarkDatabase) ReadTransaction(ctx context.Context, delegate graph.TransactionDelegate, options ...graph.TransactionOption) error { + options = append(options, pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + return s.Database.ReadTransaction(ctx, func(tx graph.Transaction) error { + if err := applyPostgreSQLShortestPathBenchmarkSettings(tx, s.planCacheMode, s.jitEnabled); err != nil { + return err + } + return delegate(&shortestExecutorBenchmarkTransaction{Transaction: tx, ctx: ctx, mapper: s.mapper, graphID: s.graph.ID, executor: s.executor}) + }, options...) +} + +func (s *shortestExecutorBenchmarkDatabase) KindMapper() pg.KindMapper { + return s.mapper +} + +func (s *shortestExecutorBenchmarkDatabase) DefaultGraph() (model.Graph, bool) { + return s.graph, true +} + +type shortestExecutorBenchmarkTransaction struct { + graph.Transaction + ctx context.Context + mapper pg.KindMapper + graphID int32 + executor optimize.ShortestPathExecutor +} + +func (s *shortestExecutorBenchmarkTransaction) Query(cypherQuery string, parameters map[string]any) graph.Result { + if !strings.Contains(strings.ToLower(cypherQuery), "shortestpath") { + return s.Transaction.Query(cypherQuery, parameters) + } + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + return graph.NewErrorResult(err) + } + translation, err := translate.TranslateForTool(s.ctx, regularQuery, s.mapper, parameters, s.graphID, translate.ToolOptions{ForceShortestPathExecutor: s.executor}) + if err != nil { + return graph.NewErrorResult(err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + return graph.NewErrorResult(err) + } + return s.Transaction.Raw(sqlQuery, translation.Parameters) +} + +// traversalPolicyBenchmarkDatabase changes only the transaction boundary used +// by a policy-path benchmark. Query translation remains in the real V2 driver, +// which therefore performs the manifest selection, SQL-anchor validation, and +// normal connection-local cache lookup. +type traversalPolicyBenchmarkDatabase struct { + graph.Database + planCacheMode string + jitEnabled bool +} + +func newTraversalPolicyBenchmarkDatabase(database graph.Database, planCacheMode string, jitEnabled bool) (graph.Database, error) { + if !benchmarkPlanCacheMode(planCacheMode) { + return nil, fmt.Errorf("unsupported PostgreSQL plan cache mode %q", planCacheMode) + } + return &traversalPolicyBenchmarkDatabase{Database: database, planCacheMode: planCacheMode, jitEnabled: jitEnabled}, nil +} + +func (s *traversalPolicyBenchmarkDatabase) ReadTransaction(ctx context.Context, delegate graph.TransactionDelegate, options ...graph.TransactionOption) error { + options = append(options, pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + return s.Database.ReadTransaction(ctx, func(tx graph.Transaction) error { + if err := applyPostgreSQLShortestPathBenchmarkSettings(tx, s.planCacheMode, s.jitEnabled); err != nil { + return err + } + return delegate(tx) + }, options...) +} + +// TranslationCacheStats preserves the PostgreSQL telemetry surface through the policy +// boundary wrapper so policy-path reports include actual cache activity. +func (s *traversalPolicyBenchmarkDatabase) TranslationCacheStats() pg.Stats { + if provider, ok := s.Database.(interface{ TranslationCacheStats() pg.Stats }); ok { + return provider.TranslationCacheStats() + } + return pg.Stats{} +} + +func applyPostgreSQLShortestPathBenchmarkSettings(tx graph.Transaction, planCacheMode string, jitEnabled bool) error { + settings := []string{"set local plan_cache_mode = " + planCacheMode} + if jitEnabled { + settings = append(settings, "set local jit = on") + } else { + settings = append(settings, "set local jit = off") + } + for _, sql := range settings { + setting := tx.Raw(sql, nil) + setting.Close() + if err := setting.Error(); err != nil { + return fmt.Errorf("apply PostgreSQL shortest-path benchmark setting: %w", err) + } + } + return nil +} diff --git a/cmd/benchmark/traversal_policy.go b/cmd/benchmark/traversal_policy.go new file mode 100644 index 00000000..fc098f54 --- /dev/null +++ b/cmd/benchmark/traversal_policy.go @@ -0,0 +1,250 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/drivers/pg/model" +) + +// traversalPolicyBenchmarkDriver exposes the production policy installation +// seam required by a policy-path benchmark. +type traversalPolicyBenchmarkDriver interface { + SetTraversalPolicy(pg.TraversalPolicy) error +} + +// benchmarkTraversalPromotionManifest retains only the manifest fields needed +// to assemble the driver's public TraversalPolicy. The driver independently +// decodes and validates the complete raw document before accepting it. +type benchmarkTraversalPromotionManifest struct { + Version int `json:"version"` + Candidate string `json:"candidate"` + SelectorVersion string `json:"selector_version"` + Caps map[string]int64 `json:"caps"` + Buckets []benchmarkPolicyBucket `json:"buckets"` +} + +type benchmarkPolicyBucket struct { + Name string `json:"name"` + QuerySHA256 []string `json:"query_sha256"` + Direction string `json:"direction"` + ObservationMode string `json:"observation_mode"` + MinimumDepth int64 `json:"minimum_depth"` + MaximumDepth int64 `json:"maximum_depth"` + RelationshipKindCount int `json:"relationship_kind_count"` + UntypedRelationship bool `json:"untyped_relationship"` +} + +// loadBenchmarkTraversalPolicy loads an immutable manifest verbatim and binds +// its exact bytes, candidate, and query authorization set into a V2 policy. +// It intentionally does not attempt to verify qualification evidence; that is +// the driver's strict installation contract and the GraphBench verifier's job. +func loadBenchmarkTraversalPolicy(path string, generation uint64) (pg.TraversalPolicy, error) { + if path == "" { + return pg.TraversalPolicy{}, fmt.Errorf("traversal policy manifest path is required") + } + if generation == 0 { + return pg.TraversalPolicy{}, fmt.Errorf("traversal policy generation must be nonzero") + } + raw, manifest, err := loadBenchmarkTraversalPromotionManifest(path) + if err != nil { + return pg.TraversalPolicy{}, err + } + if manifest.Candidate == "" { + return pg.TraversalPolicy{}, fmt.Errorf("traversal policy manifest candidate is required") + } + + queries := make([]string, 0) + for _, bucket := range manifest.Buckets { + queries = append(queries, bucket.QuerySHA256...) + } + if len(queries) == 0 { + return pg.TraversalPolicy{}, fmt.Errorf("traversal policy manifest must authorize at least one query") + } + + digest := sha256.Sum256(raw) + return pg.TraversalPolicy{ + Generation: generation, + PromotionManifestSHA256: hex.EncodeToString(digest[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: queries, + ShortestPathExecutor: optimize.ShortestPathExecutor(manifest.Candidate), + }, nil +} + +func loadBenchmarkTraversalPromotionManifest(path string) ([]byte, benchmarkTraversalPromotionManifest, error) { + if path == "" { + return nil, benchmarkTraversalPromotionManifest{}, fmt.Errorf("traversal policy manifest path is required") + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, benchmarkTraversalPromotionManifest{}, fmt.Errorf("read traversal policy manifest: %w", err) + } + var manifest benchmarkTraversalPromotionManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return nil, benchmarkTraversalPromotionManifest{}, fmt.Errorf("decode traversal policy manifest binding: %w", err) + } + return raw, manifest, nil +} + +// productionOptions derives the production translation input from the one +// exact query bucket in a provisional manifest. It is used only for SQL-anchor +// preflight; SetTraversalPolicy remains the only authorization path. +func (s benchmarkTraversalPromotionManifest) productionOptions(cypherQuery string) (translate.ProductionOptions, error) { + if s.Candidate == "" || s.SelectorVersion == "" { + return translate.ProductionOptions{}, fmt.Errorf("provisional traversal policy manifest requires candidate and selector version") + } + digest := pg.TraversalPolicyQuerySHA256(cypherQuery) + for _, bucket := range s.Buckets { + for _, allowed := range bucket.QuerySHA256 { + if allowed != digest { + continue + } + return translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutor(s.Candidate), + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: s.Caps["state_limit"], + FrontierLimit: s.Caps["frontier_limit"], + PredecessorLimit: s.Caps["predecessor_limit"], + EnumerationLimit: s.Caps["enumeration_limit"], + OutputBytesLimit: s.Caps["output_bytes_limit"], + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + }, + SelectorVersion: s.SelectorVersion, + }, nil + } + } + return translate.ProductionOptions{}, fmt.Errorf("query SHA-256 %s is absent from provisional traversal policy manifest", digest) +} + +// TraversalPolicyPreflight is a non-promotional record used to bind a formal +// manifest's SQL anchor after the benchmark graph and parameters are known. +type TraversalPolicyPreflight struct { + Candidate string `json:"candidate"` + SelectorVersion string `json:"selector_version"` + QuerySHA256 string `json:"query_sha256"` + SQLSHA256 string `json:"operational_candidate_sql_sha256"` + GraphID int32 `json:"graph_id"` + Optimization translate.OptimizationSummary `json:"optimization"` +} + +// writeTraversalPolicyPreflight creates a new provenance record without ever +// replacing a provisional manifest or an earlier capture. A repeated preflight +// must use a new path so the record's filesystem lifetime remains one-to-one +// with the invocation that produced it. +func writeTraversalPolicyPreflight(manifestPath, outputPath string, preflight TraversalPolicyPreflight) error { + manifestInfo, err := os.Stat(manifestPath) + if err != nil { + return fmt.Errorf("stat provisional traversal policy manifest: %w", err) + } + if outputInfo, err := os.Stat(outputPath); err == nil { + if os.SameFile(manifestInfo, outputInfo) { + return fmt.Errorf("preflight output must not overwrite the provisional traversal policy manifest") + } + return fmt.Errorf("preflight output already exists: %s", outputPath) + } else if !os.IsNotExist(err) { + return fmt.Errorf("stat preflight output: %w", err) + } + + encoded, err := json.MarshalIndent(preflight, "", " ") + if err != nil { + return fmt.Errorf("encode traversal policy preflight: %w", err) + } + output, err := os.OpenFile(outputPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("create preflight output: %w", err) + } + if _, err := output.Write(append(encoded, '\n')); err != nil { + _ = output.Close() + _ = os.Remove(outputPath) + return fmt.Errorf("write preflight output: %w", err) + } + if err := output.Close(); err != nil { + _ = os.Remove(outputPath) + return fmt.Errorf("close preflight output: %w", err) + } + return nil +} + +func renderTraversalPolicyPreflight(ctx context.Context, mapper pg.KindMapper, target model.Graph, scenario Scenario, manifest benchmarkTraversalPromotionManifest) (TraversalPolicyPreflight, error) { + options, err := manifest.productionOptions(scenario.Cypher) + if err != nil { + return TraversalPolicyPreflight{}, err + } + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), scenario.Cypher) + if err != nil { + return TraversalPolicyPreflight{}, fmt.Errorf("parse manifest-authorized Cypher: %w", err) + } + translation, err := translate.TranslateWithProductionOptions(ctx, regularQuery, mapper, scenario.Parameters, target.ID, options) + if err != nil { + return TraversalPolicyPreflight{}, fmt.Errorf("translate manifest-authorized Cypher: %w", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + return TraversalPolicyPreflight{}, fmt.Errorf("render manifest-authorized SQL: %w", err) + } + return TraversalPolicyPreflight{ + Candidate: manifest.Candidate, + SelectorVersion: manifest.SelectorVersion, + QuerySHA256: pg.TraversalPolicyQuerySHA256(scenario.Cypher), + SQLSHA256: sqlFingerprint(sqlQuery), + GraphID: target.ID, + Optimization: translation.Optimization, + }, nil +} + +// selectTraversalPolicyScenarios selects the sole exact Cypher query that a +// promotion manifest authorizes. Current production canaries intentionally +// authorize exactly one query, so accepting zero or multiple scenarios would +// make a benchmark's reported policy path ambiguous. +func selectTraversalPolicyScenarios(scenarios []Scenario, policy pg.TraversalPolicy) ([]Scenario, error) { + allowed := make(map[string]struct{}, len(policy.QuerySHA256Allowlist)) + for _, digest := range policy.QuerySHA256Allowlist { + allowed[digest] = struct{}{} + } + + selected := make([]Scenario, 0, 1) + for _, scenario := range scenarios { + if scenario.Cypher == "" { + continue + } + if _, found := allowed[pg.TraversalPolicyQuerySHA256(scenario.Cypher)]; found { + selected = append(selected, scenario) + } + } + if len(selected) != 1 { + return nil, fmt.Errorf("traversal policy must match exactly one scenario in the selected dataset, matched %d", len(selected)) + } + return selected, nil +} diff --git a/cmd/benchmark/traversal_policy_test.go b/cmd/benchmark/traversal_policy_test.go new file mode 100644 index 00000000..9f82fc35 --- /dev/null +++ b/cmd/benchmark/traversal_policy_test.go @@ -0,0 +1,120 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/drivers/pg" + "github.com/stretchr/testify/require" +) + +func TestLoadBenchmarkTraversalPolicyBindsExactManifestBytes(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + raw := []byte(`{"candidate":"ASP-I1-U-DAG+MAT-M0","buckets":[{"query_sha256":["` + pg.TraversalPolicyQuerySHA256(query) + `"]}]}`) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + + policy, err := loadBenchmarkTraversalPolicy(path, 42) + require.NoError(t, err) + digest := sha256.Sum256(raw) + require.Equal(t, uint64(42), policy.Generation) + require.Equal(t, hex.EncodeToString(digest[:]), policy.PromotionManifestSHA256) + require.Equal(t, raw, []byte(policy.PromotionManifestJSON)) + require.Equal(t, []string{pg.TraversalPolicyQuerySHA256(query)}, policy.QuerySHA256Allowlist) + require.Equal(t, optimize.ShortestPathExecutorASPI1DAG, policy.ShortestPathExecutor) +} + +func TestLoadBenchmarkTraversalPolicyRejectsMissingInputs(t *testing.T) { + _, err := loadBenchmarkTraversalPolicy("", 1) + require.ErrorContains(t, err, "path is required") + + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, []byte(`{"candidate":"ASP-I1-U-DAG+MAT-M0","buckets":[]}`), 0o600)) + _, err = loadBenchmarkTraversalPolicy(path, 0) + require.ErrorContains(t, err, "generation must be nonzero") + _, err = loadBenchmarkTraversalPolicy(path, 1) + require.ErrorContains(t, err, "must authorize at least one query") +} + +func TestBenchmarkTraversalPromotionManifestProductionOptions(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + manifest := benchmarkTraversalPromotionManifest{ + Candidate: string(optimize.ShortestPathExecutorASPI1DAG), + SelectorVersion: "benchmark-preflight-v1", + Caps: map[string]int64{ + "state_limit": 1000, "frontier_limit": 800, "predecessor_limit": 700, + "enumeration_limit": 600, "output_bytes_limit": 1 << 20, + }, + Buckets: []benchmarkPolicyBucket{{ + QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "outbound", ObservationMode: "all_paths", MinimumDepth: 1, MaximumDepth: 15, + UntypedRelationship: true, + }}, + } + + options, err := manifest.productionOptions(query) + require.NoError(t, err) + require.Equal(t, optimize.ShortestPathExecutorASPI1DAG, options.ShortestPathExecutor) + require.Equal(t, int64(800), options.ShortestPathCaps.FrontierLimit) + require.Equal(t, int64(700), options.ShortestPathCaps.PredecessorLimit) + require.Equal(t, "outbound", options.AuthorizedBucket.Direction) + require.True(t, options.AuthorizedBucket.UntypedRelationship) + + _, err = manifest.productionOptions("MATCH (n) RETURN n") + require.ErrorContains(t, err, "absent from provisional traversal policy manifest") +} + +func TestWriteTraversalPolicyPreflightCreatesOneNewRecord(t *testing.T) { + directory := t.TempDir() + manifestPath := filepath.Join(directory, "provisional.json") + require.NoError(t, os.WriteFile(manifestPath, []byte(`{"candidate":"ASP-I1-U-DAG+MAT-M0"}`), 0o600)) + outputPath := filepath.Join(directory, "preflight.json") + preflight := TraversalPolicyPreflight{Candidate: "ASP-I1-U-DAG+MAT-M0", SQLSHA256: "sql-digest"} + + require.NoError(t, writeTraversalPolicyPreflight(manifestPath, outputPath, preflight)) + encoded, err := os.ReadFile(outputPath) + require.NoError(t, err) + require.JSONEq(t, `{"candidate":"ASP-I1-U-DAG+MAT-M0","selector_version":"","query_sha256":"","operational_candidate_sql_sha256":"sql-digest","graph_id":0,"optimization":{}}`, string(encoded)) + + require.ErrorContains(t, writeTraversalPolicyPreflight(manifestPath, outputPath, preflight), "already exists") + require.ErrorContains(t, writeTraversalPolicyPreflight(manifestPath, manifestPath, preflight), "must not overwrite") +} + +func TestSelectTraversalPolicyScenariosRequiresOneExactMatch(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + policy := pg.TraversalPolicy{QuerySHA256Allowlist: []string{pg.TraversalPolicyQuerySHA256(query)}} + scenarios := []Scenario{ + cypherScenario("Traversal", "fixture", "other", "MATCH (n) RETURN n"), + cypherScenarioWithParameters("Shortest Paths", "fixture", "candidate", query, map[string]any{"start_id": 1, "end_id": 2}), + } + + selected, err := selectTraversalPolicyScenarios(scenarios, policy) + require.NoError(t, err) + require.Len(t, selected, 1) + require.Equal(t, "candidate", selected[0].Label) + + _, err = selectTraversalPolicyScenarios(scenarios[:1], policy) + require.ErrorContains(t, err, "matched 0") + _, err = selectTraversalPolicyScenarios(append(scenarios, scenarios[1]), policy) + require.ErrorContains(t, err, "matched 2") +} diff --git a/cmd/graphbench/README.md b/cmd/graphbench/README.md index ac530326..327864d0 100644 --- a/cmd/graphbench/README.md +++ b/cmd/graphbench/README.md @@ -5,12 +5,14 @@ It is meant for runtime gap accounting: query duration, returned row counts, PostgreSQL plan details, Neo4j plan operators, fallback reasons, and comparison summaries. -The current execution modes are: +The implemented execution modes are: - `postgres_sql`: runs DAWGS' PostgreSQL SQL translation against a PostgreSQL database. -- `local_traversal`: records explicit `not_implemented` placeholders until the local traversal executor lands. - `neo4j`: runs the same corpus against Neo4j through the DAWGS Neo4j backend. +`local_traversal` is accepted only to record explicit `not_implemented` diagnostic placeholders. It is excluded from +performance gates and must not be presented as an executor result. + Apache AGE is not an execution mode in this harness yet. AGE behavior can be captured in corpus `reference_design` notes so DAWGS can use it as design input without treating it as a direct benchmark comparison. @@ -20,11 +22,39 @@ without treating it as a direct benchmark comparison. The command loads cases from `benchmark/testdata/scale` by default and imports the fixture datasets from `integration/testdata`. +Corpus parameters support fixture IDs through `node_params` and +`node_list_params`. Tagged datetime values are decoded to `time.Time`, avoiding +lexical string comparisons in temporal cases. Mutating cases require an +explicit `write_scenario`; the runner checks matched and affected counts plus +post-state queries and rolls back warm-up, timed iterations, and PostgreSQL +plan capture. + +Read cases that return node IDs can declare `expected.id_rows` using fixture +node names. GraphBench reverse-maps backend-assigned IDs through the complete +dataset ID map and compares the rows as a multiset, preserving duplicates. + Connection strings can be supplied as flags or environment variables: - PostgreSQL: `-pg-connection`, `PG_CONNECTION_STRING`, `-connection`, or `CONNECTION_STRING`. - Neo4j: `-neo4j-connection`, `NEO4J_CONNECTION_STRING`, `-connection`, or `CONNECTION_STRING`. +Every output record includes the DAWGS source version plus source commit, +dirty-worktree hash (including untracked files), binary hash, sanitized invocation, +Go/OS/CPU/kernel/cgroup data, run UUID, arm/block/order timestamps, pool settings, +and declared memory ceilings. Use +`-dawgs-version` to override the auto-detected DAWGS version. +Portable bundle creation reconstructs that dirty-worktree hash from the exact +bundled binary patch plus sorted untracked path/content bytes and refuses a +run-environment mismatch. Verification repeats the reconstruction and rejects +malformed, duplicated, mismatched, or unchecksummed untracked entries/copies. + +GraphBench clears and reloads fixtures. A non-blocking local lock at +`.coverage/graphbench.lock` prevents overlapping processes; override it with +`-destructive-lock`. Runners on different hosts must use distinct databases +because a filesystem lock cannot coordinate across machines. Non-mutating +`-existing-graph` runs reject writes; their PostgreSQL sessions remain +read-write so temporary workspace behavior matches production. + ## Examples Run only PostgreSQL SQL translation: @@ -38,11 +68,11 @@ go run ./cmd/graphbench \ -summary-json .coverage/graphbench-postgres.json ``` -Capture PostgreSQL, local traversal placeholders, and Neo4j in one report: +Capture PostgreSQL and Neo4j in one report: ```bash go run ./cmd/graphbench \ - -modes postgres_sql,local_traversal,neo4j \ + -modes postgres_sql,neo4j \ -pg-connection "$PG_CONNECTION_STRING" \ -neo4j-connection "$NEO4J_CONNECTION_STRING" \ -jsonl-output .coverage/graphbench.jsonl \ @@ -62,6 +92,1672 @@ go run ./cmd/graphbench \ -summary .coverage/graphbench.md ``` +Capture independent rounds with 30-50 warm observations each. The PostgreSQL +runner resets its one-connection pool before every case, records the first +query execution as `cold`, and keeps connection establishment outside that +sample. Use a distinct `-round` value for every independently reloaded run: +Even-numbered rounds reverse the requested backend order to alternate which +backend runs first. + +```bash +go run ./cmd/graphbench \ + -round 1 \ + -iterations 30 \ + -modes postgres_sql,neo4j \ + -pg-connection "$PG_CONNECTION_STRING" \ + -neo4j-connection "$NEO4J_CONNECTION_STRING" \ + -jsonl-output .coverage/graphbench-round-1.jsonl +``` + +Concatenate the JSONL rounds for each version, then run the executable +confidence gate: + +```bash +make perf_gate \ + PERF_BASELINE=.coverage/graphbench-baseline.jsonl \ + PERF_CANDIDATE=.coverage/graphbench-candidate.jsonl \ + PERF_GATE_AA=.coverage/perf-aa-resolution.json +``` + +The versioned gate report includes artifact and A/A-report SHA-256 checksums, +seeded 97.5% +bootstrap intervals over matched round medians, stratified p95 intervals once +each side has at least 150 samples, and candidate-minus-baseline p95 duration +intervals. Normal and envelope timing uses the greater of the matching host +A/A resolution and the 5%/100us minimum floors. Stress timing is descriptive; +stress correctness and the independent resource gate still apply. Matched +timing artifacts must carry complementary, round-balanced arm order, block, +run UUID, and warmup evidence. The version-controlled corpus `candidate_modes` declarations are the +required-key/status manifest: a missing or non-`ok` PostgreSQL record fails +instead of disappearing through intersection-only comparison. Neo4j records +must be present and `ok`, but Neo4j latency is informational and never fails a +CySQL performance gate. Missing/malformed host A/A, tier, pairing, selection, +round, or p95 evidence fails production promotion. Diagnostic comparisons may +omit promotion evidence but cannot emit a passing promotion result. +Prioritized traversal candidates additionally require nonempty, independently +passing training and frozen-holdout cases for every concrete runtime candidate +family; a holdout from ASP, another scheduler, or another observation boundary +cannot qualify an SP candidate. + +Predeclare cases expected to improve with `PERF_TARGETS` (or +`-gate-targets`). A target passes materiality when its median-ratio upper bound +is at most `0.95` or its median-saving lower bound is at least `100us`; both +defaults are configurable. Calculate host-specific A/A resolution from a +baseline artifact with: + +```bash +make perf_aa PERF_AA_ARTIFACT=.coverage/graphbench-aa.jsonl +``` + +The schema-v4 report accepts exactly two explicitly executed A/A arms sharing +one run UUID and SQL/workload identity. It requires `block == round`, +complementary balanced order across at least five independent rounds, ten +samples per arm and round, and process timestamps proving that each complete +cohort arm ran serially in its declared position and that rounds do not +overlap. Its artifact-bound `physical_chronology` provenance prevents a later +gate from accepting label-only balance. It fingerprints the host, reports +p50/p95 ratio and absolute resolution, and keeps p99 diagnostic until each arm +has at least 10,000 samples. When append-safe capture keeps the two arm labels +in separate files, repeat `-aa-artifact` instead of concatenating them outside +GraphBench: + +```bash +graphbench \ + -aa-artifact .coverage/aa-a.jsonl \ + -aa-artifact .coverage/aa-b.jsonl \ + -aa-output .coverage/aa.json +``` + +### Targeted matched diagnostics + +`-cases` accepts exact, unambiguous case names. `-datasets`, `-categories`, and +`-tags` add exact selectors; values within one selector are alternatives and +different selector dimensions are intersected. Unknown, duplicate, ambiguous, +or empty selections fail before a fixture is changed. Filtered captures are +marked `diagnostic_only`, record both the requested and resolved selection and +the omitted declaration count, and are refused by the ordinary complete gate. +Selection-manifest schema v2 also records the count and digest of any protocol-protected +declarations removed from its runnable universe; those omissions do not by +themselves make an otherwise unfiltered run diagnostic-only. +Use `-diagnostic-gate` only to compare two artifacts with the same resolved +subset checksum. + +Configured `-warmup-iterations` run outside the recorded samples. The cold +diagnostic, exact preflight/postflight observations, and fixture reload/analyze +contract remain separate. A matched arm records `-arm`, `-arm-order`, `-block`, +`-round`, and a shared `-run-uuid`: + +```bash +go build -trimpath -o .coverage/confirm/bin/graphbench ./cmd/graphbench +.coverage/confirm/bin/graphbench \ + -modes postgres_sql \ + -cases 'LOOKUP-05_repeated_case_insensitive_prefix,GSP-D02-F016_distance' \ + -warmup-iterations 20 -iterations 50 -pool-size 1 \ + -arm candidate -arm-order 1 -block 1 -round 1 -run-uuid "$RUN_UUID" \ + -pg-connection "$PG_CONNECTION_STRING" \ + -bundle-dir .coverage/confirm/candidate-round-1 \ + -jsonl-output .coverage/confirm/candidate-round-1.jsonl +``` + +`-bundle-dir` retains the tracked patch, checksummed copies of untracked files, +`go.mod`/`go.sum`, the running executable, the complete sorted corpus +declaration and its independently recomputed identity, raw +JSONL, a sanitized manifest, and bundle checksums. It never records connection +strings or arbitrary environment variables. Add repeatable, stable-named +auxiliary evidence with `-bundle-evidence name=path`, for example +`-bundle-evidence host-aa=.coverage/host-aa.json` and +`-bundle-evidence plan-delta=.coverage/plan-delta.json`. Evidence names use only +lowercase letters, digits, `-`, and `_`; the bundle records each source digest +without retaining its host path. The destination must be new or empty so stale +payloads cannot enter its checksum inventory. + +Verify a portable bundle independently of database access. Verification rejects +missing, additional, symlinked, malformed, or checksum-mismatched payloads and +writes its report outside the bundle being checked: + +```bash +go run ./cmd/graphbench \ + -bundle-verify .coverage/confirm/candidate-round-1 \ + -bundle-verify-output .coverage/candidate-round-1-verification.json \ + -bundle-require-clean + +make perf_bundle_verify \ + PERF_BUNDLE_VERIFY_DIR=.coverage/confirm/candidate-round-1 \ + PERF_BUNDLE_REQUIRE_CLEAN=1 +``` + +Omit `-bundle-require-clean` for a diagnostic capture that deliberately carries +a source patch. Structural or checksum failures always produce a nonzero exit; +the optional clean-source policy additionally rejects any dirty capture. + +For a live capture that must be clean before it touches a database, add +`-require-clean-source`. This checks tracked and untracked source content +before target validation, fixture setup, or destructive-run locking; it is +independent of the later bundle-verification policy. + +Compare matched arms with the matching checksummed host A/A report. Only a +same-executable block/reload A/A comparison may omit this input: + +```bash +make perf_confirm \ + PERF_LEFT=.coverage/confirm/predecessor.jsonl \ + PERF_RIGHT=.coverage/confirm/candidate.jsonl \ + PERF_CONFIRM_AA=.coverage/confirm/block-aa.json \ + PERF_CASES='LOOKUP-05_repeated_case_insensitive_prefix,GSP-D02-F016_distance' +``` + +The report emits paired relative and absolute p50/p95 intervals. It classifies +fresh p95 evidence as confirmed, cleared/non-inferior, inconclusive, or a +fingerprint mismatch using a minimum 5%/0.10 ms noise floor. Comparing two +captures of the same executable produces a `block_reload_aa` report; alternate +arm order across independently reloaded rounds. + +## Concurrency and PostgreSQL references + +Serial behavior remains the default (`-pool-size 1`). An opt-in concurrency +smoke retains a physical pool and records pool wait, transaction setup, +execute/decode/drain time, backend PID, cold/warm session classification, wall +time, and QPS: + +```bash +go run ./cmd/graphbench \ + -modes postgres_sql \ + -pg-connection "$PG_CONNECTION_STRING" \ + -pool-size 8 \ + -concurrency 1,8,16 \ + -iterations 30 \ + -session-memory-ceiling-bytes 67108864 \ + -pool-memory-ceiling-bytes 536870912 \ + -jsonl-output .coverage/graphbench-concurrency.jsonl +``` + +`-postgres-references` additionally captures an identical-SQL raw-pgx boundary +(pool wait, transaction, bind/prepare, first row, remaining decode, drain, and +allocations), a raw prepared round-trip, the C1 prepared round-trip, +endpoint validation, minimum graph-access ID floor, raw ordered-ID search, +path hydration from precomputed ordered edge IDs, and complete hand-written +PostgreSQL references for the active shortest-path and fixed-suffix expansion +targets. The main +case record remains the translated-CySQL boundary rather than a fixed ordinal +among the additive references. Component floors need not match the full query's +row count; complete references do. It also records +compile-stage timings and allocations. JSON/Markdown summaries include a +versioned exclusive-boundary cost table and its unexplained residual. The waterfall marks its translation +interval as overlapping optimization, so those fields must not be summed as an +additive attribution. + +Use `-postgres-reference-arms` to run only named tournament arms; it implies +`-postgres-references` and rejects unknown or duplicate names. Generated +fixed-suffix expansion cases expose `search_ordered_ids`, +`stepwise_forward_aa_ordered_ids`, `root_reuse_*`, `late_hydration_*`, +`factored_suffix_forward_*`, `suffix_seeded_reverse_*`, and +`backward_viability_forward_*` boundaries. The `hydration_only` and +`ordered_path_ids_hydration_only` arms compare generic edge-stream path +reconstruction with direct hydration from precomputed ordered node and edge +IDs. Complete arms are exact-multiset +checked against the public CySQL observation. Ordered-ID arms retain +relationship IDs for trail uniqueness. Exactly three selected arms use a +six-round doubled Williams design that places every arm in every position twice +and balances every directed carryover pair twice. Exactly five arms use the +fixed ten-round Williams/carryover-balanced slot schedule; other arm counts +retain the historical alternating order: + +```text +0 1 4 2 3 +1 2 0 3 4 +2 3 1 4 0 +3 4 2 0 1 +4 0 3 1 2 +3 2 4 1 0 +4 3 0 2 1 +0 4 1 3 2 +1 0 2 4 3 +2 1 3 0 4 +``` + +The slots are the caller-selected arms, and rounds wrap after the tenth row. + +`-postgres-force-shortest-executor SP-S0` is the exact-incumbent control at the +same public distance or path boundary. It records selected/applied `SP-S0` and +executes the existing workspace harness, making containment regret and +candidate/reference comparisons explicit. + +`-postgres-force-shortest-executor SP-S0-DIRECT` is the tool-only direct-edge +preflight arm for structurally eligible bound-endpoint searches whose minimum +depth is one. A materialized indexed one-edge probe returns a valid singleton +witness immediately; a dependency-gated lateral branch invokes exact `SP-S0` +only when the probe is empty. Both branches share one SQL statement and +snapshot. Production `sp-static-v3` selection remains unchanged until the arm +passes exactness, zero-loop fallback, regret, resource, and concurrency gates. + +`-postgres-force-shortest-executor SP-S3-U-D` is a qualification-only seam for +eligible bounded singleton distance cases. It executes the repository-native +recursive AST directly, using compact `(next_id, depth)` state when both +endpoints are ID-only and retaining `(root_id, next_id, depth)` otherwise. It +reports the exact forced/applied target and rejects path-observed or otherwise +ineligible cases. It does not enable the executor in the public query API. + +`-postgres-force-shortest-executor SP-S3-U-E+MAT-M0` is the corresponding +qualification-only seam for eligible one-path observations. It emits +repository-native `(next_id, depth, edge_ids)` recursive state and hydrates the +ordered path directly from direction-specific edge endpoints. Distance-only, +directionless, correlated, optional, mutation, and other ineligible forms keep +the incumbent unless explicitly rejected by the tool request. Tool forcing +never broadens the structural correctness envelope. + +`-postgres-force-expansion-search EXPANSION-SUFFIX-SEEDED-REVERSE` is the +qualification-only seam for an eligible directed, bounded variable expansion +followed by exactly three fixed directed relationships. It emits the +repository-native suffix-seeded reverse recursive AST, preserves +relationship-trail uniqueness and exact suffix multiplicity, and supports +endpoint-ID and complete-path observations. The request fails closed when the +target is structurally ineligible or translation does not record the requested +strategy as applied. It is mutually exclusive with forced shortest execution. +Automatic suffix-seeded reverse dispatch remains disabled because query shape +does not bound suffix density or reverse fan-in. + +Manifest v4 is the separately versioned topology-selected production contract, +not a command-line selector. Its verifier recomputes the fixed-suffix shape and +SQL-template digests through the PostgreSQL driver contract and requires the +`topology-fixed-suffix-counts-v1` estimator, +`topology-synopsis-schema-v2`, and `topology-selected-routing-v1` cache +protocol. GraphBench retry/component captures remain development inputs; they +cannot by themselves authorize, install, or emulate a v4 route decision. + +`-postgres-suffix-route-component-closure` is a measurement-only closure for +the default-off direct fixed-suffix component preflight. It can accompany the +ordinary incumbent or `-postgres-expansion-suffix-route-component`, but it +never influences executor selection. It requires Repeatable Read, diagnostic +telemetry, pool size one, and positive session/pool workspace ceilings; it is +incompatible with reference, concurrency, guard, retry, orientation, forced, +and production-manifest modes. Each case records the client compile waterfall +and raw-PGX fresh prepared miss, same-session prepared hits, pooled prepared +miss, and same-backend release/reacquisition hits, including temporary +workspace high-water evidence. Each raw sample also carries a SHA-256 digest +of its normalized public rows; every digest must match the primary CySQL +observation and the other prepared-state strata. The frozen roster and acceptance conditions are +in [`sql_strategy_routing_component_closure_v1.json`](../../benchmark/testdata/scale/protocols/sql_strategy_routing_component_closure_v1.json). + +`-postgres-expansion-suffix-reverse-guard` enables the distinct tool-only +`suffix-reverse-guard-v1` experiment for complete-path fixed-suffix cases. It +requires `-postgres-repeatable-read`, diagnostic traversal telemetry, and pool +size one. The statement has 513-row suffix/state sentinel bounds by default +(a 512-row admissible cap plus one overflow row), no topology/degree probes, +and complementary marker-gated exact reverse and exact forward arms. Optional +`-postgres-suffix-guard-suffix-limit` and +`-postgres-suffix-guard-state-limit` overrides are diagnostic-only and cannot +be supplied without the guard. Endpoint-only observations and mutations fail +closed. + +This policy generation is now terminally stopped. Its chronology-valid capture +failed the immutable `1.10`/`100us` guard-overhead gate on both cases, so it +must not be rerun as an authorization attempt, retuned, advanced to holdout, or +given a manifest, driver policy, or rollback switch. The recipe below is +retained only for diagnostic reproduction and to make the existing failed +report and its physical-order checks auditable. + +The early stop gate was sealed to the two already-open V3 training path cases +`GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path` and +`GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path`. The archived capture used exact +forward (`incumbent`), exact suffix reverse (`reverse`), and the guarded +statement (`guarded`) for exactly six doubled-Williams rounds, with exactly five +warmups and ten timed samples per arm. Use one binary/run UUID and all six +positions +`123,231,312,321,132,213` exactly once. All three arms must retain the same +schema-v2 resolved selection/declaration identity. The position tuple is +`(incumbent,reverse,guarded)`, so the physical process order by round is +`incumbent/reverse/guarded`, `guarded/incumbent/reverse`, +`reverse/guarded/incumbent`, `guarded/reverse/incumbent`, +`incumbent/guarded/reverse`, and `reverse/incumbent/guarded`. + +Set `-round N -block N` on every arm invocation. Run the three GraphBench +processes serially in the declared order, wait for each process to exit before +starting the next, and finish all three before starting round `N+1`. Each +process must select the complete two-case cohort so its records share one +process-level `started_at`/`ended_at` interval. The feasibility gate rejects +missing or mixed intervals, an arm interval that overlaps its declared +predecessor, a later round that overlaps or predates the prior round, and run +UUID drift. Merely swapping `-arm-order` labels does not establish valid +evidence. + +Capture the incumbent A/A arms with the same physical rules: execute `aa-a` +then `aa-b` on odd rounds and `aa-b` then `aa-a` on even rounds, waiting for +each process and using `-block N -round N`, one A/A run UUID, and the complete +two-case selection. Build a schema-v4 report from both immutable arm files. +The suffix gate requires its artifact-bound physical-chronology provenance, so +an older label-balanced report cannot be reused. Then run: + +```bash +graphbench \ + -suffix-guard-incumbent-artifact .coverage/suffix-guard/incumbent.jsonl \ + -suffix-guard-reverse-artifact .coverage/suffix-guard/reverse.jsonl \ + -suffix-guard-guarded-artifact .coverage/suffix-guard/guarded.jsonl \ + -suffix-guard-aa .coverage/suffix-guard/aa.json \ + -suffix-guard-output .coverage/suffix-guard/feasibility.json +``` + +Every guarded timed sample must carry an invocation-bound runtime receipt and +the diagnostic plan must prove one selected executor and zero loops/rows in the +inactive arm. The gate requires guarded overhead versus exact reverse within +`1.10` or `100us`, regret versus the fastest exact arm within `1.10` or the A/A +floor, material median improvement versus forward (`<=0.95` or `>=100us`), and +p95 ratio `<=1.05`. Under the frozen gate contract, a pass would have authorized +creation of a fresh sealed qualification cohort only. The actual result failed, +so that conditional path is closed. + +`-postgres-force-expansion-search EXPANSION-ENDPOINT-SEEDED-REVERSE` targets the production-qualified +fixed-prefix/terminal-expansion family. Its SQL has materialized 33-row endpoint and 4097-row reverse-state probes, +then mutually exclusive reverse and incumbent branches. Generated +`generated_endpoint_seeded_expansion_v1_d_e_q_w_o_x_m1_c_p

` fixtures independently vary +matching/other endpoints, productive/unproductive lanes, cycles, and payload. Edge multiplicity is fixed at one because +DAWGS storage uniquely keys edges by start, end, kind, and graph. Structured plan metrics +report probe rows, guard overflow, and whether the incumbent branch executed. + +Traversal telemetry is disabled by default. Opt in with +`-postgres-traversal-telemetry summary` or +`-postgres-traversal-telemetry diagnostic`; both modes require +`-pool-size 1` so the recorded backend identity cannot drift. Attachment runs +after the timed case, reference, raw-PGX, and concurrency blocks. Summary mode +uses only lightweight post-timing evidence and never performs the detailed +invocation-local replay. For a function-backed B arm whose outer plan cannot +prove its branch, it serializes `runtime_outcome_available=false` and leaves +runtime/applied/fallback facts unset. Diagnostic mode additionally retains the existing +`EXPLAIN (ANALYZE, TIMING OFF, FORMAT JSON)` plan replay. For SP-B1/B2 it also +replays the exact SQL in a separate Repeatable Read transaction on that same +physical connection, guarded by a unique invocation ID and the +`begin/read/clear_bidirectional_shortest_path_diagnostic_v1` session-local API; +ASP-B1/B2 uses the corresponding +`begin/read/clear_bidirectional_all_shortest_path_diagnostic_v1` API. +Cancellation and SQL errors roll the replay transaction back; replay duration +is never added to latency samples. + +An outer PostgreSQL `Function Scan` is not treated as internal traversal work. +SP/ASP B counters are retained only when the invocation ID, connection, +scheduler, caps, exactly-one singleton search call, level rows, and runtime +outcome all validate. A B candidate that +executes exact S4 fallback retains its measured candidate/fallback evidence but +is marked incomplete because nested S4 work is still opaque. Witness SP and all +ASP executions separately require complete hydration counters. Workspace-backed +B arms also require measured per-session and pool high-water bytes; declared +memory flags alone never qualify. These counters are not yet exposed, so those +records fail closed while retaining their validated search evidence. Other +function-backed SP/ASP arms are recorded as +`hidden_counters_unavailable`, never as zero work. The resource gate requires +`counter_status=complete` for candidate architectures even when no numeric cap +was declared. + +Traversal telemetry schema v2 gives guarded inline evidence three +non-interchangeable serialized families. `ASP-I1-U-DAG+MAT-M0` emits +`asp-i1-guarded-v1` and writes bounded +relation, output, and branch evidence under `diagnostic.counters.inline_asp`. +`SP-I1-C-WE+MAT-M0` emits the distinct +`sp-i1-canonical-guarded-v1` policy and writes the same-shaped evidence under +`diagnostic.counters.inline_shortest_path`; evidence from either namespace +cannot satisfy the other family. `SP-I2-C-D` emits +`sp-i2-distance-guarded-v1` under +`diagnostic.counters.inline_shortest_distance`. Its named distance, target, and +output relations must agree with the typed counters and public row count; its +complementary markers and branch rows must select exactly one arm; and the +selected direct executor must run once while the inactive executor reports zero +loops. Evidence from the predecessor namespaces cannot satisfy this distance +contract. PostgreSQL's named candidate and fallback +marker CTEs must attribute exactly one arm, and the unselected output branch +must report zero rows. Parent-linked plan nodes also bind each branch body to +its direct inner executor; the selected executor must run and the unselected +executor must report zero loops. Canonical I1 reports `inline_canonical_witness` or +`inline_canonical_no_path` when its candidate marker executes, and +`exact_s4_fallback` with `SP-S4-C-WE+MAT-M0` when the fallback marker executes. +If any required named relation, marker, branch, or executor-loop counter is absent from the +plan replay, the diagnostic is `hidden_counters_unavailable`; absence is never +converted into a qualifying zero. +These diagnostics add fail-closed evidence for the default-off exact-query +canaries; they do not change any automatic production selector. + +An emitted `orientation-probe-v1` policy requires orientation probes, selected +ordinary expansion, and hydration families. Its exact executed-candidate and +executed-incumbent marker rows must select one arm, the other must be zero, and +each named probe may execute at most once. Attribution uses only PostgreSQL's +single `Subplan Name: CTE ...` materialization body, never repeated consumer +CTE scans; the unselected traversal branch must also report zero loops. +Plan-derived partial evidence cannot qualify. +Telemetry attaches to every reference whose declared architecture is itself a +traversal or hydration boundary. Protocol, endpoint/root validation, and other +component probes remain intentionally unannotated; their missing attachment is +not missing traversal evidence. + +`-postgres-expansion-orientation-shadow` enables the tool-only +`orientation-probe-v1` shadow statement. It always executes the exact forward +incumbent and records the mutually exclusive SQL marker result separately as +`would_select_identity`; it never relabels that hypothetical choice as the +runtime or applied arm. A marker-first runtime receipt is emitted even when the +incumbent returns zero rows, and any cap+1 probe row is reflected in the shadow +overflow summary. The shadow flag is mutually exclusive with forced shortest- +path and forced expansion selectors. + +Build the matched selector-regret and probe-overhead report from separate +true-shadow, exact incumbent, and forced suffix-reverse artifacts plus the +host A/A calibration: + +```bash +go run ./cmd/graphbench \ + -orientation-shadow-artifact .coverage/orientation-shadow.jsonl \ + -orientation-incumbent-artifact .coverage/orientation-incumbent.jsonl \ + -orientation-reverse-artifact .coverage/orientation-reverse.jsonl \ + -orientation-aa .coverage/perf-aa-resolution.json \ + -orientation-output .coverage/orientation-selector.json \ + -orientation-protocol confirmation \ + -confidence-level 0.975 -seed 1 +``` + +The report requires exact matching observations, stable workload/SQL/binary +identities, one SQL-derived `would_select_identity`, and position-balanced +three-arm rounds. Selector regret must be within a `1.10` median-ratio upper +bound or the host A/A absolute floor. Shadow probe overhead must be within +`10%` or `100us`. Training records may inform the frozen selector; holdout +records are evaluation-only; diagnostic and legacy records are serialized but +excluded from qualification. Discovery uses 5-20 rounds, five warmups, and ten +samples per arm. Confirmation uses 10-20 rounds, 20 warmups, and 50 samples per +arm. + +`orientation-probe-v2` is a separate, immutable, tool-only experiment. It does +not reinterpret the v1 report or change the v1 exact-query production seam. +The v2 selector computes +`F2 = root_rows + maximum_depth * forward_degree_rows` and +`R2 = suffix_rows + boundary_rows + reverse_degree_rows`; it selects the exact +suffix-seeded reverse arm only when every cap+1 probe is complete and +`4 * R2 < 3 * F2`. Any probe or reverse-state overflow fails closed to the exact +forward arm. Degree probes expose a scalar count over their cap+1-limited inner +stream; diagnostic telemetry attributes sample rows to that inner `Limit`, not +the one-row aggregate. This changes neither the evidence nor the overflow +boundary. The checksum-bound v3 cohort has exactly eight training cases and +four holdouts. It independently varies maximum depth, fanout, reachable and +disconnected branches, reverse fan-in, suffix multiplicity, matching-root +multiplicity, zero depth, productive-boundary cycles and self-loops, payload, +and endpoint-ID versus complete-path observation. Holdouts use previously +unused depths 7, 11, 13, and 15 and must not be opened for threshold tuning. + +The remainder of this subsection records the frozen v2 protocol for audit and +diagnostic reproducibility only. V2 failed its immutable training overhead gate, +the final manifest verifier terminally rejects it, and it must not be recaptured +or advanced to confirmation. Its protected holdouts remain unopened. Under that +historical protocol, the four artifacts used these exact arm labels and every +invocation required `-postgres-repeatable-read`, +`-postgres-traversal-telemetry summary` or `diagnostic`, and `-pool-size 1`. + +| Artifact | Exact `-arm` label | Mode-specific flags | +| --- | --- | --- | +| Shadow | `shadow` | `-postgres-expansion-orientation-shadow -postgres-expansion-orientation-policy orientation-probe-v2` | +| Exact forward | `incumbent` | no orientation or forced-expansion flag | +| Exact reverse | `reverse` | `-postgres-force-expansion-search EXPANSION-SUFFIX-SEEDED-REVERSE` | +| Guarded selector | `guarded` | `-postgres-expansion-orientation-tournament -postgres-expansion-orientation-policy orientation-probe-v2` | + +The archived recipe built GraphBench once from the clean source tree and invoked +that exact binary for every A/A, arm, and report command. Repeated `go run` +builds did not prove a single binary identity: + +```bash +CAPTURE=.coverage/orientation-v2-discovery +mkdir -p "$CAPTURE/bin" +go build -trimpath -o "$CAPTURE/bin/graphbench" ./cmd/graphbench +RUN_UUID="orientation-v2-discovery-$(git rev-parse HEAD)" +``` + +For example, the first shadow discovery round was captured with: + +```bash +"$CAPTURE/bin/graphbench" \ + -modes postgres_sql \ + -tags orientation-v2-training \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -round 1 -block 1 -run-uuid "$RUN_UUID" \ + -arm shadow -arm-order 1 \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -postgres-expansion-orientation-shadow \ + -postgres-expansion-orientation-policy orientation-probe-v2 \ + -jsonl-output "$CAPTURE/shadow.jsonl" -append-jsonl +``` + +The recipe repeated the invocation for the other table rows and rotated +`-arm-order` in each subsequent round. `-run-uuid` was one series identity: the +same value was reused across all four arms and every appended round. It changed +`-round` and `-block`, but not the UUID; append validation rejects a per-round +UUID. Discovery selects only +`orientation-v2-training` and keeps the holdout timings closed. Its four +artifacts must contain exactly the canonical eight training cases, with no +holdout or diagnostic timing. Had discovery produced a passing clean-source +freeze before the terminal decision, confirmation would have selected +`-tags orientation-v2-training,orientation-v2-holdout`, written separate +artifacts containing exactly the canonical eight training plus four holdout +cases, and used 20 warmups and 50 measured samples per arm and round. No such +freeze exists, so this path must not be executed. + +Each matched round must give the four labels distinct `-arm-order` values from +1 through 4 and share the same nonzero `-block`, `-round`, and `-run-uuid`. +Rotate the positions across rounds so every arm occupies every position evenly; +the canonical four-round rotation is +`shadow/incumbent/reverse/guarded`, +`incumbent/reverse/guarded/shadow`, +`reverse/guarded/shadow/incumbent`, then +`guarded/shadow/incumbent/reverse`. The reporter rejects a position imbalance +greater than one, missing or extra cases, mismatched round sets, observation or +SQL drift, non-Repeatable-Read records, missing timed receipts on shadow or +guarded samples, and mixed source, dirty-diff, binary, corpus, host, or +PostgreSQL identities. +The shadow receipt branch is exactly `shadow_incumbent`. Guarded reverse +execution must report `suffix_seeded_reverse`; guarded forward selection and +overflow fallback both report `exact_forward_incumbent`, with +`fallback_executed=true` required only for overflow fallback. + +The archived recipe captured the two A/A arms as separate append-safe +exact-forward artifacts using the same built binary, exact cohort tag, +Repeatable Read, diagnostic traversal telemetry, size-one pool, warmups, +samples, and fixture reload protocol as the incumbent arm. It used one A/A +series UUID and alternated the two positions across rounds. No orientation or +forced-expansion flag was permitted. GraphBench then validated the logical pair +directly: + +```bash +"$CAPTURE/bin/graphbench" \ + -aa-artifact "$CAPTURE/aa-a.jsonl" \ + -aa-artifact "$CAPTURE/aa-b.jsonl" \ + -aa-output "$CAPTURE/aa.json" \ + -confidence-level 0.975 -seed 1 +``` + +In the archived design, discovery was the only workflow that could create a +freeze. A qualifying run would have used a clean source tree, the exact +canonical eight-case training artifacts, and matching host A/A evidence. Both +output flags were mandatory: the command wrote the training-only discovery +report and, only after passing evidence and clean-source checks, a freeze +manifest binding its SHA-256 together with the policy, formula, caps, source +commit, clean dirty-diff, binary, and canonical cohort declaration: + +```bash +"$CAPTURE/bin/graphbench" \ + -orientation-v2-shadow-artifact "$CAPTURE/shadow.jsonl" \ + -orientation-v2-incumbent-artifact "$CAPTURE/incumbent.jsonl" \ + -orientation-v2-reverse-artifact "$CAPTURE/reverse.jsonl" \ + -orientation-v2-guarded-artifact "$CAPTURE/guarded.jsonl" \ + -orientation-v2-aa "$CAPTURE/aa.json" \ + -orientation-v2-output "$CAPTURE/report.json" \ + -orientation-v2-freeze-output "$CAPTURE/freeze.json" \ + -orientation-v2-protocol discovery \ + -confidence-level 0.975 -seed 1 +``` + +The unexecuted confirmation path fails closed unless it receives that exact +passing freeze manifest and the discovery report whose digest the manifest +binds. Its four timing artifacts and matching host A/A report would have had to +cover exactly the canonical eight training and four holdout cases: + +```bash +CONFIRMATION=.coverage/orientation-v2-confirmation +"$CAPTURE/bin/graphbench" \ + -orientation-v2-shadow-artifact "$CONFIRMATION/shadow.jsonl" \ + -orientation-v2-incumbent-artifact "$CONFIRMATION/incumbent.jsonl" \ + -orientation-v2-reverse-artifact "$CONFIRMATION/reverse.jsonl" \ + -orientation-v2-guarded-artifact "$CONFIRMATION/guarded.jsonl" \ + -orientation-v2-aa "$CONFIRMATION/aa.json" \ + -orientation-v2-freeze "$CAPTURE/freeze.json" \ + -orientation-v2-discovery-report "$CAPTURE/report.json" \ + -orientation-v2-output "$CONFIRMATION/report.json" \ + -orientation-v2-protocol confirmation \ + -confidence-level 0.975 -seed 1 +``` + +Every v2 A/A case carries separate checksums for its workload, the exact +PostgreSQL timing environment (including transaction isolation and normalized +ANALYZE state), and the exact validated fixture. Discovery and confirmation +reject missing or mismatched environment or fixture evidence. + +The forward-selected shadow/forward and guarded/selected overhead gates use a +`1.10` median-ratio upper bound or a `100us` absolute-gap ceiling. The +guarded/fastest regret gate uses the same ratio limit or the matching host A/A +absolute floor. Shadow overhead remains visible but is not +qualification-applicable when v2 selects reverse. The frozen confirmation +contract would have required all eight training and all four holdout cases to +pass independently. No v2 discovery or confirmation result has qualified. The +latest exact five-round +training prequalification failed selected-arm overhead on all eight cases, +with approximately 156-396 microseconds of guarded overhead against the frozen +100-microsecond limit. V2 is retained as immutable negative evidence; its +thresholds, formula, and protected holdouts must not be retuned. The flags and +schema remain for diagnostics and historical decoding; they do not provide a +qualification or production path. + +The bounded same-statement fallback and keyset-continuation experiments are +retired. They are not exposed by GraphBench or production translation. Their +negative results remain under `docs/experiments`; the active `GFSE-BOUNDARY-*` +cases are optimization-neutral cardinality holdouts. + +Independent benchmark rounds can be accumulated with `-append-jsonl`. The +append path must be supplied with `-jsonl-output`; GraphBench rejects mismatched +run UUIDs, arms, binary/diff identities, and duplicate case rounds before +writing. This is the intended input shape for paired confirmation and the +round-stratified performance gate. + +Use `-reference-closure-artifact` with a capture containing the translated +raw-pgx boundary and one exact PostgreSQL full-comparator arm to generate a +seeded production/reference closure report. The report requires 10-20 matched +rounds, at least 20 untimed warmups and 50 measured samples per side in every +round, and exact public observations. It passes when the production/reference +median-ratio upper bound is at most 1.10 or the absolute median-gap interval is +within the greater of the case's within-session A/A resolution and +`-materiality-absolute` (100 microseconds by default). The report derives and +records A/A resolution independently for the production and reference raw +boundaries by splitting alternating samples within each round. Single selected +reference captures run production first in odd rounds and the reference first +in even rounds; the order is recorded on both boundaries and enforced by the +reporter: + +Schema v2 also freezes the bootstrap count and carries the applied promotion +candidate, source commit, clean-tree digest, binary and corpus digests, and each +case's workload, normalized-query, qualification-split, and production runtime +receipt chains. The reporter rejects identity drift across rounds before it +writes a decision. Version 1 reference-closure reports cannot satisfy a final +promotion manifest. + +```bash +go run ./cmd/graphbench \ + -reference-closure-artifact .coverage/shortest-reference.jsonl \ + -reference-closure-arm s3_unidirectional_trail_cte \ + -reference-closure-output .coverage/shortest-reference-gate.json \ + -confidence-level 0.975 \ + -seed 1 +``` + +Fixed-suffix expansion JSON plans are retained in both text and structured +forms. Structured metrics include per-node planned/actual rows, loops, width, +timing, buffers, +relation/index identity, recursive rows, access-direction probe counts, and +hydration lookup loops. Derived fields state their provenance and do not present +fixture-derived per-depth counts as PostgreSQL measurements. Resource gate +version 1 applies the portable resource checks to the first upstream artifact +schema. + +The keyset-continuation v1 design and its GraphBench arm are retired. In the +10-round confirmation run, S513 had a 1.791 +median ratio (97.5% CI 1.752–1.875) and S600 had a 5.898 ratio (5.649–6.462) +against `complete_reference`. S511/S512 selected the existing bounded reverse +branch, so their improvements are not evidence for keyset continuation. The +resource gate passed without spill, local workspace, or WAL. See +`docs/experiments/guarded_suffix_keyset_continuation_v1.md` and its compact JSON +evidence. Generic `GFSE-BOUNDARY-*` holdouts preserve exact-limit, overflow, +path, multiplicity, and cyclic-trail coverage without retaining an executable +copy of the rejected arm. + +Supported generated singleton-shortest cases also run two additive comparators: +`s3_unidirectional_trail_cte` (legacy name +`complete_reference_s1_array_cte`) and `s3_bidirectional_trail_cte` (legacy name +`candidate_s2_bidirectional_cte`). New reference records declare a schema +version, architecture, implementation/state/observation shape, and semantic +validation level, raw-pgx timing boundary, normalized SQL fingerprint, and any +explicit A/A alias. A requested arm that is unavailable for a case fails the +run, and distinct architecture IDs with identical normalized SQL fail unless +the alias is declared. Full comparators are checked against untimed exact public +observations rather than row count alone. Distance S3-U uses node/depth frontier +state with no path or predecessor arrays. Historical readers preserve the old +labels in `legacy_name` while mapping them to `SP-S3-U-NE`/`SP-S3-B`. These remain +benchmark-only; S3-B is not evidence for the compact S2 architecture. + +Distance-only generated cases also expose `s1_array_bfs_distance`, a genuine +typed PL/pgSQL SP-S1 prototype. It keeps frontier and visited node IDs in +bounded arrays, records a fixed 100,000-node state ceiling, and restarts the +exact S3-U distance reference in the same statement on overflow. It is a +benchmark arm only and is never selected by production translation. + +Capture S3-U-D and SP-S1 together with 20 warmups and 50 observations, then +produce their seeded, order-balanced matched comparison with: + +```bash +go run ./cmd/graphbench \ + -reference-pair-artifact .coverage/shortest-alternatives.jsonl \ + -reference-pair-baseline s3_unidirectional_trail_cte \ + -reference-pair-candidate s1_array_bfs_distance \ + -reference-pair-output .coverage/shortest-alternatives.json \ + -confidence-level 0.975 \ + -seed 1 +``` + +The default confirmation pair reporter requires 10-20 independent rounds, 20 +warmups, 50 samples per arm per round, and distinct recorded measurement order. +`-reference-pair-protocol discovery` produces an explicitly labeled exploratory +report from 5-20 rounds, five warmups, and ten samples per arm; it cannot be +mistaken for confirmation evidence because the protocol and requirements are +written into the report. The reporter accepts two exact public-observation +comparators, two exact ordered-ID comparators, or two hydration-only arms +independently validated from the same precomputed exact path inputs; mixed +boundaries are rejected. Fixed-suffix expansion ordered-ID candidates are +checked against the canonical stepwise-forward node/edge-ID arrays before their +timing is retained. Reports show +candidate/baseline median and p95 ratios, absolute median change, and +within-session A/A resolution without turning architecture selection into a +post-hoc pass threshold. + +### Three- and five-arm reference tournaments + +Use the generic tournament reporter when a candidate family has three or five +exact PostgreSQL reference arms. The first declared arm is the incumbent: + +```bash +make perf_tournament \ + PERF_TOURNAMENT_ARTIFACT=.coverage/tournament.jsonl \ + PERF_TOURNAMENT_ARMS=expand_into_pair_join,expand_into_lower_degree_scan,expand_into_pair_cache \ + PERF_TOURNAMENT_PROTOCOL=confirmation +``` + +The reporter verifies exact public observations, immutable SQL/implementation +identity, the predeclared doubled-Williams measurement order, and per-round +sample floors. A confirmation is promotion-eligible only when one stable +candidate wins both training and frozen holdout, its median improvement clears +the configured 5% or 100us materiality floor, and its p95 ratio upper bound is +at most 1.05. Discovery reports are always non-promotional. + +Function-backed SP/ASP candidates and guarded orientation runs use a +session-local receipt around every timed invocation when `-pool-size 1` is in +effect. Arming and reading occur outside the measured interval. The receipt +binds the requested identity to the exact executed branch, fallback outcome, +and a singular record count. Multi-connection runs remain available for the +operational matrix, but their timing samples are intentionally not eligible as +per-invocation promotion evidence. + +### Operational evidence gate + +The standalone operational gate consumes one schema-v2 JSON document through +`-operational-gate-input` and writes its machine-verifiable report to the +required `-operational-gate-output`. The input has four top-level fields: +`version`, the complete `promotion_identity`, frozen `requirements`, and +candidate-bound `records`. Unknown fields, trailing JSON, unsupported versions, +dirty-source evidence, or any source/archive/binary/corpus/promotion identity +drift fail closed. + +Exactly 32 records must use one declared query, resolved parameter set +(`params`, `node_params`, and `node_list_params`), and physically validated +fixture configuration/checksum. Declared and physical node and relationship +counts must agree. The normalized Cypher digest must occur in exactly one +authorized promotion bucket; whole-case shape is checked for SP/ASP, while an +orientation policy is checked against its exact fixed-suffix optimization +target. Every record must carry the production candidate/selector/boundary, +emitted arms, and cap contract in its optimization outcome. The manifest +identity independently authorizes the production candidate SQL through +`operational_candidate_sql_sha256`; the frozen requirements must repeat that +exact anchor rather than introduce one of their own. Every non-overflow record +must both hash to and equal the manifest anchor. Only the forced-overflow +scenario may use a distinct SQL fingerprint, and +then only by lowering positive guarded cap values on the otherwise identical +translation target. It may not substitute another query, parameter set, +fixture, policy, or lowering target. The candidate runtime arm is fixed by the +registered promotion policy; it is not selected by the input document. + +The frozen candidate matrix is the complete 27-cell product of: + +- pool sizes `1`, `2`, and `8`; +- concurrency levels `1`, `8`, and `16`; and +- PostgreSQL `plan_cache_mode` values `auto`, `force_custom_plan`, and + `force_generic_plan`. + +Every cell must contain the complete native GraphBench worker/iteration block, +positive drain and wall timings, real backend PIDs within the pool bound, and +one cold-session classification per used connection. Pool-size-1 cells retain +an exact timed-invocation receipt. Pool-size-2 and pool-size-8 cells must retain +GraphBench's honest `same_case_invocation_local_replay` serial metadata with no +fabricated invocation receipt or connection ID; their measured execution is +proved by the independently validated concurrency block plus the same exact +SQL, optimization target, and plan-replay summary. The same input must also +prove receipt-bearing candidate execution with `work_mem <= 64 KiB`; SQLSTATE +`57014` cancellation in strictly less than `250 ms`, rollback, and successful +same-PID replay; Repeatable Read stability across a distinct committed writer +whose change becomes visible after the reader transaction; two-session +invocation/state isolation; and a forced-overflow receipt chain containing the +exact configured fallback. Nested exact fallback chains are retained rather +than reduced to their terminal executor. + +```bash +go run ./cmd/graphbench \ + -operational-gate-input .coverage/operational-input.json \ + -operational-gate-output .coverage/operational-report.json +``` + +This command validates an already assembled native evidence document; it does +not synthesize the 32 records. The repository currently has no standalone +operational-input producer. Release engineering must assemble the document +from the complete native GraphBench worker/iteration results and the associated +cancellation, snapshot, isolation, overflow, optimization, plan-replay, and +fixture evidence without reducing them to hand-authored summaries. The strict +`OperationalGateInput` schema and fail-closed validator are the authoritative +handoff until a capture producer is added. + +The report is written for both outcomes. Schema v2 embeds the complete +canonical input and its SHA-256; final manifest verification recomputes the +matrix, receipts, cancellation, snapshot, session isolation, overflow, and +SQL-anchor decisions from those raw records, then requires exact coverage, +record-decision, reason, and disposition equivalence. Editing either raw +evidence or only its passing summary therefore fails closed. A failing gate +exits nonzero, so CI retains the per-record reasons without mistaking the +artifact for passing promotion evidence. Operational mode is mutually +exclusive with every other standalone reporter, bundle creation, and protected +holdout authorization. + +### Promotion manifest + +Promotion is authorized only by a version-2 manifest that binds the candidate, +selector, source/binary/corpus SHA-256 digests, immutable caps, exact query +cohorts, training and frozen-holdout buckets, the exact +`operational_candidate_sql_sha256`, and checksummed A/A, +confirmation, performance, resource, reference-closure, and operational +reports. Version 1 is decoded only to reject it for new authorization. + +Every evidence report must repeat the manifest's complete authorization +identity. Generate the role-specific report first, then attach the identity +from a provisional manifest whose evidence map may still be empty. + +Final verification decodes every role against its concrete versioned schema; +unknown fields, trailing JSON, structurally shallow pass claims, and +unsupported candidate/schema combinations fail closed. Binding A/A, resource, +and reference-closure reports embeds each report's exact native producer bytes, +and verification recomputes their SHA-256 values before comparing the typed +projections. Confirmation and performance remain strictly decoded typed reports +rather than native-byte wrappers. Confirmation uses the +candidate-specific SP-I1, SP-I2, orientation-v2, or causal-ASP schema. +Both orientation schemas remain readable for diagnostics. `orientation-probe-v1` +is not promotable because its v1 report cannot bind the source, corpus, and +frozen cohort required by manifest v2. The final manifest verifier also +terminally rejects `orientation-probe-v2`: its immutable training overhead gate +failed, so authorization requires a new policy generation rather than a new v2 +report. + +Resource verification requires the candidate's exact +numeric cap map, every observed high-water mark to remain at or below its cap, +no fallback/reference case substitution, and at least 50 unique candidate +runtime receipts per case-round. SP-I1/SP-I2 confirmation must name the exact +native resource-report digest; confirmation, performance, and resource reports +must name the same candidate artifact. Every promotion case must contain the +exact performance round count in resource evidence, and the flattened resource +receipt-chain set must equal the performance receipt-chain set. Reference +closure is intentionally a +separate capture because it contains raw-pgx and comparator measurements that +the frozen SP confirmation artifacts forbid; it instead closes candidate, +source/binary/corpus, normalized query, exact dataset/name/split cohort, the +per-case workload digest against exactly one PostgreSQL A/A workload, frozen +thresholds, and independently valid production receipt chains. Reference +invocations come from their own raw-pgx/comparator capture, so their invocation +IDs are not equated with the performance/resource capture. Exact cross-role +receipt equality instead applies between performance and the complete per-round +resource set. +Confirmation and performance artifacts do not embed their raw benchmark +samples, so the verifier recomputes decisions from their typed evidence and +frozen settings but cannot independently replay every bootstrap draw. Closing +that final reproducibility gap requires a future producer-schema revision; it +does not weaken the current fail-closed identity, cohort, receipt, and decision +checks. + +Performance verification freezes seed 1, confidence 0.975, and the 5% base +regression threshold; recomputes p50/p95 noise-adjusted regressions and +effective materiality floors; validates every measured receipt terminal; and +requires its exact dataset/name/split cohort to equal confirmation. SP-I1 and +SP-I2 resolve repository-frozen canonical cohorts, while orientation-v2 +resolves the canonical eight-training/four-holdout V3 cohort and its +declaration digest. Orientation receipts may terminate only in the emitted +forward or reverse executor arm; SP/ASP receipts must terminate in the +candidate itself. +Orientation-v2 cohort and receipt validation preserves diagnostic readability; +it does not override the final verifier's terminal rejection of that policy +generation. + +SQL anchoring uses an explicit two-pass capture. First run a preflight with +`operational_candidate_sql_sha256` omitted to derive the exact SQL fingerprint. +Freeze that digest in the provisional manifest, discard the preflight as +non-promotional, and recapture every formal evidence artifact. A populated +anchor is verified by the runner against generated SQL before execution; the +final verifier and PostgreSQL driver require the same canonical anchor. Because +schema v2 carries one scalar SQL anchor, an anchored manifest must contain +exactly one unique authorized query digest (the cohort may vary parameters and +fixtures for that query). + +```bash +go run ./cmd/graphbench \ + -promotion-bind-manifest .coverage/promotion-provisional.json \ + -promotion-bind-role performance \ + -promotion-bind-input .coverage/performance-unbound.json \ + -promotion-bind-output .coverage/performance.json +``` + +Repeat this for `aa`, `confirmation`, `performance`, `resource`, +`reference_closure`, and `operational`, checksum the bound reports, and place +those digests in the final manifest. Then verify the complete closure without +opening a database connection: + +```bash +go run ./cmd/graphbench \ + -promotion-manifest .coverage/promotion.json \ + -promotion-manifest-output .coverage/promotion-verification.json +``` + +Verification fails closed for missing roles, mutated reports, path traversal, +non-passing evidence, invalid digests, absent caps, identity fields that differ +from the manifest, or buckets that do not bind the exact canonical +`["training","holdout"]` split. The evidence map must contain exactly the six +documented roles; invented roles, duplicate JSON object keys, duplicate query +digests or bucket names, duplicate allowlist entries, and symlink-based escapes +from the manifest directory are rejected. The single SQL anchor closes exactly +one globally unique query identity across the complete bucket set. This mode is +mutually exclusive with benchmark, report, bind, and bundle operations. + +### Fixed-one-hop ExpandInto study + +Build the standalone three-arm fixed-one-hop report from records captured with +the `expand_into_one_hop` category and its exact PostgreSQL references: + +```bash +go run ./cmd/graphbench \ + -expand-into-artifact .coverage/expand-into.jsonl \ + -expand-into-output .coverage/expand-into-study.json \ + -expand-into-protocol discovery \ + -confidence-level 0.975 -seed 1 + +make perf_expand_into \ + PERF_EXPAND_INTO_ARTIFACT=.coverage/expand-into.jsonl \ + PERF_EXPAND_INTO_PROTOCOL=confirmation +``` + +`discovery` requires 5-20 independently reloaded rounds, five warmups, and ten +samples per arm per round. `confirmation` requires 10-20 rounds, 20 warmups, +and 50 samples per arm per round. Both protocols require the frozen doubled +Williams order for `expand_into_pair_join`, `expand_into_lower_degree_scan`, and +`expand_into_pair_cache`, exact public observations, stable implementation/SQL +identities, and persisted plan-cache/operator evidence. Confirmation reports +also require one stable non-direct winner across training and frozen holdout, +the configured 5% or 100us materiality floor, and p95 containment at 1.05. +Even a passing report does not activate a production strategy; discovery +remains evidence-only. + +Path-observed singleton cases additionally capture benchmark-only M0 and M1 +materializer arms. Whole-query comparison uses each architecture's minimal +state: `SP-S3-U-E+MAT-M0` carries edge IDs only and derives node order from the +directed edge endpoints, while `SP-S3-U-NE+MAT-M1` carries node and edge IDs and +hydrates both streams independently by ordinality. Outbound and inbound M0 use +distinct implementation identities. Separate +hydration-only arms use precomputed IDs so search cost stays outside the timed +materializer boundary. These arms are exact-result checked but do not change +production path rendering. Odd benchmark rounds execute references in declared +order and even rounds reverse that order, balancing which M0/M1 arm runs first +across the required independently reloaded rounds. + +Every PostgreSQL dataset reload truncates the active relationship and node +partitions together. Other backends delete relationships before nodes. PostgreSQL then checks +the physical row counts in the active `node_` and `edge_` +partitions against the fixture declaration before vacuuming or measuring. A +stale/orphan row therefore fails the run instead of silently contaminating scan +and count cases. Fixture records also retain active child-partition sizes rather +than the zero-sized partitioned-parent relations. + +```bash +go run ./cmd/graphbench \ + -modes postgres_sql -postgres-references \ + -cases 'GSP-D01-F001_path,GSP-D02-F016_path,GSP-D04-F128_path,GSP-D08-F001_path_inbound,GSP-D16-F016_path,GSP-D32-F512_path,GSP-D64-F1000_path' \ + -warmup-iterations 20 -iterations 50 -pool-size 1 \ + -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output .coverage/materializer-round-1.jsonl +``` + +The optimizer also emits a typed `ShortestPathExecutorDecision` for every +shortest traversal. It records a machine-readable structural-eligibility result, +SP family and planned candidate identities, observation mode, minimum/maximum +depth, selected/fallback executor, selector version/mode, limits, and stable +fallback code. These fields are also copied into each exact target outcome. +Call count and read-only status are statement-wide, including shortest calls or +mutations separated by `WITH`. Selector `sp-static-v5-contained` chooses +`SP-S3-U-D` for qualified distance observations, bounded +`SP-S3-U-E+MAT-M0` for directed single-kind one-path observations, and +canonical `SP-S4-C-WE+MAT-M0` for deep inbound, multi-kind, or untyped witness +work. Qualification requires one directed three-element shortest-path +traversal, a supported bounded depth, one static ID equality per endpoint, no +relationship variable or predicate, no path predicate, one uncorrelated +endpoint pair, one statement-wide shortest call, and a read-only statement. +The selector also records graph direction, physical expansion +column, relationship-kind count, wildcard state, and a static topology class. +Deep `end_id` distance expansion selects canonical `SP-S4-C-D`. S4 uses compact +ID state, a bounded ceiling, and exact same-statement overflow fallback. +`asp-static-v1` selects `ASP-A1-DAG` for the narrow singleton all-shortest +envelope and retains all minimum-depth predecessor edges before enumeration. +`ASP-I1-U-DAG+MAT-M0` is a distinct inline predecessor-DAG comparator and a +default-off exact-query production canary. Its guarded statement records the +executed candidate/no-path/A1-fallback branch, uses immutable manifest caps, +and requires Repeatable Read or Serializable isolation. Forced executors +remain qualification seams. +`SP-I1-C-WE+MAT-M0` is the corresponding guarded canonical-predecessor witness +canary, with four cap+1 gates, inline M0 hydration, exact S4 fallback, and an +ordered runtime fallback event chain. Its target outcome names the exact +candidate/fallback pair and emitted `sp-i1-canonical-guarded-v1` policy, while +diagnostic resource evidence remains isolated from the ASP I1 counter family. +It remains default-off; `sp-static-v5-contained` continues to select the +automatic S3/S4 production paths. The evidence-gated `sp-static-v6` canary +identity accepts only the qualified inbound, typed, single-kind, one-path +`min=1`/`max=64` bucket. Outbound, untyped, multi-kind, and different-depth +manifests fail closed at verification, provisional capture, driver admission, +and translation. + +`SP-I2-C-D` is the guarded distance-only canary for inbound hidden fan-in. Its +reverse-physical recursive relation carries only node ID and depth, enforces +independent state and frontier caps, and selects exact `SP-S4-C-D` before output +on overflow. Provisional and final manifests require selector +`sp-static-v8-hidden-fanin`, a typed single-kind inbound distance bucket, a +stable snapshot, and exactly `state_limit` plus `frontier_limit`. Diagnostic +replay exposes candidate/fallback markers and inactive-arm loop counts under +the `inline_shortest_distance` namespace. Production evidence and activation +must use the preregistered production-form `state_limit=100000` and +`frontier_limit=100000` values; these immutable protocol inputs are not yet +qualified. V1 is now terminally rejected: selector-v8 production admission, +new V1 freezes, and V1 holdout authorization fail closed. Forced V1 execution +remains available only for diagnostics and verification of archived evidence. +The machine-readable rejection is +`benchmark/testdata/scale/protocols/sp_i2_distance_v1_rejection.json`. + +`SP-I2-C-D-V2` is the independent, default-off E1 successor. It emits policy +`sp-i2-distance-guarded-v2` and selector identity +`sp-static-v9-hidden-fanin-tail`. Its materialized admission decision executes +before target selection. When `frontier_limit >= state_limit` (including the +production `100000/100000` contract), total-state admission dominates frontier +admission and the SQL contains no depth aggregate. Unequal diagnostic caps +retain exactly one frontier aggregate. `SP-I2-C-D-V2-E0` and +`SP-I2-C-D-V2-E1` are non-promotional controls. The component identities +`SP-I2-C-D-V2-E1D`, `SP-I2-C-D-V2-E1P`, and `SP-I2-C-D-V2-E1DP` respectively +add the exact direct-edge floor, proven scalar projection elision, or both. +The direct floor lazily suppresses recursive/admission/target work after a +depth-one hit. Scalar projection forcing fails closed when entity, property, +or path hydration is consumed. GraphBench requires +`-sp-i2-generation sp-i2-distance-v2` whenever any V2 identity is forced. + +The supplemental open-corpus readiness capture requires +`-sp-i2-v2-readiness-comparison`; it alternates exact S4 distance and E0 order +across ten rounds. The component capture instead requires +`-sp-i2-v2-development-tournament` and uses the fixed ten-round Williams order +for E0, E1, E1D, E1P, and E1DP. Both modes accept only the six already-open V1 +training cases, exactly 25 warmups and 100 timed samples, Repeatable Read, and +diagnostic telemetry. Supply the forced executor identity itself as `-arm`; +`-arm-order` must match the selected mode's fixed round order. Every invocation +in a round shares `-round`, `-block`, and `-run-uuid`. These modes are mutually +exclusive, and their artifacts are diagnostic: they cannot authorize a freeze, +holdout, manifest, or production activation. + +Validate a completed raw capture before analysis. Validation requires exactly +ten complete rounds over all six open cases, the fixed arm order, 25 declared +warmups, 100 timed samples, one excluded stabilization receipt per record, +globally unique invocation receipts, and exact runtime identities: + +```bash +go run ./cmd/graphbench \ + -sp-i2-generation sp-i2-distance-v2 \ + -sp-i2-v2-development-artifact development.jsonl \ + -sp-i2-v2-development-study tournament +``` + +Use `readiness` for the two-arm E0/S4 artifact. The command exits without +database setup only after the entire artifact passes validation. + +Create the permanently diagnostic five-arm decision report from a complete +tournament artifact with: + +```bash +go run ./cmd/graphbench \ + -sp-i2-generation sp-i2-distance-v2 \ + -sp-i2-v2-development-report-artifact development.jsonl \ + -sp-i2-v2-development-report-output development-report.json +``` + +The report revalidates the raw schedule, semantic observations, canonical +plans, diagnostic receipts, resource limits, non-fallback identities, and one +source/binary identity before analysis. It uses paired round-median resampling +for median and planning-time ratios and paired-round hierarchical resampling +for p95. Eligibility applies the preregistered E0, E1, and parent contrasts; +ranking then uses plan-node score, the maximum planning-time upper bound versus +E0, the maximum p95 upper bound versus E0, and fixed arm order. If no variant +is eligible, E0 is selected. `promotion_eligible` is always false: this report +cannot authorize a freeze, holdout, manifest, or production activation. + +E1DP is fail-closed until E1D and E1P have independently passed exact semantic, +canonical-plan, resource, receipt, and fallback checks. Build one GraphBench +binary and use that same executable for both checks, authorization production, +and the subsequent E1DP tournament positions: + +```bash +graphbench \ + -sp-i2-generation sp-i2-distance-v2 \ + -sp-i2-v2-component-check \ + -modes postgres_sql -iterations 1 -warmup-iterations 1 \ + -round 1 -block 1 -arm-order 1 -run-uuid component-e1d \ + -arm SP-I2-C-D-V2-E1D -tags sp-i2-distance-v1-training \ + -jsonl-output .coverage/sp-i2-v2-component-e1d.jsonl \ + -postgres-force-shortest-executor SP-I2-C-D-V2-E1D \ + -postgres-repeatable-read -postgres-traversal-telemetry diagnostic + +# Repeat with E1P, its executor/arm identity, run UUID, and output path. + +graphbench \ + -sp-i2-generation sp-i2-distance-v2 \ + -sp-i2-v2-component-e1d-artifact .coverage/sp-i2-v2-component-e1d.jsonl \ + -sp-i2-v2-component-e1p-artifact .coverage/sp-i2-v2-component-e1p.jsonl \ + -sp-i2-v2-component-authorization-output .coverage/sp-i2-v2-components.json +``` + +Every E1DP tournament invocation must then supply +`-sp-i2-v2-component-authorization .coverage/sp-i2-v2-components.json`. +GraphBench rejects a missing, malformed, failed, protocol-mismatched, or +source/binary-mismatched authorization before database setup. Evidence paths +should remain under an ignored directory so creating them does not change the +bound working-tree fingerprint. + +The authoritative logical declaration is +`benchmark/testdata/scale/protocols/sp_i2_distance_v2.json`. It fixes 40 +rounds, 25 ordinary warmups, one excluded receipt-bearing stabilization, 100 +timed samples, 100,000 deterministic hierarchical bootstrap draws, and the +unchanged 97.5% confidence and p95 gates. The stabilization receipt is stored +separately from latency samples and timed iteration one is retained. + +V2 terminated at its mandatory prospective calibration gate before formal +A/A or discovery. Reproduce the gate from the exact clean V1 traces with: + +```bash +graphbench \ + -sp-i2-generation sp-i2-distance-v2 \ + -sp-i2-v2-simulation-baseline-trace .coverage/sp-i2-distance-v1-3865cbc/discovery-s4.jsonl \ + -sp-i2-v2-simulation-candidate-trace .coverage/sp-i2-distance-v1-3865cbc/discovery-i2.jsonl \ + -sp-i2-v2-simulation-output .coverage/sp-i2-distance-v2-power-simulation.json +``` + +The producer verifies both trace hashes and their source identity, reconstructs +the frozen p50/p95 round-drift vectors, expands the 20-round cycle-control +trace to the fixed 40-by-100 design with the declared log-quantile transform, +and replays the exact 100,000-draw hierarchical bootstrap used to calibrate the +20,000-run scenarios. It writes the report and exits nonzero because identity +A/A admission power, target power, control power, and both order-stratified +power cases miss the frozen 90% Wilson-lower requirement. This is a terminal +protocol result: do not run V2 A/A, discovery, holdout, capture-plan, sealed +manifest, confirmation, operational, or promotion phases. Any renewed work +requires a separately identified successor protocol with a new corpus and +prospectively fixed design. The checked-in +`benchmark/testdata/scale/protocols/sp_i2_distance_v2_rejection.json` +tombstone binds the implementation commit, protocol, generated report, failed +Wilson bounds, and the fact that no formal A/A, capture plan, sealed +preregistration, holdout, or production activation occurred. + +In less technical terms, the benchmark was too noisy for this fixed study to +make a dependable yes/no decision. Comparing the same implementation with +itself produced a plausible p95 range of about `0.946x` to `1.063x` and an +absolute range of `-116us` to `+133us`; admission required those entire ranges +to stay within about `0.952x` to `1.05x` and `-100us` to `+100us`. Across +20,000 trials, the target decision succeeded 47.94% of the time and the control +decision 51.23%, versus the required 90% lower confidence bound. Interval +coverage and false-positive checks passed, so the analysis was behaving as +designed—the study was simply not precise enough. Changing the sample count or +limits after seeing that result would invalidate the frozen plan. + +### Frozen canonical-I1 qualification + +The `sp-i1-inbound-v1` study is a dedicated two-arm comparison between exact +forced `SP-S4-C-WE+MAT-M0` and guarded forced +`SP-I1-C-WE+MAT-M0`. Its fresh cohort contains four training cases at depths 4 +and 16 and three unopened holdouts at depths 8 and 32. Every case uses the +same typed inbound one-path query with `min=1`, `max=64`, one `Traverse` kind, +exact path observations, and forbidden fallback. GraphBench excludes these +protocol-only holdouts from ordinary default, category, dataset, and generic-tag +selection. Only the exact holdout protocol tag (or an exact holdout case name) +enters the protected authorization path. Exact-name selection still fails +closed because the only executable confirmation selection is the complete +four-training/three-holdout cohort with a passing training freeze. The frozen +performance study executes PostgreSQL only; Neo4j remains part of the declared +cross-backend semantic contract, not an authorized holdout timing arm. + +Build GraphBench once from a clean committed tree. Keep the binary and all +outputs under ignored `.coverage`; repeated `go run` invocations have different +binary identities and cannot satisfy the freeze: + +```bash +CAPTURE=.coverage/sp-i1-inbound-v1 +mkdir -p "$CAPTURE/bin" +go build -trimpath -o "$CAPTURE/bin/graphbench" ./cmd/graphbench +BIN="$CAPTURE/bin/graphbench" +DISCOVERY_UUID="sp-i1-discovery-$(git rev-parse HEAD)" +``` + +Discovery opens only the four training declarations. Capture 5-20 paired +rounds with at least 5 warmups and 10 samples per arm per round. Use the same +UUID for both artifacts and all rounds. Odd rounds put S4 first; even rounds +put canonical I1 first. For round 1, the two commands are: + +```bash +"$BIN" \ + -modes postgres_sql \ + -tags sp-i1-inbound-v1-training \ + -round 1 -block 1 -run-uuid "$DISCOVERY_UUID" \ + -arm sp-i1-s4 -arm-order 1 \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -postgres-force-shortest-executor SP-S4-C-WE+MAT-M0 \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output "$CAPTURE/discovery-s4.jsonl" -append-jsonl + +"$BIN" \ + -modes postgres_sql \ + -tags sp-i1-inbound-v1-training \ + -round 1 -block 1 -run-uuid "$DISCOVERY_UUID" \ + -arm sp-i1-candidate -arm-order 2 \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -postgres-force-shortest-executor SP-I1-C-WE+MAT-M0 \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output "$CAPTURE/discovery-i1.jsonl" -append-jsonl +``` + +After all training rounds, bind resource-gate v5 to the exact candidate JSONL, +then write the discovery report and freeze: + +```bash +"$BIN" \ + -resource-artifact "$CAPTURE/discovery-i1.jsonl" \ + -resource-output "$CAPTURE/discovery-i1-resource.json" + +"$BIN" \ + -sp-i1-baseline-artifact "$CAPTURE/discovery-s4.jsonl" \ + -sp-i1-candidate-artifact "$CAPTURE/discovery-i1.jsonl" \ + -sp-i1-resource-report "$CAPTURE/discovery-i1-resource.json" \ + -sp-i1-protocol discovery \ + -sp-i1-output "$CAPTURE/discovery-report.json" \ + -sp-i1-freeze-output "$CAPTURE/discovery-freeze.json" +``` + +For structurally valid evidence, the reporter preserves the discovery result +and freeze even when a statistical or resource disposition fails. Identity, +path, and source-validation failures do not write an artifact. A failed freeze +cannot authorize holdout capture. A passing freeze binds the clean source archive, commit, +binary, query, training/full declarations and resolved selections, training +artifacts, resource report, and the promotion-form cap names +`state_limit`, `predecessor_limit`, `enumeration_limit`, and +`output_bytes_limit`. Resource evidence uses the corresponding telemetry names +`state_rows`, `predecessor_rows`, `output_rows`, and `output_bytes`. The CLI +fixes the bootstrap seed at `1` and confidence at `0.975`, uses 10,000 +resamples, and freezes all three settings. Schedule validation checks the +recorded invocation timestamps as well as the declared alternating order. +Resource-gate v5 binds every decision to the exact candidate arm, round, +block, run UUID, runtime receipt, and diagnostic counters. +The qualification validator requires `planned_candidates` to preserve the +translator's complete shortest-path executor search space. The exact study +arms are bound independently through selected, applied, emitted, and timed +runtime-receipt identities; a reduced two-arm planned list is invalid evidence. +Every warm sample also carries a unique session-local runtime invocation ID, +repeated on its receipt events; duplicate reuse anywhere in the paired study +is rejected. Fixture +and PostgreSQL comparison is deliberately strict, including byte-identical +node and edge relation sizes across paired arms and rounds. + +Only after discovery passes may confirmation open the full four-training and +three-holdout cohort. Every capture command must provide the freeze and its +checksummed discovery report before database setup. Confirmation requires +10-20 paired rounds, at least 20 warmups, 50 samples per arm per round, pool +size 1, diagnostic telemetry, Repeatable Read, an explicit shared UUID, block +equal to round, and the exact alternating labels/order. For confirmation round +1, create a fresh series UUID, add these authorization and cohort flags to the +two discovery commands, increase the sample settings, and write separate +artifacts. Reuse that confirmation UUID across both arms and every confirmation +round: + +```text +CONFIRMATION_UUID="sp-i1-confirmation-$(git rev-parse HEAD)" +-run-uuid "$CONFIRMATION_UUID" +-tags sp-i1-inbound-v1-training,sp-i1-inbound-v1-holdout +-sp-i1-freeze .coverage/sp-i1-inbound-v1/discovery-freeze.json +-sp-i1-discovery-report .coverage/sp-i1-inbound-v1/discovery-report.json +-sp-i1-training-baseline-artifact .coverage/sp-i1-inbound-v1/discovery-s4.jsonl +-sp-i1-training-candidate-artifact .coverage/sp-i1-inbound-v1/discovery-i1.jsonl +-sp-i1-training-resource-report .coverage/sp-i1-inbound-v1/discovery-i1-resource.json +-warmup-iterations 20 -iterations 50 +``` + +Use `sp-i1-s4` at order 1 and `sp-i1-candidate` at order 2 on odd rounds; +reverse those orders on even rounds. Rounds after the first must use +`-append-jsonl`. GraphBench rejects partial or extra cohorts, a changed tag or +case declaration, source/binary drift, insufficient capture settings, path +aliasing with freeze inputs, supplemental arms, and any attempt to enter an +unrelated report mode with holdout authorization flags. +Before every protected capture, GraphBench reloads those three training inputs, +checks their frozen digests, and recomputes the discovery statistics and +resource decisions before opening the database. + +Create resource-gate v5 from the complete confirmation I1 artifact, then issue +the final report with the frozen discovery inputs: + +```bash +"$BIN" \ + -resource-artifact "$CAPTURE/confirmation-i1.jsonl" \ + -resource-output "$CAPTURE/confirmation-i1-resource.json" + +"$BIN" \ + -sp-i1-baseline-artifact "$CAPTURE/confirmation-s4.jsonl" \ + -sp-i1-candidate-artifact "$CAPTURE/confirmation-i1.jsonl" \ + -sp-i1-resource-report "$CAPTURE/confirmation-i1-resource.json" \ + -sp-i1-freeze "$CAPTURE/discovery-freeze.json" \ + -sp-i1-discovery-report "$CAPTURE/discovery-report.json" \ + -sp-i1-training-baseline-artifact "$CAPTURE/discovery-s4.jsonl" \ + -sp-i1-training-candidate-artifact "$CAPTURE/discovery-i1.jsonl" \ + -sp-i1-training-resource-report "$CAPTURE/discovery-i1-resource.json" \ + -sp-i1-protocol confirmation \ + -sp-i1-output "$CAPTURE/confirmation-report.json" +``` + +Each case passes only when the candidate has complete per-sample timed runtime +receipts with no fallback or overflow, exact observations match S4, resource +evidence passes all four limits, the median-ratio upper bound is at most `0.95` +or the median-saving lower bound is at least `100us`, and the p95-ratio upper +bound is at most `1.05`. The study does not change the automatic production +selector; a passing report is input to later canary, rollback, and promotion +closure. + +### Frozen SP-I2 distance qualification + +The commands below document the archived V1 workflow. They cannot create a +new freeze or authorize a holdout; all V1 evidence use must explicitly pass +`-sp-i2-generation sp-i2-distance-v1` and is verification-only. + +The `sp-i2-distance-v1` study compares exact forced `SP-S4-C-D` with guarded +forced `SP-I2-C-D`. Its sealed cohort contains six training cases at fixture +depths 3, 6, 8, and 16 and four unopened holdouts at depths 5, 13, and 21. +The matrix covers full-depth and early targets, disconnected exhaustion, +hidden root/intermediate fan-in, and a cycle control. Every declaration uses +the same typed inbound distance query with `min=1`, `max=64`, one `Traverse` +kind, exact scalar observations, and forbidden fallback. Ordinary selection +excludes the four holdouts; only the exact holdout tag or exact case name can +enter the protected path, and timing still requires a valid discovery freeze. + +Build one binary from a clean committed source and capture 5-20 alternating +discovery rounds with at least 5 warmups and 10 samples per arm. Use a fresh +capture directory. Every invocation must set `block` equal to `round`, and the +commands must actually run in their declared arm order: S4 first on odd rounds +and I2 first on even rounds. Merely swapping the `-arm-order` labels while +running S4 first in every round produces invalid chronology. This example +captures the minimum five rounds: + +```bash +CAPTURE=.coverage/sp-i2-distance-v1 +mkdir -p "$CAPTURE/bin" +go build -trimpath -o "$CAPTURE/bin/graphbench" ./cmd/graphbench +BIN="$CAPTURE/bin/graphbench" +DISCOVERY_UUID="sp-i2-discovery-$(git rev-parse HEAD)" + +capture_sp_i2_arm() { + local round="$1" arm="$2" arm_order="$3" executor="$4" output="$5" + "$BIN" -modes postgres_sql -tags sp-i2-distance-v1-training \ + -round "$round" -block "$round" -run-uuid "$DISCOVERY_UUID" \ + -arm "$arm" -arm-order "$arm_order" \ + -warmup-iterations 5 -iterations 10 -pool-size 1 \ + -postgres-force-shortest-executor "$executor" -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic -pg-connection "$PG_CONNECTION_STRING" \ + -jsonl-output "$output" -append-jsonl +} + +for round in 1 2 3 4 5; do + if (( round % 2 == 1 )); then + capture_sp_i2_arm "$round" sp-i2-s4 1 SP-S4-C-D \ + "$CAPTURE/discovery-s4.jsonl" + capture_sp_i2_arm "$round" sp-i2-candidate 2 SP-I2-C-D \ + "$CAPTURE/discovery-i2.jsonl" + else + capture_sp_i2_arm "$round" sp-i2-candidate 1 SP-I2-C-D \ + "$CAPTURE/discovery-i2.jsonl" + capture_sp_i2_arm "$round" sp-i2-s4 2 SP-S4-C-D \ + "$CAPTURE/discovery-s4.jsonl" + fi +done +``` + +After discovery capture, bind the resource report and create the freeze: + +```bash +"$BIN" -resource-artifact "$CAPTURE/discovery-i2.jsonl" \ + -resource-output "$CAPTURE/discovery-i2-resource.json" + +"$BIN" -sp-i2-baseline-artifact "$CAPTURE/discovery-s4.jsonl" \ + -sp-i2-candidate-artifact "$CAPTURE/discovery-i2.jsonl" \ + -sp-i2-resource-report "$CAPTURE/discovery-i2-resource.json" \ + -sp-i2-protocol discovery -sp-i2-output "$CAPTURE/discovery-report.json" \ + -sp-i2-freeze-output "$CAPTURE/discovery-freeze.json" +``` + +Only a passing clean-source freeze authorizes the protected cohort. Capture +10-20 confirmation rounds with at least 20 warmups and 50 samples per arm, +using a fresh shared UUID and the same alternating schedule. For every round, +set `-round "$round" -block "$round"`; physically execute S4 before I2 on odd +rounds and I2 before S4 on even rounds, with arm orders `1` then `2`. Rounds +after the first must use `-append-jsonl`. Add these frozen authorization inputs +to both arm commands: + +```text +-tags sp-i2-distance-v1-training,sp-i2-distance-v1-holdout +-sp-i2-freeze .coverage/sp-i2-distance-v1/discovery-freeze.json +-sp-i2-discovery-report .coverage/sp-i2-distance-v1/discovery-report.json +-sp-i2-training-baseline-artifact .coverage/sp-i2-distance-v1/discovery-s4.jsonl +-sp-i2-training-candidate-artifact .coverage/sp-i2-distance-v1/discovery-i2.jsonl +-sp-i2-training-resource-report .coverage/sp-i2-distance-v1/discovery-i2-resource.json +-warmup-iterations 20 -iterations 50 +``` + +Then bind the complete candidate artifact and issue confirmation: + +```bash +"$BIN" -resource-artifact "$CAPTURE/confirmation-i2.jsonl" \ + -resource-output "$CAPTURE/confirmation-i2-resource.json" + +"$BIN" -sp-i2-baseline-artifact "$CAPTURE/confirmation-s4.jsonl" \ + -sp-i2-candidate-artifact "$CAPTURE/confirmation-i2.jsonl" \ + -sp-i2-resource-report "$CAPTURE/confirmation-i2-resource.json" \ + -sp-i2-freeze "$CAPTURE/discovery-freeze.json" \ + -sp-i2-discovery-report "$CAPTURE/discovery-report.json" \ + -sp-i2-training-baseline-artifact "$CAPTURE/discovery-s4.jsonl" \ + -sp-i2-training-candidate-artifact "$CAPTURE/discovery-i2.jsonl" \ + -sp-i2-training-resource-report "$CAPTURE/discovery-i2-resource.json" \ + -sp-i2-protocol confirmation -sp-i2-output "$CAPTURE/confirmation-report.json" +``` + +Each normal case requires exact observations, unique timed receipt chains, +`SP-I2-C-D` with zero fallback/overflow and zero inactive fallback loops, +state/frontier evidence within the frozen 100,000-row ceilings (the portable +`queue_rows` observation conservatively aliases the frontier), median +ratio upper bound at most `0.95` or median saving lower bound at least `100us`, +and p95 ratio upper bound at most `1.05`. The preregistered cycle case is an +adverse control: its median ratio upper bound may be at most `1.10`, or its +absolute overhead upper bound at most `100us`, while the same p95 ceiling +continues to apply. Any failed training case prevents a +freeze; any holdout failure rejects this policy generation without retuning. + +The completed local rehearsal is therefore diagnostic, not a failed formal +discovery decision. Five target cases were strong and receipt-complete, while +the cycle control missed its bounds on point estimates. Clean-source validation +stopped the workflow before either report or freeze creation, so only a fresh +clean recapture can make the authoritative discovery decision. + +### Existing canonical SP-I1 qualification result + +The clean `6d56a609` confirmation completed 10 paired rounds and 500 timed +samples per arm/case. All four training and three holdout cases passed with +zero candidate fallbacks; median reductions were 75.9-94.2% and p95 reductions +were 70.2-89.7%. Resource-gate v5 passed all 70 candidate case-round records, +with maxima of 281 state rows, 280 predecessor rows, 33 output rows, and 9,075 +output bytes. This closes the frozen cohort; it does not replace the production +statement, reference-closure, and operational evidence required by a promotion +manifest. + +### Production-manifest statement capture + +Use `-postgres-production-manifest` to measure the exact guarded production +statement from a provisional version-2 manifest before the evidence map can be +closed. The runner validates the candidate/fallback pair, selector, +family-specific immutable caps, unique exact query digests, and bucket match. +Guarded SP-I1/ASP-I1 candidates require their four positive shortest-path +caps; SP-I2 instead requires the preregistered, production-form +`state_limit=100000` and `frontier_limit=100000` contract and no unrelated cap +dimensions; those values do not imply that SP-I2 has qualified. +`orientation-probe-v1` and `orientation-probe-v2` staging instead requires the +optimizer's exact +`root_row_limit=512`, `reverse_seed_row_limit=512`, +`directional_degree_row_limit=16384`, and `state_limit=4096` contract, the +`EXPANSION-STEPWISE-FORWARD` fallback, and the `guarded_dual_arm` boundary; their +production options enable expansion orientation without selecting a +shortest-path executor. This preserves exact diagnostic statement capture; it +does not make either generation final-authorizable. Final verification rejects +v1 because its evidence schema cannot bind the required source/corpus/cohort, +and terminally rejects v2 because its immutable training overhead gate failed. +The runner executes each statement under Repeatable Read and retains per-sample +runtime receipts. This flag is mutually exclusive with tool-forced and shadow +modes; +evidence may be empty only because the capture is producing that evidence. +The initial preflight manifest may also omit +`operational_candidate_sql_sha256` solely to derive it. Formal capture must set +that digest; the runner then fails before execution if generated SQL differs. +Preflight artifacts cannot be bound into final promotion evidence. +Final rollout still requires the ordinary complete manifest verifier. +Use `-postgres-repeatable-read` on the incumbent arm so a matched comparison +measures both sides under the stable-snapshot admission contract. A production +manifest implies this option and cannot be combined with it explicitly. + +## Existing graph non-mutating mode + +`-existing-graph` runs a selected PostgreSQL corpus without asserting schema, +clearing/loading fixtures, vacuuming, or creating persistent helpers. It +requires a versioned logical-key anchor manifest and refuses `write_scenario` +or mutation keywords before runner construction. It deliberately uses +read-write PostgreSQL sessions so session-local workspace setup, reset, and +statistics match production. Example: + +```json +{ + "version": 1, + "graph": "integration_test", + "content_identity": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "anchors": { + "outbound_source": {"logical_key": "sanitized-source", "kind": "Source"}, + "outbound_target": {"logical_key": "sanitized-target", "kind": "Target"} + } +} +``` + +```bash +go run ./cmd/graphbench \ + -existing-graph \ + -anchor-manifest anchors.json \ + -cases LIVE-outbound-distance \ + -checkpoint .coverage/live/checkpoint.json \ + -progress .coverage/live/progress.jsonl \ + -jsonl-output .coverage/live/results.jsonl +``` + +Anchor values are used only at runtime. Durable records replace them with +one-way hashes, omit rendered parameters and Cypher, and redact observed-row +and error payloads in both primary and nested reference outcomes. The runner captures +before/after graph cardinalities, relation sizes, PostgreSQL settings, and +schema/index fingerprints. Artifact schema v2 records a digest of the complete +workload, fixture identity, corpus, and run configuration. Each completed record +is checkpointed by stable backend/dataset/case/workload identity using an atomic +rename; `-resume` accepts only a matching manifest, corpus, and run identity and +preserves the original run UUID. + +Legacy graphs without `logical_key` properties may instead use a runtime-only +physical anchor with a content proof: + +```json +{"physical_id": 42, "content_sha256": "sha256:<64 lowercase hex characters>", "kind": "Entity"} +``` + +The digest is SHA-256 over PostgreSQL's canonical `kind_ids::text`, a newline, +and `properties::text` for that node. The runner accepts the ID only after the +digest and optional kind match, then removes the ID and manifest values from +durable records. Each anchor must use exactly one of `logical_key` or +`physical_id`; a physical anchor always requires its content digest. +For exact path observations on a legacy graph, include content-proved anchors +for intermediate nodes as well as parameter endpoints so stable path identity +can be reconstructed without persisting physical IDs. + +Existing-graph runs require the target database to have the DAWGS schema and +workspace functions from the current checkout already deployed. The runner +does not assert or upgrade schema in this mode because doing so would violate +its non-mutating existing-graph contract. + +Adaptive discovery is explicit: + +```bash +go run ./cmd/graphbench \ + -existing-graph -anchor-manifest anchors.json \ + -discovery -timeout-classes 100ms,1s,10s \ + -discovery-sample-floor 1 \ + -checkpoint .coverage/live/checkpoint.json +``` + +Every timeout and sample reduction stays in the case record. Adaptive artifacts +are refused by the complete performance gate. Confirmation omits `-discovery` +and uses fixed timeouts, arm order, warmups, and samples. + +The independent state/resource report is produced with +`-resource-artifact results.jsonl -resource-output resources.json`. Schema v5 +records the SHA-256 digest of the exact input JSONL so +promotion evidence can verify that resource decisions remain bound to their +capture. For non-stress portable PostgreSQL candidates it rejects temp spill, +local workspace, and WAL for non-mutating reads. S4 and ASP explicitly permit their +session-local compact workspace but still reject executor temp-file spill and +WAL; exact incumbent fallback retains its documented temporary-workspace +contract. `SP-S0-DIRECT` records are +attributed from the measured fallback function loops, so workspace use is +accepted only when the incumbent branch actually ran. Exact full-comparator +reference arms receive independent resource cases rather than inheriting the +outer production result. + +Shortest tournament references are independently selectable with +`-postgres-reference-arms s4_canonical_source_distance`, +`s4_canonical_source_witness_m0`, `sp_b1_strict_alternating_distance`, +`sp_b1_strict_alternating_witness_m0`, +`sp_b2_smaller_frontier_distance`, +`sp_b2_smaller_frontier_witness_m0`, `asp_a1_stored_helper_m0`, +`asp_i1_inline_predecessor_dag_m0`, +`asp_b1_bidirectional_dag_strict_m0`, and +`asp_b2_bidirectional_dag_smaller_frontier_m0`. They are exact full-query comparators at the same +public observation boundary, not production selectors. S4 canonicalizes inbound +search to physical `start_id -> end_id`; B1 alternates one accepted node per +side, while B2 expands the smaller complete current level with a deterministic +forward tie-break. Both candidates retain ID-only state, reconstruct one stable +witness late, and fall back to exact S4 before output if a seen, frontier, or +predecessor cap overflows. Their multi-statement functions reject Read Committed; +GraphBench runs any selected B1/B2 production or reference arm at Repeatable +Read so candidate search and fallback share one transaction snapshot. The ASP +arms retain every relationship-distinct shortest-depth predecessor, select one +canonical completed meeting cut, and separately cap discovery state, frontier, +predecessors, saturating path count, enumerated rows, and output bytes before +exact A1 fallback. SP and ASP identities are forceable with +`-postgres-force-shortest-executor`; automatic selection remains on S3/S4 for +SP and A1 for ASP. Activation evidence still requires the saved +plan/resource, holdout, concurrency, cancellation, and reference-closure gates. + +`-backend-delta-artifact combined.jsonl -backend-delta-output deltas.json` +produces matched PostgreSQL/Neo4j median and p95 ratios only when both records +exist, reports logical-observation agreement, and emits a descending `outliers` +list for repeated successful stable-observation regressions. Each outlier +retains round count, runtime/applied identities, branch and fallback metadata, +SQL fingerprints, direction, state class, observation mode, and selector +versions. The report is explicitly descriptive and never participates in +PostgreSQL pass/fail selection. +Every other shape retains `SP-S0` and its specific fallback code. + +Ordinary variable expansions with fixed continuations similarly emit a typed +`ExpansionSearchStrategyDecision`. It records suffix bounds, logical direction, +observation mode, depth bounds, structural facts, selection mode, and stable +fallback codes. It also reports the fixed-suffix expansion family, planned +candidate set, selector version, and distinct correlated-suffix/cross-region +fallback reasons. +Factored-suffix and backward-viability SQL remains reference-only. +`EXPANSION-SUFFIX-SEEDED-REVERSE` has a repository-native emitter for +qualification, but it is not selected by the public query API. Structurally +eligible forms select `EXPANSION-STEPWISE-FORWARD` with +`tournament_unqualified`. + ## Outputs JSONL output contains one `CaseResult` record per case and execution mode. @@ -69,6 +1765,58 @@ Markdown and JSON summaries aggregate mode status counts, per-case timings, row counts, fallback reasons, and baseline regressions or improvements when a baseline capture is supplied. -PostgreSQL records include translated SQL and `EXPLAIN (ANALYZE, BUFFERS, -TIMING OFF, FORMAT JSON)` metrics. Neo4j records include plan operator names +PostgreSQL case records also include aggregate query-text-free parse-cache +counters. Optimization diagnostics retain target-specific selected, applied, +and skipped identities; compile-time records do not claim a runtime branch. + +Each timing record retains the unsorted cold and warm latency samples with round, +iteration, case, dataset, backend, and connection/session fields so confidence +interval and regression tooling does not have to reconstruct observations from +summary percentiles. Read cases run untimed preflight and postflight queries +and compare their complete row multisets, including duplicate rows, around the +timed block. For declared `id_rows`, `path_set`, and scalar results, recorded +`observed_rows` use stable fixture identities, retain relationship order, +kinds, and properties, and reject relationship reuse within a path. GraphBench +compares those stable result kinds across backends. Other result kinds still +receive per-backend preflight/postflight checks, but are not compared across +backends because they may contain backend-generated relationship IDs. +PostgreSQL fixture loads are followed by `VACUUM (ANALYZE)` through +the pool; a maintenance failure aborts the benchmark. +The PostgreSQL runner defaults to a one-connection pool and records +`pg_backend_pid()` as the serial sample connection identifier. Concurrency +blocks record the physical PID of every direct pool acquisition so per-session +cold state and pool queuing remain visible. + +Write records additionally report matched and affected counts and each +post-state observation. The recorded duration covers the mutation query; setup, +verification, and rollback are outside that duration. + +PostgreSQL records include translated SQL and its fingerprint, server settings, +fixture checksum/cardinalities, and `EXPLAIN (ANALYZE, BUFFERS, TIMING OFF)` +shared/local/temp metrics plus `EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, +FORMAT JSON)` for reads. Neo4j records include plan operator names when an `EXPLAIN` plan can be captured. + +## PostgreSQL scale-plan correctness gate + +The PostgreSQL-only `TestPostgreSQLScalePlanInvariants` test loads the same +scale corpus and fixture as the command. It executes all required Cypher scale +representatives, requires their declared cardinalities and mutation post-state, +and verifies that the captured plan came from `EXPLAIN ANALYZE`. Stable +assertions cover relationship/node mutation targets, branch-local logical +structure, temporal filtering, and anchored edge-index orientation. The test +uses rollback isolation for writes and runs automatically under +`make test_all` when `CONNECTION_STRING` selects PostgreSQL. + +Run only the scale-plan gate with: + +```bash +CONNECTION_STRING="$PG_CONNECTION_STRING" \ + go test -tags manual_integration ./cmd/graphbench \ + -run 'Test(PostgreSQLScalePlanInvariants|ScaleCorpusRequiredRepresentativesDeclareCardinality)' \ + -count=1 +``` + +The non-integration cardinality test also guarantees that every required stable +query-form ID remains represented in the scale corpus and declares an expected +read or write cardinality. diff --git a/cmd/graphbench/aa_report.go b/cmd/graphbench/aa_report.go new file mode 100644 index 00000000..657a8770 --- /dev/null +++ b/cmd/graphbench/aa_report.go @@ -0,0 +1,601 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + // aaReportVersion identifies the serialized schema revision for A/A report. + aaReportVersion = 4 + // aaPhysicalChronologyVersion identifies process-interval chronology proof. + aaPhysicalChronologyVersion = 1 +) + +// AAMetricResolution captures relative and absolute within-arm noise for one latency quantile. +type AAMetricResolution struct { + // Ratio reports the candidate-to-baseline latency ratio. + Ratio RatioInterval `json:"ratio"` + // RatioResolution supplies the ratio resolution input to the AAMetricResolution contract. + RatioResolution float64 `json:"ratio_resolution"` + // AbsoluteChange reports the paired candidate-minus-baseline A/A duration interval. + AbsoluteChange DurationInterval `json:"absolute_change"` + // AbsoluteResolution supplies the absolute resolution input to the AAMetricResolution contract. + AbsoluteResolution time.Duration `json:"absolute_resolution"` +} + +// AAResolutionCase reports matched sample counts and median and P95 noise floors for one case. +type AAResolutionCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // WorkloadSHA256 binds the resolution to the exact logical workload declaration. + WorkloadSHA256 string `json:"workload_sha256"` + // PostgresEnvironmentSHA256 binds PostgreSQL A/A noise to the exact timing + // environment, including transaction isolation and normalized analyze state. + PostgresEnvironmentSHA256 string `json:"postgres_environment_sha256,omitempty"` + // FixtureSHA256 binds PostgreSQL A/A noise to the exact validated fixture. + FixtureSHA256 string `json:"fixture_sha256,omitempty"` + // Rounds records the number of rounds. + Rounds int `json:"rounds"` + // SamplesPerArm records matched timing samples available from each A/A arm. + SamplesPerArm int `json:"samples_per_arm"` + // P50 records relative and absolute A/A noise at median latency. + P50 AAMetricResolution `json:"p50"` + // P95 records relative and absolute A/A noise at 95th-percentile latency. + P95 AAMetricResolution `json:"p95"` + // P99Gated reports whether the sample count is sufficient to enforce the P99 noise threshold. + P99Gated bool `json:"p99_gated"` + // P99Reason explains why P99 gating was applied or omitted. + P99Reason string `json:"p99_reason,omitempty"` +} + +// AAPhysicalChronology records that the report builder validated the physical +// process intervals in the immutable A/A source artifacts, rather than only +// accepting their round and arm-order labels. +type AAPhysicalChronology struct { + // Version identifies the chronology-validation contract. + Version int `json:"version"` + // Validated reports that all arm and round intervals passed validation. + Validated bool `json:"validated"` + // ArtifactSHA256 binds this proof to the same immutable artifact set as the + // statistical report. + ArtifactSHA256 string `json:"artifact_sha256"` + // Rounds records the contiguous physically validated round count. + Rounds int `json:"rounds"` + // Arms records the two explicit arm identities in stable order. + Arms []string `json:"arms"` +} + +// AAResolutionReport contains per-case A/A noise floors and the artifact identity used to derive them. +type AAResolutionReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // ArtifactSHA256 identifies the exact input artifact summarized by the report. + ArtifactSHA256 string `json:"artifact_sha256"` + // HostFingerprint identifies the host whose timing noise this report measures. + HostFingerprint string `json:"host_fingerprint"` + // MinimumRounds records the number of minimum rounds. + MinimumRounds int `json:"minimum_rounds"` + // MinimumSamplesPerArmPerRound supplies the minimum samples per arm per round input to the AAResolutionReport contract. + MinimumSamplesPerArmPerRound int `json:"minimum_samples_per_arm_per_round"` + // OrderBalanced reports that the two explicitly executed A/A arms have complementary balanced first position. + OrderBalanced bool `json:"order_balanced"` + // PhysicalChronology proves that source process timestamps follow those + // balanced labels without arm or round overlap. + PhysicalChronology *AAPhysicalChronology `json:"physical_chronology"` + // MinimumP99SamplesPerArm sets the per-arm sample floor required before P99 gating. + MinimumP99SamplesPerArm int `json:"minimum_p99_samples_per_arm"` + // Cases contains per-workload A/A noise estimates and resolution thresholds. + Cases []AAResolutionCase `json:"cases"` +} + +// buildAAResolutionReport splits matched A/A samples and estimates per-case median and P95 noise floors. +func buildAAResolutionReport(records []CaseResult, options PerfGateOptions) (AAResolutionReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return AAResolutionReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return AAResolutionReport{}, fmt.Errorf("bootstrap count must be positive") + } + hostFingerprint, err := artifactHostFingerprint(records) + if err != nil { + return AAResolutionReport{}, err + } + + all, err := collectExplicitAASeries(records) + if err != nil { + return AAResolutionReport{}, err + } + physicalChronology, err := validateAAPhysicalChronology(records) + if err != nil { + return AAResolutionReport{}, err + } + keys := make([]performanceKey, 0, len(all)) + for key := range all { + if key.backend == ModePostgresSQL { + keys = append(keys, key) + } + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + if len(keys) == 0 { + return AAResolutionReport{}, fmt.Errorf("artifact has no successful PostgreSQL warm samples") + } + + report := AAResolutionReport{ + Version: aaReportVersion, + Seed: options.Seed, + Confidence: options.Confidence, + HostFingerprint: hostFingerprint, + MinimumRounds: minimumGateRounds, + MinimumSamplesPerArmPerRound: 10, + OrderBalanced: true, + PhysicalChronology: physicalChronology, + MinimumP99SamplesPerArm: 10_000, + } + for idx, key := range keys { + var ( + armA, armB = all[key][0], all[key][1] + seed = options.Seed + int64(idx)*7919 + ) + + armA, armB = matchedRounds(armA, armB) + if len(armA) < minimumGateRounds { + return AAResolutionReport{}, fmt.Errorf("%s/%s requires at least %d A/A rounds, got %d", key.dataset, key.name, minimumGateRounds, len(armA)) + } + for _, round := range sortedRounds(armA) { + if len(armA[round]) < report.MinimumSamplesPerArmPerRound || len(armB[round]) < report.MinimumSamplesPerArmPerRound { + return AAResolutionReport{}, fmt.Errorf("%s/%s round %d requires at least %d samples per A/A arm, got %d/%d", key.dataset, key.name, round, report.MinimumSamplesPerArmPerRound, len(armA[round]), len(armB[round])) + } + } + + var ( + p50 = bootstrapRoundMedianRatio(armA, armB, seed, options) + p95 = bootstrapStratifiedP95Ratio(armA, armB, seed+1, options) + p50Change = negateDurationInterval(bootstrapRoundMedianSaving(armA, armB, seed+2, options)) + p95Change = bootstrapStratifiedQuantileChange(armA, armB, 0.95, seed+3, options) + armSamples = min(sampleCount(armA), sampleCount(armB)) + ) + + workloadSHA256, err := workloadSHA256ForKey(records, key) + if err != nil { + return AAResolutionReport{}, err + } + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(records, key) + if err != nil { + return AAResolutionReport{}, err + } + fixtureSHA256, err := fixtureSHA256ForKey(records, key) + if err != nil { + return AAResolutionReport{}, err + } + entry := AAResolutionCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + WorkloadSHA256: workloadSHA256, + PostgresEnvironmentSHA256: postgresEnvironmentSHA256, + FixtureSHA256: fixtureSHA256, + Rounds: len(armA), + SamplesPerArm: armSamples, + P50: aaMetricResolution(p50, p50Change), + P95: aaMetricResolution(p95, p95Change), + P99Gated: armSamples >= 10_000, + } + if !entry.P99Gated { + entry.P99Reason = fmt.Sprintf("diagnostic only: need at least 10000 samples per A/A arm, got %d", armSamples) + } + + report.Cases = append(report.Cases, entry) + } + + return report, nil +} + +// aaPhysicalInvocationIdentity binds one A/A arm to the GraphBench process +// interval that executed the complete selected cohort for one round. +type aaPhysicalInvocationIdentity struct { + round int + block int + order int + arm string + runUUID string + startedAt time.Time + endedAt time.Time +} + +// validateAAPhysicalChronology verifies the process intervals behind an A/A +// report. Every arm/round must execute the same exact cohort in one process; +// declared arm order and round order must agree with non-overlapping timestamps. +func validateAAPhysicalChronology(records []CaseResult) (*AAPhysicalChronology, error) { + invocations := map[string]map[int]aaPhysicalInvocationIdentity{} + cases := map[string]map[int]map[performanceKey]struct{}{} + expectedKeys := map[performanceKey]struct{}{} + + for _, record := range records { + if record.Status != StatusOK || record.ExecutionMode != ModePostgresSQL || !hasWarmLatencySample(record) { + continue + } + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s A/A record lacks physical invocation chronology", record.Dataset, record.Name) + } + environment := record.Environment + identity := aaPhysicalInvocationIdentity{ + round: environment.Round, + block: environment.Block, + order: environment.ArmOrder, + arm: environment.Arm, + runUUID: environment.RunUUID, + startedAt: environment.StartedAt, + endedAt: environment.EndedAt, + } + if identity.round < 1 || identity.block != identity.round { + return nil, fmt.Errorf("%s/%s A/A arm requires block equal to round", record.Dataset, record.Name) + } + if identity.arm == "" || identity.arm == "unlabeled" || identity.order < 1 || identity.order > 2 || strings.TrimSpace(identity.runUUID) == "" { + return nil, fmt.Errorf("%s/%s A/A record has malformed physical arm metadata", record.Dataset, record.Name) + } + if identity.startedAt.IsZero() || identity.endedAt.IsZero() || identity.endedAt.Before(identity.startedAt) { + return nil, fmt.Errorf("%s/%s A/A arm %q round %d has malformed invocation timestamps", record.Dataset, record.Name, identity.arm, identity.round) + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.Round != identity.round || sample.Block != identity.block || sample.Arm != identity.arm || + sample.ArmOrder != identity.order || sample.RunUUID != identity.runUUID { + return nil, fmt.Errorf("%s/%s A/A warm sample is outside its physical arm invocation", record.Dataset, record.Name) + } + } + + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + expectedKeys[key] = struct{}{} + if invocations[identity.arm] == nil { + invocations[identity.arm] = map[int]aaPhysicalInvocationIdentity{} + cases[identity.arm] = map[int]map[performanceKey]struct{}{} + } + if prior, found := invocations[identity.arm][identity.round]; found && prior != identity { + return nil, fmt.Errorf("A/A arm %q round %d mixes invocation identities across the selected cohort", identity.arm, identity.round) + } + invocations[identity.arm][identity.round] = identity + if cases[identity.arm][identity.round] == nil { + cases[identity.arm][identity.round] = map[performanceKey]struct{}{} + } + if _, duplicate := cases[identity.arm][identity.round][key]; duplicate { + return nil, fmt.Errorf("A/A arm %q round %d duplicates case %s/%s", identity.arm, identity.round, key.dataset, key.name) + } + cases[identity.arm][identity.round][key] = struct{}{} + } + + if len(invocations) != 2 || len(expectedKeys) == 0 { + return nil, fmt.Errorf("A/A physical chronology requires exactly two explicit arms over a nonempty cohort") + } + armNames := make([]string, 0, 2) + for arm := range invocations { + armNames = append(armNames, arm) + } + sort.Strings(armNames) + rounds := len(invocations[armNames[0]]) + if rounds < minimumGateRounds || len(invocations[armNames[1]]) != rounds { + return nil, fmt.Errorf("A/A physical chronology does not contain one complete matched round schedule") + } + + firstArmFirst := 0 + runUUID := "" + var priorEnded time.Time + for round := 1; round <= rounds; round++ { + left, leftFound := invocations[armNames[0]][round] + right, rightFound := invocations[armNames[1]][round] + if !leftFound || !rightFound { + return nil, fmt.Errorf("A/A physical chronology must use contiguous rounds starting at 1") + } + for _, arm := range armNames { + current := invocations[arm][round] + if current.block != round { + return nil, fmt.Errorf("A/A round %d requires block equal to round", round) + } + if len(cases[arm][round]) != len(expectedKeys) { + return nil, fmt.Errorf("A/A arm %q round %d does not contain the exact selected cohort", arm, round) + } + for key := range expectedKeys { + if _, found := cases[arm][round][key]; !found { + return nil, fmt.Errorf("A/A arm %q round %d does not contain the exact selected cohort", arm, round) + } + } + if runUUID == "" { + runUUID = current.runUUID + } else if runUUID != current.runUUID { + return nil, fmt.Errorf("A/A physical chronology mixes run UUIDs across arms or rounds") + } + } + if !((left.order == 1 && right.order == 2) || (left.order == 2 && right.order == 1)) { + return nil, fmt.Errorf("A/A round %d lacks one complete physical two-arm order", round) + } + first, second := left, right + if right.order == 1 { + first, second = right, left + } else { + firstArmFirst++ + } + if first.endedAt.After(second.startedAt) { + return nil, fmt.Errorf("A/A round %d arm timestamps contradict the declared execution order", round) + } + if !priorEnded.IsZero() && priorEnded.After(first.startedAt) { + return nil, fmt.Errorf("A/A round %d overlaps or predates the prior round", round) + } + priorEnded = second.endedAt + } + if secondArmFirst := rounds - firstArmFirst; firstArmFirst-secondArmFirst > 1 || secondArmFirst-firstArmFirst > 1 { + return nil, fmt.Errorf("A/A physical arm order is not balanced: %d/%d", firstArmFirst, secondArmFirst) + } + return &AAPhysicalChronology{ + Version: aaPhysicalChronologyVersion, + Validated: true, + Rounds: rounds, + Arms: armNames, + }, nil +} + +// collectExplicitAASeries requires two independently executed arms with +// identical SQL and balanced block order. Splitting one timing stream into +// synthetic labels understates reload, connection, and first-order carryover +// noise and is therefore deliberately refused by the promotion-grade report. +func collectExplicitAASeries(records []CaseResult) (map[performanceKey][2]roundSamples, error) { + // armIdentity binds one A/A arm to its exact SQL and workload content. + type armIdentity struct { + // SQLFingerprint supplies the sql fingerprint input to the armIdentity contract. + SQLFingerprint string + // WorkloadSHA256 binds the referenced workload content by SHA-256 digest. + WorkloadSHA256 string + } + + // armSeries accumulates balanced samples and scheduling identity for one arm. + type armSeries struct { + // identity retains the identity while armSeries is assembled or evaluated. + identity armIdentity + // samples retains the samples while armSeries is assembled or evaluated. + samples roundSamples + // orders retains the orders while armSeries is assembled or evaluated. + orders map[int]int + // blocks retains the blocks while armSeries is assembled or evaluated. + blocks map[int]int + // runUUIDs retains the run uui ds while armSeries is assembled or evaluated. + runUUIDs map[int]string + } + + byKey := map[performanceKey]map[string]*armSeries{} + for _, record := range records { + if record.Status != StatusOK || record.ExecutionMode != ModePostgresSQL { + continue + } + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.Round < 1 || sample.Block < 1 || sample.Arm == "" || sample.Arm == "unlabeled" || sample.ArmOrder < 1 || sample.RunUUID == "" { + return nil, fmt.Errorf("%s/%s has A/A sample without explicit round, block, arm, order, and run UUID", key.dataset, key.name) + } + arms := byKey[key] + if arms == nil { + arms = map[string]*armSeries{} + byKey[key] = arms + } + arm := arms[sample.Arm] + if arm == nil { + arm = &armSeries{ + identity: armIdentity{ + SQLFingerprint: record.SQLFingerprint, + WorkloadSHA256: record.WorkloadSHA256, + }, + samples: roundSamples{}, + orders: map[int]int{}, + blocks: map[int]int{}, + runUUIDs: map[int]string{}, + } + arms[sample.Arm] = arm + } + identity := armIdentity{ + SQLFingerprint: record.SQLFingerprint, + WorkloadSHA256: record.WorkloadSHA256, + } + if arm.identity != identity || identity.SQLFingerprint == "" || identity.WorkloadSHA256 == "" { + return nil, fmt.Errorf("%s/%s arm %q changes or lacks executable/workload identity", key.dataset, key.name, sample.Arm) + } + if prior, found := arm.orders[sample.Round]; found && prior != sample.ArmOrder { + return nil, fmt.Errorf("%s/%s arm %q round %d changes order", key.dataset, key.name, sample.Arm, sample.Round) + } + if prior, found := arm.blocks[sample.Round]; found && prior != sample.Block { + return nil, fmt.Errorf("%s/%s arm %q round %d changes block", key.dataset, key.name, sample.Arm, sample.Round) + } + if prior, found := arm.runUUIDs[sample.Round]; found && prior != sample.RunUUID { + return nil, fmt.Errorf("%s/%s arm %q round %d changes run UUID", key.dataset, key.name, sample.Arm, sample.Round) + } + arm.orders[sample.Round] = sample.ArmOrder + arm.blocks[sample.Round] = sample.Block + arm.runUUIDs[sample.Round] = sample.RunUUID + arm.samples[sample.Round] = append(arm.samples[sample.Round], sample.Duration) + } + } + + result := map[performanceKey][2]roundSamples{} + for key, arms := range byKey { + if len(arms) != 2 { + return nil, fmt.Errorf("%s/%s requires exactly two explicit A/A arms, got %d", key.dataset, key.name, len(arms)) + } + names := make([]string, 0, 2) + for name := range arms { + names = append(names, name) + } + sort.Strings(names) + left, right := arms[names[0]], arms[names[1]] + if left.identity != right.identity { + return nil, fmt.Errorf("%s/%s A/A arms do not have identical SQL and workload identities", key.dataset, key.name) + } + leftSamples, rightSamples := matchedRounds(left.samples, right.samples) + leftFirst := 0 + for _, round := range sortedRounds(leftSamples) { + if left.blocks[round] != right.blocks[round] || left.runUUIDs[round] != right.runUUIDs[round] { + return nil, fmt.Errorf("%s/%s round %d has mismatched A/A block or run identity", key.dataset, key.name, round) + } + if !((left.orders[round] == 1 && right.orders[round] == 2) || (left.orders[round] == 2 && right.orders[round] == 1)) { + return nil, fmt.Errorf("%s/%s round %d lacks a complete two-arm A/A order", key.dataset, key.name, round) + } + if left.orders[round] == 1 { + leftFirst++ + } + } + if rightFirst := len(leftSamples) - leftFirst; leftFirst-rightFirst > 1 || rightFirst-leftFirst > 1 { + return nil, fmt.Errorf("%s/%s A/A order is not balanced: %d/%d", key.dataset, key.name, leftFirst, rightFirst) + } + result[key] = [2]roundSamples{leftSamples, rightSamples} + } + return result, nil +} + +// aaMetricResolution returns the larger relative and absolute confidence-bound deviations observed between paired A/A samples. +func aaMetricResolution(interval RatioInterval, absoluteChange DurationInterval) AAMetricResolution { + resolution := math.Max(math.Abs(1-interval.Lower), math.Abs(interval.Upper-1)) + return AAMetricResolution{ + Ratio: interval, + RatioResolution: resolution, + AbsoluteChange: absoluteChange, + AbsoluteResolution: max(absDuration(absoluteChange.Lower), absDuration(absoluteChange.Upper)), + } +} + +// writeAAResolutionReport writes an A/A resolution report as indented JSON. +func writeAAResolutionReport(path string, report AAResolutionReport) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +// createAAResolutionReport loads one or more immutable arm artifacts, builds +// their joint A/A resolution report, and writes the result. +func createAAResolutionReport(artifactPaths []string, outputPath string, options PerfGateOptions) error { + records, artifactSHA256, err := loadAAResolutionArtifacts(artifactPaths) + if err != nil { + return err + } + report, err := buildAAResolutionReport(records, options) + if err != nil { + return err + } + report.ArtifactSHA256 = artifactSHA256 + report.PhysicalChronology.ArtifactSHA256 = artifactSHA256 + return writeAAResolutionReport(outputPath, report) +} + +// loadAAResolutionArtifacts combines separately captured A/A arms without +// weakening appendJSONLFile's one-arm run-series identity. A single input keeps +// the historical raw-file checksum. Multiple inputs use a domain-separated, +// order-independent digest of their exact file checksums. +func loadAAResolutionArtifacts(paths []string) ([]CaseResult, string, error) { + if len(paths) == 0 { + return nil, "", fmt.Errorf("at least one A/A artifact is required") + } + + var ( + records []CaseResult + digests = make([]string, 0, len(paths)) + seen = make(map[string]struct{}, len(paths)) + ) + for _, path := range paths { + path = strings.TrimSpace(path) + if path == "" { + return nil, "", fmt.Errorf("A/A artifact path must not be empty") + } + cleaned := filepath.Clean(path) + if _, duplicate := seen[cleaned]; duplicate { + return nil, "", fmt.Errorf("duplicate A/A artifact %q", path) + } + seen[cleaned] = struct{}{} + + current, err := readJSONLFile(path) + if err != nil { + return nil, "", fmt.Errorf("read A/A artifact %q: %w", path, err) + } + digest, err := fileSHA256(path) + if err != nil { + return nil, "", fmt.Errorf("checksum A/A artifact %q: %w", path, err) + } + records = append(records, current...) + digests = append(digests, digest) + } + if len(digests) == 1 { + return records, digests[0], nil + } + + sort.Strings(digests) + hasher := sha256.New() + _, _ = hasher.Write([]byte("graphbench-aa-artifact-set-v1\n")) + for _, digest := range digests { + _, _ = hasher.Write([]byte(digest)) + _, _ = hasher.Write([]byte{'\n'}) + } + return records, hex.EncodeToString(hasher.Sum(nil)), nil +} + +// loadAAResolutionReport decodes a host A/A report and returns the report file's checksum. +func loadAAResolutionReport(path string) (*AAResolutionReport, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + report := &AAResolutionReport{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, "", fmt.Errorf("decode A/A report: %w", err) + } + digest := sha256.Sum256(raw) + return report, hex.EncodeToString(digest[:]), nil +} diff --git a/cmd/graphbench/aa_report_test.go b/cmd/graphbench/aa_report_test.go new file mode 100644 index 00000000..5d8db4b3 --- /dev/null +++ b/cmd/graphbench/aa_report_test.go @@ -0,0 +1,194 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildAAResolutionReportUsesExplicitMatchedArmsAndKeepsP99Diagnostic verifies that independently executed balanced arms produce a promotion-grade noise floor while P99 remains explicitly non-gating. +func TestBuildAAResolutionReportUsesExplicitMatchedArmsAndKeepsP99Diagnostic(t *testing.T) { + records := explicitAARecords(t, 5, 20) + report, err := buildAAResolutionReport(records, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 100, + }) + + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.Equal(t, aaReportVersion, report.Version) + require.True(t, validSHA256(report.HostFingerprint)) + require.True(t, report.OrderBalanced) + require.NotNil(t, report.PhysicalChronology) + require.True(t, report.PhysicalChronology.Validated) + require.Equal(t, 5, report.PhysicalChronology.Rounds) + require.Equal(t, 100, report.Cases[0].SamplesPerArm) + require.InDelta(t, 1, report.Cases[0].P50.Ratio.Estimate, 0.0001) + require.False(t, report.Cases[0].P99Gated) + require.Contains(t, report.Cases[0].P99Reason, "diagnostic only") +} + +// TestBuildAAResolutionReportRejectsPhysicalChronologyTampering verifies an +// alternating label schedule cannot hide fixed-order, overlapping, or detached +// process execution in the immutable A/A source artifacts. +func TestBuildAAResolutionReportRejectsPhysicalChronologyTampering(t *testing.T) { + tests := []struct { + name string + mutate func([]CaseResult) + problem string + }{ + { + name: "fixed physical order behind alternating labels", + mutate: func(records []CaseResult) { + for index := range records { + environment := records[index].Environment + roundStarted := time.Unix(1_700_000_000+int64(environment.Round)*10, 0).UTC() + if environment.Arm == "aa-b" { + roundStarted = roundStarted.Add(2 * time.Second) + } + environment.StartedAt = roundStarted + environment.EndedAt = roundStarted.Add(time.Second) + } + }, + problem: "arm timestamps contradict the declared execution order", + }, + { + name: "block differs from round", + mutate: func(records []CaseResult) { + for recordIndex := range records { + if records[recordIndex].Environment.Round != 2 { + continue + } + records[recordIndex].Environment.Block = 1 + for sampleIndex := range records[recordIndex].Stats.Samples { + records[recordIndex].Stats.Samples[sampleIndex].Block = 1 + } + } + }, + problem: "requires block equal to round", + }, + { + name: "sample labels detached from process", + mutate: func(records []CaseResult) { + records[0].Environment.ArmOrder = 2 + }, + problem: "outside its physical arm invocation", + }, + { + name: "round overlaps prior round", + mutate: func(records []CaseResult) { + priorEnded := time.Time{} + for index := range records { + if records[index].Environment.Round == 1 && records[index].Environment.ArmOrder == 2 { + priorEnded = records[index].Environment.EndedAt + } + } + firstStarted := priorEnded.Add(-500 * time.Millisecond) + for index := range records { + environment := records[index].Environment + if environment.Round != 2 { + continue + } + environment.StartedAt = firstStarted.Add(time.Duration(environment.ArmOrder-1) * 2 * time.Second) + environment.EndedAt = environment.StartedAt.Add(time.Second) + } + }, + problem: "overlaps or predates the prior round", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + records := explicitAARecords(t, 5, 20) + test.mutate(records) + _, err := buildAAResolutionReport(records, PerfGateOptions{ + Seed: 1, Confidence: 0.95, BootstrapCount: 10, + }) + require.ErrorContains(t, err, test.problem) + }) + } +} + +// TestBuildAAResolutionReportRejectsSyntheticSingleStream verifies unlabeled samples cannot be relabeled after timing to manufacture A/A evidence. +func TestBuildAAResolutionReportRejectsSyntheticSingleStream(t *testing.T) { + record := perfGateRecord("case", ModePostgresSQL, time.Millisecond, 5, 40) + _, err := buildAAResolutionReport([]CaseResult{record}, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 100, + }) + require.ErrorContains(t, err, "without explicit round, block, arm, order, and run UUID") +} + +// TestCreateAAResolutionReportAcceptsSeparateArmArtifacts verifies the native +// multi-input path combines two immutable append-series arms and binds both +// exact files into one report checksum. +func TestCreateAAResolutionReportAcceptsSeparateArmArtifacts(t *testing.T) { + paths := []string{filepath.Join(t.TempDir(), "aa-a.jsonl"), filepath.Join(t.TempDir(), "aa-b.jsonl")} + records := explicitAARecords(t, 5, 10) + var left, right []CaseResult + for _, record := range records { + if record.Stats.Samples[0].Arm == "aa-a" { + left = append(left, record) + } else { + right = append(right, record) + } + } + require.NoError(t, writeJSONLFile(paths[0], left)) + require.NoError(t, writeJSONLFile(paths[1], right)) + + output := filepath.Join(t.TempDir(), "aa.json") + require.NoError(t, createAAResolutionReport(paths, output, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 100, + })) + + report, _, err := loadAAResolutionReport(output) + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.True(t, validSHA256(report.ArtifactSHA256)) + require.NotNil(t, report.PhysicalChronology) + require.Equal(t, report.ArtifactSHA256, report.PhysicalChronology.ArtifactSHA256) + leftDigest, err := fileSHA256(paths[0]) + require.NoError(t, err) + require.NotEqual(t, leftDigest, report.ArtifactSHA256) +} + +// explicitAARecords prepares or inspects test evidence for explicit aa records. +func explicitAARecords(t *testing.T, rounds, samples int) []CaseResult { + t.Helper() + var records []CaseResult + for round := 1; round <= rounds; round++ { + for armIndex, arm := range []string{"aa-a", "aa-b"} { + record := perfGateRecord("case", ModePostgresSQL, time.Millisecond, 1, samples) + record.SQLFingerprint = "identical-sql" + record.WorkloadSHA256 = "identical-workload" + armOrder := 1 + (armIndex+round-1)%2 + roundStarted := time.Unix(1_700_000_000+int64(round)*10, 0).UTC() + record.Environment.Round = round + record.Environment.Block = round + record.Environment.Arm = arm + record.Environment.ArmOrder = armOrder + record.Environment.RunUUID = "aa-run" + record.Environment.StartedAt = roundStarted.Add(time.Duration(armOrder-1) * 2 * time.Second) + record.Environment.EndedAt = record.Environment.StartedAt.Add(time.Second) + for idx := range record.Stats.Samples { + record.Stats.Samples[idx].Round = round + record.Stats.Samples[idx].Block = round + record.Stats.Samples[idx].Arm = arm + record.Stats.Samples[idx].ArmOrder = armOrder + record.Stats.Samples[idx].RunUUID = "aa-run" + } + records = append(records, record) + } + } + return records +} diff --git a/cmd/graphbench/backend_delta.go b/cmd/graphbench/backend_delta.go new file mode 100644 index 00000000..4cad3e7f --- /dev/null +++ b/cmd/graphbench/backend_delta.go @@ -0,0 +1,327 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" +) + +// backendDeltaKey identifies one dataset, case, and round during backend +// matching and repeated-round aggregation. +type backendDeltaKey struct { + dataset string + name string + round int +} + +// BackendDeltaReport contains descriptive PostgreSQL-to-Neo4j correctness and latency deltas for matched records. +type BackendDeltaReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Notice states that backend deltas are descriptive and not release-gate evidence. + Notice string `json:"notice"` + // Cases contains matched PostgreSQL-to-Neo4j comparisons in deterministic report order. + Cases []BackendDeltaCase `json:"cases"` + // Outliers aggregates complete repeated-round PostgreSQL losses in descending + // PostgreSQL-to-Neo4j latency-ratio order. It is a diagnostic work ledger, + // not a release gate. + Outliers []BackendDeltaOutlier `json:"outliers,omitempty"` +} + +// BackendDeltaOutlier aggregates one matched workload across repeated rounds +// and preserves the runtime facts needed to route optimization work. +type BackendDeltaOutlier struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the workload case. + Name string `json:"name"` + // Category identifies the workload family declared by the corpus. + Category string `json:"category,omitempty"` + // Rounds counts complete, successful, observation-matching backend pairs. + Rounds int `json:"rounds"` + // MedianPostgresOverNeo4j is the median of matched per-round latency ratios. + MedianPostgresOverNeo4j float64 `json:"median_postgres_over_neo4j"` + // P95PostgresOverNeo4j is the median of matched per-round P95 ratios. + P95PostgresOverNeo4j float64 `json:"p95_postgres_over_neo4j,omitempty"` + // PostgresMedian is the median repeated-round PostgreSQL p50. + PostgresMedian time.Duration `json:"postgres_median"` + // Neo4jMedian is the median repeated-round Neo4j p50. + Neo4jMedian time.Duration `json:"neo4j_median"` + // RuntimeIdentities lists every observed PostgreSQL runtime identity. + RuntimeIdentities []string `json:"runtime_identities,omitempty"` + // AppliedIdentities lists every observed PostgreSQL applied identity. + AppliedIdentities []string `json:"applied_identities,omitempty"` + // RuntimeBranches lists every observed PostgreSQL runtime branch. + RuntimeBranches []string `json:"runtime_branches,omitempty"` + // FallbackReasons lists every translation fallback reason. + FallbackReasons []string `json:"fallback_reasons,omitempty"` + // SQLFingerprints lists the SQL identities observed across repeated rounds. + SQLFingerprints []string `json:"sql_fingerprints,omitempty"` + // Direction records the declared traversal direction when available. + Direction string `json:"direction,omitempty"` + // ExpectedStateClass records the diagnostic topology classification. It is + // never suitable as a production selector input by itself. + ExpectedStateClass string `json:"expected_state_class,omitempty"` + // ObservationMode records the PostgreSQL executor observation boundary. + ObservationMode string `json:"observation_mode,omitempty"` + // SelectorVersion records the PostgreSQL runtime selector version. + SelectorVersions []string `json:"selector_versions,omitempty"` +} + +// BackendDeltaCase compares one matched PostgreSQL and Neo4j case round without assigning release-gate status. +type BackendDeltaCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Round identifies the measurement round. + Round int `json:"round,omitempty"` + // Complete reports whether both backend records were present. + Complete bool `json:"complete"` + // IncompleteReason identifies the absent backend side. + IncompleteReason string `json:"incomplete_reason,omitempty"` + // PostgresStatus supplies the postgres status input to the BackendDeltaCase contract. + PostgresStatus string `json:"postgres_status"` + // Neo4jStatus supplies the neo4j status input to the BackendDeltaCase contract. + Neo4jStatus string `json:"neo4j_status"` + // PostgresMedian records PostgreSQL median latency for the matched round. + PostgresMedian time.Duration `json:"postgres_median,omitempty"` + // PostgresP95 records PostgreSQL P95 latency for the matched round. + PostgresP95 time.Duration `json:"postgres_p95,omitempty"` + // Neo4jMedian records Neo4j median latency for the matched round. + Neo4jMedian time.Duration `json:"neo4j_median,omitempty"` + // Neo4jP95 records Neo4j P95 latency for the matched round. + Neo4jP95 time.Duration `json:"neo4j_p95,omitempty"` + // MedianNeo4jOverPG reports the Neo4j-to-PostgreSQL median latency ratio. + MedianNeo4jOverPG float64 `json:"median_neo4j_over_postgres,omitempty"` + // P95Neo4jOverPG reports the Neo4j-to-PostgreSQL P95 latency ratio. + P95Neo4jOverPG float64 `json:"p95_neo4j_over_postgres,omitempty"` + // ObservationsComparable reports whether both backend records contain stable observations at the same boundary. + ObservationsComparable bool `json:"observations_comparable"` + // ObservationsMatch reports whether comparable backend row counts and normalized observations are equal. + ObservationsMatch bool `json:"observations_match"` +} + +// createBackendDeltaReport matches PostgreSQL and Neo4j records and writes descriptive latency and correctness deltas. +func createBackendDeltaReport(artifact, output string) error { + records, err := readJSONLFile(artifact) + if err != nil { + return err + } + + postgres, neo4j := map[backendDeltaKey]CaseResult{}, map[backendDeltaKey]CaseResult{} + for _, record := range records { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + nextKey := backendDeltaKey{ + dataset: record.Dataset, + name: record.Name, + round: round, + } + + switch record.ExecutionMode { + case ModePostgresSQL: + if _, duplicate := postgres[nextKey]; duplicate { + return fmt.Errorf("backend-delta artifact has duplicate PostgreSQL record for %s/%s round %d", nextKey.dataset, nextKey.name, nextKey.round) + } + postgres[nextKey] = record + case ModeNeo4j: + if _, duplicate := neo4j[nextKey]; duplicate { + return fmt.Errorf("backend-delta artifact has duplicate Neo4j record for %s/%s round %d", nextKey.dataset, nextKey.name, nextKey.round) + } + neo4j[nextKey] = record + } + } + + report := BackendDeltaReport{ + Version: 2, + Notice: "Descriptive only: PostgreSQL release gates compare PostgreSQL predecessors and exact PostgreSQL references, not Neo4j latency.", + } + keys := make(map[backendDeltaKey]struct{}, len(postgres)+len(neo4j)) + for nextKey := range postgres { + keys[nextKey] = struct{}{} + } + for nextKey := range neo4j { + keys[nextKey] = struct{}{} + } + for nextKey := range keys { + pgRecord, pgFound := postgres[nextKey] + neoRecord, neoFound := neo4j[nextKey] + observationsComparable := pgRecord.StableObservation && neoRecord.StableObservation + next := BackendDeltaCase{ + Dataset: nextKey.dataset, + Name: nextKey.name, + Round: nextKey.round, + Complete: pgFound && neoFound, + PostgresStatus: pgRecord.Status, + Neo4jStatus: neoRecord.Status, + PostgresMedian: pgRecord.Stats.Median, + PostgresP95: pgRecord.Stats.P95, + Neo4jMedian: neoRecord.Stats.Median, + Neo4jP95: neoRecord.Stats.P95, + ObservationsComparable: observationsComparable, + ObservationsMatch: observationsComparable && pgRecord.RowCount == neoRecord.RowCount && slices.Equal(pgRecord.ObservedRows, neoRecord.ObservedRows), + } + switch { + case !pgFound: + next.IncompleteReason = "missing_postgres" + case !neoFound: + next.IncompleteReason = "missing_neo4j" + } + + if next.Complete && next.PostgresMedian > 0 && next.Neo4jMedian > 0 { + next.MedianNeo4jOverPG = float64(next.Neo4jMedian) / float64(next.PostgresMedian) + } + if next.Complete && next.PostgresP95 > 0 && next.Neo4jP95 > 0 { + next.P95Neo4jOverPG = float64(next.Neo4jP95) / float64(next.PostgresP95) + } + report.Cases = append(report.Cases, next) + } + report.Outliers = backendDeltaOutliers(postgres, neo4j) + + if len(report.Cases) == 0 { + return fmt.Errorf("backend-delta artifact has no PostgreSQL or Neo4j cases") + } + sort.Slice(report.Cases, func(i, j int) bool { + if report.Cases[i].Dataset != report.Cases[j].Dataset { + return report.Cases[i].Dataset < report.Cases[j].Dataset + } + if report.Cases[i].Name != report.Cases[j].Name { + return report.Cases[i].Name < report.Cases[j].Name + } + return report.Cases[i].Round < report.Cases[j].Round + }) + raw, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if output == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + return err + } + return os.WriteFile(output, append(raw, '\n'), 0o644) +} + +// backendDeltaOutliers aggregates only complete, successful, semantically +// matching backend pairs. Missing and mismatching pairs remain visible in +// BackendDeltaReport.Cases but cannot be ranked as performance work. +func backendDeltaOutliers(postgres, neo4j map[backendDeltaKey]CaseResult) []BackendDeltaOutlier { + type aggregate struct { + outlier BackendDeltaOutlier + medianRatios []float64 + p95Ratios []float64 + postgresTimes []float64 + neo4jTimes []float64 + runtime map[string]struct{} + applied map[string]struct{} + branches map[string]struct{} + fallbacks map[string]struct{} + fingerprints map[string]struct{} + selectors map[string]struct{} + } + + aggregates := map[string]*aggregate{} + for nextKey, pgRecord := range postgres { + neoRecord, found := neo4j[nextKey] + if !found || pgRecord.Status != StatusOK || neoRecord.Status != StatusOK || + !pgRecord.StableObservation || !neoRecord.StableObservation || + pgRecord.RowCount != neoRecord.RowCount || !slices.Equal(pgRecord.ObservedRows, neoRecord.ObservedRows) || + pgRecord.Stats.Median <= 0 || neoRecord.Stats.Median <= 0 { + continue + } + + caseKey := nextKey.dataset + "\x00" + nextKey.name + next := aggregates[caseKey] + if next == nil { + next = &aggregate{ + outlier: BackendDeltaOutlier{ + Dataset: nextKey.dataset, + Name: nextKey.name, + Category: pgRecord.Category, + Direction: pgRecord.Shape.Direction, + ExpectedStateClass: pgRecord.Shape.ExpectedStateClass, + }, + runtime: map[string]struct{}{}, + applied: map[string]struct{}{}, + branches: map[string]struct{}{}, + fallbacks: map[string]struct{}{}, + fingerprints: map[string]struct{}{}, + selectors: map[string]struct{}{}, + } + aggregates[caseKey] = next + } + next.outlier.Rounds++ + next.medianRatios = append(next.medianRatios, float64(pgRecord.Stats.Median)/float64(neoRecord.Stats.Median)) + next.postgresTimes = append(next.postgresTimes, float64(pgRecord.Stats.Median)) + next.neo4jTimes = append(next.neo4jTimes, float64(neoRecord.Stats.Median)) + if pgRecord.Stats.P95 > 0 && neoRecord.Stats.P95 > 0 { + next.p95Ratios = append(next.p95Ratios, float64(pgRecord.Stats.P95)/float64(neoRecord.Stats.P95)) + } + addBackendDeltaValue(next.fallbacks, pgRecord.FallbackReason) + addBackendDeltaValue(next.fingerprints, pgRecord.SQLFingerprint) + if pgRecord.TraversalTelemetry != nil { + summary := pgRecord.TraversalTelemetry.Summary + addBackendDeltaValue(next.runtime, summary.RuntimeIdentity) + addBackendDeltaValue(next.applied, summary.AppliedIdentity) + addBackendDeltaValue(next.branches, summary.RuntimeBranch) + addBackendDeltaValue(next.selectors, summary.SelectorVersion) + if next.outlier.ObservationMode == "" { + next.outlier.ObservationMode = summary.ObservationMode + } + } + } + + outliers := make([]BackendDeltaOutlier, 0, len(aggregates)) + for _, next := range aggregates { + if next.outlier.Rounds < 2 { + continue + } + next.outlier.MedianPostgresOverNeo4j = quantile(next.medianRatios, 0.5) + if next.outlier.MedianPostgresOverNeo4j <= 1 { + continue + } + next.outlier.P95PostgresOverNeo4j = quantile(next.p95Ratios, 0.5) + next.outlier.PostgresMedian = time.Duration(quantile(next.postgresTimes, 0.5)) + next.outlier.Neo4jMedian = time.Duration(quantile(next.neo4jTimes, 0.5)) + next.outlier.RuntimeIdentities = sortedBackendDeltaValues(next.runtime) + next.outlier.AppliedIdentities = sortedBackendDeltaValues(next.applied) + next.outlier.RuntimeBranches = sortedBackendDeltaValues(next.branches) + next.outlier.FallbackReasons = sortedBackendDeltaValues(next.fallbacks) + next.outlier.SQLFingerprints = sortedBackendDeltaValues(next.fingerprints) + next.outlier.SelectorVersions = sortedBackendDeltaValues(next.selectors) + outliers = append(outliers, next.outlier) + } + sort.Slice(outliers, func(i, j int) bool { + if outliers[i].MedianPostgresOverNeo4j != outliers[j].MedianPostgresOverNeo4j { + return outliers[i].MedianPostgresOverNeo4j > outliers[j].MedianPostgresOverNeo4j + } + if outliers[i].Dataset != outliers[j].Dataset { + return outliers[i].Dataset < outliers[j].Dataset + } + return outliers[i].Name < outliers[j].Name + }) + return outliers +} + +func addBackendDeltaValue(values map[string]struct{}, value string) { + if value != "" { + values[value] = struct{}{} + } +} + +func sortedBackendDeltaValues(values map[string]struct{}) []string { + result := make([]string, 0, len(values)) + for value := range values { + result = append(result, value) + } + sort.Strings(result) + return result +} diff --git a/cmd/graphbench/backend_delta_test.go b/cmd/graphbench/backend_delta_test.go new file mode 100644 index 00000000..e6ac68e1 --- /dev/null +++ b/cmd/graphbench/backend_delta_test.go @@ -0,0 +1,249 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBackendDeltaReportIsDescriptiveAndRequiresMatchedObservations verifies that equal stable rows make backend timings comparable while the report remains explicitly non-gating. +func TestBackendDeltaReportIsDescriptiveAndRequiresMatchedObservations(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"one"}, + Stats: DurationStats{ + Median: time.Millisecond, + P95: 2 * time.Millisecond, + }, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"one"}, + Stats: DurationStats{ + Median: 2 * time.Millisecond, + P95: 3 * time.Millisecond, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 1) + require.True(t, report.Cases[0].ObservationsComparable) + require.True(t, report.Cases[0].ObservationsMatch) + require.Equal(t, 2.0, report.Cases[0].MedianNeo4jOverPG) + require.Contains(t, report.Notice, "Descriptive only") +} + +// TestBackendDeltaReportComparesPersistedObservations verifies that differing canonical row payloads are reported as a semantic mismatch even when row counts agree. +func TestBackendDeltaReportComparesPersistedObservations(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"postgres"}, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"neo4j"}, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 1) + require.False(t, report.Cases[0].ObservationsMatch) +} + +// TestBackendDeltaReportDoesNotTreatAbsentObservationsAsMatching verifies that matching cardinalities cannot establish comparability without persisted stable row observations. +func TestBackendDeltaReportDoesNotTreatAbsentObservationsAsMatching(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + RowCount: 1, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.False(t, report.Cases[0].ObservationsComparable) + require.False(t, report.Cases[0].ObservationsMatch) +} + +// TestBackendDeltaReportPreservesRepeatedRounds verifies that matched backend observations remain separate, ordered report cases for each measurement round. +func TestBackendDeltaReportPreservesRepeatedRounds(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + var records []CaseResult + for round := 1; round <= 2; round++ { + for _, mode := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + records = append(records, CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: mode, + Status: StatusOK, + StableObservation: true, + ObservedRows: []string{"one"}, + RowCount: 1, + Environment: &RunEnvironment{Round: round}, + Stats: DurationStats{Median: time.Duration(round) * time.Millisecond}, + }) + } + } + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 2) + require.Equal(t, 1, report.Cases[0].Round) + require.Equal(t, 2, report.Cases[1].Round) +} + +// TestBackendDeltaReportPreservesIncompletePairs verifies a missing backend +// remains visible instead of disappearing from an intersection-only report. +func TestBackendDeltaReportPreservesIncompletePairs(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + records := []CaseResult{{ + Dataset: "fixture", + Name: "postgres-only", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + }} + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Equal(t, 2, report.Version) + require.Len(t, report.Cases, 1) + require.False(t, report.Cases[0].Complete) + require.Equal(t, "missing_neo4j", report.Cases[0].IncompleteReason) + require.Zero(t, report.Cases[0].MedianNeo4jOverPG) + require.Zero(t, report.Cases[0].P95Neo4jOverPG) +} + +// TestBackendDeltaReportRanksRepeatedRoundOutliers verifies the descriptive +// report turns matched rounds into a runtime-attributed optimization ledger. +func TestBackendDeltaReportRanksRepeatedRoundOutliers(t *testing.T) { + root := t.TempDir() + artifact, output := filepath.Join(root, "records.jsonl"), filepath.Join(root, "delta.json") + var records []CaseResult + for round, postgresMedian := range []time.Duration{8 * time.Millisecond, 12 * time.Millisecond} { + environment := &RunEnvironment{Round: round + 1} + records = append(records, + CaseResult{ + Dataset: "fixture", + Name: "slow", + Category: "shortest_path", + Shape: WorkloadShape{Direction: "inbound", ExpectedStateClass: "hidden_fanin"}, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"one"}, + Environment: environment, + Stats: DurationStats{Median: postgresMedian, P95: postgresMedian + time.Millisecond}, + SQLFingerprint: "sql-fingerprint", + FallbackReason: "tournament_unqualified", + TraversalTelemetry: &TraversalExecutionTelemetry{Summary: TraversalExecutionSummary{ + RuntimeIdentity: "SP-S4-C-D", + AppliedIdentity: "SP-S4-C-D", + RuntimeBranch: "compact_distance", + ObservationMode: "distance", + SelectorVersion: "sp-static-v5-contained", + }}, + }, + CaseResult{ + Dataset: "fixture", + Name: "slow", + Category: "shortest_path", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + RowCount: 1, + StableObservation: true, + ObservedRows: []string{"one"}, + Environment: environment, + Stats: DurationStats{Median: 2 * time.Millisecond, P95: 3 * time.Millisecond}, + }, + ) + } + // A PostgreSQL win remains in the complete case report but not the outlier ledger. + records = append(records, + CaseResult{Dataset: "fixture", Name: "fast", ExecutionMode: ModePostgresSQL, Status: StatusOK, RowCount: 1, StableObservation: true, ObservedRows: []string{"one"}, Stats: DurationStats{Median: time.Millisecond}}, + CaseResult{Dataset: "fixture", Name: "fast", ExecutionMode: ModeNeo4j, Status: StatusOK, RowCount: 1, StableObservation: true, ObservedRows: []string{"one"}, Stats: DurationStats{Median: 2 * time.Millisecond}}, + ) + + require.NoError(t, writeJSONLFile(artifact, records)) + require.NoError(t, createBackendDeltaReport(artifact, output)) + raw, err := os.ReadFile(output) + require.NoError(t, err) + var report BackendDeltaReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Outliers, 1) + outlier := report.Outliers[0] + require.Equal(t, "slow", outlier.Name) + require.Equal(t, 2, outlier.Rounds) + require.Equal(t, 4.0, outlier.MedianPostgresOverNeo4j) + require.Equal(t, 8*time.Millisecond, outlier.PostgresMedian) + require.Equal(t, 2*time.Millisecond, outlier.Neo4jMedian) + require.Equal(t, []string{"SP-S4-C-D"}, outlier.AppliedIdentities) + require.Equal(t, []string{"compact_distance"}, outlier.RuntimeBranches) + require.Equal(t, []string{"tournament_unqualified"}, outlier.FallbackReasons) + require.Equal(t, "hidden_fanin", outlier.ExpectedStateClass) +} diff --git a/cmd/graphbench/bundle.go b/cmd/graphbench/bundle.go new file mode 100644 index 00000000..b6b76db7 --- /dev/null +++ b/cmd/graphbench/bundle.go @@ -0,0 +1,933 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" +) + +// captureBundleVersion identifies the serialized schema revision for capture bundle. +const captureBundleVersion = 3 + +// captureBundleChecksumFile reserves the stable protocol value used to recognize capture bundle checksum file across artifacts and executions. +const captureBundleChecksumFile = "checksums.sha256" + +// CaptureBundleManifest inventories the benchmark artifacts and source provenance copied into a portable bundle. +type CaptureBundleManifest struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Environment captures the environment in which the measurement ran. + Environment RunEnvironment `json:"environment"` + // RecordCount records case-result records included in the capture bundle. + RecordCount int `json:"record_count"` + // CorpusDeclaration contains the exact selected corpus declaration bundled for replay. + CorpusDeclaration string `json:"corpus_declaration"` + // RawArtifact identifies the uncopied artifact used as bundle input. + RawArtifact string `json:"raw_artifact"` + // Executable captures executable path, digest, and build metadata. + Executable string `json:"executable"` + // SourcePatch contains the tracked working-tree patch preserved as source provenance. + SourcePatch string `json:"source_patch"` + // UntrackedManifest names the bundle-relative JSON inventory of copied untracked sources. + UntrackedManifest string `json:"untracked_manifest"` + // SourceClean reports whether the captured working-tree fingerprint contains no tracked or untracked changes. + SourceClean bool `json:"source_clean"` + // Evidence contains named, checksummed gate and plan artifacts copied into the bundle. + Evidence []CaptureBundleEvidence `json:"evidence,omitempty"` +} + +// CaptureCorpusDeclaration preserves every selected workload field needed to +// reconstruct the exact benchmark corpus rather than only its backend index. +type CaptureCorpusDeclaration struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []ScaleCase `json:"cases"` +} + +// CaptureBundleEvidence identifies one auxiliary plan, A/A, correctness, resource, or decision artifact. +type CaptureBundleEvidence struct { + // Name is a stable, user-supplied evidence identity. + Name string `json:"name"` + // SourceSHA256 identifies the exact input bytes before copying. + SourceSHA256 string `json:"source_sha256"` + // Copy names the bundle-relative payload path. + Copy string `json:"copy"` +} + +// CaptureBundleEvidenceInput supplies one auxiliary artifact to a capture bundle. +type CaptureBundleEvidenceInput struct { + // Name is serialized as the evidence identity and file name stem. + Name string + // Path locates the source artifact copied into the bundle. + Path string +} + +// CaptureBundleVerification is the fail-closed result of validating a portable bundle. +type CaptureBundleVerification struct { + // Version identifies this verification result schema. + Version int `json:"version"` + // ManifestVersion is the bundle schema version read from manifest.json. + ManifestVersion int `json:"manifest_version"` + // SourceClean reports the source state declared by the bundle manifest. + SourceClean bool `json:"source_clean"` + // CheckedFiles records how many checksummed payload files were verified. + CheckedFiles int `json:"checked_files"` + // RecordCount records how many JSONL case records were decoded and matched to the manifest. + RecordCount int `json:"record_count"` + // Passed reports whether every structural, checksum, and provenance invariant succeeded. + Passed bool `json:"passed"` + // Reasons contains stable validation failures when Passed is false. + Reasons []string `json:"reasons,omitempty"` +} + +// UntrackedSource describes an untracked source file copied into an artifact bundle. +type UntrackedSource struct { + // Path identifies the filesystem path. + Path string `json:"path"` + // SHA256 verifies the copied file's contents without depending on its path. + SHA256 string `json:"sha256"` + // Copy identifies the bundle-relative copy of an untracked source file. + Copy string `json:"copy"` +} + +// writeCaptureBundle copies run artifacts and provenance into a checksummed portable bundle. +func writeCaptureBundle(root string, corpus ScaleCorpus, records []CaseResult, environment RunEnvironment) error { + return writeCaptureBundleWithEvidence(root, corpus, records, environment, nil) +} + +// writeCaptureBundleWithEvidence copies run artifacts, auxiliary evidence, and provenance into a checksummed portable bundle. +func writeCaptureBundleWithEvidence(root string, corpus ScaleCorpus, records []CaseResult, environment RunEnvironment, evidenceInputs []CaptureBundleEvidenceInput) error { + root = filepath.Clean(root) + if root == "." || root == string(filepath.Separator) { + return fmt.Errorf("bundle directory must be a dedicated path") + } + if err := validateCaptureBundleDestination(root); err != nil { + return err + } + currentFingerprint, err := calculateWorkingTreeSHA256(root) + if err != nil { + return fmt.Errorf("fingerprint current source before bundle capture: %w", err) + } + if !isLowerHexSHA256(environment.DirtyDiffSHA256) || currentFingerprint != environment.DirtyDiffSHA256 { + return fmt.Errorf("current source fingerprint %s differs from run environment fingerprint %s", currentFingerprint, environment.DirtyDiffSHA256) + } + untracked, err := listUntrackedSources(root) + if err != nil { + return err + } + for _, dir := range []string{root, filepath.Join(root, "artifacts"), filepath.Join(root, "bin"), filepath.Join(root, "source-untracked")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + } + + patch, err := exec.Command("git", "diff", "--binary", "HEAD", "--").Output() + if err != nil { + return fmt.Errorf("capture tracked source patch: %w", err) + } + if err := os.WriteFile(filepath.Join(root, "source.patch"), patch, 0o644); err != nil { + return err + } + + untrackedManifest := make([]UntrackedSource, 0, len(untracked)) + for _, source := range untracked { + destination := filepath.Join(root, "source-untracked", source) + if err := copyRegularFile(source, destination, 0o644); err != nil { + return fmt.Errorf("copy untracked source %s: %w", source, err) + } + checksum, err := fileSHA256(source) + if err != nil { + return err + } + untrackedManifest = append(untrackedManifest, UntrackedSource{ + Path: filepath.ToSlash(source), + SHA256: checksum, + Copy: filepath.ToSlash(filepath.Join("source-untracked", source)), + }) + } + if err := writeIndentedJSON(filepath.Join(root, "source-untracked-manifest.json"), untrackedManifest); err != nil { + return err + } + capturedFingerprint, err := capturedWorkingTreeSHA256(patch, untrackedManifest, root) + if err != nil { + return fmt.Errorf("fingerprint captured source: %w", err) + } + if !isLowerHexSHA256(environment.DirtyDiffSHA256) || capturedFingerprint != environment.DirtyDiffSHA256 { + return fmt.Errorf("captured source fingerprint %s differs from run environment fingerprint %s", capturedFingerprint, environment.DirtyDiffSHA256) + } + currentFingerprint, err = calculateWorkingTreeSHA256(root) + if err != nil { + return fmt.Errorf("fingerprint current source after bundle capture: %w", err) + } + if currentFingerprint != environment.DirtyDiffSHA256 { + return fmt.Errorf("source changed during bundle capture: current fingerprint %s differs from run environment fingerprint %s", currentFingerprint, environment.DirtyDiffSHA256) + } + + executable, err := os.Executable() + if err != nil { + return err + } + binaryName := "graphbench-" + environment.BinarySHA256 + if err := copyRegularFile(executable, filepath.Join(root, "bin", binaryName), 0o755); err != nil { + return fmt.Errorf("copy executable: %w", err) + } + if err := copyRegularFile("go.mod", filepath.Join(root, "go.mod"), 0o644); err != nil { + return err + } + if err := copyRegularFile("go.sum", filepath.Join(root, "go.sum"), 0o644); err != nil { + return err + } + cases := append([]ScaleCase(nil), corpus.Cases...) + sort.Slice(cases, func(i, j int) bool { + if cases[i].Source != cases[j].Source { + return cases[i].Source < cases[j].Source + } + if cases[i].Dataset != cases[j].Dataset { + return cases[i].Dataset < cases[j].Dataset + } + return cases[i].Name < cases[j].Name + }) + if err := writeIndentedJSON(filepath.Join(root, "corpus-declaration.json"), CaptureCorpusDeclaration{ + Version: 2, + Cases: cases, + }); err != nil { + return err + } + if err := writeBundleJSONL(filepath.Join(root, "combined.jsonl"), records); err != nil { + return err + } + evidence, err := copyCaptureBundleEvidence(root, evidenceInputs) + if err != nil { + return err + } + + manifest := CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: len(records), + CorpusDeclaration: "corpus-declaration.json", + RawArtifact: "combined.jsonl", + Executable: filepath.ToSlash(filepath.Join("bin", binaryName)), + SourcePatch: "source.patch", + UntrackedManifest: "source-untracked-manifest.json", + SourceClean: environment.DirtyDiffSHA256 == cleanWorkingTreeSHA256(), + Evidence: evidence, + } + if err := writeIndentedJSON(filepath.Join(root, "manifest.json"), manifest); err != nil { + return err + } + if err := writeBundleChecksums(root); err != nil { + return err + } + verification, err := verifyCaptureBundle(root, false) + if err != nil { + return err + } + if !verification.Passed { + return fmt.Errorf("verify capture bundle: %s", strings.Join(verification.Reasons, "; ")) + } + return nil +} + +// capturedWorkingTreeSHA256 reconstructs the exact byte framing used by +// workingTreeSHA256 from the patch and copied untracked payloads in a bundle. +func capturedWorkingTreeSHA256(patch []byte, untracked []UntrackedSource, root string) (string, error) { + digest := sha256.New() + writeWorkingTreePatchFingerprint(digest, patch) + entries := append([]UntrackedSource(nil), untracked...) + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + seenPaths := map[string]struct{}{} + seenCopies := map[string]struct{}{} + for index, source := range entries { + if !validUntrackedSourcePath(source.Path) { + return "", fmt.Errorf("untracked source %d has invalid path %q", index, source.Path) + } + if _, duplicate := seenPaths[source.Path]; duplicate { + return "", fmt.Errorf("untracked source path %q is duplicated", source.Path) + } + seenPaths[source.Path] = struct{}{} + if !isLowerHexSHA256(source.SHA256) { + return "", fmt.Errorf("untracked source %q has invalid SHA-256", source.Path) + } + expectedCopy := filepath.ToSlash(filepath.Join("source-untracked", filepath.FromSlash(source.Path))) + if source.Copy != expectedCopy { + return "", fmt.Errorf("untracked source %q has noncanonical copy %q; expected %q", source.Path, source.Copy, expectedCopy) + } + copyPath, err := resolveBundlePath(root, source.Copy) + if err != nil { + return "", fmt.Errorf("untracked source %q copy: %w", source.Path, err) + } + if _, duplicate := seenCopies[source.Copy]; duplicate { + return "", fmt.Errorf("untracked source copy %q is duplicated", source.Copy) + } + seenCopies[source.Copy] = struct{}{} + content, err := os.ReadFile(copyPath) + if err != nil { + return "", fmt.Errorf("read untracked source %q copy: %w", source.Path, err) + } + actual := fmt.Sprintf("%x", sha256.Sum256(content)) + if actual != source.SHA256 { + return "", fmt.Errorf("untracked source %q digest does not match its copy", source.Path) + } + writeWorkingTreeUntrackedFingerprint(digest, source.Path, content) + } + return fmt.Sprintf("%x", digest.Sum(nil)), nil +} + +// validUntrackedSourcePath reports whether a relative path can be copied into a capture bundle safely. +func validUntrackedSourcePath(path string) bool { + if path == "" || filepath.IsAbs(path) || path != filepath.ToSlash(path) { + return false + } + clean := filepath.Clean(filepath.FromSlash(path)) + return clean != "." && clean != ".." && !strings.HasPrefix(clean, ".."+string(filepath.Separator)) && filepath.ToSlash(clean) == path +} + +// validateCaptureBundleDestination rejects symlinks, non-directories, and stale +// payloads so every checksum inventory is constructed in a fresh destination. +func validateCaptureBundleDestination(root string) error { + info, err := os.Lstat(root) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return fmt.Errorf("inspect bundle destination: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("bundle destination must be a directory") + } + entries, err := os.ReadDir(root) + if err != nil { + return fmt.Errorf("inspect bundle destination: %w", err) + } + if len(entries) != 0 { + return fmt.Errorf("bundle destination must not already contain files") + } + return nil +} + +// copyCaptureBundleEvidence validates stable names and copies every auxiliary artifact into the bundle. +func copyCaptureBundleEvidence(root string, inputs []CaptureBundleEvidenceInput) ([]CaptureBundleEvidence, error) { + seen := map[string]struct{}{} + evidence := make([]CaptureBundleEvidence, 0, len(inputs)) + for _, input := range inputs { + name := strings.TrimSpace(input.Name) + if !validBundleEvidenceName(name) { + return nil, fmt.Errorf("invalid capture bundle evidence name %q", input.Name) + } + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("duplicate capture bundle evidence name %q", name) + } + seen[name] = struct{}{} + info, err := os.Lstat(input.Path) + if err != nil { + return nil, fmt.Errorf("stat capture bundle evidence %q: %w", name, err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("capture bundle evidence %q is not a regular file", name) + } + extension := strings.ToLower(filepath.Ext(input.Path)) + if extension == "" || len(extension) > 10 { + extension = ".artifact" + } + relative := filepath.ToSlash(filepath.Join("artifacts", name+extension)) + if err := copyRegularFile(input.Path, filepath.Join(root, filepath.FromSlash(relative)), 0o644); err != nil { + return nil, fmt.Errorf("copy capture bundle evidence %q: %w", name, err) + } + digest, err := fileSHA256(input.Path) + if err != nil { + return nil, err + } + evidence = append(evidence, CaptureBundleEvidence{ + Name: name, + SourceSHA256: digest, + Copy: relative, + }) + } + sort.Slice(evidence, func(i, j int) bool { return evidence[i].Name < evidence[j].Name }) + return evidence, nil +} + +// validBundleEvidenceName accepts stable path-independent artifact identities. +func validBundleEvidenceName(name string) bool { + if name == "" { + return false + } + for _, char := range name { + if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' && char != '_' { + return false + } + } + return true +} + +// cleanWorkingTreeSHA256 returns the fingerprint emitted by workingTreeSHA256 for a clean source tree. +func cleanWorkingTreeSHA256() string { + return fmt.Sprintf("%x", sha256.Sum256(nil)) +} + +// listUntrackedSources returns untracked repository files eligible for inclusion in the bundle. +func listUntrackedSources(bundleRoot string) ([]string, error) { + gitPaths, err := gitUntrackedPaths() + if err != nil { + return nil, err + } + absRoot, _ := filepath.Abs(bundleRoot) + var paths []string + for _, path := range gitPaths { + absPath, err := filepath.Abs(path) + if err != nil { + return nil, err + } + if absPath == absRoot || strings.HasPrefix(absPath, absRoot+string(filepath.Separator)) { + continue + } + info, err := os.Lstat(path) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("untracked source %q is not a regular file", path) + } + paths = append(paths, filepath.Clean(path)) + } + return paths, nil +} + +// copyRegularFile copies one regular file to a newly created bundle path with the requested mode. +func copyRegularFile(source, destination string, mode os.FileMode) (err error) { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("source is not a regular file") + } + input, err := os.Open(source) + if err != nil { + return err + } + defer input.Close() + if err := os.MkdirAll(filepath.Dir(destination), 0o755); err != nil { + return err + } + output, err := os.OpenFile(destination, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + _, err = io.Copy(output, input) + return err +} + +// writeIndentedJSON writes one value as indented JSON with a trailing newline. +func writeIndentedJSON(path string, value any) (err error) { + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} + +// writeBundleJSONL writes case records as JSON Lines inside an artifact bundle. +func writeBundleJSONL(path string, records []CaseResult) (err error) { + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + return writeJSONL(output, records) +} + +// writeBundleChecksums writes sorted SHA-256 entries for every bundled file except the checksum file. +func writeBundleChecksums(root string) error { + var paths []string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() || path == filepath.Join(root, captureBundleChecksumFile) { + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return err + } + sort.Strings(paths) + var lines strings.Builder + for _, path := range paths { + checksum, err := fileSHA256(path) + if err != nil { + return err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + fmt.Fprintf(&lines, "%s %s\n", checksum, filepath.ToSlash(relative)) + } + return os.WriteFile(filepath.Join(root, captureBundleChecksumFile), []byte(lines.String()), 0o644) +} + +// verifyCaptureBundle validates bundle structure, every payload checksum, source provenance, and record count. +// When requireCleanSource is true, diagnostic bundles carrying a patch or untracked source are rejected. +func verifyCaptureBundle(root string, requireCleanSource bool) (CaptureBundleVerification, error) { + report := CaptureBundleVerification{ + Version: 1, + Passed: true, + } + root = filepath.Clean(root) + rootInfo, err := os.Stat(root) + if err != nil { + return report, fmt.Errorf("stat capture bundle: %w", err) + } + if !rootInfo.IsDir() { + return report, fmt.Errorf("capture bundle path is not a directory: %s", root) + } + + checksums, reasons, err := readBundleChecksums(root) + if err != nil { + return report, err + } + report.Reasons = append(report.Reasons, reasons...) + for relative, expected := range checksums { + path, pathErr := resolveBundlePath(root, relative) + if pathErr != nil { + report.Reasons = append(report.Reasons, pathErr.Error()) + continue + } + info, statErr := os.Lstat(path) + if statErr != nil { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksummed file %q is unavailable: %v", relative, statErr)) + continue + } + if !info.Mode().IsRegular() { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksummed path %q is not a regular file", relative)) + continue + } + actual, digestErr := fileSHA256(path) + if digestErr != nil { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksum %q: %v", relative, digestErr)) + continue + } + if actual != expected { + report.Reasons = append(report.Reasons, fmt.Sprintf("checksum mismatch for %q", relative)) + continue + } + report.CheckedFiles++ + } + + listedReasons, err := verifyBundleFileInventory(root, checksums) + if err != nil { + return report, err + } + report.Reasons = append(report.Reasons, listedReasons...) + + manifest, reasons := verifyBundleManifest(root, checksums) + report.ManifestVersion = manifest.Version + report.SourceClean = manifest.SourceClean + report.Reasons = append(report.Reasons, reasons...) + report.Reasons = append(report.Reasons, verifyBundleCorpus(root, manifest)...) + if requireCleanSource && !manifest.SourceClean { + report.Reasons = append(report.Reasons, "bundle source is not clean") + } + + recordCount, reasons := verifyBundleRecords(root, manifest) + report.RecordCount = recordCount + report.Reasons = append(report.Reasons, reasons...) + report.Passed = len(report.Reasons) == 0 + return report, nil +} + +// verifyBundleCorpus verifies bundle corpus. +func verifyBundleCorpus(root string, manifest CaptureBundleManifest) []string { + path, err := resolveBundlePath(root, manifest.CorpusDeclaration) + if err != nil { + return []string{err.Error()} + } + content, err := os.ReadFile(path) + if err != nil { + return []string{fmt.Sprintf("read corpus declaration: %v", err)} + } + var declaration CaptureCorpusDeclaration + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&declaration); err != nil { + return []string{fmt.Sprintf("decode corpus declaration: %v", err)} + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return []string{"corpus declaration contains trailing JSON data"} + } + if declaration.Version != 2 { + return []string{fmt.Sprintf("unsupported corpus declaration version %d", declaration.Version)} + } + identity := corpusIdentity(ScaleCorpus{Cases: declaration.Cases}) + if identity != manifest.Environment.CorpusSHA256 { + return []string{fmt.Sprintf("corpus declaration identity %s differs from manifest %s", identity, manifest.Environment.CorpusSHA256)} + } + return nil +} + +// createCaptureBundleVerification validates a portable bundle, writes its complete +// verification result, and reports whether it passed every requested invariant. +func createCaptureBundleVerification(root, outputPath string, requireCleanSource bool) (passed bool, err error) { + if outputPath != "" { + absoluteRoot, rootErr := filepath.Abs(filepath.Clean(root)) + absoluteOutput, outputErr := filepath.Abs(filepath.Clean(outputPath)) + if rootErr != nil { + return false, rootErr + } + if outputErr != nil { + return false, outputErr + } + if absoluteOutput == absoluteRoot || strings.HasPrefix(absoluteOutput, absoluteRoot+string(filepath.Separator)) { + return false, fmt.Errorf("bundle verification output must be outside the verified bundle") + } + } + report, err := verifyCaptureBundle(root, requireCleanSource) + if err != nil { + return false, err + } + + var output *os.File + if outputPath == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(outputPath); err != nil { + return false, err + } + output, err = os.Create(outputPath) + if err != nil { + return false, err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + passed = false + } + }() + } + + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return false, err + } + return report.Passed, nil +} + +// readBundleChecksums parses the deterministic SHA-256 manifest without trusting its paths. +func readBundleChecksums(root string) (map[string]string, []string, error) { + path := filepath.Join(root, captureBundleChecksumFile) + input, err := os.Open(path) + if err != nil { + return nil, nil, fmt.Errorf("open capture bundle checksums: %w", err) + } + defer input.Close() + + checksums := map[string]string{} + var reasons []string + scanner := bufio.NewScanner(input) + lineNumber := 0 + for scanner.Scan() { + lineNumber++ + line := scanner.Text() + separator := strings.Index(line, " ") + if separator != 64 || len(line) <= separator+2 { + reasons = append(reasons, fmt.Sprintf("malformed checksum line %d", lineNumber)) + continue + } + digest := line[:separator] + relative := line[separator+2:] + if !isLowerHexSHA256(digest) { + reasons = append(reasons, fmt.Sprintf("invalid SHA-256 on checksum line %d", lineNumber)) + continue + } + if _, duplicate := checksums[relative]; duplicate { + reasons = append(reasons, fmt.Sprintf("duplicate checksum path %q", relative)) + continue + } + checksums[relative] = digest + } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("read capture bundle checksums: %w", err) + } + if len(checksums) == 0 { + reasons = append(reasons, "capture bundle checksum manifest is empty") + } + return checksums, reasons, nil +} + +// isLowerHexSHA256 reports whether value is one canonical lowercase SHA-256 digest. +func isLowerHexSHA256(value string) bool { + if len(value) != 64 { + return false + } + for _, char := range value { + if (char < '0' || char > '9') && (char < 'a' || char > 'f') { + return false + } + } + return true +} + +// resolveBundlePath rejects absolute, parent, platform-ambiguous, and checksum-self references. +func resolveBundlePath(root, relative string) (string, error) { + if relative == "" || filepath.IsAbs(relative) || relative != filepath.ToSlash(relative) { + return "", fmt.Errorf("invalid bundle-relative path %q", relative) + } + clean := filepath.Clean(filepath.FromSlash(relative)) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) || relative == captureBundleChecksumFile { + return "", fmt.Errorf("invalid bundle-relative path %q", relative) + } + return filepath.Join(root, clean), nil +} + +// verifyBundleFileInventory rejects unchecksummed payload files and missing checksum entries. +func verifyBundleFileInventory(root string, checksums map[string]string) ([]string, error) { + var reasons []string + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + if path != root { + info, err := entry.Info() + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + reasons = append(reasons, fmt.Sprintf("bundle contains symlink directory %q", path)) + return filepath.SkipDir + } + } + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return err + } + relative = filepath.ToSlash(relative) + if relative == captureBundleChecksumFile { + return nil + } + if _, listed := checksums[relative]; !listed { + reasons = append(reasons, fmt.Sprintf("unchecksummed bundle file %q", relative)) + } + return nil + }) + return reasons, err +} + +// verifyBundleManifest decodes the manifest and validates every referenced payload identity. +func verifyBundleManifest(root string, checksums map[string]string) (CaptureBundleManifest, []string) { + var manifest CaptureBundleManifest + var reasons []string + manifestPath, present := checksums["manifest.json"] + if !present || manifestPath == "" { + return manifest, []string{"manifest.json is not checksummed"} + } + content, err := os.ReadFile(filepath.Join(root, "manifest.json")) + if err != nil { + return manifest, []string{fmt.Sprintf("read manifest.json: %v", err)} + } + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil { + return manifest, []string{fmt.Sprintf("decode manifest.json: %v", err)} + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return manifest, []string{"manifest.json contains trailing JSON data"} + } + if manifest.Version != captureBundleVersion { + reasons = append(reasons, fmt.Sprintf("unsupported capture bundle version %d", manifest.Version)) + } + for label, relative := range map[string]string{ + "corpus declaration": manifest.CorpusDeclaration, + "raw artifact": manifest.RawArtifact, + "executable": manifest.Executable, + "source patch": manifest.SourcePatch, + "untracked manifest": manifest.UntrackedManifest, + } { + if _, err := resolveBundlePath(root, relative); err != nil { + reasons = append(reasons, fmt.Sprintf("%s: %v", label, err)) + continue + } + if _, exists := checksums[relative]; !exists { + reasons = append(reasons, fmt.Sprintf("%s %q is not checksummed", label, relative)) + } + } + evidenceNames := map[string]struct{}{} + for _, artifact := range manifest.Evidence { + if !validBundleEvidenceName(artifact.Name) { + reasons = append(reasons, fmt.Sprintf("invalid evidence name %q", artifact.Name)) + } + if _, duplicate := evidenceNames[artifact.Name]; duplicate { + reasons = append(reasons, fmt.Sprintf("duplicate evidence name %q", artifact.Name)) + } + evidenceNames[artifact.Name] = struct{}{} + path, pathErr := resolveBundlePath(root, artifact.Copy) + if pathErr != nil { + reasons = append(reasons, fmt.Sprintf("evidence %q: %v", artifact.Name, pathErr)) + continue + } + listedDigest, listed := checksums[artifact.Copy] + if !listed { + reasons = append(reasons, fmt.Sprintf("evidence %q copy %q is not checksummed", artifact.Name, artifact.Copy)) + continue + } + if !isLowerHexSHA256(artifact.SourceSHA256) || listedDigest != artifact.SourceSHA256 { + reasons = append(reasons, fmt.Sprintf("evidence %q source identity does not match its bundled copy", artifact.Name)) + continue + } + if digest, digestErr := fileSHA256(path); digestErr != nil || digest != artifact.SourceSHA256 { + reasons = append(reasons, fmt.Sprintf("evidence %q payload identity is invalid", artifact.Name)) + } + } + if manifest.Environment.BinarySHA256 == "" || manifest.Environment.BinarySHA256 == "unknown" { + reasons = append(reasons, "manifest has no concrete executable SHA-256") + } else if executablePath, err := resolveBundlePath(root, manifest.Executable); err == nil { + if digest, digestErr := fileSHA256(executablePath); digestErr != nil || digest != manifest.Environment.BinarySHA256 { + reasons = append(reasons, "manifest executable identity does not match bundled executable") + } + } + if manifest.Environment.SourceCommit == "" || manifest.Environment.SourceCommit == "unknown" { + reasons = append(reasons, "manifest has no concrete source commit") + } + if manifest.SourceClean && manifest.Environment.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + reasons = append(reasons, "clean-source declaration contradicts dirty source fingerprint") + } + if manifest.SourceClean { + patchPath, patchErr := resolveBundlePath(root, manifest.SourcePatch) + if patchErr == nil { + if patchInfo, err := os.Stat(patchPath); err != nil || patchInfo.Size() != 0 { + reasons = append(reasons, "clean-source bundle contains a non-empty source patch") + } + } + untrackedPath, untrackedErr := resolveBundlePath(root, manifest.UntrackedManifest) + if untrackedErr == nil { + var untracked []UntrackedSource + content, err := os.ReadFile(untrackedPath) + if err != nil || json.Unmarshal(content, &untracked) != nil || len(untracked) != 0 { + reasons = append(reasons, "clean-source bundle contains untracked source entries") + } + } + } + patchPath, patchErr := resolveBundlePath(root, manifest.SourcePatch) + untrackedPath, untrackedErr := resolveBundlePath(root, manifest.UntrackedManifest) + if patchErr == nil && untrackedErr == nil { + patch, readPatchErr := os.ReadFile(patchPath) + untracked, decodeReasons := readUntrackedSourceManifest(untrackedPath) + reasons = append(reasons, decodeReasons...) + if readPatchErr != nil { + reasons = append(reasons, fmt.Sprintf("read bundled source patch: %v", readPatchErr)) + } else if len(decodeReasons) == 0 { + fingerprint, fingerprintErr := capturedWorkingTreeSHA256(patch, untracked, root) + if fingerprintErr != nil { + reasons = append(reasons, "reconstruct bundled source fingerprint: "+fingerprintErr.Error()) + } else { + if !isLowerHexSHA256(manifest.Environment.DirtyDiffSHA256) || fingerprint != manifest.Environment.DirtyDiffSHA256 { + reasons = append(reasons, "manifest dirty source fingerprint does not match bundled patch and untracked sources") + } + if manifest.SourceClean != (fingerprint == cleanWorkingTreeSHA256()) { + reasons = append(reasons, "source_clean declaration does not match bundled source fingerprint") + } + } + } + manifestCopies := map[string]struct{}{} + for _, source := range untracked { + manifestCopies[source.Copy] = struct{}{} + if _, listed := checksums[source.Copy]; !listed { + reasons = append(reasons, fmt.Sprintf("untracked source %q copy %q is not checksummed", source.Path, source.Copy)) + } + } + for relative := range checksums { + if strings.HasPrefix(relative, "source-untracked/") { + if _, declared := manifestCopies[relative]; !declared { + reasons = append(reasons, fmt.Sprintf("checksummed untracked source copy %q has no manifest entry", relative)) + } + } + } + } + return manifest, reasons +} + +// readUntrackedSourceManifest reads untracked source manifest. +func readUntrackedSourceManifest(path string) ([]UntrackedSource, []string) { + content, err := os.ReadFile(path) + if err != nil { + return nil, []string{fmt.Sprintf("read untracked source manifest: %v", err)} + } + if len(strings.TrimSpace(string(content))) == 0 || strings.TrimSpace(string(content))[0] != '[' { + return nil, []string{"untracked source manifest must be a JSON array"} + } + var sources []UntrackedSource + decoder := json.NewDecoder(strings.NewReader(string(content))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&sources); err != nil { + return nil, []string{fmt.Sprintf("decode untracked source manifest: %v", err)} + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + return nil, []string{"untracked source manifest contains trailing JSON data"} + } + return sources, nil +} + +// verifyBundleRecords decodes the JSONL payload and binds every record to the manifest environment. +func verifyBundleRecords(root string, manifest CaptureBundleManifest) (int, []string) { + artifactPath, err := resolveBundlePath(root, manifest.RawArtifact) + if err != nil { + return 0, []string{err.Error()} + } + records, err := readJSONLFile(artifactPath) + if err != nil { + return 0, []string{fmt.Sprintf("decode bundled records: %v", err)} + } + var reasons []string + if len(records) != manifest.RecordCount { + reasons = append(reasons, fmt.Sprintf("manifest record count %d does not match artifact count %d", manifest.RecordCount, len(records))) + } + for index, record := range records { + if record.Environment == nil { + reasons = append(reasons, fmt.Sprintf("record %d has no environment provenance", index)) + continue + } + if record.Environment.BinarySHA256 != manifest.Environment.BinarySHA256 || + record.Environment.SourceCommit != manifest.Environment.SourceCommit || + record.Environment.DirtyDiffSHA256 != manifest.Environment.DirtyDiffSHA256 || + record.Environment.CorpusSHA256 != manifest.Environment.CorpusSHA256 { + reasons = append(reasons, fmt.Sprintf("record %d provenance does not match bundle manifest", index)) + } + } + return len(records), reasons +} diff --git a/cmd/graphbench/bundle_test.go b/cmd/graphbench/bundle_test.go new file mode 100644 index 00000000..193ce6f7 --- /dev/null +++ b/cmd/graphbench/bundle_test.go @@ -0,0 +1,341 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestVerifyCaptureBundleValidatesChecksumsAndProvenance exercises the portable bundle verifier without depending on a built graphbench executable. +func TestVerifyCaptureBundleValidatesChecksumsAndProvenance(t *testing.T) { + root := t.TempDir() + environment := RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: corpusIdentity(ScaleCorpus{}), + SourceCommit: "commit", + DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: "placeholder", + } + record := CaseResult{ + Environment: &environment, + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + } + require.NoError(t, os.MkdirAll(filepath.Join(root, "bin"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "bin", "graphbench"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "bin", "graphbench")) + require.NoError(t, err) + environment.BinarySHA256 = binarySHA + record.Environment.BinarySHA256 = binarySHA + + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), nil, 0o644)) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "source-untracked-manifest.json"), []UntrackedSource{})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus-declaration.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeBundleJSONL(filepath.Join(root, "combined.jsonl"), []CaseResult{record})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: 1, + CorpusDeclaration: "corpus-declaration.json", + RawArtifact: "combined.jsonl", + Executable: "bin/graphbench", + SourcePatch: "source.patch", + UntrackedManifest: "source-untracked-manifest.json", + SourceClean: true, + })) + require.NoError(t, writeBundleChecksums(root)) + + report, err := verifyCaptureBundle(root, true) + require.NoError(t, err) + require.True(t, report.Passed, report.Reasons) + require.Equal(t, 6, report.CheckedFiles) + require.Equal(t, 1, report.RecordCount) + + outputPath := filepath.Join(t.TempDir(), "verification.json") + passed, err := createCaptureBundleVerification(root, outputPath, true) + require.NoError(t, err) + require.True(t, passed) + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var written CaptureBundleVerification + require.NoError(t, json.Unmarshal(content, &written)) + require.Equal(t, report, written) + + _, err = createCaptureBundleVerification(root, filepath.Join(root, "verification.json"), true) + require.ErrorContains(t, err, "must be outside the verified bundle") + + manifestPath := filepath.Join(root, "manifest.json") + manifestContent, err := os.ReadFile(manifestPath) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, append(manifestContent, []byte("{}\n")...), 0o644)) + require.NoError(t, writeBundleChecksums(root)) + report, err = verifyCaptureBundle(root, true) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "manifest.json contains trailing JSON data") +} + +// TestVerifyCaptureBundleFailsClosedOnTamperingDirtySourceAndUnlistedFiles covers the three qualification boundaries a checksum-only writer cannot enforce. +func TestVerifyCaptureBundleFailsClosedOnTamperingDirtySourceAndUnlistedFiles(t *testing.T) { + root := t.TempDir() + environment := RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: corpusIdentity(ScaleCorpus{}), + SourceCommit: "commit", + DirtyDiffSHA256: "dirty", + BinarySHA256: "placeholder", + } + require.NoError(t, os.WriteFile(filepath.Join(root, "binary"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "binary")) + require.NoError(t, err) + environment.BinarySHA256 = binarySHA + recordEnvironment := environment + record := CaseResult{ + Environment: &recordEnvironment, + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + } + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), []byte("diff"), 0o644)) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "untracked.json"), []UntrackedSource{})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeBundleJSONL(filepath.Join(root, "records.jsonl"), []CaseResult{record})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: 1, + CorpusDeclaration: "corpus.json", + RawArtifact: "records.jsonl", + Executable: "binary", + SourcePatch: "source.patch", + UntrackedManifest: "untracked.json", + SourceClean: false, + })) + require.NoError(t, writeBundleChecksums(root)) + require.NoError(t, os.WriteFile(filepath.Join(root, "records.jsonl"), []byte("tampered\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "unlisted"), []byte("payload"), 0o644)) + + report, err := verifyCaptureBundle(root, true) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "checksum mismatch for \"records.jsonl\"") + require.Contains(t, report.Reasons, "unchecksummed bundle file \"unlisted\"") + require.Contains(t, report.Reasons, "bundle source is not clean") + + outputPath := filepath.Join(t.TempDir(), "failed-verification.json") + passed, err := createCaptureBundleVerification(root, outputPath, true) + require.NoError(t, err) + require.False(t, passed) + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var written CaptureBundleVerification + require.NoError(t, json.Unmarshal(content, &written)) + require.False(t, written.Passed) + require.NotEmpty(t, written.Reasons) +} + +// TestResolveBundlePathRejectsTraversal verifies checksum manifests cannot escape the capture root. +func TestResolveBundlePathRejectsTraversal(t *testing.T) { + _, err := resolveBundlePath(t.TempDir(), "../escape") + require.ErrorContains(t, err, "invalid bundle-relative path") +} + +// TestCopyCaptureBundleEvidenceUsesStableNamesAndDigests verifies auxiliary plan/gate inputs are copied without retaining host paths. +func TestCopyCaptureBundleEvidenceUsesStableNamesAndDigests(t *testing.T) { + root := t.TempDir() + input := filepath.Join(t.TempDir(), "aa-report.json") + require.NoError(t, os.WriteFile(input, []byte(`{"version":1}`), 0o644)) + + evidence, err := copyCaptureBundleEvidence(root, []CaptureBundleEvidenceInput{{ + Name: "host-aa", + Path: input, + }}) + require.NoError(t, err) + require.Len(t, evidence, 1) + require.Equal(t, "host-aa", evidence[0].Name) + require.Equal(t, "artifacts/host-aa.json", evidence[0].Copy) + require.FileExists(t, filepath.Join(root, "artifacts", "host-aa.json")) + require.NotContains(t, evidence[0].Copy, filepath.Dir(input)) + + _, err = copyCaptureBundleEvidence(root, []CaptureBundleEvidenceInput{{ + Name: "../escape", + Path: input, + }}) + require.ErrorContains(t, err, "invalid capture bundle evidence name") + + symlink := filepath.Join(t.TempDir(), "outside.json") + require.NoError(t, os.Symlink(input, symlink)) + _, err = copyCaptureBundleEvidence(root, []CaptureBundleEvidenceInput{{ + Name: "symlink", + Path: symlink, + }}) + require.ErrorContains(t, err, "is not a regular file") +} + +// TestWriteCaptureBundleRejectsNonemptyDestination verifies stale payloads cannot leak into a newly checksummed bundle inventory. +func TestWriteCaptureBundleRejectsNonemptyDestination(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(root, "stale.json"), []byte("stale"), 0o644)) + + err := writeCaptureBundleWithEvidence(root, ScaleCorpus{}, nil, RunEnvironment{}, nil) + require.ErrorContains(t, err, "must not already contain files") + require.FileExists(t, filepath.Join(root, "stale.json")) +} + +// TestWriteCaptureBundleRejectsStaleRunEnvironmentFingerprint verifies write capture bundle rejects stale run environment fingerprint behavior. +func TestWriteCaptureBundleRejectsStaleRunEnvironmentFingerprint(t *testing.T) { + root := filepath.Join(t.TempDir(), "bundle") + err := writeCaptureBundleWithEvidence(root, ScaleCorpus{}, nil, RunEnvironment{ + DirtyDiffSHA256: strings.Repeat("0", 64), + }, nil) + require.ErrorContains(t, err, "current source fingerprint") + require.NoDirExists(t, root) +} + +// TestParseNULTerminatedPathsPreservesWhitespace verifies parse nul terminated paths preserves whitespace behavior. +func TestParseNULTerminatedPathsPreservesWhitespace(t *testing.T) { + require.Equal(t, []string{"dir/name with spaces.go", "line\nbreak.go"}, parseNULTerminatedPaths([]byte("dir/name with spaces.go\x00line\nbreak.go\x00"))) +} + +// TestCopyRegularFileRejectsSymlink verifies the shared source copier cannot follow an untracked-source symlink outside the repository. +func TestCopyRegularFileRejectsSymlink(t *testing.T) { + source := filepath.Join(t.TempDir(), "outside") + link := filepath.Join(t.TempDir(), "untracked-link") + require.NoError(t, os.WriteFile(source, []byte("outside"), 0o644)) + require.NoError(t, os.Symlink(source, link)) + + err := copyRegularFile(link, filepath.Join(t.TempDir(), "copy"), 0o644) + require.ErrorContains(t, err, "source is not a regular file") +} + +// TestVerifyCaptureBundleBindsDirtyFingerprintToPatchAndUntrackedCopies verifies verify capture bundle binds dirty fingerprint to patch and untracked copies behavior. +func TestVerifyCaptureBundleBindsDirtyFingerprintToPatchAndUntrackedCopies(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "source-untracked", "pkg"), 0o755)) + patch := []byte("diff --git a/a.go b/a.go\n") + content := []byte("package pkg\n") + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), patch, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source-untracked", "pkg", "new.go"), content, 0o644)) + contentSHA := fmt.Sprintf("%x", sha256.Sum256(content)) + untracked := []UntrackedSource{{ + Path: "pkg/new.go", + SHA256: contentSHA, + Copy: "source-untracked/pkg/new.go", + }} + require.NoError(t, writeIndentedJSON(filepath.Join(root, "untracked.json"), untracked)) + fingerprint, err := capturedWorkingTreeSHA256(patch, untracked, root) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(filepath.Join(root, "binary"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "binary")) + require.NoError(t, err) + environment := RunEnvironment{ + SourceCommit: "commit", + DirtyDiffSHA256: fingerprint, + BinarySHA256: binarySHA, + CorpusSHA256: corpusIdentity(ScaleCorpus{}), + } + recordEnvironment := environment + require.NoError(t, writeBundleJSONL(filepath.Join(root, "records.jsonl"), []CaseResult{{Environment: &recordEnvironment}})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: 1, + CorpusDeclaration: "corpus.json", + RawArtifact: "records.jsonl", + Executable: "binary", + SourcePatch: "source.patch", + UntrackedManifest: "untracked.json", + SourceClean: false, + })) + require.NoError(t, writeBundleChecksums(root)) + + report, err := verifyCaptureBundle(root, false) + require.NoError(t, err) + require.True(t, report.Passed, report.Reasons) + + manifestPath := filepath.Join(root, "manifest.json") + var manifest CaptureBundleManifest + raw, err := os.ReadFile(manifestPath) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &manifest)) + manifest.Environment.DirtyDiffSHA256 = strings.Repeat("0", 64) + require.NoError(t, writeIndentedJSON(manifestPath, manifest)) + require.NoError(t, writeBundleChecksums(root)) + report, err = verifyCaptureBundle(root, false) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "manifest dirty source fingerprint does not match bundled patch and untracked sources") +} + +// TestVerifyCaptureBundleRejectsMalformedOrUnchecksummedUntrackedEntries verifies verify capture bundle rejects malformed or unchecksummed untracked entries behavior. +func TestVerifyCaptureBundleRejectsMalformedOrUnchecksummedUntrackedEntries(t *testing.T) { + root := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(root, "source-untracked"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source.patch"), nil, 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(root, "source-untracked", "new.go"), []byte("package p\n"), 0o644)) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "untracked.json"), []UntrackedSource{{ + Path: "../escape.go", + SHA256: "bad", + Copy: "source-untracked/new.go", + }})) + require.NoError(t, os.WriteFile(filepath.Join(root, "binary"), []byte("binary"), 0o755)) + binarySHA, err := fileSHA256(filepath.Join(root, "binary")) + require.NoError(t, err) + environment := RunEnvironment{ + SourceCommit: "commit", + DirtyDiffSHA256: strings.Repeat("0", 64), + BinarySHA256: binarySHA, + CorpusSHA256: corpusIdentity(ScaleCorpus{}), + } + recordEnvironment := environment + require.NoError(t, writeBundleJSONL(filepath.Join(root, "records.jsonl"), []CaseResult{{Environment: &recordEnvironment}})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "corpus.json"), CaptureCorpusDeclaration{Version: 2})) + require.NoError(t, writeIndentedJSON(filepath.Join(root, "manifest.json"), CaptureBundleManifest{ + Version: captureBundleVersion, + Environment: environment, + RecordCount: 1, + CorpusDeclaration: "corpus.json", + RawArtifact: "records.jsonl", + Executable: "binary", + SourcePatch: "source.patch", + UntrackedManifest: "untracked.json", + SourceClean: false, + })) + require.NoError(t, writeBundleChecksums(root)) + checksums, _, err := readBundleChecksums(root) + require.NoError(t, err) + delete(checksums, "source-untracked/new.go") + var lines strings.Builder + paths := make([]string, 0, len(checksums)) + for path := range checksums { + paths = append(paths, path) + } + sort.Strings(paths) + for _, path := range paths { + fmt.Fprintf(&lines, "%s %s\n", checksums[path], path) + } + require.NoError(t, os.WriteFile(filepath.Join(root, captureBundleChecksumFile), []byte(lines.String()), 0o644)) + + report, err := verifyCaptureBundle(root, false) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, strings.Join(report.Reasons, "\n"), "invalid path") + require.Contains(t, report.Reasons, "untracked source \"../escape.go\" copy \"source-untracked/new.go\" is not checksummed") +} diff --git a/cmd/graphbench/concurrency.go b/cmd/graphbench/concurrency.go new file mode 100644 index 00000000..d6c5d8c6 --- /dev/null +++ b/cmd/graphbench/concurrency.go @@ -0,0 +1,181 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "sort" + "strconv" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// measurePostgresConcurrency runs requested concurrency levels and records latency and connection reuse. +func measurePostgresConcurrency( + ctx context.Context, + pool *pgxpool.Pool, + sqlQuery string, + parameters map[string]any, + poolSize int, + levels []int, + iterations int, + isolation ...pgx.TxIsoLevel, +) ([]ConcurrencyBlock, error) { + blocks := make([]ConcurrencyBlock, 0, len(levels)) + for _, concurrency := range levels { + block, err := measurePostgresConcurrencyBlock(ctx, pool, sqlQuery, parameters, poolSize, concurrency, iterations, isolation...) + if err != nil { + return nil, fmt.Errorf("concurrency %d: %w", concurrency, err) + } + blocks = append(blocks, block) + } + return blocks, nil +} + +// measurePostgresConcurrencyBlock coordinates workers for one concurrency level and aggregates their samples. +func measurePostgresConcurrencyBlock( + ctx context.Context, + pool *pgxpool.Pool, + sqlQuery string, + parameters map[string]any, + poolSize, concurrency, iterations int, + isolation ...pgx.TxIsoLevel, +) (ConcurrencyBlock, error) { + var ( + startBarrier = make(chan struct{}) + wg sync.WaitGroup + mutex sync.Mutex + samples = make([]ConcurrencySample, 0, concurrency*iterations) + errorsSeen []error + seenPID = map[uint32]struct{}{} + ) + blockStart := time.Now() + for worker := range concurrency { + wg.Add(1) + go func() { + defer wg.Done() + <-startBarrier + for iteration := range iterations { + sample, pid, err := measurePostgresConcurrentIteration(ctx, pool, sqlQuery, parameters, worker+1, iteration+1, isolation...) + mutex.Lock() + if err != nil { + errorsSeen = append(errorsSeen, err) + mutex.Unlock() + return + } + if _, found := seenPID[pid]; found { + sample.Classification = "warm-session" + } else { + seenPID[pid] = struct{}{} + sample.Classification = "cold-session" + } + samples = append(samples, sample) + mutex.Unlock() + } + }() + } + close(startBarrier) + wg.Wait() + wall := time.Since(blockStart) + if len(errorsSeen) > 0 { + return ConcurrencyBlock{}, errorsSeen[0] + } + sort.Slice(samples, func(i, j int) bool { + if samples[i].Worker != samples[j].Worker { + return samples[i].Worker < samples[j].Worker + } + return samples[i].Iteration < samples[j].Iteration + }) + return ConcurrencyBlock{ + Concurrency: concurrency, + PoolSize: poolSize, + Operations: len(samples), + Wall: wall, + QPS: float64(len(samples)) / wall.Seconds(), + Samples: samples, + }, nil +} + +// measurePostgresConcurrentIteration executes one timed query in a transaction and records its backend process ID. +func measurePostgresConcurrentIteration( + ctx context.Context, + pool *pgxpool.Pool, + sqlQuery string, + parameters map[string]any, + worker, iteration int, + isolation ...pgx.TxIsoLevel, +) (ConcurrencySample, uint32, error) { + totalStart := time.Now() + acquireStart := time.Now() + conn, err := pool.Acquire(ctx) + if err != nil { + return ConcurrencySample{}, 0, err + } + defer conn.Release() + + poolWait := time.Since(acquireStart) + pid := conn.Conn().PgConn().PID() + + txStart := time.Now() + // DAWGS read queries may create and reset session-local workspace tables. + // Keep the transaction read-write, matching drivers/pg ReadTransaction, + // while rolling it back after the measurement. + txOptions := postgresConcurrencyTxOptions(isolation...) + tx, err := conn.BeginTx(ctx, txOptions) + if err != nil { + return ConcurrencySample{}, 0, err + } + defer func() { _ = tx.Rollback(ctx) }() + + transactionDuration := time.Since(txStart) + + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}} + if len(parameters) > 0 { + queryArgs = append(queryArgs, pgx.NamedArgs(parameters)) + } + executeStart := time.Now() + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + if err != nil { + return ConcurrencySample{}, 0, err + } + for rows.Next() { + if _, err := rows.Values(); err != nil { + rows.Close() + return ConcurrencySample{}, 0, err + } + } + rows.Close() + if err := rows.Err(); err != nil { + return ConcurrencySample{}, 0, err + } + executeDuration := time.Since(executeStart) + if err := tx.Rollback(ctx); err != nil { + return ConcurrencySample{}, 0, err + } + + return ConcurrencySample{ + Worker: worker, + Iteration: iteration, + ConnectionID: strconv.FormatUint(uint64(pid), 10), + PoolWait: poolWait, + Transaction: transactionDuration, + ExecuteDrain: executeDuration, + Total: time.Since(totalStart), + }, pid, nil +} + +// postgresConcurrencyTxOptions returns transaction options that preserve session-local workspace maintenance. +func postgresConcurrencyTxOptions(isolation ...pgx.TxIsoLevel) pgx.TxOptions { + options := pgx.TxOptions{AccessMode: pgx.ReadWrite} + if len(isolation) > 0 { + options.IsoLevel = isolation[0] + } + return options +} diff --git a/cmd/graphbench/concurrency_test.go b/cmd/graphbench/concurrency_test.go new file mode 100644 index 00000000..fbfdc75d --- /dev/null +++ b/cmd/graphbench/concurrency_test.go @@ -0,0 +1,20 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/require" +) + +// TestPostgresConcurrencyTransactionsPermitSessionWorkspaceMaintenance verifies that concurrent benchmark transactions are read-write so session-scoped workspace tables can be maintained. +func TestPostgresConcurrencyTransactionsPermitSessionWorkspaceMaintenance(t *testing.T) { + require.Equal(t, pgx.ReadWrite, postgresConcurrencyTxOptions().AccessMode) + require.Empty(t, postgresConcurrencyTxOptions().IsoLevel) + require.Equal(t, pgx.RepeatableRead, postgresConcurrencyTxOptions(pgx.RepeatableRead).IsoLevel) +} diff --git a/cmd/graphbench/confirm_report.go b/cmd/graphbench/confirm_report.go new file mode 100644 index 00000000..e706e271 --- /dev/null +++ b/cmd/graphbench/confirm_report.go @@ -0,0 +1,619 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand" + "os" + "regexp" + "sort" + "strings" + "time" +) + +// confirmationReportVersion identifies the JSON schema emitted by confirmation reports. +const confirmationReportVersion = 4 + +// ConfirmationOptions selects the paired artifacts, cases, confidence level, and bootstrap seed used for confirmation. +type ConfirmationOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // CaseNames restricts confirmation to the named workloads when nonempty. + CaseNames []string +} + +// ConfirmationMetric combines ratio, absolute-change, noise-floor, and classification evidence for one metric. +type ConfirmationMetric struct { + // Ratio reports the candidate-to-baseline latency ratio. + Ratio RatioInterval `json:"ratio"` + // AbsoluteChange reports the estimated absolute duration change and confidence bounds. + AbsoluteChange DurationInterval `json:"absolute_change"` + // NoiseRatio supplies the noise ratio input to the ConfirmationMetric contract. + NoiseRatio float64 `json:"noise_ratio"` + // NoiseAbsolute supplies the noise absolute input to the ConfirmationMetric contract. + NoiseAbsolute time.Duration `json:"noise_absolute"` + // Classification supplies the classification input to the ConfirmationMetric contract. + Classification string `json:"classification"` +} + +// ConfirmationCase reports comparability, timing deltas, and the final disposition for one confirmed case. +type ConfirmationCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // Tier identifies whether latency is promotion-gated or stress-diagnostic. + Tier string `json:"tier"` + // QualificationSplit identifies training, frozen holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // TimingGated reports whether timing evidence contributes to promotion. + TimingGated bool `json:"timing_gated"` + // MatchedRounds records rounds containing both left- and right-arm samples. + MatchedRounds int `json:"matched_rounds"` + // LeftSamples records warm samples accepted from the left confirmation arm. + LeftSamples int `json:"left_samples"` + // RightSamples records warm samples accepted from the right confirmation arm. + RightSamples int `json:"right_samples"` + // Comparable reports whether the paired measurements satisfy comparison prerequisites. + Comparable bool `json:"comparable"` + // Comparability lists reasons paired confirmation records are or are not comparable. + Comparability []string `json:"comparability_reasons,omitempty"` + // P50 contains median ratio, absolute-change, noise, and classification evidence. + P50 ConfirmationMetric `json:"p50"` + // P95 contains 95th-percentile ratio, absolute-change, noise, and classification evidence. + P95 ConfirmationMetric `json:"p95"` + // Disposition supplies the disposition input to the ConfirmationCase contract. + Disposition string `json:"disposition"` + // RightRuntimeReceiptChains preserves the candidate/right arm's complete + // measured runtime branch chains. + RightRuntimeReceiptChains [][]RuntimeReceiptEvent `json:"right_runtime_receipt_chains,omitempty"` +} + +// ConfirmationReport contains paired-arm identities, A/A noise evidence, and per-case confirmation decisions. +type ConfirmationReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Kind identifies the serialized confirmation-report format. + Kind string `json:"kind"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // LeftArm identifies the artifact treated as the left confirmation arm. + LeftArm string `json:"left_arm"` + // RightArm identifies the artifact treated as the right confirmation arm. + RightArm string `json:"right_arm"` + // LeftSHA256 identifies the exact left-arm artifact evaluated by the report. + LeftSHA256 string `json:"left_sha256"` + // RightSHA256 identifies the exact right-arm artifact evaluated by the report. + RightSHA256 string `json:"right_sha256"` + // AAReport contains A/A noise evidence used to classify confirmation differences. + AAReport string `json:"aa_report,omitempty"` + // AAReportSHA256 identifies the exact A/A report used for classification. + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + // PromotionEligible reports whether every timing-gated causal case is comparable and P95-non-inferior. + PromotionEligible bool `json:"promotion_eligible"` + // QualificationRequired reports whether the artifact contains a prioritized traversal candidate that requires independent training and frozen-holdout confirmation. + QualificationRequired bool `json:"qualification_required"` + // TrainingCases records prioritized traversal cases confirmed on the selector-training partition. + TrainingCases int `json:"training_cases"` + // HoldoutCases records prioritized traversal cases confirmed on the frozen topology holdout. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed reports whether every observed prioritized training case cleared confirmation. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed reports whether every observed prioritized holdout case cleared confirmation. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed reports whether nonempty training and holdout partitions independently cleared confirmation. + QualificationPassed bool `json:"qualification_passed"` + // QualificationFamilies contains the independent split disposition for each concrete traversal candidate family. + QualificationFamilies []TraversalQualificationStatus `json:"qualification_families,omitempty"` + // Cases contains paired-arm evidence and the resulting disposition for each confirmed workload. + Cases []ConfirmationCase `json:"cases"` +} + +// createConfirmationReport loads both benchmark arms and optional A/A evidence, builds their comparison, and writes the resulting report. +func createConfirmationReport(leftPath, rightPath, aaPath, outputPath string, options ConfirmationOptions) error { + left, err := readJSONLFile(leftPath) + if err != nil { + return fmt.Errorf("read left artifact: %w", err) + } + right, err := readJSONLFile(rightPath) + if err != nil { + return fmt.Errorf("read right artifact: %w", err) + } + var aa *AAResolutionReport + aaSHA256 := "" + if aaPath != "" { + aa, aaSHA256, err = loadAAResolutionReport(aaPath) + if err != nil { + return fmt.Errorf("read A/A report: %w", err) + } + } + report, err := buildConfirmationReport(left, right, aa, options) + if err != nil { + return err + } + report.LeftSHA256, err = fileSHA256(leftPath) + if err != nil { + return err + } + report.RightSHA256, err = fileSHA256(rightPath) + if err != nil { + return err + } + report.AAReport = aaPath + report.AAReportSHA256 = aaSHA256 + return writeConfirmationReport(outputPath, report) +} + +// buildConfirmationReport pairs comparable cases, derives confidence intervals and noise-adjusted classifications, and records why incomparable cases were skipped. +func buildConfirmationReport(left, right []CaseResult, aa *AAResolutionReport, options ConfirmationOptions) (ConfirmationReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return ConfirmationReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return ConfirmationReport{}, fmt.Errorf("bootstrap count must be positive") + } + leftSeries, rightSeries := collectWarmSeries(left), collectWarmSeries(right) + blockAA := sameExecutable(left, right) + if !blockAA && len(options.CaseNames) == 0 { + return ConfirmationReport{}, fmt.Errorf("causal confirmation requires exact primary case names") + } + if len(options.CaseNames) > 0 && len(options.CaseNames) <= 2 && options.Confidence < 0.975 { + options.Confidence = 0.975 + } + keys := make([]performanceKey, 0) + for key := range leftSeries { + if key.backend == ModePostgresSQL && rightSeries[key] != nil { + keys = append(keys, key) + } + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + if len(options.CaseNames) > 0 { + requested := map[string]bool{} + for _, name := range options.CaseNames { + requested[name] = false + } + filtered := keys[:0] + for _, key := range keys { + if _, ok := requested[key.name]; ok { + requested[key.name] = true + filtered = append(filtered, key) + } + } + for name, found := range requested { + if !found { + return ConfirmationReport{}, fmt.Errorf("unknown confirmation case %q", name) + } + } + keys = filtered + } + if len(keys) == 0 { + return ConfirmationReport{}, fmt.Errorf("artifacts have no matched PostgreSQL warm series") + } + tiers := make(map[performanceKey]string, len(keys)) + splits := make(map[performanceKey]string, len(keys)) + requiresAA := false + for _, key := range keys { + tier, err := timingTier(key, left, right) + if err != nil { + return ConfirmationReport{}, err + } + tiers[key] = tier + split, err := qualificationSplit(key, left, right) + if err != nil { + return ConfirmationReport{}, err + } + splits[key] = split + if !blockAA && tier != "stress" && promotionTimingSplit(split) { + requiresAA = true + } + } + if requiresAA { + if err := validateAAResolutionEvidence(aa, left, options.Confidence); err != nil { + return ConfirmationReport{}, fmt.Errorf("left-arm A/A evidence: %w", err) + } + if err := validateAAResolutionEvidence(aa, right, options.Confidence); err != nil { + return ConfirmationReport{}, fmt.Errorf("right-arm A/A evidence: %w", err) + } + } else if aa != nil { + if err := validateAAResolutionEvidence(aa, left, options.Confidence); err != nil { + return ConfirmationReport{}, err + } + } + + report := ConfirmationReport{ + Version: confirmationReportVersion, + Kind: "causal_confirmation", + Seed: options.Seed, + Confidence: options.Confidence, + } + report.LeftArm = artifactArm(left) + report.RightArm = artifactArm(right) + if blockAA { + report.Kind = "block_reload_aa" + } + report.PromotionEligible = !blockAA && requiresAA + report.TrainingPassed = true + report.HoldoutPassed = true + qualification := map[string]*TraversalQualificationStatus{} + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for idx, key := range keys { + leftRounds, rightRounds := matchedRounds(leftSeries[key], rightSeries[key]) + timingGated := tiers[key] != "stress" && promotionTimingSplit(splits[key]) && !blockAA + if timingGated && (len(leftRounds) < 10 || len(leftRounds) > 20) { + return ConfirmationReport{}, fmt.Errorf("%s/%s requires 10-20 matched rounds, got %d", key.dataset, key.name, len(leftRounds)) + } + for _, round := range sortedRounds(leftRounds) { + if timingGated && (len(leftRounds[round]) < 50 || len(rightRounds[round]) < 50) { + return ConfirmationReport{}, fmt.Errorf("%s/%s round %d requires at least 50 warm samples per arm", key.dataset, key.name, round) + } + } + if timingGated { + if err := validatePairedOrderEvidence(left, right, key, sortedRounds(leftRounds), 20); err != nil { + return ConfirmationReport{}, fmt.Errorf("invalid confirmation evidence: %w", err) + } + } + seed := options.Seed + int64(idx)*7919 + p50Ratio := bootstrapRoundMedianRatio(leftRounds, rightRounds, seed, gateOptions) + p50Change := negateDurationInterval(bootstrapRoundMedianSaving(leftRounds, rightRounds, seed+1, gateOptions)) + p95Ratio := bootstrapStratifiedP95Ratio(leftRounds, rightRounds, seed+2, gateOptions) + p95Change := bootstrapStratifiedQuantileChange(leftRounds, rightRounds, 0.95, seed+3, gateOptions) + p50NoiseRatio, p50NoiseAbsolute := minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + p95NoiseRatio, p95NoiseAbsolute := minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + if aa != nil { + if ratio, absolute, floorErr := aaTimingFloor(aa, key, false, 0); floorErr == nil { + p50NoiseRatio, p50NoiseAbsolute = ratio, absolute + } else if timingGated { + return ConfirmationReport{}, floorErr + } + if ratio, absolute, floorErr := aaTimingFloor(aa, key, true, 0); floorErr == nil { + p95NoiseRatio, p95NoiseAbsolute = ratio, absolute + } else if timingGated { + return ConfirmationReport{}, floorErr + } + } + comparable, reasons := confirmationComparable(left, right, key) + entry := ConfirmationCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + Tier: tiers[key], + QualificationSplit: splits[key], + TimingGated: timingGated, + MatchedRounds: len(leftRounds), + LeftSamples: sampleCount(leftRounds), + RightSamples: sampleCount(rightRounds), + Comparable: comparable, + Comparability: reasons, + RightRuntimeReceiptChains: caseRuntimeReceiptChains(right, key), + P50: classifyConfirmationMetric(p50Ratio, p50Change, p50NoiseRatio, p50NoiseAbsolute), + P95: classifyConfirmationMetric(p95Ratio, p95Change, p95NoiseRatio, p95NoiseAbsolute), + } + entry.Disposition = entry.P95.Classification + if tiers[key] == "stress" { + entry.Disposition = "stress_diagnostic" + } + if splits[key] == "diagnostic" { + entry.Disposition = "qualification_diagnostic" + } + if !comparable { + entry.Disposition = "fingerprint_mismatch" + } + if entry.TimingGated && (!entry.Comparable || entry.P95.Classification != "cleared_non_inferior") { + report.PromotionEligible = false + } + if prioritizedTraversalKey(key, left, right) && entry.TimingGated { + report.QualificationRequired = true + passed := entry.Comparable && entry.P95.Classification == "cleared_non_inferior" + family := traversalQualificationFamily(key, left, right) + status := qualification[family] + if status == nil { + status = &TraversalQualificationStatus{ + Family: family, + TrainingPassed: true, + HoldoutPassed: true, + } + qualification[family] = status + } + switch entry.QualificationSplit { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && passed + status.TrainingCases++ + status.TrainingPassed = status.TrainingPassed && passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && passed + status.HoldoutCases++ + status.HoldoutPassed = status.HoldoutPassed && passed + } + } + report.Cases = append(report.Cases, entry) + } + if report.QualificationRequired { + families := make([]string, 0, len(qualification)) + for family := range qualification { + families = append(families, family) + } + sort.Strings(families) + for _, family := range families { + status := qualification[family] + status.TrainingPassed = status.TrainingPassed && status.TrainingCases > 0 + status.HoldoutPassed = status.HoldoutPassed && status.HoldoutCases > 0 + status.Passed = status.TrainingPassed && status.HoldoutPassed + report.TrainingPassed = report.TrainingPassed && status.TrainingPassed + report.HoldoutPassed = report.HoldoutPassed && status.HoldoutPassed + report.QualificationFamilies = append(report.QualificationFamilies, *status) + } + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + report.PromotionEligible = report.PromotionEligible && report.QualificationPassed + } else { + report.TrainingPassed = false + report.HoldoutPassed = false + } + return report, nil +} + +// classifyConfirmationMetric labels a confidence interval as regression, improvement, or inconclusive only when both relative and absolute noise floors are crossed. +func classifyConfirmationMetric(ratio RatioInterval, change DurationInterval, noiseRatio float64, noiseAbsolute time.Duration) ConfirmationMetric { + classification := "inconclusive" + if ratio.Lower > 1+noiseRatio && change.Lower > noiseAbsolute { + classification = "confirmed" + } + if ratio.Upper <= 1+noiseRatio && change.Upper <= noiseAbsolute { + classification = "cleared_non_inferior" + } + return ConfirmationMetric{ + Ratio: ratio, + AbsoluteChange: change, + NoiseRatio: noiseRatio, + NoiseAbsolute: noiseAbsolute, + Classification: classification, + } +} + +// bootstrapStratifiedQuantileChange estimates a quantile delta and confidence interval by resampling within matching benchmark rounds. +func bootstrapStratifiedQuantileChange(left, right roundSamples, probability float64, seed int64, options PerfGateOptions) DurationInterval { + rounds := sortedRounds(left) + estimate := durationQuantile(flattenSamples(right, rounds), probability) - durationQuantile(flattenSamples(left, rounds), probability) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + changes := make([]float64, options.BootstrapCount) + for idx := range changes { + var sampledLeft, sampledRight []time.Duration + for _, round := range rounds { + sampledLeft = append(sampledLeft, resampleDurations(rng, left[round])...) + sampledRight = append(sampledRight, resampleDurations(rng, right[round])...) + } + changes[idx] = durationQuantile(sampledRight, probability) - durationQuantile(sampledLeft, probability) + } + interval := confidenceInterval(estimate, changes, options.Confidence) + return DurationInterval{ + Estimate: time.Duration(interval.Estimate), + Lower: time.Duration(interval.Lower), + Upper: time.Duration(interval.Upper), + } +} + +// negateDurationInterval reverses interval direction and swaps its bounds so left/right arm normalization preserves a valid ordered interval. +func negateDurationInterval(value DurationInterval) DurationInterval { + return DurationInterval{ + Estimate: -value.Estimate, + Lower: -value.Upper, + Upper: -value.Lower, + } +} + +// confirmationComparable compares two confirmation records and returns every reason they cannot be paired. +func confirmationComparable(left, right []CaseResult, key performanceKey) (bool, []string) { + leftRecords := matchingRecords(left, key) + rightRecords := matchingRecords(right, key) + var reasons []string + if len(leftRecords) == 0 || len(rightRecords) == 0 { + reasons = append(reasons, "missing record") + return false, reasons + } + leftRecord, rightRecord := leftRecords[0], rightRecords[0] + reasons = append(reasons, confirmationArmConsistency(leftRecords)...) + reasons = append(reasons, confirmationArmConsistency(rightRecords)...) + if leftRecord.Status != StatusOK || rightRecord.Status != StatusOK { + reasons = append(reasons, "non-ok status") + } + if leftRecord.Fixture == nil || rightRecord.Fixture == nil || leftRecord.Fixture.Checksum != rightRecord.Fixture.Checksum { + reasons = append(reasons, "fixture checksum differs") + } + if fmt.Sprint(leftRecord.ObservedRows) != fmt.Sprint(rightRecord.ObservedRows) { + reasons = append(reasons, "exact observations differ") + } + if leftRecord.RowCount != rightRecord.RowCount { + reasons = append(reasons, "row count differs") + } + if !comparablePostgresEnvironment(leftRecord.PostgresEnvironment, rightRecord.PostgresEnvironment) { + reasons = append(reasons, "PostgreSQL settings or relation sizes differ") + } + return len(reasons) == 0, uniqueStrings(reasons) +} + +// confirmationArmConsistency reports within-arm drift in environment, executable, and normalized PostgreSQL plan shape. +func confirmationArmConsistency(records []CaseResult) []string { + if len(records) == 0 { + return []string{"missing record"} + } + + baseline := records[0] + var reasons []string + for _, record := range records[1:] { + if record.Status != StatusOK { + reasons = append(reasons, "non-ok status") + } + if record.SQLFingerprint != baseline.SQLFingerprint { + reasons = append(reasons, "SQL fingerprint changes within arm") + } + if record.Fixture == nil || baseline.Fixture == nil || record.Fixture.Checksum != baseline.Fixture.Checksum { + reasons = append(reasons, "fixture checksum differs") + } + if fmt.Sprint(record.ObservedRows) != fmt.Sprint(baseline.ObservedRows) { + reasons = append(reasons, "exact observations differ") + } + if record.RowCount != baseline.RowCount { + reasons = append(reasons, "row count differs") + } + if !comparablePostgresEnvironment(baseline.PostgresEnvironment, record.PostgresEnvironment) { + reasons = append(reasons, "PostgreSQL settings or relation sizes differ") + } + if postgresPlanShapeSHA256(record.PostgresPlan) != postgresPlanShapeSHA256(baseline.PostgresPlan) { + reasons = append(reasons, "intended plan shape changes within arm") + } + } + return reasons +} + +var ( + // volatilePlanDetails matches planner cost and runtime annotations that do not define structural plan shape. + volatilePlanDetails = regexp.MustCompile(`\s+\((?:cost|actual)[^)]*\)`) + + // volatilePlanIDs matches generated bigint constants so dataset-specific IDs do not perturb plan-shape hashes. + volatilePlanIDs = regexp.MustCompile(`'[0-9]+'::bigint`) + + // volatilePlanLine matches resource and timing summary lines excluded from structural plan-shape hashes. + volatilePlanLine = regexp.MustCompile(`^(?:Buffers|Planning Time|Execution Time):`) +) + +// postgresPlanShapeSHA256 hashes structural EXPLAIN lines after removing costs, runtime counters, transient IDs, and timing details; confirmation compares plan shape without treating volatile measurements as structural changes. +func postgresPlanShapeSHA256(plan []string) string { + digest := sha256.New() + for _, line := range plan { + line = volatilePlanDetails.ReplaceAllString(line, "") + line = volatilePlanIDs.ReplaceAllString(line, "'$id'::bigint") + line = strings.TrimSpace(line) + if line == "" || volatilePlanLine.MatchString(line) { + continue + } + fmt.Fprintln(digest, line) + } + return hex.EncodeToString(digest.Sum(nil)) +} + +// matchingRecords selects successful measured records for one dataset, case, backend, and executor identity. +func matchingRecords(records []CaseResult, key performanceKey) []CaseResult { + var matched []CaseResult + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend { + matched = append(matched, record) + } + } + return matched +} + +// comparablePostgresEnvironment requires server version and normalized settings to match while tolerating absent environment metadata on both arms. +func comparablePostgresEnvironment(left, right *PostgresEnvironment) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return left.PlanCacheMode == right.PlanCacheMode && left.TransactionIsolation == right.TransactionIsolation && left.WorkMem == right.WorkMem && left.TempFileLimit == right.TempFileLimit && + left.GraphPartitionCount == right.GraphPartitionCount && left.NodeRelationBytes == right.NodeRelationBytes && left.EdgeRelationBytes == right.EdgeRelationBytes + +} + +// uniqueStrings removes duplicate diagnostic reasons while preserving their first-seen order. +func uniqueStrings(values []string) []string { + seen := map[string]struct{}{} + result := make([]string, 0, len(values)) + for _, value := range values { + if _, found := seen[value]; found { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +// artifactArm returns the first recorded benchmark arm label, or "unknown" when an artifact lacks environment metadata. +func artifactArm(records []CaseResult) string { + for _, record := range records { + if record.Environment != nil { + return record.Environment.Arm + } + } + return "unknown" +} + +// sameExecutable identifies a true block/reload A/A treatment. A shared +// executable alone is insufficient because one GraphBench binary can emit +// different forced executors and SQL statements. +func sameExecutable(left, right []CaseResult) bool { + leftIdentity := effectiveTreatmentIdentity(left) + return leftIdentity != "" && leftIdentity == effectiveTreatmentIdentity(right) +} + +// effectiveTreatmentIdentity derives the stable identity used to compare effective treatment. +func effectiveTreatmentIdentity(records []CaseResult) string { + if len(records) == 0 || records[0].Environment == nil || records[0].Environment.BinarySHA256 == "" { + return "" + } + identity := []string{"binary=" + records[0].Environment.BinarySHA256} + for _, argument := range records[0].Environment.Invocation { + if strings.Contains(argument, "postgres-force-shortest-executor") || + strings.Contains(argument, "postgres-force-expansion-strategy") || + strings.Contains(argument, "postgres-expansion-orientation") || + strings.Contains(argument, "reference-arm") { + identity = append(identity, "option="+argument) + } + } + fingerprints := make([]string, 0, len(records)) + for _, record := range records { + fingerprints = append(fingerprints, record.Dataset+"/"+record.Name+"="+record.SQLFingerprint) + } + sort.Strings(fingerprints) + identity = append(identity, fingerprints...) + digest := sha256.Sum256([]byte(strings.Join(identity, "\n"))) + return hex.EncodeToString(digest[:]) +} + +// writeConfirmationReport emits indented JSON to stdout or atomically replaces the requested output file. +func writeConfirmationReport(path string, report ConfirmationReport) (err error) { + output := os.Stdout + if path != "" { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/confirm_report_test.go b/cmd/graphbench/confirm_report_test.go new file mode 100644 index 00000000..134c810e --- /dev/null +++ b/cmd/graphbench/confirm_report_test.go @@ -0,0 +1,245 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildConfirmationReportClassifiesFreshMatchedP95 verifies that distinct predecessor and candidate binaries with a measurable P95 increase produce a comparable causal confirmation. +func TestBuildConfirmationReportClassifiesFreshMatchedP95(t *testing.T) { + left := []CaseResult{confirmationRecord("alert", "predecessor", "binary-a", 10*time.Millisecond)} + right := []CaseResult{confirmationRecord("alert", "candidate", "binary-b", 13*time.Millisecond)} + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 7, + Confidence: 0.95, + BootstrapCount: 100, + CaseNames: []string{"alert"}, + }) + + require.NoError(t, err) + require.Equal(t, "causal_confirmation", report.Kind) + require.Equal(t, "confirmed", report.Cases[0].P95.Classification) + require.Equal(t, 3*time.Millisecond, report.Cases[0].P95.AbsoluteChange.Estimate) + require.True(t, report.Cases[0].Comparable) + require.False(t, report.PromotionEligible) +} + +// TestBuildConfirmationReportRecognizesSameBinaryBlockAA verifies that identical binaries are classified as a reload control and clear a non-inferior result. +func TestBuildConfirmationReportRecognizesSameBinaryBlockAA(t *testing.T) { + left := []CaseResult{confirmationRecord("control", "block-a", "same", 10*time.Millisecond)} + right := []CaseResult{confirmationRecord("control", "block-b", "same", 10*time.Millisecond)} + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 50, + }) + require.NoError(t, err) + require.Equal(t, "block_reload_aa", report.Kind) + require.Equal(t, "cleared_non_inferior", report.Cases[0].Disposition) +} + +// TestBuildConfirmationReportAllowsIntentionalCrossArmSQLAndPlanChanges verifies that implementation changes between predecessor and candidate arms do not invalidate an otherwise controlled comparison. +func TestBuildConfirmationReportAllowsIntentionalCrossArmSQLAndPlanChanges(t *testing.T) { + left := []CaseResult{confirmationRecord("changed", "predecessor", "binary-a", 10*time.Millisecond)} + right := []CaseResult{confirmationRecord("changed", "candidate", "binary-b", 5*time.Millisecond)} + left[0].SQLFingerprint = "incumbent-sql" + right[0].SQLFingerprint = "candidate-sql" + left[0].PostgresPlan = []string{"CTE Scan on incumbent"} + right[0].PostgresPlan = []string{"Recursive Union"} + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 50, + CaseNames: []string{"changed"}, + }) + require.NoError(t, err) + require.True(t, report.Cases[0].Comparable) + require.True(t, report.PromotionEligible) +} + +// TestConfirmationComparableRejectsFingerprintChangeWithinArm verifies that SQL drift among repetitions of one arm makes the confirmation comparison invalid. +func TestConfirmationComparableRejectsFingerprintChangeWithinArm(t *testing.T) { + left := []CaseResult{ + confirmationRecord("changed", "predecessor", "binary-a", 10*time.Millisecond), + confirmationRecord("changed", "predecessor", "binary-a", 10*time.Millisecond), + } + right := []CaseResult{confirmationRecord("changed", "candidate", "binary-b", 5*time.Millisecond)} + left[1].SQLFingerprint = "unstable-sql" + + comparable, reasons := confirmationComparable(left, right, performanceKey{ + dataset: left[0].Dataset, + name: "changed", + backend: ModePostgresSQL, + }) + require.False(t, comparable) + require.Contains(t, reasons, "SQL fingerprint changes within arm") +} + +// TestPostgresPlanShapeIgnoresReloadedEntityIDs verifies that literal database IDs and timing noise do not alter the normalized PostgreSQL plan fingerprint. +func TestPostgresPlanShapeIgnoresReloadedEntityIDs(t *testing.T) { + left := []string{"Index Cond: (id = '4624444'::bigint)", "Planning Time: 0.408 ms", "Execution Time: 0.224 ms"} + right := []string{"Index Cond: (id = '4630087'::bigint)", "Planning Time: 0.189 ms", "Execution Time: 0.093 ms"} + require.Equal(t, postgresPlanShapeSHA256(left), postgresPlanShapeSHA256(right)) +} + +// TestBuildConfirmationReportRejectsUnknownExactCase verifies that an exact selector must resolve to an observed case instead of yielding an empty confirmation report. +func TestBuildConfirmationReportRejectsUnknownExactCase(t *testing.T) { + record := confirmationRecord("present", "arm", "binary", time.Millisecond) + _, err := buildConfirmationReport([]CaseResult{record}, []CaseResult{record}, nil, ConfirmationOptions{ + Seed: 1, + Confidence: 0.95, + BootstrapCount: 10, + CaseNames: []string{"missing"}, + }) + require.ErrorContains(t, err, "unknown confirmation case") +} + +// TestBuildConfirmationReportRequiresHostAAForCausalPromotion verifies a fresh binary comparison fails closed without per-case host calibration. +func TestBuildConfirmationReportRequiresHostAAForCausalPromotion(t *testing.T) { + left := []CaseResult{confirmationRecord("changed", "predecessor", "binary-a", time.Millisecond)} + right := []CaseResult{confirmationRecord("changed", "candidate", "binary-b", 900*time.Microsecond)} + stampPairedEvidence(left, right, 20) + + _, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + CaseNames: []string{"changed"}, + }) + + require.ErrorContains(t, err, "host A/A resolution report is required") +} + +// TestSameExecutableRequiresSameEffectiveTreatment verifies same executable requires same effective treatment behavior. +func TestSameExecutableRequiresSameEffectiveTreatment(t *testing.T) { + left := []CaseResult{confirmationRecord("changed", "a1", "shared-binary", time.Millisecond)} + right := []CaseResult{confirmationRecord("changed", "i1", "shared-binary", time.Millisecond)} + left[0].SQLFingerprint = "a1-sql" + right[0].SQLFingerprint = "i1-sql" + require.False(t, sameExecutable(left, right)) + + right[0].SQLFingerprint = left[0].SQLFingerprint + right[0].Environment.Invocation = append(right[0].Environment.Invocation, "--postgres-force-shortest-executor=ASP-I1-U-DAG+MAT-M0") + require.False(t, sameExecutable(left, right)) + + right[0].Environment.Invocation = append([]string(nil), left[0].Environment.Invocation...) + require.True(t, sameExecutable(left, right)) +} + +// TestBuildConfirmationReportKeepsStressTimingDiagnostic verifies stress comparisons remain descriptive and need no promotion calibration. +func TestBuildConfirmationReportKeepsStressTimingDiagnostic(t *testing.T) { + left := []CaseResult{confirmationRecord("stress", "predecessor", "binary-a", time.Millisecond)} + right := []CaseResult{confirmationRecord("stress", "candidate", "binary-b", 10*time.Millisecond)} + left[0].Shape.FixtureTier = "stress" + right[0].Shape.FixtureTier = "stress" + + report, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + CaseNames: []string{"stress"}, + }) + + require.NoError(t, err) + require.False(t, report.PromotionEligible) + require.False(t, report.Cases[0].TimingGated) + require.Equal(t, "stress_diagnostic", report.Cases[0].Disposition) +} + +// TestBuildConfirmationReportKeepsDiagnosticSplitOutOfPromotion verifies a +// normal-tier boundary case remains evaluation-only by declaration. +func TestBuildConfirmationReportKeepsDiagnosticSplitOutOfPromotion(t *testing.T) { + left := []CaseResult{confirmationRecord("boundary", "predecessor", "binary-a", time.Millisecond)} + right := []CaseResult{confirmationRecord("boundary", "candidate", "binary-b", 10*time.Millisecond)} + left[0].Shape.QualificationSplit = "diagnostic" + right[0].Shape.QualificationSplit = "diagnostic" + + report, err := buildConfirmationReport(left, right, nil, ConfirmationOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + CaseNames: []string{"boundary"}, + }) + + require.NoError(t, err) + require.False(t, report.PromotionEligible) + require.False(t, report.Cases[0].TimingGated) + require.Equal(t, "qualification_diagnostic", report.Cases[0].Disposition) +} + +// TestBuildConfirmationReportRequiresIndependentTraversalHoldout verifies a +// clean training result cannot qualify a traversal candidate without an +// independently named frozen-holdout case. +func TestBuildConfirmationReportRequiresIndependentTraversalHoldout(t *testing.T) { + left := []CaseResult{ + confirmationRecord("sp-training", "predecessor", "binary-a", 10*time.Millisecond), + confirmationRecord("sp-holdout", "predecessor", "binary-a", 10*time.Millisecond), + } + right := []CaseResult{ + confirmationRecord("sp-training", "candidate", "binary-b", 5*time.Millisecond), + confirmationRecord("sp-holdout", "candidate", "binary-b", 5*time.Millisecond), + } + for _, records := range [][]CaseResult{left, right} { + records[0].Category = "generated_shortest_path_v2" + records[0].Shape.QualificationSplit = "training" + records[1].Category = "generated_shortest_path_v2" + records[1].Shape.QualificationSplit = "holdout" + } + stampPairedEvidence(left, right, 20) + + report, err := buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + CaseNames: []string{"sp-training", "sp-holdout"}, + }) + require.NoError(t, err) + require.True(t, report.QualificationRequired) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.True(t, report.PromotionEligible) + + left = left[:1] + right = right[:1] + report, err = buildConfirmationReport(left, right, testAAReportForRecords(t, left), ConfirmationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + CaseNames: []string{"sp-training"}, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.False(t, report.PromotionEligible) +} + +// confirmationRecord returns a stable PostgreSQL observation annotated with the requested arm and binary identity. +func confirmationRecord(name, arm, binary string, duration time.Duration) CaseResult { + record := perfGateRecord(name, ModePostgresSQL, duration, 10, 50) + record.SQLFingerprint = "sql" + record.ObservedRows = []string{"[1]"} + record.Fixture = &FixtureMetadata{Checksum: "fixture"} + record.Environment = &RunEnvironment{ + Arm: arm, + BinarySHA256: binary, + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + } + return record +} diff --git a/cmd/graphbench/corpus.go b/cmd/graphbench/corpus.go index 7d1c9075..1fc5dea5 100644 --- a/cmd/graphbench/corpus.go +++ b/cmd/graphbench/corpus.go @@ -21,9 +21,12 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" + "strings" ) +// loadScaleCorpus loads all scale-case JSON files and rejects duplicate or invalid declarations. func loadScaleCorpus(root string) (ScaleCorpus, error) { casePaths, err := filepath.Glob(filepath.Join(root, "cases", "*.json")) if err != nil { @@ -45,6 +48,7 @@ func loadScaleCorpus(root string) (ScaleCorpus, error) { source := filepath.ToSlash(path) for idx, testCase := range file.Cases { testCase.Source = source + normalizeFallbackExpectation(&testCase) if err := validateScaleCase(testCase); err != nil { return ScaleCorpus{}, fmt.Errorf("%s case %d: %w", source, idx, err) } @@ -56,6 +60,25 @@ func loadScaleCorpus(root string) (ScaleCorpus, error) { return corpus, nil } +// normalizeFallbackExpectation normalizes fallback expectation. +func normalizeFallbackExpectation(testCase *ScaleCase) { + if testCase == nil || testCase.Shape.FallbackExpectation != "" || !requiresQualificationSplit(*testCase) { + return + } + testCase.Shape.FallbackExpectation = "forbidden" + if testCase.Shape.FixtureTier == "stress" { + testCase.Shape.FallbackExpectation = "allowed" + } + for _, tag := range testCase.Tags { + normalized := strings.ToLower(tag) + if strings.Contains(normalized, "fallback") || strings.Contains(normalized, "overflow") { + testCase.Shape.FallbackExpectation = "required" + return + } + } +} + +// validateScaleCase checks case identity, modes, parameters, expectations, and workload shape. func validateScaleCase(testCase ScaleCase) error { if testCase.Name == "" { return fmt.Errorf("name is required") @@ -78,10 +101,141 @@ func validateScaleCase(testCase ScaleCase) error { return fmt.Errorf("unsupported candidate mode %q", mode) } } + for mode, reason := range testCase.UnsupportedModes { + if !mode.Valid() { + return fmt.Errorf("invalid unsupported mode %q", mode) + } + if reason == "" { + return fmt.Errorf("unsupported mode %q requires a reason", mode) + } + if testCase.Supports(mode) { + return fmt.Errorf("mode %q cannot be both candidate and unsupported", mode) + } + } + if testCase.Shape.RelationshipKindCount < 0 { + return fmt.Errorf("shape.relationship_kind_count must not be negative") + } + if tier := testCase.Shape.FixtureTier; tier != "" && tier != "normal" && tier != "envelope" && tier != "stress" { + return fmt.Errorf("shape.fixture_tier must be normal, envelope, or stress") + } + if split := testCase.Shape.QualificationSplit; split != "" && split != "training" && split != "holdout" && split != "diagnostic" { + return fmt.Errorf("shape.qualification_split must be training, holdout, or diagnostic") + } + if role := testCase.Shape.QualificationRole; role != "" && role != "adverse_control" && role != "efficacy_target" { + return fmt.Errorf("shape.qualification_role must be adverse_control or efficacy_target") + } + if slices.Contains(testCase.Tags, "sp-i2-distance-v2-training") || slices.Contains(testCase.Tags, "sp-i2-distance-v2-holdout") { + if testCase.Shape.QualificationRole == "" { + return fmt.Errorf("formal SP-I2 V2 cases require shape.qualification_role") + } + } + if expectation := testCase.Shape.FallbackExpectation; expectation != "" && expectation != "forbidden" && expectation != "required" && expectation != "allowed" { + return fmt.Errorf("shape.fallback_expectation must be forbidden, required, or allowed") + } + if requiresQualificationSplit(testCase) && testCase.Shape.QualificationSplit == "" { + return fmt.Errorf("shape.qualification_split is required for traversal qualification cases") + } + if testCase.Shape.FixtureTier == "stress" && testCase.Shape.QualificationSplit != "diagnostic" && requiresQualificationSplit(testCase) { + return fmt.Errorf("stress traversal qualification cases must use shape.qualification_split diagnostic") + } + if slices.Contains(testCase.Tags, "holdout") && testCase.Shape.QualificationSplit != "holdout" { + return fmt.Errorf("holdout-tagged cases must use shape.qualification_split holdout") + } + if testCase.Shape.QualificationSplit == "holdout" && !slices.Contains(testCase.Tags, "holdout") { + return fmt.Errorf("shape.qualification_split holdout requires the holdout tag") + } + if direction := testCase.Shape.Direction; direction != "" && direction != "outbound" && direction != "inbound" && direction != "directionless" && direction != "mirrored" { + return fmt.Errorf("shape.direction must be outbound, inbound, directionless, or mirrored") + } + + if len(testCase.Expected.IDRows) > 0 { + if testCase.Expected.ResultKind != "id_rows" { + return fmt.Errorf("expected.id_rows requires result_kind id_rows") + } + if testCase.Expected.RowCount == nil || int64(len(testCase.Expected.IDRows)) != *testCase.Expected.RowCount { + return fmt.Errorf("expected.id_rows must contain exactly row_count rows") + } + } + if len(testCase.Expected.PathRows) > 0 { + if testCase.Expected.ResultKind != "path_set" { + return fmt.Errorf("expected.path_rows requires result_kind path_set") + } + if testCase.Expected.RowCount == nil || int64(len(testCase.Expected.PathRows)) != *testCase.Expected.RowCount { + return fmt.Errorf("expected.path_rows must contain exactly row_count rows") + } + for idx, path := range testCase.Expected.PathRows { + if len(path.Nodes) != len(path.RelationshipKinds)+1 { + return fmt.Errorf("expected.path_rows[%d] must have one more node than relationship kind", idx) + } + if slices.Contains(testCase.Tags, "fixed-suffix-expansion-v3") && len(path.RelationshipKeys) != len(path.RelationshipKinds) { + return fmt.Errorf("fixed-suffix v3 expected.path_rows[%d] must identify every relationship", idx) + } + } + } + if slices.Contains(testCase.Tags, "fixed-suffix-expansion-v3") && testCase.Expected.ResultKind == "path_set" && len(testCase.Expected.PathRows) == 0 { + return fmt.Errorf("fixed-suffix v3 path_set cases require exact expected.path_rows") + } + + if testCase.WriteScenario != nil { + if err := validateWriteScenario(*testCase.WriteScenario); err != nil { + return err + } + } + + return nil +} + +// requiresQualificationSplit identifies traversal-program declarations whose +// training/holdout boundary is part of their immutable workload identity. +// Older general-purpose scale cases remain loadable while each prioritized +// traversal family is migrated deliberately. +func requiresQualificationSplit(testCase ScaleCase) bool { + switch testCase.Category { + case "generated_shortest_path_v2", "expand_into_one_hop", "generated_endpoint_seeded_expansion": + return true + case "generated_fixed_suffix_expansion": + return slices.Contains(testCase.Tags, "fixed-suffix-expansion-v2") || + slices.Contains(testCase.Tags, "fixed-suffix-expansion-v3") || + slices.Contains(testCase.Tags, "fixed-suffix-expansion-boundary") + default: + return slices.Contains(testCase.Tags, "traversal-qualification") + } +} + +// validateWriteScenario checks mutation expectations and post-state query completeness. +func validateWriteScenario(scenario WriteScenario) error { + if scenario.SelectionCypher == "" { + return fmt.Errorf("write_scenario.selection_cypher is required") + } + if scenario.ExpectedMatched == nil { + return fmt.Errorf("write_scenario.expected_matched is required") + } + if scenario.ExpectedAffected == nil { + return fmt.Errorf("write_scenario.expected_affected is required") + } + if scenario.AffectedEntity != "node" && scenario.AffectedEntity != "relationship" { + return fmt.Errorf("write_scenario.affected_entity must be node or relationship") + } + if len(scenario.PostState) == 0 { + return fmt.Errorf("write_scenario.post_state is required") + } + + for idx, postState := range scenario.PostState { + if postState.Name == "" { + return fmt.Errorf("write_scenario.post_state[%d].name is required", idx) + } + if postState.Cypher == "" { + return fmt.Errorf("write_scenario.post_state[%d].cypher is required", idx) + } + if postState.Expected.RowCount == nil && postState.Expected.ScalarInt == nil { + return fmt.Errorf("write_scenario.post_state[%d].expected requires row_count or scalar_int", idx) + } + } return nil } +// decodeJSONFile reads a JSON file and decodes it into the supplied destination. func decodeJSONFile(path string, target any) error { raw, err := os.ReadFile(path) if err != nil { @@ -94,6 +248,7 @@ func decodeJSONFile(path string, target any) error { return nil } +// scaleCorpusDatasets returns unique corpus dataset names in sorted order. func scaleCorpusDatasets(corpus ScaleCorpus) []string { var ( seen = map[string]struct{}{} @@ -113,6 +268,7 @@ func scaleCorpusDatasets(corpus ScaleCorpus) []string { return datasets } +// scaleCasesByDataset indexes scale cases by dataset while preserving corpus order. func scaleCasesByDataset(corpus ScaleCorpus) map[string][]ScaleCase { grouped := map[string][]ScaleCase{} for _, testCase := range corpus.Cases { diff --git a/cmd/graphbench/corpus_test.go b/cmd/graphbench/corpus_test.go index 211b2084..231071bd 100644 --- a/cmd/graphbench/corpus_test.go +++ b/cmd/graphbench/corpus_test.go @@ -17,11 +17,15 @@ package main import ( + "fmt" "testing" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/testutil" "github.com/stretchr/testify/require" ) +// TestLoadScaleCorpus verifies that every loaded case identifies its source, declares PostgreSQL support status, and excludes the reference-only AGE mode. func TestLoadScaleCorpus(t *testing.T) { corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") require.NoError(t, err) @@ -29,17 +33,286 @@ func TestLoadScaleCorpus(t *testing.T) { for _, testCase := range corpus.Cases { require.NotEqual(t, "", testCase.Source) - require.True(t, testCase.Supports(ModePostgresSQL), "postgres_sql should be part of the initial corpus for %s", testCase.Name) + _, explicitlyUnsupported := testCase.UnsupportedReason(ModePostgresSQL) + require.True(t, testCase.Supports(ModePostgresSQL) || explicitlyUnsupported, + "postgres_sql should be a candidate or explicitly unsupported for %s", testCase.Name) require.False(t, testCase.Supports(ExecutionMode("age")), "AGE is a reference design only for %s", testCase.Name) } } -func TestScaleCorpusDatasets(t *testing.T) { - corpus := ScaleCorpus{Cases: []ScaleCase{ - {Name: "a", Dataset: "base", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, - {Name: "b", Dataset: "adcs_fanout", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, - {Name: "c", Dataset: "base", Category: "counts", Cypher: "return 1", CandidateModes: []ExecutionMode{ModePostgresSQL}}, +// TestValidateScaleCaseRequiresConsistentUnsupportedModes verifies that a backend cannot be both runnable and unsupported and that every exclusion has a reason. +func TestValidateScaleCaseRequiresConsistentUnsupportedModes(t *testing.T) { + testCase := ScaleCase{ + Name: "directionless", + Dataset: "base", + Category: "shortest_path", + Cypher: "MATCH p = shortestPath((a)-[*]-(b)) RETURN p", + CandidateModes: []ExecutionMode{ModeNeo4j}, + UnsupportedModes: map[ExecutionMode]string{ModePostgresSQL: "translator does not support this form"}, + } + + require.NoError(t, validateScaleCase(testCase)) + testCase.CandidateModes = append(testCase.CandidateModes, ModePostgresSQL) + require.ErrorContains(t, validateScaleCase(testCase), "both candidate and unsupported") + testCase.CandidateModes = []ExecutionMode{ModeNeo4j} + testCase.UnsupportedModes[ModePostgresSQL] = "" + require.ErrorContains(t, validateScaleCase(testCase), "requires a reason") +} + +// TestValidateScaleCaseFreezesTraversalQualificationSplit verifies prioritized +// traversal cases cannot silently move between training, holdout, and +// diagnostic evidence after selector thresholds are chosen. +func TestValidateScaleCaseFreezesTraversalQualificationSplit(t *testing.T) { + testCase := ScaleCase{ + Name: "qualified", + Dataset: "generated", + Category: "generated_shortest_path_v2", + Cypher: "MATCH p = shortestPath((s)-[*]->(e)) RETURN p", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + Shape: WorkloadShape{FixtureTier: "normal"}, + } + + require.ErrorContains(t, validateScaleCase(testCase), "qualification_split is required") + testCase.Shape.QualificationSplit = "training" + require.NoError(t, validateScaleCase(testCase)) + + testCase.Tags = []string{"holdout"} + require.ErrorContains(t, validateScaleCase(testCase), "holdout-tagged") + testCase.Shape.QualificationSplit = "holdout" + require.NoError(t, validateScaleCase(testCase)) + + testCase.Tags = nil + require.ErrorContains(t, validateScaleCase(testCase), "requires the holdout tag") + testCase.Shape = WorkloadShape{ + FixtureTier: "stress", + QualificationSplit: "training", + } + require.ErrorContains(t, validateScaleCase(testCase), "stress traversal") + testCase.Shape.QualificationSplit = "diagnostic" + require.NoError(t, validateScaleCase(testCase)) +} + +// TestValidateScaleCaseRequiresExactFixedSuffixV3Paths prevents a costly v2 +// capture from reaching report time without an independent stable path oracle. +func TestValidateScaleCaseRequiresExactFixedSuffixV3Paths(t *testing.T) { + rowCount := int64(1) + testCase := ScaleCase{ + Name: "v3-path", + Dataset: "generated", + Category: "generated_fixed_suffix_expansion", + Cypher: "MATCH p = (s)-[*]->(e) RETURN p", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + Tags: []string{"fixed-suffix-expansion-v3"}, + Shape: WorkloadShape{ + FixtureTier: "normal", + QualificationSplit: "training", + }, + Expected: ExpectedResult{ + RowCount: &rowCount, + ResultKind: "path_set", + }, + } + + require.ErrorContains(t, validateScaleCase(testCase), "require exact expected.path_rows") + testCase.Expected.PathRows = []ExpectedPath{{ + Nodes: []string{"s", "e"}, + RelationshipKinds: []string{"Expand"}, }} + require.ErrorContains(t, validateScaleCase(testCase), "identify every relationship") + testCase.Expected.PathRows[0].RelationshipKeys = []string{"expand-1"} + require.NoError(t, validateScaleCase(testCase)) +} + +// TestScaleCorpusDatasets verifies that corpus dataset discovery removes repeated names and returns a deterministic lexical order. +func TestScaleCorpusDatasets(t *testing.T) { + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "a", + Dataset: "base", + Category: "counts", + Cypher: "return 1", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "b", + Dataset: "fixed_suffix_expansion_fanout", + Category: "counts", + Cypher: "return 1", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "c", + Dataset: "base", + Category: "counts", + Cypher: "return 1", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + require.Equal(t, []string{"base", "fixed_suffix_expansion_fanout"}, scaleCorpusDatasets(corpus)) +} + +// TestGeneratedReconciliationDatasetRegistersThirtyKinds verifies that reconciliation generation exposes every RecKind01 through RecKind30 relationship kind. +func TestGeneratedReconciliationDatasetRegistersThirtyKinds(t *testing.T) { + doc, err := parseDataset("unused", testutil.ReconciliationScaleDataset) + require.NoError(t, err) + _, edgeKinds := doc.Graph.Kinds() + + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("RecKind%02d", idx))) + } +} + +// TestGeneratedTrustPruningDatasetRegistersProductionShapes verifies that trust-pruning fixtures contain the domain, candidate, same-forest, cross-forest, and batch labels used by production queries. +func TestGeneratedTrustPruningDatasetRegistersProductionShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.TrustPruningScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("Domain")) + require.Contains(t, nodeKinds, graph.StringKind("PruneCandidate")) + require.Contains(t, edgeKinds, graph.StringKind("SameForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("CrossForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("PruneBatch")) +} + +// TestGeneratedHopDatasetRegistersThirtyKindsAndEndpointSets verifies that hop fixtures expose both endpoint node classes, all thirty numbered relationship kinds, and the set-membership edge. +func TestGeneratedHopDatasetRegistersThirtyKindsAndEndpointSets(t *testing.T) { + doc, err := parseDataset("unused", testutil.HopScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("HopIDEndpoint")) + require.Contains(t, nodeKinds, graph.StringKind("HopTemplate")) + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("HopKind%02d", idx))) + } + require.Contains(t, edgeKinds, graph.StringKind("HopSetEdge")) +} + +// TestGeneratedScanLookupDatasetRegistersWideAndLargeShapes verifies that scan fixtures contain the base, role, and hydration nodes plus every relationship kind used by wide lookup plans. +func TestGeneratedScanLookupDatasetRegistersWideAndLargeShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.ScanLookupScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("ADBase")) + require.Contains(t, nodeKinds, graph.StringKind("AZRole")) + require.Contains(t, nodeKinds, graph.StringKind("Hydrate")) + require.Contains(t, edgeKinds, graph.StringKind("ScanPostProcessed")) + require.Contains(t, edgeKinds, graph.StringKind("Contains")) + for idx := 1; idx <= 9; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("ScanEdge%02d", idx))) + } +} + +// TestGeneratedShortestPathDatasetRegistersMatrixShapes verifies that shortest-path generation produces nonempty nodes and both generic and typed traversal relationships. +func TestGeneratedShortestPathDatasetRegistersMatrixShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.ShortestPathScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() + + require.Contains(t, nodeKinds, graph.StringKind("ShortestNode")) + require.Contains(t, edgeKinds, graph.StringKind("Traverse")) + require.Contains(t, edgeKinds, graph.StringKind("TypedTraverse")) + require.NotEmpty(t, doc.Graph.Nodes) +} + +// TestGeneratedFixedSuffixExpansionDatasetRegistersSuffixAndDecoyShapes verifies that fixed-suffix fixtures register all path stages and the wrong-entry decoy needed to detect over-broad matching. +func TestGeneratedFixedSuffixExpansionDatasetRegistersSuffixAndDecoyShapes(t *testing.T) { + doc, err := parseDataset("unused", testutil.FixedSuffixExpansionScaleDataset) + require.NoError(t, err) + nodeKinds, edgeKinds := doc.Graph.Kinds() - require.Equal(t, []string{"adcs_fanout", "base"}, scaleCorpusDatasets(corpus)) + for _, kind := range []string{"ExpansionRoot", "ExpansionNode", "SuffixHead", "SuffixMiddle", "SuffixTerminal"} { + require.Contains(t, nodeKinds, graph.StringKind(kind)) + } + for _, kind := range []string{"Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix", "WrongEnterSuffix"} { + require.Contains(t, edgeKinds, graph.StringKind(kind)) + } +} + +// TestValidateScaleCaseRequiresCompleteWriteScenario verifies that destructive cases include at least one post-state assertion after selection and affected-count expectations. +func TestValidateScaleCaseRequiresCompleteWriteScenario(t *testing.T) { + zero := int64(0) + testCase := ScaleCase{ + Name: "write", + Dataset: "base", + Category: "delete", + Cypher: "MATCH (n) DELETE n", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + WriteScenario: &WriteScenario{ + SelectionCypher: "MATCH (n) RETURN n", + AffectedEntity: "node", + ExpectedMatched: &zero, + ExpectedAffected: &zero, + PostState: []ScaleStateQuery{{ + Name: "survivors", + Cypher: "MATCH (n) RETURN n", + Expected: ExpectedResult{RowCount: &zero}, + }}, + }, + } + + require.NoError(t, validateScaleCase(testCase)) + testCase.WriteScenario.PostState = nil + require.ErrorContains(t, validateScaleCase(testCase), "post_state is required") +} + +// TestSelectScaleCorpusUsesExactSelectorsAndMarksDiagnostics verifies that partial dataset/tag selection records omitted declarations, marks the manifest diagnostic-only, and rejects unresolved exact selectors. +func TestSelectScaleCorpusUsesExactSelectorsAndMarksDiagnostics(t *testing.T) { + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "lookup", + Dataset: "base", + Category: "lookup", + Tags: []string{"primary"}, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "control", + Dataset: "base", + Category: "lookup", + Tags: []string{"control"}, + CandidateModes: []ExecutionMode{ModePostgresSQL, ModeNeo4j}, + }, + { + Name: "other", + Dataset: "other", + Category: "count", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + selected, manifest, err := selectScaleCorpus(corpus, CorpusSelectors{ + Datasets: []string{"base"}, + Tags: []string{"primary", "control"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 2) + require.True(t, manifest.DiagnosticOnly) + require.Equal(t, 1, manifest.OmittedDeclarationCount) + require.NotEmpty(t, manifest.DeclarationSHA256) + + _, _, err = selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"missing"}}) + require.ErrorContains(t, err, "unknown case selector") +} + +// TestSelectScaleCorpusRejectsAmbiguousExactNames verifies that a bare case selector cannot choose between identically named cases from different datasets. +func TestSelectScaleCorpusRejectsAmbiguousExactNames(t *testing.T) { + corpus := ScaleCorpus{ + Cases: []ScaleCase{{ + Name: "same", + Dataset: "one", + }, { + Name: "same", + Dataset: "two", + }}, + } + _, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"same"}}) + require.ErrorContains(t, err, "ambiguous case selector") } diff --git a/cmd/graphbench/datasets.go b/cmd/graphbench/datasets.go index af400ca8..e08abca9 100644 --- a/cmd/graphbench/datasets.go +++ b/cmd/graphbench/datasets.go @@ -18,16 +18,24 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" "fmt" "os" "path/filepath" + "strings" + "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) +// defaultGraphName names the isolated graph populated with benchmark fixtures. const defaultGraphName = "integration_test" +// scanDatasetKinds enumerates dataset kinds without changing the source data. func scanDatasetKinds(datasetDir string, datasetNames []string) (graph.Kinds, graph.Kinds, error) { var nodeKinds, edgeKinds graph.Kinds @@ -45,7 +53,12 @@ func scanDatasetKinds(datasetDir string, datasetNames []string) (graph.Kinds, gr return nodeKinds, edgeKinds, nil } +// parseDataset decodes a fixture document or dispatches to the requested generated dataset builder. func parseDataset(datasetDir, name string) (opengraph.Document, error) { + if fixture := generatedDataset(name); fixture != nil { + return opengraph.Document{Graph: *fixture}, nil + } + path := filepath.Join(datasetDir, name+".json") f, err := os.Open(path) if err != nil { @@ -61,7 +74,12 @@ func parseDataset(datasetDir, name string) (opengraph.Document, error) { return doc, nil } +// loadDataset decodes and loads a named fixture dataset into an empty graph. func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string) (opengraph.IDMap, error) { + if fixture := generatedDataset(name); fixture != nil { + return opengraph.WriteGraph(ctx, db, fixture) + } + path := filepath.Join(datasetDir, name+".json") f, err := os.Open(path) if err != nil { @@ -77,12 +95,651 @@ func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string return idMap, nil } +// generatedDataset constructs a named generated fixture and its shape-specific expectations. +func generatedDataset(name string) *opengraph.Graph { + if config, ok := parseEndpointSeededExpansionDatasetName(name); ok { + return testutil.NewEndpointSeededExpansionScaleFixture(config) + } + if config, ok := parseShortestPathV2DatasetName(name); ok { + return testutil.NewShortestPathScaleV2Fixture(config) + } + var shortestDepth, shortestFanout int + if matched, _ := fmt.Sscanf(name, testutil.ShortestPathScaleDataset+"_d%d_f%d", &shortestDepth, &shortestFanout); matched == 2 && shortestDepth >= 1 && shortestFanout >= 1 && name == fmt.Sprintf(testutil.ShortestPathScaleDataset+"_d%d_f%d", shortestDepth, shortestFanout) { + return testutil.NewShortestPathScaleFixture(testutil.ShortestPathScaleConfig{ + Depth: shortestDepth, + Fanout: shortestFanout, + }) + } + var expansionDepth, expansionFanout, validSuffixEvery, expansionPayload int + if matched, _ := fmt.Sscanf(name, testutil.FixedSuffixExpansionScaleDataset+"_d%d_f%d_v%d_p%d", &expansionDepth, &expansionFanout, &validSuffixEvery, &expansionPayload); matched == 4 && expansionDepth >= 0 && expansionFanout >= 1 && validSuffixEvery >= 1 && expansionPayload >= 0 && name == fmt.Sprintf(testutil.FixedSuffixExpansionScaleDataset+"_d%d_f%d_v%d_p%d", expansionDepth, expansionFanout, validSuffixEvery, expansionPayload) { + return testutil.NewFixedSuffixExpansionScaleFixture(testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: expansionDepth, + Fanout: expansionFanout, + ValidSuffixEvery: validSuffixEvery, + PropertyPayloadSize: expansionPayload, + }) + } + if config, ok := parseFixedSuffixExpansionV3DatasetName(name); ok { + return testutil.NewFixedSuffixExpansionScaleFixture(config) + } + if config, ok := parseFixedSuffixExpansionV2DatasetName(name); ok { + return testutil.NewFixedSuffixExpansionScaleFixture(config) + } + switch name { + case testutil.ReconciliationScaleDataset: + return testutil.NewReconciliationScaleFixture(128) + case testutil.TrustPruningScaleDataset: + return testutil.NewTrustPruningScaleFixture(128) + case testutil.HopScaleDataset: + return testutil.NewHopScaleFixture(128) + case testutil.ScanLookupScaleDataset: + return testutil.NewScanLookupScaleFixture(128) + case testutil.ShortestPathScaleDataset: + return testutil.NewShortestPathScaleFixture(testutil.ShortestPathScaleConfig{ + Depth: 16, + Fanout: 128, + }) + case testutil.FixedSuffixExpansionScaleDataset: + return testutil.NewFixedSuffixExpansionScaleFixture(testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 8, + Fanout: 100, + ValidSuffixEvery: 10, + PropertyPayloadSize: 4096, + }) + default: + return nil + } +} + +// FixtureMetadata captures fixture cardinalities, checksums, and generated-shape expectations. +type FixtureMetadata struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Checksum identifies the fixture's canonical logical node and relationship contents. + Checksum string `json:"checksum"` + // NodeCount records logical fixture nodes declared or loaded. + NodeCount int `json:"node_count"` + // EdgeCount records logical fixture relationships declared or loaded. + EdgeCount int `json:"edge_count"` + // PhysicalValidated reports whether live database counts and checksum matched fixture metadata. + PhysicalValidated bool `json:"physical_cardinality_validated,omitempty"` + // PhysicalNodeCount records physical node rows present in the backend fixture. + PhysicalNodeCount int64 `json:"physical_node_count,omitempty"` + // PhysicalEdgeCount records physical relationship rows present in the backend fixture. + PhysicalEdgeCount int64 `json:"physical_edge_count,omitempty"` + // NodeRelationBytes supplies the node relation bytes input to the FixtureMetadata contract. + NodeRelationBytes int64 `json:"node_relation_bytes,omitempty"` + // EdgeRelationBytes supplies the edge relation bytes input to the FixtureMetadata contract. + EdgeRelationBytes int64 `json:"edge_relation_bytes,omitempty"` + // Configuration captures the generator parameters that define the fixture shape. + Configuration string `json:"configuration,omitempty"` + // Shortest contains expectations derived from a generated shortest-path fixture. + Shortest *ShortestFixtureExpectations `json:"shortest,omitempty"` + // FixedSuffixExpansion contains expectations derived from a fixed-suffix expansion fixture. + FixedSuffixExpansion *FixedSuffixExpansionFixtureExpectations `json:"fixed_suffix_expansion,omitempty"` + // EndpointSeededExpansion contains expectations derived from an endpoint-seeded expansion fixture. + EndpointSeededExpansion *EndpointSeededExpansionFixtureExpectations `json:"endpoint_seeded_expansion,omitempty"` +} + +// ShortestFixtureExpectations records expected distances, witnesses, and intermediate state for shortest-path fixtures. +type ShortestFixtureExpectations struct { + // RootForwardDegree records outgoing relationships incident to the traversal root. + RootForwardDegree int64 `json:"root_forward_degree"` + // RootReverseDegree records incoming relationships incident to the traversal root. + RootReverseDegree int64 `json:"root_reverse_degree"` + // MaximumIntermediateForwardByLevel maps traversal depth to the largest expected forward frontier. + MaximumIntermediateForwardByLevel map[string]int64 `json:"maximum_intermediate_forward_by_level"` + // MaximumIntermediateReverseByLevel maps traversal depth to the largest expected reverse frontier. + MaximumIntermediateReverseByLevel map[string]int64 `json:"maximum_intermediate_reverse_by_level"` + // PhysicalTraversableEdgesByKind maps relationship kind to physical traversable edge count. + PhysicalTraversableEdgesByKind map[string]int64 `json:"physical_traversable_edges_by_kind"` + // DistinctReachableNodesByLevel maps traversal depth to distinct reachable node count. + DistinctReachableNodesByLevel map[string]int64 `json:"distinct_reachable_nodes_by_level"` + // ExpectedMinimumDistance supplies the expected minimum distance input to the ShortestFixtureExpectations contract. + ExpectedMinimumDistance int64 `json:"expected_minimum_distance"` + // ExpectedOnePathCardinality supplies the expected one path cardinality input to the ShortestFixtureExpectations contract. + ExpectedOnePathCardinality int64 `json:"expected_one_path_cardinality"` + // ExpectedAllShortestCardinality supplies the expected all shortest cardinality input to the ShortestFixtureExpectations contract. + ExpectedAllShortestCardinality int64 `json:"expected_all_shortest_cardinality"` + // ExpectedPredecessorEdges records predecessor edges expected in the shortest-path DAG. + ExpectedPredecessorEdges int64 `json:"expected_relationship_distinct_predecessor_edges"` + // DisconnectedStateCardinality records recursive states belonging to disconnected shortest-path regions. + DisconnectedStateCardinality int64 `json:"disconnected_state_cardinality"` + // ParallelPhysicalEdges records physical parallel relationships in the generated fixture. + ParallelPhysicalEdges int64 `json:"parallel_physical_edges"` + // ParallelDistinctTargets records distinct targets reached by parallel fixture edges. + ParallelDistinctTargets int64 `json:"parallel_distinct_targets"` +} + +// FixedSuffixExpansionFixtureExpectations records expected state and output sizes for fixed-suffix expansion fixtures. +type FixedSuffixExpansionFixtureExpectations struct { + // RootSourceRows records rows selected as expansion roots. + RootSourceRows int64 `json:"root_source_rows"` + // DistinctRoots records unique root nodes in the generated fixture. + DistinctRoots int64 `json:"distinct_roots"` + // ForwardExpansionStates records recursive states visited by forward fixed-suffix expansion. + ForwardExpansionStates int64 `json:"forward_expansion_states"` + // SuffixRows records rows belonging to the fixed suffix of generated paths. + SuffixRows int64 `json:"suffix_rows"` + // DistinctBoundaries records unique terminal boundaries in the generated fixture. + DistinctBoundaries int64 `json:"distinct_boundaries"` + // ReachableBoundaries records terminal boundaries reachable in the generated fixture. + ReachableBoundaries int64 `json:"reachable_boundaries"` + // DisconnectedBoundaries records terminal boundaries intentionally disconnected from traversal roots. + DisconnectedBoundaries int64 `json:"disconnected_boundaries"` + // ExpectedReverseStates supplies the expected reverse states input to the FixedSuffixExpansionFixtureExpectations contract. + ExpectedReverseStates int64 `json:"expected_reverse_states"` + // CompleteOutputTrails records output trails before fixture eligibility filters are applied. + CompleteOutputTrails int64 `json:"complete_output_trails"` + // ProductiveBoundaryCycleEdges records the two relationship-distinct Expand + // relationships forming the optional productive-boundary cycle. + ProductiveBoundaryCycleEdges int64 `json:"productive_boundary_cycle_edges,omitempty"` + // ProductiveBoundarySelfLoopEdges records the optional productive-boundary + // Expand self-loop. + ProductiveBoundarySelfLoopEdges int64 `json:"productive_boundary_self_loop_edges,omitempty"` +} + +// EndpointSeededExpansionFixtureExpectations records expected state and output sizes for endpoint-seeded expansion fixtures. +type EndpointSeededExpansionFixtureExpectations struct { + // MatchingEndpoints records endpoints satisfying the generated fixture predicate. + MatchingEndpoints int64 `json:"matching_endpoints"` + // OtherEndpoints records nonmatching endpoint nodes in an endpoint-seeded fixture. + OtherEndpoints int64 `json:"other_endpoints"` + // EligiblePrefixRows records prefix rows that can connect to the required suffix. + EligiblePrefixRows int64 `json:"eligible_prefix_rows"` + // MatchingIneligibleLanes records matching lanes excluded by endpoint eligibility filters. + MatchingIneligibleLanes int64 `json:"matching_ineligible_lanes"` + // ExpectedReverseStates supplies the expected reverse states input to the EndpointSeededExpansionFixtureExpectations contract. + ExpectedReverseStates int64 `json:"expected_reverse_states"` + // ExpectedOutputTrails records result trails expected from the generated expansion fixture. + ExpectedOutputTrails int64 `json:"expected_output_trails"` +} + +// fixtureMetadata derives fixture counts, checksums, and generated-shape expectations from a graph. +func fixtureMetadata(datasetDir, name string) (FixtureMetadata, error) { + doc, err := parseDataset(datasetDir, name) + if err != nil { + return FixtureMetadata{}, err + } + raw, err := json.Marshal(doc.Graph) + if err != nil { + return FixtureMetadata{}, fmt.Errorf("encode dataset %s for checksum: %w", name, err) + } + digest := sha256.Sum256(raw) + configuration := "file" + if generatedDataset(name) != nil { + configuration = name + } + metadata := FixtureMetadata{ + Dataset: name, + Checksum: hex.EncodeToString(digest[:]), + NodeCount: len(doc.Graph.Nodes), + EdgeCount: len(doc.Graph.Edges), + Configuration: configuration, + } + if config, ok := parseFixedSuffixExpansionV3DatasetName(name); ok { + metadata.FixedSuffixExpansion = fixedSuffixExpansionV3FixtureExpectations(doc.Graph, config) + } else if config, ok := parseFixedSuffixExpansionV2DatasetName(name); ok { + metadata.FixedSuffixExpansion = fixedSuffixExpansionV2FixtureExpectations(config) + } + if config, ok := parseShortestPathV2DatasetName(name); ok { + metadata.Shortest = shortestFixtureExpectations(doc.Graph, config) + } + if config, ok := parseEndpointSeededExpansionDatasetName(name); ok { + metadata.EndpointSeededExpansion = endpointSeededExpansionFixtureExpectations(doc.Graph, config) + } + return metadata, nil +} + +// parseEndpointSeededExpansionDatasetName decodes and validates every scale parameter embedded in an endpoint-seeded dataset name. +func parseEndpointSeededExpansionDatasetName(name string) (testutil.EndpointSeededExpansionScaleConfig, bool) { + var depth, matchingEndpoints, otherEndpoints, matchingEligible, otherEligible, matchingIneligible, parallel, cycle, payload int + format := testutil.EndpointSeededExpansionScaleDataset + "_d%d_e%d_q%d_w%d_o%d_x%d_m%d_c%d_p%d" + matched, _ := fmt.Sscanf(name, format, &depth, &matchingEndpoints, &otherEndpoints, &matchingEligible, &otherEligible, &matchingIneligible, ¶llel, &cycle, &payload) + config := testutil.EndpointSeededExpansionScaleConfig{ + Depth: depth, + MatchingEndpoints: matchingEndpoints, + OtherEndpoints: otherEndpoints, + MatchingEligibleLanes: matchingEligible, + OtherEligibleLanes: otherEligible, + MatchingIneligibleLanes: matchingIneligible, + ParallelEdges: parallel, + AddCycle: cycle == 1, + PropertyPayloadSize: payload, + } + if matched != 9 || (cycle != 0 && cycle != 1) || testutil.ValidateEndpointSeededExpansionScaleConfig(config) != nil || name != endpointSeededExpansionDatasetName(config) { + return testutil.EndpointSeededExpansionScaleConfig{}, false + } + return config, true +} + +// endpointSeededExpansionDatasetName encodes endpoint-seeded scale parameters in their canonical dataset name. +func endpointSeededExpansionDatasetName(config testutil.EndpointSeededExpansionScaleConfig) string { + cycle := 0 + if config.AddCycle { + cycle = 1 + } + return fmt.Sprintf(testutil.EndpointSeededExpansionScaleDataset+"_d%d_e%d_q%d_w%d_o%d_x%d_m%d_c%d_p%d", + config.Depth, config.MatchingEndpoints, config.OtherEndpoints, config.MatchingEligibleLanes, + config.OtherEligibleLanes, config.MatchingIneligibleLanes, config.ParallelEdges, cycle, config.PropertyPayloadSize) +} + +// endpointSeededExpansionFixtureExpectations derives reverse-search state and output counts from an endpoint-seeded fixture. +func endpointSeededExpansionFixtureExpectations(fixture opengraph.Graph, config testutil.EndpointSeededExpansionScaleConfig) *EndpointSeededExpansionFixtureExpectations { + incoming := map[string][]int{} + matching := map[string]bool{} + eligibleUsers := map[string]bool{} + for _, node := range fixture.Nodes { + if objectID, ok := node.Properties["objectid"].(string); ok && strings.HasSuffix(objectID, "-512") { + matching[node.ID] = true + } + } + for edgeIdx, edge := range fixture.Edges { + if edge.Kind == "MemberOf" { + incoming[edge.EndID] = append(incoming[edge.EndID], edgeIdx) + } else if edge.Kind == "HasSession" { + eligibleUsers[edge.EndID] = true + } + } + var ( + states, outputs int64 + visit func(string, int, map[int]bool) + ) + visit = func(nodeID string, depth int, used map[int]bool) { + states++ + if depth > 0 && eligibleUsers[nodeID] { + outputs++ + } + if depth == 64 { + return + } + for _, edgeIdx := range incoming[nodeID] { + if used[edgeIdx] { + continue + } + used[edgeIdx] = true + visit(fixture.Edges[edgeIdx].StartID, depth+1, used) + delete(used, edgeIdx) + } + } + for endpoint := range matching { + visit(endpoint, 0, map[int]bool{}) + } + return &EndpointSeededExpansionFixtureExpectations{ + MatchingEndpoints: int64(config.MatchingEndpoints), + OtherEndpoints: int64(config.OtherEndpoints), + EligiblePrefixRows: int64(config.MatchingEligibleLanes + config.OtherEligibleLanes), + MatchingIneligibleLanes: int64(config.MatchingIneligibleLanes), + ExpectedReverseStates: states, + ExpectedOutputTrails: outputs, + } +} + +// parseShortestPathV2DatasetName decodes and validates every scale parameter embedded in a shortest-path dataset name. +func parseShortestPathV2DatasetName(name string) (testutil.ShortestPathScaleV2Config, bool) { + var ( + depth, rootOut, rootIn, intermediateOut, intermediateIn, level int + kinds, targets, diamond, disconnected, payload, cycle, selfLoop int + ) + + format := testutil.ShortestPathScaleV2Dataset + "_d%d_o%d_r%d_fo%d_fi%d_l%d_k%d_t%d_w%d_x%d_p%d_c%d_s%d" + matched, _ := fmt.Sscanf(name, format, &depth, &rootOut, &rootIn, &intermediateOut, &intermediateIn, &level, &kinds, &targets, &diamond, &disconnected, &payload, &cycle, &selfLoop) + if matched != 13 || (cycle != 0 && cycle != 1) || (selfLoop != 0 && selfLoop != 1) { + return testutil.ShortestPathScaleV2Config{}, false + } + config := testutil.ShortestPathScaleV2Config{ + Depth: depth, + ForwardRootFanOut: rootOut, + ReverseRootFanIn: rootIn, + IntermediateFanOut: intermediateOut, + IntermediateReverseFanIn: intermediateIn, + FanInLevel: level, + ParallelKindCount: kinds, + ParallelTargetCount: targets, + DiamondWidth: diamond, + DisconnectedWidth: disconnected, + PropertyPayloadSize: payload, + AddCycle: cycle == 1, + AddSelfLoop: selfLoop == 1, + } + if err := testutil.ValidateShortestPathScaleV2Config(config); err != nil || name != shortestPathV2DatasetName(config) { + return testutil.ShortestPathScaleV2Config{}, false + } + return config, true +} + +// shortestPathV2DatasetName encodes shortest-path scale parameters in their canonical dataset name. +func shortestPathV2DatasetName(config testutil.ShortestPathScaleV2Config) string { + cycle, selfLoop := 0, 0 + if config.AddCycle { + cycle = 1 + } + if config.AddSelfLoop { + selfLoop = 1 + } + return fmt.Sprintf(testutil.ShortestPathScaleV2Dataset+"_d%d_o%d_r%d_fo%d_fi%d_l%d_k%d_t%d_w%d_x%d_p%d_c%d_s%d", + config.Depth, config.ForwardRootFanOut, config.ReverseRootFanIn, + config.IntermediateFanOut, config.IntermediateReverseFanIn, config.FanInLevel, + config.ParallelKindCount, config.ParallelTargetCount, config.DiamondWidth, + config.DisconnectedWidth, config.PropertyPayloadSize, cycle, selfLoop) +} + +// shortestFixtureExpectations derives shortest distance, path cardinality, and intermediate-state expectations from a fixture. +func shortestFixtureExpectations(fixture opengraph.Graph, config testutil.ShortestPathScaleV2Config) *ShortestFixtureExpectations { + expectations := &ShortestFixtureExpectations{ + MaximumIntermediateForwardByLevel: map[string]int64{}, + MaximumIntermediateReverseByLevel: map[string]int64{}, + PhysicalTraversableEdgesByKind: map[string]int64{}, + DistinctReachableNodesByLevel: map[string]int64{}, + ExpectedMinimumDistance: int64(config.Depth), + ExpectedOnePathCardinality: 1, + ExpectedAllShortestCardinality: 1, + ExpectedPredecessorEdges: int64(config.Depth), + DisconnectedStateCardinality: int64(config.DisconnectedWidth + 1), + ParallelPhysicalEdges: int64(config.ParallelKindCount * config.ParallelTargetCount), + ParallelDistinctTargets: int64(config.ParallelTargetCount), + } + outgoing, incoming := map[string][]string{}, map[string][]string{} + for _, edge := range fixture.Edges { + expectations.PhysicalTraversableEdgesByKind[edge.Kind]++ + outgoing[edge.StartID] = append(outgoing[edge.StartID], edge.EndID) + incoming[edge.EndID] = append(incoming[edge.EndID], edge.StartID) + } + expectations.RootForwardDegree = int64(len(outgoing["sp-v2-start"])) + expectations.RootReverseDegree = int64(len(incoming["sp-v2-inbound-root"])) + for level := 1; level < config.Depth; level++ { + id, key := fmt.Sprintf("sp-v2-linear-%02d", level), fmt.Sprintf("%d", level) + expectations.MaximumIntermediateForwardByLevel[key] = int64(len(outgoing[id])) + expectations.MaximumIntermediateReverseByLevel[key] = int64(len(incoming[fmt.Sprintf("sp-v2-inbound-linear-%02d", level)])) + } + seen := map[string]bool{"sp-v2-start": true} + frontier := []string{"sp-v2-start"} + for level := 0; len(frontier) > 0 && level <= 64; level++ { + expectations.DistinctReachableNodesByLevel[fmt.Sprintf("%d", level)] = int64(len(frontier)) + next := []string{} + for _, source := range frontier { + for _, target := range outgoing[source] { + if !seen[target] { + seen[target] = true + next = append(next, target) + } + } + } + frontier = next + } + return expectations +} + +// parseFixedSuffixExpansionV3DatasetName decodes the exact fixed-suffix +// grammar with independently encoded matching roots and productive-boundary +// cycle/self-loop controls. +func parseFixedSuffixExpansionV3DatasetName(name string) (testutil.FixedSuffixExpansionScaleConfig, bool) { + var depth, fanout, reachable, disconnected, fanIn, multiplicity, roots, zeroDepth, cycle, selfLoop, payload int + format := testutil.FixedSuffixExpansionScaleV3Dataset + "_d%d_f%d_r%d_x%d_i%d_m%d_q%d_z%d_c%d_s%d_p%d" + matched, _ := fmt.Sscanf(name, format, &depth, &fanout, &reachable, &disconnected, &fanIn, &multiplicity, &roots, &zeroDepth, &cycle, &selfLoop, &payload) + if matched != 11 || (zeroDepth != 0 && zeroDepth != 1) || (cycle != 0 && cycle != 1) || (selfLoop != 0 && selfLoop != 1) { + return testutil.FixedSuffixExpansionScaleConfig{}, false + } + + rootSuffix := zeroDepth == 1 + config := testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: depth, + Fanout: fanout, + ExactReachableSuffixSources: &reachable, + DisconnectedSuffixSources: disconnected, + ReverseFanIn: fanIn, + SuffixPathsPerBoundary: multiplicity, + RootMatchCount: roots, + RootHasZeroDepthSuffix: &rootSuffix, + AddProductiveBoundaryCycle: cycle == 1, + AddProductiveBoundarySelfLoop: selfLoop == 1, + PropertyPayloadSize: payload, + } + if testutil.ValidateFixedSuffixExpansionScaleV3Config(config) != nil || name != fixedSuffixExpansionV3DatasetName(config) { + return testutil.FixedSuffixExpansionScaleConfig{}, false + } + return config, true +} + +// fixedSuffixExpansionV3DatasetName encodes every v3 fixture dimension in its +// canonical, round-trippable dataset name. +func fixedSuffixExpansionV3DatasetName(config testutil.FixedSuffixExpansionScaleConfig) string { + reachable, zeroDepth, cycle, selfLoop := 0, 0, 0, 0 + if config.ExactReachableSuffixSources != nil { + reachable = *config.ExactReachableSuffixSources + } + if config.RootHasZeroDepthSuffix != nil && *config.RootHasZeroDepthSuffix { + zeroDepth = 1 + } + if config.AddProductiveBoundaryCycle { + cycle = 1 + } + if config.AddProductiveBoundarySelfLoop { + selfLoop = 1 + } + return fmt.Sprintf(testutil.FixedSuffixExpansionScaleV3Dataset+"_d%d_f%d_r%d_x%d_i%d_m%d_q%d_z%d_c%d_s%d_p%d", + config.ExpansionDepth, config.Fanout, reachable, config.DisconnectedSuffixSources, + config.ReverseFanIn, config.SuffixPathsPerBoundary, config.RootMatchCount, + zeroDepth, cycle, selfLoop, config.PropertyPayloadSize) +} + +// parseFixedSuffixExpansionV2DatasetName decodes and validates every scale parameter embedded in a fixed-suffix dataset name. +func parseFixedSuffixExpansionV2DatasetName(name string) (testutil.FixedSuffixExpansionScaleConfig, bool) { + var depth, fanout, reachable, disconnected, fanIn, multiplicity, zeroDepth, payload int + format := testutil.FixedSuffixExpansionScaleDataset + "_v2_d%d_f%d_r%d_x%d_i%d_m%d_z%d_p%d" + matched, _ := fmt.Sscanf(name, format, &depth, &fanout, &reachable, &disconnected, &fanIn, &multiplicity, &zeroDepth, &payload) + if matched != 8 || depth < 0 || fanout < 1 || reachable < 0 || reachable > fanout || disconnected < 0 || fanIn < 0 || multiplicity < 1 || (zeroDepth != 0 && zeroDepth != 1) || payload < 0 || name != fmt.Sprintf(format, depth, fanout, reachable, disconnected, fanIn, multiplicity, zeroDepth, payload) { + return testutil.FixedSuffixExpansionScaleConfig{}, false + } + rootSuffix := zeroDepth == 1 + return testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: depth, + Fanout: fanout, + ExactReachableSuffixSources: &reachable, + DisconnectedSuffixSources: disconnected, + ReverseFanIn: fanIn, + SuffixPathsPerBoundary: multiplicity, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &rootSuffix, + PropertyPayloadSize: payload, + }, true +} + +// fixedSuffixExpansionV2FixtureExpectations preserves the exact v2 metadata +// contract for every existing fixture name. +func fixedSuffixExpansionV2FixtureExpectations(config testutil.FixedSuffixExpansionScaleConfig) *FixedSuffixExpansionFixtureExpectations { + reachable := 0 + if config.ExactReachableSuffixSources != nil { + reachable = *config.ExactReachableSuffixSources + } + rootSuffix := config.RootHasZeroDepthSuffix != nil && *config.RootHasZeroDepthSuffix + zero := 0 + if rootSuffix { + zero = 1 + } + multiplicity := max(config.SuffixPathsPerBoundary, 1) + rootCount := max(config.RootMatchCount, 1) + productiveFanIn := 0 + if zero+reachable > 0 { + productiveFanIn = config.ReverseFanIn + } + return &FixedSuffixExpansionFixtureExpectations{ + RootSourceRows: int64(rootCount), + DistinctRoots: int64(rootCount), + ForwardExpansionStates: int64(rootCount + config.Fanout*config.ExpansionDepth), + SuffixRows: int64((zero + reachable + config.DisconnectedSuffixSources) * multiplicity), + DistinctBoundaries: int64(zero + reachable + config.DisconnectedSuffixSources), + ReachableBoundaries: int64(zero + reachable), + DisconnectedBoundaries: int64(config.DisconnectedSuffixSources), + ExpectedReverseStates: int64(zero + reachable*(config.ExpansionDepth+1) + config.DisconnectedSuffixSources + productiveFanIn), + CompleteOutputTrails: int64((zero + reachable) * multiplicity), + } +} + +// fixedSuffixExpansionV3FixtureExpectations derives exact forward and reverse +// relationship-distinct states and output trails from a v3 fixture graph. +func fixedSuffixExpansionV3FixtureExpectations(fixture opengraph.Graph, config testutil.FixedSuffixExpansionScaleConfig) *FixedSuffixExpansionFixtureExpectations { + // adjacentEdge identifies one indexed transition in the generated fixture. + type adjacentEdge struct { + // index retains the index while adjacentEdge is assembled or evaluated. + index int + // next retains the next while adjacentEdge is assembled or evaluated. + next string + } + + nodeKinds := map[string]map[string]bool{} + roots := []string{} + for _, node := range fixture.Nodes { + kinds := map[string]bool{} + for _, kind := range node.Kinds { + kinds[kind] = true + } + nodeKinds[node.ID] = kinds + if kinds["ExpansionRoot"] && node.Properties["root_key"] == "generated-fse-root" { + roots = append(roots, node.ID) + } + } + + expandForward := map[string][]adjacentEdge{} + expandReverse := map[string][]adjacentEdge{} + edgesByStart := map[string][]int{} + for edgeIdx, edge := range fixture.Edges { + edgesByStart[edge.StartID] = append(edgesByStart[edge.StartID], edgeIdx) + if edge.Kind == "Expand" { + expandForward[edge.StartID] = append(expandForward[edge.StartID], adjacentEdge{ + index: edgeIdx, + next: edge.EndID, + }) + expandReverse[edge.EndID] = append(expandReverse[edge.EndID], adjacentEdge{ + index: edgeIdx, + next: edge.StartID, + }) + } + } + + suffixPaths := map[string]int64{} + for _, enter := range fixture.Edges { + if enter.Kind != "EnterSuffix" || !nodeKinds[enter.EndID]["SuffixHead"] { + continue + } + for _, continueIdx := range edgesByStart[enter.EndID] { + continuation := fixture.Edges[continueIdx] + if continuation.Kind != "ContinueSuffix" || !nodeKinds[continuation.EndID]["SuffixMiddle"] { + continue + } + for _, completeIdx := range edgesByStart[continuation.EndID] { + completion := fixture.Edges[completeIdx] + if completion.Kind == "CompleteSuffix" && nodeKinds[completion.EndID]["SuffixTerminal"] { + suffixPaths[enter.StartID]++ + } + } + } + } + + used := make([]bool, len(fixture.Edges)) + var enumerate func(map[string][]adjacentEdge, string, int, func(string)) int64 + enumerate = func(adjacency map[string][]adjacentEdge, nodeID string, depth int, observe func(string)) int64 { + states := int64(1) + observe(nodeID) + if depth == config.ExpansionDepth { + return states + } + for _, edge := range adjacency[nodeID] { + if used[edge.index] { + continue + } + used[edge.index] = true + states += enumerate(adjacency, edge.next, depth+1, observe) + used[edge.index] = false + } + return states + } + + boundaryVisits := map[string]int64{} + forwardStates := int64(0) + for _, root := range roots { + forwardStates += enumerate(expandForward, root, 0, func(nodeID string) { + if suffixPaths[nodeID] > 0 { + boundaryVisits[nodeID]++ + } + }) + } + + reverseStates := int64(0) + for boundary := range suffixPaths { + reverseStates += enumerate(expandReverse, boundary, 0, func(string) {}) + } + + suffixRows, outputTrails := int64(0), int64(0) + for boundary, pathCount := range suffixPaths { + suffixRows += pathCount + outputTrails += boundaryVisits[boundary] * pathCount + } + cycleEdges, selfLoopEdges := int64(0), int64(0) + if config.AddProductiveBoundaryCycle { + cycleEdges = 2 + } + if config.AddProductiveBoundarySelfLoop { + selfLoopEdges = 1 + } + return &FixedSuffixExpansionFixtureExpectations{ + RootSourceRows: int64(len(roots)), + DistinctRoots: int64(len(roots)), + ForwardExpansionStates: forwardStates, + SuffixRows: suffixRows, + DistinctBoundaries: int64(len(suffixPaths)), + ReachableBoundaries: int64(len(boundaryVisits)), + DisconnectedBoundaries: int64(len(suffixPaths) - len(boundaryVisits)), + ExpectedReverseStates: reverseStates, + CompleteOutputTrails: outputTrails, + ProductiveBoundaryCycleEdges: cycleEdges, + ProductiveBoundarySelfLoopEdges: selfLoopEdges, + } +} + +// clearGraph removes relationships before nodes, using PostgreSQL partition truncation when available. func clearGraph(ctx context.Context, db graph.Database) error { + if pgDriver, isPostgres := db.(*pg.Driver); isPostgres { + graphTarget, hasDefaultGraph := pgDriver.DefaultGraph() + if !hasDefaultGraph { + return fmt.Errorf("PostgreSQL default graph is not set") + } + + return clearPostgresGraph(ctx, db, graphTarget.ID) + } + + return db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if err := tx.Relationships().Delete(); err != nil { + return fmt.Errorf("delete relationships: %w", err) + } + + if err := tx.Nodes().Delete(); err != nil { + return fmt.Errorf("delete nodes: %w", err) + } + + return nil + }) +} + +// clearPostgresGraph truncates one PostgreSQL graph's edge and node partitions in a transaction. +func clearPostgresGraph(ctx context.Context, db graph.Database, graphID int32) error { return db.WriteTransaction(ctx, func(tx graph.Transaction) error { - return tx.Nodes().Delete() + // Truncate the active child partitions together. The high-level + // relationship query cannot see an already-orphaned edge, while DELETE + // leaves heap and index size dependent on earlier benchmark fixtures. + // Naming the children also avoids the node parent's cross-graph trigger. + statement := fmt.Sprintf("truncate table edge_%d, node_%d", graphID, graphID) + result := tx.Raw(statement, nil) + result.Close() + if err := result.Error(); err != nil { + return fmt.Errorf("execute PostgreSQL graph reset: %w", err) + } + + return nil }) } +// benchmarkSchema returns the SQL schema used to isolate benchmark preparation from timed execution. func benchmarkSchema(nodeKinds, edgeKinds graph.Kinds) graph.Schema { return graph.Schema{ Graphs: []graph.Graph{{ @@ -94,24 +751,102 @@ func benchmarkSchema(nodeKinds, edgeKinds graph.Kinds) graph.Schema { } } +// resolveCaseParams resolves a scale case's scalar, node-key, node-list, and generated-node parameters. func resolveCaseParams(testCase ScaleCase, idMap opengraph.IDMap) (map[string]any, error) { - params := make(map[string]any, len(testCase.Params)+len(testCase.NodeParams)) - for key, value := range testCase.Params { + return resolveParams(testCase.Name, testCase.Params, testCase.NodeParams, testCase.NodeListParams, testCase.GeneratedNodeListParams, idMap) +} + +// resolveParams copies literal parameters and replaces symbolic node keys with database identifiers. +func resolveParams(caseName string, rawParams map[string]any, nodeParams map[string]string, nodeListParams map[string][]string, generatedNodeListParams map[string]testutil.GeneratedNodeListParam, idMap opengraph.IDMap) (map[string]any, error) { + params := make(map[string]any, len(rawParams)+len(nodeParams)+len(nodeListParams)+len(generatedNodeListParams)) + for key, value := range rawParams { params[key] = value } - for paramName, nodeName := range testCase.NodeParams { + for paramName, nodeName := range nodeParams { id, found := idMap[nodeName] if !found { - return nil, fmt.Errorf("case %s references unknown dataset node %q", testCase.Name, nodeName) + return nil, fmt.Errorf("case %s references unknown dataset node %q", caseName, nodeName) } params[paramName] = id.Int64() } + for paramName, nodeNames := range nodeListParams { + ids := make([]int64, len(nodeNames)) + for idx, nodeName := range nodeNames { + id, found := idMap[nodeName] + if !found { + return nil, fmt.Errorf("case %s references unknown dataset node %q in list parameter %q", caseName, nodeName, paramName) + } + + ids[idx] = id.Int64() + } + + params[paramName] = ids + } + + for paramName, spec := range generatedNodeListParams { + if spec.Count < 0 { + return nil, fmt.Errorf("case %s generated node list parameter %q has negative count", caseName, paramName) + } + + nodeNames := append([]string(nil), spec.Include...) + nodeNames = append(nodeNames, testutil.FixtureNames(spec.Prefix, spec.Count)...) + ids := make([]int64, len(nodeNames)) + for idx, nodeName := range nodeNames { + id, found := idMap[nodeName] + if !found { + return nil, fmt.Errorf("case %s references unknown dataset node %q in generated list parameter %q", caseName, nodeName, paramName) + } + ids[idx] = id.Int64() + } + params[paramName] = ids + } + if len(params) == 0 { return nil, nil } return params, nil } + +// resolveWriteScenario resolves selection and post-state parameters while preserving the write expectation contract. +func resolveWriteScenario(testCase ScaleCase, idMap opengraph.IDMap) (resolvedWriteScenario, error) { + if testCase.WriteScenario == nil { + return resolvedWriteScenario{}, nil + } + + scenario := testCase.WriteScenario + if scenario.ExpectedMatched == nil || scenario.ExpectedAffected == nil { + return resolvedWriteScenario{}, fmt.Errorf("case %s has an incomplete write scenario", testCase.Name) + } + selectionParams, err := resolveParams(testCase.Name+" selection", scenario.Params, scenario.NodeParams, scenario.NodeListParams, scenario.GeneratedNodeListParams, idMap) + if err != nil { + return resolvedWriteScenario{}, err + } + + resolved := resolvedWriteScenario{ + SelectionCypher: scenario.SelectionCypher, + SelectionParams: selectionParams, + AffectedEntity: scenario.AffectedEntity, + ExpectedMatched: *scenario.ExpectedMatched, + ExpectedAffected: *scenario.ExpectedAffected, + } + + for _, postState := range scenario.PostState { + params, err := resolveParams(testCase.Name+" post-state "+postState.Name, postState.Params, postState.NodeParams, postState.NodeListParams, postState.GeneratedNodeListParams, idMap) + if err != nil { + return resolvedWriteScenario{}, err + } + + resolved.PostState = append(resolved.PostState, resolvedStateQuery{ + Name: postState.Name, + Cypher: postState.Cypher, + Params: params, + Expected: postState.Expected, + }) + } + + return resolved, nil +} diff --git a/cmd/graphbench/datasets_test.go b/cmd/graphbench/datasets_test.go new file mode 100644 index 00000000..c650e4a1 --- /dev/null +++ b/cmd/graphbench/datasets_test.go @@ -0,0 +1,419 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestGeneratedFixedSuffixExpansionV2DatasetCarriesExactExpectations verifies that a canonical encoded name derives exact forward, reverse, boundary, suffix-row, and output-trail counts. +func TestGeneratedFixedSuffixExpansionV2DatasetCarriesExactExpectations(t *testing.T) { + name := "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m2_z1_p0" + config, ok := parseFixedSuffixExpansionV2DatasetName(name) + require.True(t, ok) + require.Equal(t, 16, config.ExpansionDepth) + require.Equal(t, 1, *config.ExactReachableSuffixSources) + require.Equal(t, 2, config.SuffixPathsPerBoundary) + + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.FixedSuffixExpansion) + require.Equal(t, int64(16_001), metadata.FixedSuffixExpansion.ForwardExpansionStates) + require.Equal(t, int64(6), metadata.FixedSuffixExpansion.SuffixRows) + require.Equal(t, int64(3), metadata.FixedSuffixExpansion.DistinctBoundaries) + require.Equal(t, int64(19), metadata.FixedSuffixExpansion.ExpectedReverseStates) + require.Equal(t, int64(4), metadata.FixedSuffixExpansion.CompleteOutputTrails) +} + +// TestGeneratedFixedSuffixExpansionV3DatasetRoundTripsAllBoundaryControls +// verifies independent root multiplicity and every canonical cycle/self-loop +// combination, including exact relationship-distinct state and output counts. +func TestGeneratedFixedSuffixExpansionV3DatasetRoundTripsAllBoundaryControls(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // cycle indicates whether cycle applies. + cycle bool + // selfLoop indicates whether self loop applies. + selfLoop bool + // forwardStates retains the forward states while anonymous record is assembled or evaluated. + forwardStates int64 + // reverseStates retains the reverse states while anonymous record is assembled or evaluated. + reverseStates int64 + // outputTrails retains the output trails while anonymous record is assembled or evaluated. + outputTrails int64 + }{ + { + name: "neither", + forwardStates: 5, + reverseStates: 1, + outputTrails: 1, + }, + { + name: "cycle", + cycle: true, + forwardStates: 7, + reverseStates: 3, + outputTrails: 2, + }, + { + name: "self-loop", + selfLoop: true, + forwardStates: 7, + reverseStates: 2, + outputTrails: 2, + }, + { + name: "both", + cycle: true, + selfLoop: true, + forwardStates: 10, + reverseStates: 5, + outputTrails: 3, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + reachable := 0 + zeroDepth := true + config := testutil.FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 1, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 3, + RootHasZeroDepthSuffix: &zeroDepth, + AddProductiveBoundaryCycle: testCase.cycle, + AddProductiveBoundarySelfLoop: testCase.selfLoop, + } + name := fixedSuffixExpansionV3DatasetName(config) + parsed, ok := parseFixedSuffixExpansionV3DatasetName(name) + require.True(t, ok) + require.Equal(t, config, parsed) + + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.FixedSuffixExpansion) + require.Equal(t, int64(3), metadata.FixedSuffixExpansion.RootSourceRows) + require.Equal(t, testCase.forwardStates, metadata.FixedSuffixExpansion.ForwardExpansionStates) + require.Equal(t, testCase.reverseStates, metadata.FixedSuffixExpansion.ExpectedReverseStates) + require.Equal(t, testCase.outputTrails, metadata.FixedSuffixExpansion.CompleteOutputTrails) + if testCase.cycle { + require.Equal(t, int64(2), metadata.FixedSuffixExpansion.ProductiveBoundaryCycleEdges) + } else { + require.Zero(t, metadata.FixedSuffixExpansion.ProductiveBoundaryCycleEdges) + } + if testCase.selfLoop { + require.Equal(t, int64(1), metadata.FixedSuffixExpansion.ProductiveBoundarySelfLoopEdges) + } else { + require.Zero(t, metadata.FixedSuffixExpansion.ProductiveBoundarySelfLoopEdges) + } + }) + } +} + +// TestGeneratedEndpointSeededExpansionDatasetRoundTripsWithExactExpectations verifies lossless name encoding and the expected endpoint, prefix, output, and reverse-search cardinalities. +func TestGeneratedEndpointSeededExpansionDatasetRoundTripsWithExactExpectations(t *testing.T) { + config := testutil.EndpointSeededExpansionScaleConfig{ + Depth: 3, + MatchingEndpoints: 2, + OtherEndpoints: 1, + MatchingEligibleLanes: 2, + OtherEligibleLanes: 1, + MatchingIneligibleLanes: 1, + ParallelEdges: 1, + AddCycle: false, + PropertyPayloadSize: 8, + } + name := endpointSeededExpansionDatasetName(config) + parsed, ok := parseEndpointSeededExpansionDatasetName(name) + require.True(t, ok) + require.Equal(t, config, parsed) + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.EndpointSeededExpansion) + require.Equal(t, int64(2), metadata.EndpointSeededExpansion.MatchingEndpoints) + require.Equal(t, int64(3), metadata.EndpointSeededExpansion.EligiblePrefixRows) + require.Equal(t, int64(2), metadata.EndpointSeededExpansion.ExpectedOutputTrails) + require.Greater(t, metadata.EndpointSeededExpansion.ExpectedReverseStates, metadata.EndpointSeededExpansion.ExpectedOutputTrails) +} + +// TestGeneratedEndpointSeededExpansionRejectsInvalidNames verifies that zero dimensions, padded numbers, invalid booleans, and inconsistent parallelism cannot select a generated fixture. +func TestGeneratedEndpointSeededExpansionRejectsInvalidNames(t *testing.T) { + for _, name := range []string{ + "generated_endpoint_seeded_expansion_v1_d0_e1_q0_w1_o0_x0_m1_c0_p0", + "generated_endpoint_seeded_expansion_v1_d3_e0_q0_w1_o0_x0_m1_c0_p0", + "generated_endpoint_seeded_expansion_v1_d3_e1_q0_w1_o0_x0_m0_c0_p0", + "generated_endpoint_seeded_expansion_v1_d03_e1_q0_w1_o0_x0_m1_c0_p0", + "generated_endpoint_seeded_expansion_v1_d3_e1_q0_w1_o0_x0_m1_c2_p0", + "generated_endpoint_seeded_expansion_v1_d3_e1_q0_w1_o0_x0_m2_c0_p0", + } { + _, ok := parseEndpointSeededExpansionDatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestGeneratedShortestPathV2DatasetRoundTripsAndCarriesExactExpectations verifies lossless configuration naming and exact topology metrics for branching, parallel, disconnected, cyclic, and self-loop shapes. +func TestGeneratedShortestPathV2DatasetRoundTripsAndCarriesExactExpectations(t *testing.T) { + config := testutil.ShortestPathScaleV2Config{ + Depth: 3, + ForwardRootFanOut: 2, + ReverseRootFanIn: 2, + IntermediateFanOut: 1, + IntermediateReverseFanIn: 4, + FanInLevel: 2, + ParallelKindCount: 3, + ParallelTargetCount: 2, + DiamondWidth: 2, + DisconnectedWidth: 3, + PropertyPayloadSize: 8, + AddCycle: true, + AddSelfLoop: true, + } + name := shortestPathV2DatasetName(config) + parsed, ok := parseShortestPathV2DatasetName(name) + require.True(t, ok) + require.Equal(t, config, parsed) + + metadata, err := fixtureMetadata("unused", name) + require.NoError(t, err) + require.NotNil(t, metadata.Shortest) + require.Equal(t, 32, metadata.NodeCount) + require.Equal(t, 33, metadata.EdgeCount) + require.Equal(t, int64(5), metadata.Shortest.RootForwardDegree) + require.Equal(t, int64(3), metadata.Shortest.RootReverseDegree) + require.Equal(t, int64(2), metadata.Shortest.MaximumIntermediateForwardByLevel["2"]) + require.Equal(t, int64(5), metadata.Shortest.MaximumIntermediateReverseByLevel["2"]) + require.Equal(t, int64(23), metadata.Shortest.PhysicalTraversableEdgesByKind["Traverse"]) + require.Equal(t, int64(6), metadata.Shortest.ParallelPhysicalEdges) + require.Equal(t, int64(2), metadata.Shortest.ParallelDistinctTargets) + require.Equal(t, int64(3), metadata.Shortest.ExpectedMinimumDistance) + require.Equal(t, int64(3), metadata.Shortest.ExpectedPredecessorEdges) + require.Equal(t, int64(4), metadata.Shortest.DisconnectedStateCardinality) + require.Equal(t, int64(5), metadata.Shortest.DistinctReachableNodesByLevel["1"]) + require.NotEmpty(t, metadata.Checksum) +} + +// TestGeneratedShortestPathV2DatasetRejectsInvalidOrNonCanonicalNames verifies that inconsistent levels, empty required dimensions, padded or negative values, invalid booleans, and trailing tokens are rejected. +func TestGeneratedShortestPathV2DatasetRejectsInvalidOrNonCanonicalNames(t *testing.T) { + for _, name := range []string{ + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l3_k3_t2_w2_x3_p8_c1_s1", + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l2_k3_t0_w2_x3_p8_c1_s1", + "generated_shortest_paths_v2_d03_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c1_s1", + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c2_s1", + "generated_shortest_paths_v2_d3_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c1_s1_unknown", + "generated_shortest_paths_v2_d-1_o2_r2_fo1_fi4_l2_k3_t2_w2_x3_p8_c1_s1", + } { + _, ok := parseShortestPathV2DatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestGeneratedFixedSuffixExpansionV2DatasetRejectsInvalidOrNonCanonicalNames verifies that impossible reachability, zero multiplicity, invalid booleans, and padded dimensions cannot identify a fixture. +func TestGeneratedFixedSuffixExpansionV2DatasetRejectsInvalidOrNonCanonicalNames(t *testing.T) { + for _, name := range []string{ + "generated_fixed_suffix_expansion_v2_d16_f1000_r1001_x1_i0_m1_z1_p0", + "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m0_z1_p0", + "generated_fixed_suffix_expansion_v2_d16_f1000_r1_x1_i0_m1_z2_p0", + "generated_fixed_suffix_expansion_v2_d016_f1000_r1_x1_i0_m1_z1_p0", + } { + _, ok := parseFixedSuffixExpansionV2DatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestGeneratedFixedSuffixExpansionV3DatasetRejectsInvalidOrNonCanonicalNames +// verifies strict roots, booleans, productive-boundary requirements, exact +// depth-zero reachability, canonical numbers, and complete token consumption. +func TestGeneratedFixedSuffixExpansionV3DatasetRejectsInvalidOrNonCanonicalNames(t *testing.T) { + for _, name := range []string{ + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q0_z1_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z1_c2_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z1_c0_s2_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z0_c1_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i1_m1_q1_z0_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d0_f1_r1_x0_i0_m1_q1_z0_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d02_f1_r0_x0_i0_m1_q1_z1_c0_s0_p0", + "generated_fixed_suffix_expansion_v3_d2_f1_r0_x0_i0_m1_q1_z1_c0_s0_p0_unknown", + } { + _, ok := parseFixedSuffixExpansionV3DatasetName(name) + require.False(t, ok, name) + require.Nil(t, generatedDataset(name), name) + } +} + +// TestClearGraphDeletesRelationshipsBeforeNodes verifies that cleanup removes relationships before nodes so attached edges cannot block node deletion. +func TestClearGraphDeletesRelationshipsBeforeNodes(t *testing.T) { + database := &clearGraphTestDatabase{} + + require.NoError(t, clearGraph(context.Background(), database)) + require.Equal(t, []string{"relationships", "nodes"}, database.deletes) +} + +// TestClearGraphStopsWhenRelationshipDeleteFails verifies that a relationship deletion error is wrapped and prevents the subsequent node deletion. +func TestClearGraphStopsWhenRelationshipDeleteFails(t *testing.T) { + database := &clearGraphTestDatabase{relationshipError: errors.New("relationship failure")} + + err := clearGraph(context.Background(), database) + require.ErrorContains(t, err, "delete relationships: relationship failure") + require.Equal(t, []string{"relationships"}, database.deletes) +} + +// TestClearGraphReportsNodeDeleteFailure verifies that cleanup reports a node deletion failure only after relationships have been removed. +func TestClearGraphReportsNodeDeleteFailure(t *testing.T) { + database := &clearGraphTestDatabase{nodeError: errors.New("node failure")} + + err := clearGraph(context.Background(), database) + require.ErrorContains(t, err, "delete nodes: node failure") + require.Equal(t, []string{"relationships", "nodes"}, database.deletes) +} + +// TestClearPostgresGraphTruncatesPhysicalPartitionsTogether verifies that PostgreSQL cleanup issues one parameter-free TRUNCATE for both graph-specific physical tables. +func TestClearPostgresGraphTruncatesPhysicalPartitionsTogether(t *testing.T) { + database := &clearPostgresGraphTestDatabase{} + + require.NoError(t, clearPostgresGraph(context.Background(), database, 42)) + require.Equal(t, []string{"truncate table edge_42, node_42"}, database.statements) + require.Equal(t, []map[string]any{nil}, database.parameters) +} + +// TestClearPostgresGraphRollsBackAfterRawDeleteFailure verifies that a failed physical-table reset is surfaced from the enclosing write transaction. +func TestClearPostgresGraphRollsBackAfterRawDeleteFailure(t *testing.T) { + database := &clearPostgresGraphTestDatabase{failAt: 1} + + err := clearPostgresGraph(context.Background(), database, 42) + require.ErrorContains(t, err, "execute PostgreSQL graph reset") + require.Equal(t, []string{"truncate table edge_42, node_42"}, database.statements) +} + +// clearGraphTestDatabase supplies a fake transaction for graph-cleanup tests. +type clearGraphTestDatabase struct { + // Database supplies methods irrelevant to the cleanup interaction under test. + graph.Database + + // deletes records whether relationship or node deletion was requested first. + deletes []string + + // relationshipError is returned when cleanup attempts relationship deletion. + relationshipError error + + // nodeError is returned when cleanup attempts node deletion. + nodeError error +} + +// WriteTransaction routes cleanup through a transaction that shares the deletion trace and injected failures. +func (s *clearGraphTestDatabase) WriteTransaction(_ context.Context, delegate graph.TransactionDelegate, _ ...graph.TransactionOption) error { + return delegate(&clearGraphTestTransaction{database: s}) +} + +// clearGraphTestTransaction routes relationship and node queries to graph-cleanup fakes. +type clearGraphTestTransaction struct { + // Transaction supplies methods outside the cleanup query surface. + graph.Transaction + + // database owns the call trace and injected deletion failures. + database *clearGraphTestDatabase +} + +// Relationships returns a deletion recorder backed by the owning database trace. +func (s *clearGraphTestTransaction) Relationships() graph.RelationshipQuery { + return &clearGraphTestRelationshipQuery{database: s.database} +} + +// Nodes returns a deletion recorder backed by the owning database trace. +func (s *clearGraphTestTransaction) Nodes() graph.NodeQuery { + return &clearGraphTestNodeQuery{database: s.database} +} + +// clearGraphTestRelationshipQuery records relationship deletion and injects configured failures. +type clearGraphTestRelationshipQuery struct { + // RelationshipQuery supplies methods other than the deletion operation under test. + graph.RelationshipQuery + + // database receives the relationship deletion trace and supplies its error. + database *clearGraphTestDatabase +} + +// Delete prepares or inspects test evidence for delete. +func (s *clearGraphTestRelationshipQuery) Delete() error { + s.database.deletes = append(s.database.deletes, "relationships") + return s.database.relationshipError +} + +// clearGraphTestNodeQuery records node deletion and injects configured failures. +type clearGraphTestNodeQuery struct { + // NodeQuery supplies methods other than the deletion operation under test. + graph.NodeQuery + + // database receives the node deletion trace and supplies its error. + database *clearGraphTestDatabase +} + +// Delete prepares or inspects test evidence for delete. +func (s *clearGraphTestNodeQuery) Delete() error { + s.database.deletes = append(s.database.deletes, "nodes") + return s.database.nodeError +} + +// clearPostgresGraphTestDatabase supplies a fake raw transaction for PostgreSQL cleanup tests. +type clearPostgresGraphTestDatabase struct { + // Database supplies methods irrelevant to the raw cleanup interaction. + graph.Database + + // statements records every raw SQL statement issued by cleanup. + statements []string + + // parameters retains the parameters while clearPostgresGraphTestDatabase is assembled or evaluated. + parameters []map[string]any + + // failAt selects the one-based raw call that returns a terminal result error. + failAt int +} + +// WriteTransaction routes PostgreSQL cleanup through a raw-SQL recorder owned by the fake database. +func (s *clearPostgresGraphTestDatabase) WriteTransaction(_ context.Context, delegate graph.TransactionDelegate, _ ...graph.TransactionOption) error { + return delegate(&clearPostgresGraphTestTransaction{database: s}) +} + +// clearPostgresGraphTestTransaction records PostgreSQL cleanup SQL and returns a configured result. +type clearPostgresGraphTestTransaction struct { + // Transaction supplies methods outside the raw SQL cleanup surface. + graph.Transaction + + // database owns the captured statements, parameters, and failure injection. + database *clearPostgresGraphTestDatabase +} + +// Raw captures one statement and its parameters, injecting a terminal error on the selected call. +func (s *clearPostgresGraphTestTransaction) Raw(statement string, parameters map[string]any) graph.Result { + s.database.statements = append(s.database.statements, statement) + s.database.parameters = append(s.database.parameters, parameters) + if s.database.failAt > 0 && len(s.database.statements) == s.database.failAt { + return &clearPostgresGraphTestResult{err: errors.New("raw delete failure")} + } + + return &clearPostgresGraphTestResult{} +} + +// clearPostgresGraphTestResult exposes a configured terminal raw-statement failure to cleanup code. +type clearPostgresGraphTestResult struct { + // Result supplies result methods that cleanup does not exercise. + graph.Result + + // err is exposed as the terminal raw-statement failure. + err error +} + +// Error returns the configured terminal iterator error. +func (s *clearPostgresGraphTestResult) Error() error { + return s.err +} + +// Close satisfies graph.Result; this fake has no close state to record. +func (s *clearPostgresGraphTestResult) Close() {} diff --git a/cmd/graphbench/dormant_forms_guard_test.go b/cmd/graphbench/dormant_forms_guard_test.go new file mode 100644 index 00000000..63482cc2 --- /dev/null +++ b/cmd/graphbench/dormant_forms_guard_test.go @@ -0,0 +1,44 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDormantFormsStayOutOfScaleCorpus verifies that active scale-case names and tags never publish FUTURE-prefixed query forms. +func TestDormantFormsStayOutOfScaleCorpus(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + for _, testCase := range corpus.Cases { + requireNoDormantQueryFormID(t, testCase.Source+" name", testCase.Name) + for _, tag := range testCase.Tags { + requireNoDormantQueryFormID(t, testCase.Source+" tag", tag) + } + } +} + +// requireNoDormantQueryFormID rejects a case field containing the reserved FUTURE marker, independent of letter case. +func requireNoDormantQueryFormID(t *testing.T, field, value string) { + t.Helper() + require.False(t, strings.Contains(strings.ToUpper(value), "FUTURE-"), + "%s %q places a dormant query form in the active scale corpus", field, value) +} diff --git a/cmd/graphbench/environment.go b/cmd/graphbench/environment.go new file mode 100644 index 00000000..8b14f4d8 --- /dev/null +++ b/cmd/graphbench/environment.go @@ -0,0 +1,360 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" +) + +// RunEnvironment captures source, host, invocation, fixture, and protocol identity for a benchmark run. +type RunEnvironment struct { + // ArtifactSchemaVersion identifies the benchmark artifact schema emitted by the run. + ArtifactSchemaVersion int `json:"artifact_schema_version"` + // CorpusSHA256 binds run provenance to the exact canonical workload declarations. + CorpusSHA256 string `json:"corpus_sha256,omitempty"` + // RunIdentitySHA256 binds resumable records to execution settings that affect comparability. + RunIdentitySHA256 string `json:"run_identity_sha256,omitempty"` + // SourceCommit identifies the source commit used to build the benchmark executable. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 identifies uncommitted source changes present during the run. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 identifies the benchmark executable used for the run. + BinarySHA256 string `json:"binary_sha256"` + // GOOS supplies the goos input to the RunEnvironment contract. + GOOS string `json:"goos"` + // GOARCH supplies the goarch input to the RunEnvironment contract. + GOARCH string `json:"goarch"` + // GoVersion identifies the schema version for go version. + GoVersion string `json:"go_version"` + // CPUCount records logical CPUs visible to the benchmark process. + CPUCount int `json:"cpu_count"` + // CPUModel supplies the cpu model input to the RunEnvironment contract. + CPUModel string `json:"cpu_model,omitempty"` + // Kernel supplies the kernel input to the RunEnvironment contract. + Kernel string `json:"kernel,omitempty"` + // CgroupCPU supplies the cgroup cpu input to the RunEnvironment contract. + CgroupCPU string `json:"cgroup_cpu,omitempty"` + // CgroupMemory supplies the cgroup memory input to the RunEnvironment contract. + CgroupMemory string `json:"cgroup_memory,omitempty"` + // CPUGovernor supplies the cpu governor input to the RunEnvironment contract. + CPUGovernor string `json:"cpu_governor,omitempty"` + // CPUFrequency supplies the cpu frequency input to the RunEnvironment contract. + CPUFrequency string `json:"cpu_frequency,omitempty"` + // HostLoad records host load averages observed during the run. + HostLoad string `json:"host_load,omitempty"` + // Invocation supplies the invocation input to the RunEnvironment contract. + Invocation []string `json:"invocation"` + // BuildCommand supplies the build command input to the RunEnvironment contract. + BuildCommand string `json:"build_command"` + // RunUUID groups records produced by the same resumable benchmark run series. + RunUUID string `json:"run_uuid"` + // Arm identifies the measurement arm that produced the sample. + Arm string `json:"arm"` + // ArmOrder supplies the arm order input to the RunEnvironment contract. + ArmOrder int `json:"arm_order,omitempty"` + // Block identifies the measurement block used to control carryover effects. + Block int `json:"block"` + // Round identifies the measurement round. + Round int `json:"round"` + // StartedAt records when the benchmark run began. + StartedAt time.Time `json:"started_at"` + // EndedAt records when the benchmark run finished. + EndedAt time.Time `json:"ended_at"` + // WarmupIterations records the number of warmup iterations. + WarmupIterations int `json:"warmup_iterations"` + // Selection captures the exact workload selection applied to the run. + Selection *SelectionManifest `json:"selection,omitempty"` + // PoolSize sets the database connection-pool size. + PoolSize int `json:"pool_size"` + // Concurrency supplies the concurrency input to the RunEnvironment contract. + Concurrency []int `json:"concurrency,omitempty"` + // SessionMemoryCeilingBytes sets the per-session memory ceiling in bytes. + SessionMemoryCeilingBytes int64 `json:"session_memory_ceiling_bytes,omitempty"` + // PoolMemoryCeilingBytes sets the aggregate pool memory ceiling in bytes. + PoolMemoryCeilingBytes int64 `json:"pool_memory_ceiling_bytes,omitempty"` + // ExistingGraph selects read-only execution against a pre-existing graph. + ExistingGraph bool `json:"existing_graph,omitempty"` + // Protocol identifies the measurement protocol. + Protocol string `json:"protocol,omitempty"` +} + +// PostgresEnvironment captures PostgreSQL settings, relation sizes, and schema fingerprints required for comparability. +type PostgresEnvironment struct { + // Version identifies the serialized schema revision. + Version string `json:"version"` + // Database names the PostgreSQL database whose settings and schema were captured. + Database string `json:"database"` + // PlanCacheMode records PostgreSQL plan_cache_mode for environment comparability. + PlanCacheMode string `json:"plan_cache_mode"` + // TransactionIsolation records the isolation applied to measured read + // transactions. Tool and provisional guarded orientation evidence uses + // Repeatable Read even when the server default differs. + TransactionIsolation string `json:"transaction_isolation"` + // WorkMem records PostgreSQL work_mem for environment comparability. + WorkMem string `json:"work_mem"` + // TempFileLimit supplies the temp file limit input to the PostgresEnvironment contract. + TempFileLimit string `json:"temp_file_limit"` + // GraphPartitionCount records physical PostgreSQL graph partitions included in relation-size evidence. + GraphPartitionCount int64 `json:"graph_partition_count"` + // PostmasterStartedAt records PostgreSQL server start time for restart detection. + PostmasterStartedAt time.Time `json:"postmaster_started_at,omitempty"` + // DatabaseOID identifies the PostgreSQL database across environment and restart comparisons. + DatabaseOID int64 `json:"database_oid,omitempty"` + // Autovacuum records PostgreSQL autovacuum settings relevant to comparability. + Autovacuum string `json:"autovacuum,omitempty"` + // NodeRelationBytes supplies the node relation bytes input to the PostgresEnvironment contract. + NodeRelationBytes int64 `json:"node_relation_bytes,omitempty"` + // EdgeRelationBytes supplies the edge relation bytes input to the PostgresEnvironment contract. + EdgeRelationBytes int64 `json:"edge_relation_bytes,omitempty"` + // AnalyzeState records PostgreSQL analyze statistics state for the fixture. + AnalyzeState string `json:"analyze_state,omitempty"` + // SchemaFingerprint identifies the normalized PostgreSQL graph schema definition. + SchemaFingerprint string `json:"schema_fingerprint,omitempty"` + // IndexFingerprint identifies the normalized database index configuration. + IndexFingerprint string `json:"index_fingerprint,omitempty"` +} + +// resolveRunEnvironment captures reproducibility metadata, invocation, fixture selection, and run timestamps. +func resolveRunEnvironment(cfg config, args []string, selection SelectionManifest, startedAt, endedAt time.Time) RunEnvironment { + runUUID := cfg.RunUUID + if runUUID == "" { + runUUID = newRunUUID() + } + return RunEnvironment{ + ArtifactSchemaVersion: 2, + SourceCommit: commandOutput("git", "rev-parse", "HEAD"), + DirtyDiffSHA256: workingTreeSHA256(), + BinarySHA256: executableSHA256(), + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + GoVersion: runtime.Version(), + CPUCount: runtime.NumCPU(), + CPUModel: cpuModel(), + Kernel: commandOutput("uname", "-srvm"), + CgroupCPU: firstReadableFile("/sys/fs/cgroup/cpu.max", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"), + CgroupMemory: firstReadableFile("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes"), + CPUGovernor: firstReadableFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"), + CPUFrequency: firstReadableFile("/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq"), + HostLoad: firstReadableFile("/proc/loadavg"), + Invocation: sanitizedInvocation(args), + BuildCommand: cfg.BuildCommand, + RunUUID: runUUID, + Arm: cfg.Arm, + ArmOrder: cfg.ArmOrder, + Block: cfg.Block, + Round: cfg.Round, + StartedAt: startedAt.UTC(), + EndedAt: endedAt.UTC(), + WarmupIterations: cfg.WarmupIterations, + Selection: &selection, + PoolSize: cfg.PoolSize, + Concurrency: append([]int(nil), cfg.Concurrency...), + SessionMemoryCeilingBytes: cfg.SessionMemoryCeilingBytes, + PoolMemoryCeilingBytes: cfg.PoolMemoryCeilingBytes, + ExistingGraph: cfg.ExistingGraph, + Protocol: benchmarkProtocol(cfg), + } +} + +// benchmarkProtocol returns the stable name of the measurement protocol selected by the command. +func benchmarkProtocol(cfg config) string { + if cfg.Discovery { + return "adaptive_discovery" + } + return "fixed_confirmation" +} + +// newRunUUID generates a random RFC 4122 version 4 run identifier. +func newRunUUID() string { + var value [16]byte + if _, err := rand.Read(value[:]); err != nil { + return fmt.Sprintf("fallback-%d", time.Now().UnixNano()) + } + value[6] = (value[6] & 0x0f) | 0x40 + value[8] = (value[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", value[0:4], value[4:6], value[6:8], value[8:10], value[10:16]) +} + +// cpuModel returns the host CPU model reported by the operating system. +func cpuModel() string { + raw, err := os.ReadFile("/proc/cpuinfo") + if err != nil { + return "unknown" + } + for _, line := range strings.Split(string(raw), "\n") { + if name, value, found := strings.Cut(line, ":"); found && strings.TrimSpace(name) == "model name" { + return strings.TrimSpace(value) + } + } + return "unknown" +} + +// firstReadableFile returns trimmed contents of the first readable path. +func firstReadableFile(paths ...string) string { + for _, path := range paths { + if raw, err := os.ReadFile(path); err == nil { + return strings.TrimSpace(string(raw)) + } + } + return "unknown" +} + +// sanitizedInvocation returns command arguments with connection-string credentials redacted. +func sanitizedInvocation(args []string) []string { + const redacted = "" + connectionFlags := []string{"-connection", "-pg-connection", "-neo4j-connection"} + result := append([]string(nil), args...) + for idx := range result { + for _, name := range connectionFlags { + if result[idx] == name && idx+1 < len(result) { + result[idx+1] = redacted + break + } + if strings.HasPrefix(result[idx], name+"=") { + result[idx] = name + "=" + redacted + break + } + } + } + return result +} + +// commandOutput runs a provenance command and returns its trimmed standard output. +func commandOutput(name string, args ...string) string { + output, err := exec.Command(name, args...).Output() + if err != nil { + return "unknown" + } + return strings.TrimSpace(string(output)) +} + +// workingTreeSHA256 hashes the tracked Git diff together with sorted untracked paths and contents. +func workingTreeSHA256() string { + fingerprint, err := calculateWorkingTreeSHA256("") + if err != nil { + return "unknown" + } + return fingerprint +} + +// requireCleanSourceCapture refuses a live capture when either tracked edits +// or untracked source files would make the binary's provenance non-reproducible. +// It is intentionally evaluated before GraphBench validates a destructive +// target or opens its run lock. +func requireCleanSourceCapture() error { + return validateCleanSourceFingerprint(workingTreeSHA256()) +} + +// validateCleanSourceFingerprint isolates the fail-closed clean-source rule +// so capture wiring can be tested without consulting the caller's repository. +func validateCleanSourceFingerprint(fingerprint string) error { + if fingerprint != cleanWorkingTreeSHA256() { + return fmt.Errorf("clean-source capture requires a clean committed source tree") + } + return nil +} + +// calculateWorkingTreeSHA256 computes working tree sha256. +func calculateWorkingTreeSHA256(excludedRoot string) (string, error) { + digest := sha256.New() + output, err := exec.Command("git", "diff", "--binary", "HEAD", "--").Output() + if err != nil { + return "", fmt.Errorf("capture tracked source diff: %w", err) + } + writeWorkingTreePatchFingerprint(digest, output) + paths, err := gitUntrackedPaths() + if err != nil { + return "", err + } + excludedAbsolute := "" + if excludedRoot != "" { + excludedAbsolute, err = filepath.Abs(excludedRoot) + if err != nil { + return "", fmt.Errorf("resolve excluded source root: %w", err) + } + } + for _, path := range paths { + if excludedAbsolute != "" { + absolute, err := filepath.Abs(path) + if err != nil { + return "", fmt.Errorf("resolve untracked source %q: %w", path, err) + } + if absolute == excludedAbsolute || strings.HasPrefix(absolute, excludedAbsolute+string(filepath.Separator)) { + continue + } + } + content, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read untracked source %q: %w", path, err) + } + writeWorkingTreeUntrackedFingerprint(digest, filepath.ToSlash(path), content) + } + return hex.EncodeToString(digest.Sum(nil)), nil +} + +// gitUntrackedPaths supports benchmark evidence processing for git untracked paths. +func gitUntrackedPaths() ([]string, error) { + output, err := exec.Command("git", "ls-files", "-z", "--others", "--exclude-standard").Output() + if err != nil { + return nil, fmt.Errorf("list untracked source: %w", err) + } + paths := parseNULTerminatedPaths(output) + sort.Strings(paths) + return paths, nil +} + +// parseNULTerminatedPaths parses nul terminated paths. +func parseNULTerminatedPaths(output []byte) []string { + fields := strings.Split(string(output), "\x00") + paths := make([]string, 0, len(fields)) + for _, path := range fields { + if path != "" { + paths = append(paths, path) + } + } + return paths +} + +// writeWorkingTreePatchFingerprint writes working tree patch fingerprint. +func writeWorkingTreePatchFingerprint(digest io.Writer, patch []byte) { + _, _ = digest.Write(patch) +} + +// writeWorkingTreeUntrackedFingerprint writes working tree untracked fingerprint. +func writeWorkingTreeUntrackedFingerprint(digest io.Writer, path string, content []byte) { + _, _ = fmt.Fprintf(digest, "untracked:%s\x00", path) + _, _ = digest.Write(content) +} + +// executableSHA256 returns the SHA-256 digest of the running benchmark executable. +func executableSHA256() string { + path, err := os.Executable() + if err != nil { + return "unknown" + } + checksum, err := fileSHA256(path) + if err != nil { + return "unknown" + } + return checksum +} + +// sqlFingerprint returns the SHA-256 digest of the supplied SQL text exactly as provided. +func sqlFingerprint(sql string) string { + digest := sha256.Sum256([]byte(sql)) + return hex.EncodeToString(digest[:]) +} diff --git a/cmd/graphbench/environment_test.go b/cmd/graphbench/environment_test.go new file mode 100644 index 00000000..35624c9e --- /dev/null +++ b/cmd/graphbench/environment_test.go @@ -0,0 +1,39 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestSQLFingerprintIsStableAndContentSensitive verifies that identical SQL yields a repeatable 256-bit digest while a query change alters that digest. +func TestSQLFingerprintIsStableAndContentSensitive(t *testing.T) { + require.Equal(t, sqlFingerprint("select 1"), sqlFingerprint("select 1")) + require.NotEqual(t, sqlFingerprint("select 1"), sqlFingerprint("select 2")) + require.Len(t, sqlFingerprint("select 1"), 64) +} + +// TestSanitizedInvocationRedactsConnectionStrings verifies redaction for split and inline connection flags while preserving unrelated arguments and the caller's input slice. +func TestSanitizedInvocationRedactsConnectionStrings(t *testing.T) { + args := []string{ + "graphbench", + "-connection", "postgres://user:secret@host/database", + "-pg-connection=postgres://user:secret@host/database", + "-neo4j-connection", "neo4j://user:secret@host", + "-iterations", "30", + } + + require.Equal(t, []string{ + "graphbench", + "-connection", "", + "-pg-connection=", + "-neo4j-connection", "", + "-iterations", "30", + }, sanitizedInvocation(args)) + require.Contains(t, args[2], "secret", "the caller's argument slice must not be mutated") +} diff --git a/cmd/graphbench/expand_into_report.go b/cmd/graphbench/expand_into_report.go new file mode 100644 index 00000000..4d156e0f --- /dev/null +++ b/cmd/graphbench/expand_into_report.go @@ -0,0 +1,599 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "time" +) + +// expandIntoStudyReportVersion reserves the stable protocol value used to recognize expand into study report version across artifacts and executions. +const expandIntoStudyReportVersion = 2 + +// expandIntoStudyArms contains the frozen expand into study arms declaration consulted by package validation. +var expandIntoStudyArms = []string{ + "expand_into_pair_join", + "expand_into_lower_degree_scan", + "expand_into_pair_cache", +} + +// ExpandIntoStudyOptions selects the discovery or confirmation evidence protocol. +type ExpandIntoStudyOptions struct { + // Seed makes randomized statistical procedures reproducible. + Seed int64 + // Confidence sets the requested statistical confidence level. + Confidence float64 + // BootstrapCount records the number of bootstrap count. + BootstrapCount int + // Protocol identifies the protocol. + Protocol string + // MaterialityRatio supplies the materiality ratio input to the ExpandIntoStudyOptions contract. + MaterialityRatio float64 + // MaterialityAbsolute supplies the materiality absolute input to the ExpandIntoStudyOptions contract. + MaterialityAbsolute time.Duration + // P95RatioLimit supplies the p95 ratio limit input to the ExpandIntoStudyOptions contract. + P95RatioLimit float64 +} + +// ExpandIntoStudyReport contains exact three-arm fixed-one-hop evidence. +type ExpandIntoStudyReport struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // ArtifactSHA256 binds the referenced artifact content by SHA-256 digest. + ArtifactSHA256 string `json:"artifact_sha256"` + // Protocol identifies the protocol. + Protocol string `json:"protocol"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // TrainingCases supplies the training cases input to the ExpandIntoStudyReport contract. + TrainingCases int `json:"training_cases"` + // HoldoutCases supplies the holdout cases input to the ExpandIntoStudyReport contract. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed indicates whether qualification passed applies. + QualificationPassed bool `json:"qualification_passed"` + // PromotionEligible indicates whether promotion eligible applies. + PromotionEligible bool `json:"promotion_eligible"` + // Winner supplies the winner input to the ExpandIntoStudyReport contract. + Winner string `json:"winner,omitempty"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []ExpandIntoStudyCase `json:"cases"` +} + +// ExpandIntoStudyCase reports exactness, order balance, plan shape, and latency for one pair workload. +type ExpandIntoStudyCase struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset"` + // Name identifies the name. + Name string `json:"name"` + // Tier supplies the tier input to the ExpandIntoStudyCase contract. + Tier string `json:"tier,omitempty"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // Rounds records the number of rounds. + Rounds int `json:"rounds"` + // Winner supplies the winner input to the ExpandIntoStudyCase contract. + Winner string `json:"descriptive_median_winner,omitempty"` + // QualifiedWinner supplies the qualified winner input to the ExpandIntoStudyCase contract. + QualifiedWinner string `json:"qualified_winner,omitempty"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` + // ArmResults supplies the arm results input to the ExpandIntoStudyCase contract. + ArmResults []ExpandIntoStudyArmEvidence `json:"arms"` +} + +// ExpandIntoStudyArmEvidence records one exact plan-study arm and its ratio to the direct pair join. +type ExpandIntoStudyArmEvidence struct { + // Name identifies the name. + Name string `json:"name"` + // Architecture supplies the architecture input to the ExpandIntoStudyArmEvidence contract. + Architecture string `json:"architecture"` + // ImplementationID identifies the implementation id. + ImplementationID string `json:"implementation_id"` + // SQLFingerprint supplies the sql fingerprint input to the ExpandIntoStudyArmEvidence contract. + SQLFingerprint string `json:"sql_fingerprint"` + // Samples supplies the samples input to the ExpandIntoStudyArmEvidence contract. + Samples int `json:"samples"` + // Median supplies the median input to the ExpandIntoStudyArmEvidence contract. + Median time.Duration `json:"median"` + // P95 supplies the p95 input to the ExpandIntoStudyArmEvidence contract. + P95 time.Duration `json:"p95"` + // MedianRatioToDirect supplies the median ratio to direct input to the ExpandIntoStudyArmEvidence contract. + MedianRatioToDirect *RatioInterval `json:"median_ratio_to_direct,omitempty"` + // MedianSavingToDirect supplies the median saving to direct input to the ExpandIntoStudyArmEvidence contract. + MedianSavingToDirect *DurationInterval `json:"median_saving_to_direct,omitempty"` + // P95RatioToDirect supplies the p95 ratio to direct input to the ExpandIntoStudyArmEvidence contract. + P95RatioToDirect *RatioInterval `json:"p95_ratio_to_direct,omitempty"` + // Material indicates whether material applies. + Material bool `json:"material"` + // P95Contained indicates whether p95 contained applies. + P95Contained bool `json:"p95_contained"` + // QualifiedWinner indicates whether qualified winner applies. + QualifiedWinner bool `json:"qualified_winner"` + // PlanModes supplies the plan modes input to the ExpandIntoStudyArmEvidence contract. + PlanModes []ExpandIntoPlanMode `json:"plan_modes"` +} + +// ExpandIntoPlanMode summarizes the PostgreSQL shapes observed under one plan-cache mode. +type ExpandIntoPlanMode struct { + // PlanCacheMode identifies the plan cache mode. + PlanCacheMode string `json:"plan_cache_mode"` + // Fingerprints supplies the fingerprints input to the ExpandIntoPlanMode contract. + Fingerprints []string `json:"plan_fingerprints"` + // OperatorFamilies supplies the operator families input to the ExpandIntoPlanMode contract. + OperatorFamilies []string `json:"operator_families"` + // ParameterizedIndex indicates whether parameterized index applies. + ParameterizedIndex bool `json:"parameterized_index"` + // Memoize indicates whether memoize applies. + Memoize bool `json:"memoize"` + // HashJoin indicates whether hash join applies. + HashJoin bool `json:"hash_join"` +} + +// expandIntoArmSeries accumulates matched observations used to evaluate expand into arm. +type expandIntoArmSeries struct { + // identity retains the identity while expandIntoArmSeries is assembled or evaluated. + identity postgresReferenceSpec + // samples retains the samples while expandIntoArmSeries is assembled or evaluated. + samples roundSamples + // plans retains the plans while expandIntoArmSeries is assembled or evaluated. + plans map[string]map[string][]string +} + +// buildExpandIntoStudyReport validates all three exact arms and constructs descriptive crossover evidence. +func buildExpandIntoStudyReport(records []CaseResult, options ExpandIntoStudyOptions) (ExpandIntoStudyReport, error) { + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolDiscovery + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 5, 5, 20, 10 + if protocol == referencePairProtocolConfirmation { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 20, 10, 20, 50 + } else if protocol != referencePairProtocolDiscovery { + return ExpandIntoStudyReport{}, fmt.Errorf("unsupported ExpandInto study protocol %q", protocol) + } + if options.Confidence <= 0 || options.Confidence >= 1 { + return ExpandIntoStudyReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.MaterialityRatio == 0 { + options.MaterialityRatio = .95 + } + if options.MaterialityRatio <= 0 || options.MaterialityRatio >= 1 { + return ExpandIntoStudyReport{}, fmt.Errorf("materiality ratio must be between 0 and 1") + } + if options.MaterialityAbsolute == 0 { + options.MaterialityAbsolute = 100 * time.Microsecond + } + if options.MaterialityAbsolute < 0 { + return ExpandIntoStudyReport{}, fmt.Errorf("materiality absolute must not be negative") + } + if options.P95RatioLimit == 0 { + options.P95RatioLimit = 1.05 + } + if options.P95RatioLimit <= 0 { + return ExpandIntoStudyReport{}, fmt.Errorf("p95 ratio limit must be positive") + } + + // key identifies a dataset and workload pair in the collected evidence. + type key struct { + // dataset identifies the fixture dataset containing the workload. + dataset string + + // name identifies the workload within the dataset. + name string + } + + // caseSeries accumulates the evidence arms and rounds for one workload. + type caseSeries struct { + // tier retains the tier while caseSeries is assembled or evaluated. + tier string + // split retains the split while caseSeries is assembled or evaluated. + split string + // arms retains the arms while caseSeries is assembled or evaluated. + arms map[string]*expandIntoArmSeries + // rounds retains the rounds while caseSeries is assembled or evaluated. + rounds map[int]struct{} + // planModes retains the plan modes while caseSeries is assembled or evaluated. + planModes map[string]struct{} + // problems retains the problems while caseSeries is assembled or evaluated. + problems map[string]struct{} + } + series := map[key]*caseSeries{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL || record.Category != "expand_into_one_hop" { + continue + } + if record.Status != StatusOK || record.Environment == nil || record.Environment.WarmupIterations < minimumWarmups { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s lacks a successful %d-warmup PostgreSQL record", record.Dataset, record.Name, minimumWarmups) + } + caseKey := key{record.Dataset, record.Name} + current := series[caseKey] + if current == nil { + current = &caseSeries{ + tier: record.Shape.FixtureTier, + split: record.Shape.QualificationSplit, + arms: map[string]*expandIntoArmSeries{}, + rounds: map[int]struct{}{}, + planModes: map[string]struct{}{}, + problems: map[string]struct{}{}, + } + series[caseKey] = current + } else if current.tier != record.Shape.FixtureTier { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s changes fixture tier across rounds", record.Dataset, record.Name) + } else if current.split != record.Shape.QualificationSplit { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s changes qualification split across rounds", record.Dataset, record.Name) + } + if current.split != "training" && current.split != "holdout" { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s requires a training or holdout qualification split", record.Dataset, record.Name) + } + if record.Environment.Round < 1 { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s has an invalid measurement round %d", record.Dataset, record.Name, record.Environment.Round) + } + if _, duplicate := current.rounds[record.Environment.Round]; duplicate { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s has duplicate round %d", record.Dataset, record.Name, record.Environment.Round) + } + current.rounds[record.Environment.Round] = struct{}{} + cacheMode := "" + if record.PostgresEnvironment != nil { + cacheMode = record.PostgresEnvironment.PlanCacheMode + } + if cacheMode != "auto" && cacheMode != "force_custom_plan" && cacheMode != "force_generic_plan" { + current.problems[fmt.Sprintf("round %d has missing or unsupported plan_cache_mode %q", record.Environment.Round, cacheMode)] = struct{}{} + } else { + current.planModes[cacheMode] = struct{}{} + } + for _, armName := range expandIntoStudyArms { + reference := findReference(record.PostgresReferences, armName) + if reference == nil { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s round %d lacks ExpandInto arm %s", record.Dataset, record.Name, record.Environment.Round, armName) + } + if !reference.FullComparator || reference.SemanticValidation != "exact_public_observation" || reference.RowCount != record.RowCount || !equalStrings(reference.ObservedRows, record.ObservedRows) { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s arm %s is not an exact public comparator", record.Dataset, record.Name, armName) + } + if reference.Stats.WarmupIterations < minimumWarmups { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s arm %s has fewer than %d warmups", record.Dataset, record.Name, armName, minimumWarmups) + } + arm := current.arms[armName] + identity := normalizedReferenceSpec(postgresReferenceSpec{ + name: reference.Name, + architecture: reference.Architecture, + implementationID: reference.ImplementationID, + stateShape: reference.StateShape, + observationShape: reference.ObservationShape, + semanticValidation: reference.SemanticValidation, + boundary: reference.Boundary, + fullComparator: reference.FullComparator, + timingBoundary: reference.TimingBoundary, + sql: reference.SQL, + }) + if arm == nil { + arm = &expandIntoArmSeries{ + identity: identity, + samples: roundSamples{}, + plans: map[string]map[string][]string{}, + } + current.arms[armName] = arm + } else if arm.identity.architecture != identity.architecture || arm.identity.implementationID != identity.implementationID || normalizedSQLFingerprint(arm.identity.sql) != normalizedSQLFingerprint(identity.sql) { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s arm %s identity changed across rounds", record.Dataset, record.Name, armName) + } + for _, sample := range reference.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + arm.samples[record.Environment.Round] = append(arm.samples[record.Environment.Round], sample.Duration) + } + } + if len(reference.PostgresPlan) == 0 { + current.problems[fmt.Sprintf("%s round %d has no persisted PostgreSQL plan", armName, record.Environment.Round)] = struct{}{} + } + planModeKey := cacheMode + if planModeKey == "" { + planModeKey = "unknown" + } + fingerprint := normalizedSQLFingerprint(strings.Join(reference.PostgresPlan, "\n")) + if arm.plans[planModeKey] == nil { + arm.plans[planModeKey] = map[string][]string{} + } + arm.plans[planModeKey][fingerprint] = append([]string(nil), reference.PostgresPlan...) + } + if err := validateExpandIntoRoundOrder(record.Environment.Round, record.PostgresReferences); err != nil { + return ExpandIntoStudyReport{}, fmt.Errorf("%s/%s: %w", record.Dataset, record.Name, err) + } + } + if len(series) == 0 { + return ExpandIntoStudyReport{}, fmt.Errorf("artifact has no PostgreSQL ExpandInto study records") + } + + report := ExpandIntoStudyReport{ + Version: expandIntoStudyReportVersion, + Protocol: protocol, + Confidence: options.Confidence, + Passed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + keys := make([]key, 0, len(series)) + for caseKey := range series { + keys = append(keys, caseKey) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + qualifiedWinners := map[string]struct{}{} + for caseIndex, caseKey := range keys { + current := series[caseKey] + entry := ExpandIntoStudyCase{ + Dataset: caseKey.dataset, + Name: caseKey.name, + Tier: current.tier, + QualificationSplit: current.split, + Rounds: len(current.rounds), + Passed: true, + } + for problem := range current.problems { + entry.Reasons = append(entry.Reasons, problem) + } + sort.Strings(entry.Reasons) + if len(entry.Reasons) > 0 { + entry.Passed = false + } + if entry.Rounds < minimumRounds || entry.Rounds > maximumRounds { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("requires %d-%d rounds, got %d", minimumRounds, maximumRounds, entry.Rounds)) + } + if protocol == referencePairProtocolConfirmation { + for _, mode := range []string{"auto", "force_custom_plan", "force_generic_plan"} { + if _, present := current.planModes[mode]; !present { + entry.Passed = false + entry.Reasons = append(entry.Reasons, "confirmation requires plan_cache_mode="+mode) + } + } + } + direct := current.arms[expandIntoStudyArms[0]].samples + winnerMedian := time.Duration(1<<63 - 1) + qualifiedWinnerMedian := time.Duration(1<<63 - 1) + for armIndex, armName := range expandIntoStudyArms { + arm := current.arms[armName] + for _, round := range sortedRoundSet(current.rounds) { + if len(arm.samples[round]) < minimumSamples { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s round %d requires %d samples, got %d", armName, round, minimumSamples, len(arm.samples[round]))) + } + } + flat := flattenSamples(arm.samples, sortedRounds(arm.samples)) + evidence := ExpandIntoStudyArmEvidence{ + Name: armName, + Architecture: arm.identity.architecture, + ImplementationID: arm.identity.implementationID, + SQLFingerprint: normalizedSQLFingerprint(arm.identity.sql), + Samples: len(flat), + Median: time.Duration(durationQuantile(flat, .50)), + P95: time.Duration(durationQuantile(flat, .95)), + PlanModes: expandIntoPlanModes(arm.plans), + } + if evidence.Median < winnerMedian { + winnerMedian, entry.Winner = evidence.Median, armName + } + if armName != expandIntoStudyArms[0] { + baseline, candidate := matchedRounds(direct, arm.samples) + if len(baseline) > 0 { + seed := options.Seed + int64(caseIndex*31+armIndex)*7919 + median := bootstrapRoundMedianRatio(baseline, candidate, seed, gateOptions) + evidence.MedianRatioToDirect = &median + saving := bootstrapRoundMedianSaving(baseline, candidate, seed+1, gateOptions) + evidence.MedianSavingToDirect = &saving + if sampleCount(baseline) >= minimumP95Samples && sampleCount(candidate) >= minimumP95Samples { + p95 := bootstrapStratifiedP95Ratio(baseline, candidate, seed+2, gateOptions) + evidence.P95RatioToDirect = &p95 + } + evidence.Material = median.Upper <= options.MaterialityRatio || saving.Lower >= options.MaterialityAbsolute + evidence.P95Contained = evidence.P95RatioToDirect != nil && evidence.P95RatioToDirect.Upper <= options.P95RatioLimit + evidence.QualifiedWinner = evidence.Material && evidence.P95Contained + if evidence.QualifiedWinner && evidence.Median < qualifiedWinnerMedian { + qualifiedWinnerMedian, entry.QualifiedWinner = evidence.Median, armName + } + } + } + entry.ArmResults = append(entry.ArmResults, evidence) + } + if protocol == referencePairProtocolConfirmation && entry.QualifiedWinner == "" { + entry.Passed = false + entry.Reasons = append(entry.Reasons, "no non-incumbent arm materially beats the direct pair join with p95 containment") + } + if protocol == referencePairProtocolConfirmation && entry.Passed { + qualifiedWinners[entry.QualifiedWinner] = struct{}{} + } + if !entry.Passed { + report.Passed = false + } + switch entry.QualificationSplit { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && entry.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && entry.Passed + } + report.Cases = append(report.Cases, entry) + } + report.TrainingPassed = report.TrainingCases > 0 && report.TrainingPassed + report.HoldoutPassed = report.HoldoutCases > 0 && report.HoldoutPassed + report.QualificationPassed = protocol == referencePairProtocolConfirmation && + report.TrainingPassed && report.HoldoutPassed && len(qualifiedWinners) == 1 + if len(qualifiedWinners) == 1 { + for winner := range qualifiedWinners { + report.Winner = winner + } + } + report.PromotionEligible = report.QualificationPassed + if protocol == referencePairProtocolConfirmation && !report.QualificationPassed { + report.Passed = false + } + return report, nil +} + +// sortedRoundSet returns declared measurement rounds in stable order, including +// rounds whose arms contain no usable warm sample. +func sortedRoundSet(rounds map[int]struct{}) []int { + ordered := make([]int, 0, len(rounds)) + for round := range rounds { + ordered = append(ordered, round) + } + sort.Ints(ordered) + return ordered +} + +// equalStrings reports whether two ordered string collections contain identical values. +func equalStrings(left, right []string) bool { + if len(left) != len(right) { + return false + } + for idx := range left { + if left[idx] != right[idx] { + return false + } + } + return true +} + +// validateExpandIntoRoundOrder enforces the predeclared doubled Williams schedule relative to the three selected arms. +func validateExpandIntoRoundOrder(round int, references []PostgresReferenceResult) error { + base := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + base[idx] = postgresReferenceSpec{name: name} + } + expected := referenceSpecsForRound(base, round) + byName := map[string]int{} + for _, reference := range references { + if containsString(expandIntoStudyArms, reference.Name) { + byName[reference.Name] = reference.MeasurementOrder + } + } + for _, name := range expandIntoStudyArms { + if byName[name] <= 0 { + return fmt.Errorf("round %d is missing measurement order for %s", round, name) + } + } + for idx := 1; idx < len(expected); idx++ { + if byName[expected[idx-1].name] >= byName[expected[idx].name] { + return fmt.Errorf("round %d lacks the declared three-arm carryover order", round) + } + } + return nil +} + +// containsString reports whether an exact string occurs in a collection. +func containsString(values []string, value string) bool { + for _, candidate := range values { + if candidate == value { + return true + } + } + return false +} + +// expandIntoPlanModes classifies parameterized index, Memoize, and hash alternatives per plan-cache mode. +func expandIntoPlanModes(plans map[string]map[string][]string) []ExpandIntoPlanMode { + var modes []ExpandIntoPlanMode + for mode, byFingerprint := range plans { + evidence := ExpandIntoPlanMode{PlanCacheMode: mode} + operators := map[string]struct{}{} + for fingerprint, plan := range byFingerprint { + evidence.Fingerprints = append(evidence.Fingerprints, fingerprint) + joined := strings.ToLower(strings.Join(plan, "\n")) + evidence.ParameterizedIndex = evidence.ParameterizedIndex || strings.Contains(joined, "index scan") && (strings.Contains(joined, "start_id") || strings.Contains(joined, "end_id")) + evidence.Memoize = evidence.Memoize || strings.Contains(joined, "memoize") + evidence.HashJoin = evidence.HashJoin || strings.Contains(joined, "hash join") + for _, line := range plan { + operator := expandIntoPlanOperator(line) + if operator != "" { + operators[operator] = struct{}{} + } + } + } + for operator := range operators { + evidence.OperatorFamilies = append(evidence.OperatorFamilies, operator) + } + sort.Strings(evidence.Fingerprints) + sort.Strings(evidence.OperatorFamilies) + modes = append(modes, evidence) + } + sort.Slice(modes, func(i, j int) bool { return modes[i].PlanCacheMode < modes[j].PlanCacheMode }) + return modes +} + +// expandIntoPlanOperator removes EXPLAIN decorations while retaining the physical operator family. +func expandIntoPlanOperator(line string) string { + line = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "->")) + if line == "" || strings.HasPrefix(line, "Filter:") || strings.HasPrefix(line, "Index Cond:") || strings.HasPrefix(line, "Join Filter:") { + return "" + } + if index := strings.Index(line, " ("); index >= 0 { + line = line[:index] + } + if index := strings.Index(line, " on "); index >= 0 { + line = line[:index] + } + if index := strings.Index(line, " using "); index >= 0 { + line = line[:index] + } + return strings.TrimSpace(line) +} + +// createExpandIntoStudyReport reads, validates, fingerprints, and writes a three-arm study artifact. +func createExpandIntoStudyReport(artifactPath, outputPath string, options ExpandIntoStudyOptions) error { + records, err := readJSONLFile(artifactPath) + if err != nil { + return err + } + report, err := buildExpandIntoStudyReport(records, options) + if err != nil { + return err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return err + } + var output *os.File + if outputPath == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(outputPath); err != nil { + return err + } + output, err = os.Create(outputPath) + if err != nil { + return err + } + defer output.Close() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return err + } + if !report.Passed { + return fmt.Errorf("ExpandInto %s evidence did not pass its declared protocol", report.Protocol) + } + return nil +} diff --git a/cmd/graphbench/expand_into_report_test.go b/cmd/graphbench/expand_into_report_test.go new file mode 100644 index 00000000..7da87ac0 --- /dev/null +++ b/cmd/graphbench/expand_into_report_test.go @@ -0,0 +1,319 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildExpandIntoStudyReportValidatesThreeArmEvidence verifies exactness, Williams order, ratios, winners, and physical-plan classification. +func TestBuildExpandIntoStudyReportValidatesThreeArmEvidence(t *testing.T) { + var records []CaseResult + for round := 1; round <= 5; round++ { + orderSpecs := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + orderSpecs[idx].name = name + } + ordered := referenceSpecsForRound(orderSpecs, round) + orders := map[string]int{} + for idx, spec := range ordered { + orders[spec.name] = idx + 2 + } + record := CaseResult{ + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 5, + }, + PostgresEnvironment: &PostgresEnvironment{PlanCacheMode: "force_custom_plan"}, + Dataset: "expand_into", + Name: "pair", + Category: "expand_into_one_hop", + Shape: WorkloadShape{ + FixtureTier: "normal", + QualificationSplit: "training", + }, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{`["edge"]`}, + } + for idx, name := range expandIntoStudyArms { + duration := time.Duration(100-idx*10) * time.Microsecond + var samples []LatencySample + for sample := 0; sample < 10; sample++ { + samples = append(samples, LatencySample{ + Classification: "warm", + Duration: duration + time.Duration(sample), + }) + } + plan := []string{"Nested Loop (cost=0.00..1.00 rows=1 width=8)", " -> Index Scan using edge_start_id_idx on edge (cost=0.00..1.00 rows=1 width=8)", " Index Cond: (start_id = input_pairs.start_id)"} + if name == "expand_into_pair_cache" { + plan = []string{"Hash Join (cost=0.00..1.00 rows=1 width=8)", " -> Memoize (cost=0.00..1.00 rows=1 width=8)"} + } + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + SchemaVersion: postgresReferenceSchemaVersion, + Name: name, + Architecture: "architecture-" + name, + ImplementationID: name + "-v1", + StateShape: "state", + ObservationShape: "relationships", + SemanticValidation: "exact_public_observation", + Boundary: "relationships", + TimingBoundary: "raw_pgx", + FullComparator: true, + MeasurementOrder: orders[name], + SQL: "select '" + name + "'", + SQLFingerprint: name, + RowCount: 1, + ObservedRows: []string{`["edge"]`}, + Stats: DurationStats{ + WarmupIterations: 5, + Samples: samples, + }, + PostgresPlan: plan, + }) + } + records = append(records, record) + } + + report, err := buildExpandIntoStudyReport(records, ExpandIntoStudyOptions{ + Seed: 1, + Confidence: .975, + BootstrapCount: 100, + Protocol: referencePairProtocolDiscovery, + }) + require.NoError(t, err) + require.True(t, report.Passed) + require.Equal(t, 1, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.Len(t, report.Cases, 1) + entry := report.Cases[0] + require.Equal(t, "expand_into_pair_cache", entry.Winner) + require.Len(t, entry.ArmResults, 3) + require.Nil(t, entry.ArmResults[0].MedianRatioToDirect) + require.NotNil(t, entry.ArmResults[1].MedianRatioToDirect) + require.True(t, entry.ArmResults[0].PlanModes[0].ParameterizedIndex) + require.True(t, entry.ArmResults[2].PlanModes[0].Memoize) + require.True(t, entry.ArmResults[2].PlanModes[0].HashJoin) + require.Equal(t, "training", entry.QualificationSplit) + + artifactPath := filepath.Join(t.TempDir(), "expand-into.jsonl") + outputPath := filepath.Join(t.TempDir(), "expand-into.json") + require.NoError(t, writeJSONLFile(artifactPath, records)) + require.NoError(t, createExpandIntoStudyReport(artifactPath, outputPath, ExpandIntoStudyOptions{ + Seed: 1, + Confidence: .975, + BootstrapCount: 100, + Protocol: referencePairProtocolDiscovery, + })) + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var written ExpandIntoStudyReport + require.NoError(t, json.Unmarshal(content, &written)) + require.True(t, written.Passed) + require.True(t, validSHA256(written.ArtifactSHA256)) + require.Equal(t, referencePairProtocolDiscovery, written.Protocol) + + var confirmationRecords []CaseResult + for round := 1; round <= 10; round++ { + record := records[(round-1)%len(records)] + record.Environment = &RunEnvironment{ + Round: round, + WarmupIterations: 20, + } + planModes := []string{"auto", "force_custom_plan", "force_generic_plan"} + record.PostgresEnvironment = &PostgresEnvironment{PlanCacheMode: planModes[(round-1)%len(planModes)]} + record.PostgresReferences = append([]PostgresReferenceResult(nil), record.PostgresReferences...) + orderSpecs := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + orderSpecs[idx].name = name + } + orders := map[string]int{} + for idx, spec := range referenceSpecsForRound(orderSpecs, round) { + orders[spec.name] = idx + 2 + } + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + reference.MeasurementOrder = orders[reference.Name] + reference.Stats.WarmupIterations = 20 + duration := reference.Stats.Samples[0].Duration + reference.Stats.Samples = make([]LatencySample, 50) + for sample := range reference.Stats.Samples { + reference.Stats.Samples[sample] = LatencySample{ + Classification: "warm", + Duration: duration + time.Duration(sample), + } + } + } + confirmationRecords = append(confirmationRecords, record) + holdout := record + holdout.Name = "pair-holdout" + holdout.Shape.QualificationSplit = "holdout" + confirmationRecords = append(confirmationRecords, holdout) + } + confirmation, err := buildExpandIntoStudyReport(confirmationRecords, ExpandIntoStudyOptions{ + Seed: 1, + Confidence: .975, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.True(t, confirmation.Passed) + require.Equal(t, 1, confirmation.TrainingCases) + require.Equal(t, 1, confirmation.HoldoutCases) + require.True(t, confirmation.TrainingPassed) + require.True(t, confirmation.HoldoutPassed) + require.True(t, confirmation.QualificationPassed) + require.Equal(t, referencePairProtocolConfirmation, confirmation.Protocol) + var trainingOnly []CaseResult + for _, record := range confirmationRecords { + if record.Shape.QualificationSplit == "training" { + trainingOnly = append(trainingOnly, record) + } + } + trainingOnlyReport, err := buildExpandIntoStudyReport(trainingOnly, ExpandIntoStudyOptions{ + Seed: 1, + Confidence: .975, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.False(t, trainingOnlyReport.Passed) + require.True(t, trainingOnlyReport.TrainingPassed) + require.False(t, trainingOnlyReport.HoldoutPassed) + require.False(t, trainingOnlyReport.QualificationPassed) + + for idx := range confirmationRecords { + confirmationRecords[idx].PostgresEnvironment = &PostgresEnvironment{PlanCacheMode: "force_custom_plan"} + } + incompleteModes, err := buildExpandIntoStudyReport(confirmationRecords, ExpandIntoStudyOptions{ + Seed: 1, + Confidence: .975, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.False(t, incompleteModes.Passed) + require.Contains(t, incompleteModes.Cases[0].Reasons, "confirmation requires plan_cache_mode=auto") + require.Contains(t, incompleteModes.Cases[0].Reasons, "confirmation requires plan_cache_mode=force_generic_plan") +} + +// TestBuildExpandIntoStudyReportFailsClosedOnObservationOrOrderMismatch verifies plan evidence cannot qualify without exact rows and declared carryover order. +func TestBuildExpandIntoStudyReportFailsClosedOnObservationOrOrderMismatch(t *testing.T) { + record := CaseResult{ + Environment: &RunEnvironment{ + Round: 1, + WarmupIterations: 5, + }, + Dataset: "expand_into", + Name: "pair", + Category: "expand_into_one_hop", + Shape: WorkloadShape{ + FixtureTier: "normal", + QualificationSplit: "training", + }, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"public"}, + } + for _, name := range expandIntoStudyArms { + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + Name: name, + Architecture: name, + ImplementationID: name, + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"different"}, + Stats: DurationStats{WarmupIterations: 5}, + MeasurementOrder: 2, + }) + } + _, err := buildExpandIntoStudyReport([]CaseResult{record}, ExpandIntoStudyOptions{ + Confidence: .975, + Protocol: referencePairProtocolDiscovery, + }) + require.ErrorContains(t, err, "not an exact public comparator") +} + +// TestCreateExpandIntoStudyReportPersistsAndRejectsIncompleteEvidence verifies +// a durable diagnostic report cannot be mistaken for a successful gate. +func TestCreateExpandIntoStudyReportPersistsAndRejectsIncompleteEvidence(t *testing.T) { + record := CaseResult{ + Environment: &RunEnvironment{ + Round: 1, + WarmupIterations: 5, + }, + Dataset: "expand_into", + Name: "pair", + Category: "expand_into_one_hop", + Shape: WorkloadShape{ + FixtureTier: "normal", + QualificationSplit: "training", + }, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{`["edge"]`}, + } + orderSpecs := make([]postgresReferenceSpec, len(expandIntoStudyArms)) + for idx, name := range expandIntoStudyArms { + orderSpecs[idx].name = name + } + orders := map[string]int{} + for idx, spec := range referenceSpecsForRound(orderSpecs, 1) { + orders[spec.name] = idx + 2 + } + for _, name := range expandIntoStudyArms { + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + Name: name, + Architecture: name, + ImplementationID: name + "-v1", + StateShape: "state", + ObservationShape: "relationships", + Boundary: "relationships", + TimingBoundary: "raw_pgx", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{`["edge"]`}, + SQL: "select '" + name + "'", + MeasurementOrder: orders[name], + Stats: DurationStats{ + WarmupIterations: 5, + Samples: []LatencySample{{ + Classification: "warm", + Duration: time.Millisecond, + }}, + }, + }) + } + artifactPath := filepath.Join(t.TempDir(), "incomplete.jsonl") + outputPath := filepath.Join(t.TempDir(), "report.json") + require.NoError(t, writeJSONLFile(artifactPath, []CaseResult{record})) + require.ErrorContains(t, createExpandIntoStudyReport(artifactPath, outputPath, ExpandIntoStudyOptions{ + Seed: 1, + Confidence: .975, + BootstrapCount: 100, + Protocol: referencePairProtocolDiscovery, + }), "did not pass") + + content, err := os.ReadFile(outputPath) + require.NoError(t, err) + var report ExpandIntoStudyReport + require.NoError(t, json.Unmarshal(content, &report)) + require.False(t, report.Passed) +} diff --git a/cmd/graphbench/live_mode.go b/cmd/graphbench/live_mode.go new file mode 100644 index 00000000..a7a1fd54 --- /dev/null +++ b/cmd/graphbench/live_mode.go @@ -0,0 +1,619 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bufio" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" +) + +// existingGraphCheckpointVersion identifies the serialized schema revision for existing graph checkpoint. +const existingGraphCheckpointVersion = 2 + +// mutationKeyword matches Cypher keywords that can mutate an existing graph. +var mutationKeyword = regexp.MustCompile(`(?i)\b(create|merge|delete|detach|set|remove|drop|alter|truncate|grant|revoke|call|foreach|load\s+csv)\b`) + +// ExistingGraphAnchorManifest authorizes read-only live-graph workloads against validated logical or redacted physical anchors. +type ExistingGraphAnchorManifest struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Graph identifies the graph addressed by the artifact. + Graph string `json:"graph"` + // ContentIdentity binds resumable work to the logical contents of the live graph. + ContentIdentity string `json:"content_identity"` + // Anchors maps manifest anchor names to their logical or redacted physical identities. + Anchors map[string]ExistingGraphAnchor `json:"anchors"` + // Checksum supplies the checksum input to the ExistingGraphAnchorManifest contract. + Checksum string `json:"-"` +} + +// ExistingGraphAnchor maps a logical fixture key to either a logical or redacted physical identity. +type ExistingGraphAnchor struct { + // LogicalKey identifies an anchor using a corpus-visible fixture key. + LogicalKey string `json:"logical_key,omitempty"` + // PhysicalID selects a backend node directly when no corpus-visible logical key is available. + PhysicalID *int64 `json:"physical_id,omitempty"` + // ContentSHA256 identifies scrubbed physical anchor content without exposing it. + ContentSHA256 string `json:"content_sha256,omitempty"` + // Kind optionally requires the resolved anchor node to carry this graph kind. + Kind string `json:"kind,omitempty"` +} + +// ExistingGraphAttempt captures the applied deadline, collected samples, and outcome of one live-graph execution. +type ExistingGraphAttempt struct { + // Timeout supplies the timeout input to the ExistingGraphAttempt contract. + Timeout time.Duration `json:"timeout"` + // WarmupSamples records untimed samples collected before live-graph measurement. + WarmupSamples int `json:"warmup_samples"` + // MeasuredSamples records timed samples collected for the live-graph attempt. + MeasuredSamples int `json:"measured_samples"` + // Status supplies the status input to the ExistingGraphAttempt contract. + Status string `json:"status"` + // Error supplies the error input to the ExistingGraphAttempt contract. + Error string `json:"error,omitempty"` +} + +// ExistingGraphRun describes a resumable live-graph run and all attempts made in it. +type ExistingGraphRun struct { + // ManifestSHA256 identifies the anchor manifest that authorized the run. + ManifestSHA256 string `json:"manifest_sha256"` + // ContentIdentity binds resumable work to the logical contents of the live graph. + ContentIdentity string `json:"content_identity"` + // Protocol identifies the measurement protocol. + Protocol string `json:"protocol"` + // Adaptive indicates that adaptive discovery, rather than a fixed protocol, produced the record. + Adaptive bool `json:"adaptive"` + // Attempts lists live-graph attempts in execution order. + Attempts []ExistingGraphAttempt `json:"attempts,omitempty"` + // PreNodeCount records graph nodes present before the live-graph run. + PreNodeCount int64 `json:"pre_node_count"` + // PreEdgeCount records graph relationships present before the live-graph run. + PreEdgeCount int64 `json:"pre_edge_count"` + // PostNodeCount records graph nodes present after the live-graph run. + PostNodeCount int64 `json:"post_node_count"` + // PostEdgeCount records graph relationships present after the live-graph run. + PostEdgeCount int64 `json:"post_edge_count"` +} + +// ExistingGraphProgress is one append-only progress event emitted during a live-graph run. +type ExistingGraphProgress struct { + // At records when the progress event was emitted. + At time.Time `json:"at"` + // Stage identifies the stage reached by a live-graph progress event. + Stage string `json:"stage"` + // CaseKey identifies the dataset/case pair addressed by a progress event. + CaseKey string `json:"case_key,omitempty"` + // Detail contains the progress or failure detail safe to persist. + Detail string `json:"detail,omitempty"` +} + +// existingGraphCheckpoint binds completed live-graph cases to a corpus, run configuration, and fixture identity. +type existingGraphCheckpoint struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // ManifestSHA256 identifies the anchor manifest that authorized the run. + ManifestSHA256 string `json:"manifest_sha256"` + // CorpusSHA256 binds checkpoint records to the exact canonical workload declarations. + CorpusSHA256 string `json:"corpus_sha256"` + // RunSHA256 binds completed records to the exact resumable run configuration. + RunSHA256 string `json:"run_sha256"` + // Records contains completed CaseResults retained for resumable execution. + Records []CaseResult `json:"records"` +} + +// loadExistingGraphAnchorManifest reads and validates a live-graph anchor manifest and records its checksum. +func loadExistingGraphAnchorManifest(path string) (ExistingGraphAnchorManifest, error) { + raw, err := os.ReadFile(path) + if err != nil { + return ExistingGraphAnchorManifest{}, fmt.Errorf("read anchor manifest: %w", err) + } + var manifest ExistingGraphAnchorManifest + if err := json.Unmarshal(raw, &manifest); err != nil { + return ExistingGraphAnchorManifest{}, fmt.Errorf("decode anchor manifest: %w", err) + } + if manifest.Version != 1 { + return ExistingGraphAnchorManifest{}, fmt.Errorf("unsupported anchor manifest version %d", manifest.Version) + } + if len(manifest.Anchors) == 0 { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor manifest must contain anchors") + } + if strings.TrimSpace(manifest.Graph) == "" { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor manifest graph must not be empty") + } + if matched, _ := regexp.MatchString(`^sha256:[0-9a-f]{64}$`, manifest.ContentIdentity); !matched { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor manifest content_identity must be a lowercase sha256 digest") + } + for name, anchor := range manifest.Anchors { + if strings.TrimSpace(name) == "" { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor names must not be empty") + } + hasLogicalKey := strings.TrimSpace(anchor.LogicalKey) != "" + hasPhysicalID := anchor.PhysicalID != nil + if hasLogicalKey == hasPhysicalID { + return ExistingGraphAnchorManifest{}, fmt.Errorf("anchor %s must declare exactly one of logical_key or physical_id", name) + } + if hasPhysicalID { + if matched, _ := regexp.MatchString(`^sha256:[0-9a-f]{64}$`, anchor.ContentSHA256); !matched { + return ExistingGraphAnchorManifest{}, fmt.Errorf("physical anchor %s content_sha256 must be a lowercase sha256 digest", name) + } + } else if anchor.ContentSHA256 != "" { + return ExistingGraphAnchorManifest{}, fmt.Errorf("logical-key anchor %s must not declare content_sha256", name) + } + } + digest := sha256.Sum256(raw) + manifest.Checksum = hex.EncodeToString(digest[:]) + return manifest, nil +} + +// validateExistingGraphCorpus rejects mutations and anchors absent from the live-graph manifest. +func validateExistingGraphCorpus(corpus ScaleCorpus, manifest ExistingGraphAnchorManifest) error { + for _, testCase := range corpus.Cases { + if testCase.WriteScenario != nil { + return fmt.Errorf("existing-graph mode rejects write_scenario in case %s", testCase.Name) + } + if mutationKeyword.MatchString(stripCypherStringLiterals(testCase.Cypher)) { + return fmt.Errorf("existing-graph mode rejects mutation keyword in case %s", testCase.Name) + } + for _, anchor := range testCase.NodeParams { + if _, found := manifest.Anchors[anchor]; !found { + return fmt.Errorf("case %s references anchor %q absent from the manifest", testCase.Name, anchor) + } + } + for _, anchors := range testCase.NodeListParams { + for _, anchor := range anchors { + if _, found := manifest.Anchors[anchor]; !found { + return fmt.Errorf("case %s references anchor %q absent from the manifest", testCase.Name, anchor) + } + } + } + } + return nil +} + +// stripCypherStringLiterals replaces quoted Cypher contents with spaces before mutation-keyword scanning. +func stripCypherStringLiterals(query string) string { + var ( + result strings.Builder + quote rune + escaped bool + ) + + for _, value := range query { + if quote != 0 { + if escaped { + escaped = false + continue + } + if value == '\\' { + escaped = true + continue + } + if value == quote { + quote = 0 + } + result.WriteRune(' ') + continue + } + + if value == '\'' || value == '"' { + quote = value + result.WriteRune(' ') + continue + } + result.WriteRune(value) + } + + return result.String() +} + +// existingGraphCaseKey joins execution mode, dataset, and case name into the checkpoint lookup key. +func existingGraphCaseKey(mode ExecutionMode, testCase ScaleCase) string { + return strings.Join([]string{string(mode), testCase.Dataset, testCase.Name}, "/") +} + +// corpusIdentity hashes the canonical corpus declaration used to bind checkpoints to workloads. +func corpusIdentity(corpus ScaleCorpus) string { + cases := append([]ScaleCase(nil), corpus.Cases...) + sort.Slice(cases, func(i, j int) bool { + if cases[i].Source != cases[j].Source { + return cases[i].Source < cases[j].Source + } + if cases[i].Dataset != cases[j].Dataset { + return cases[i].Dataset < cases[j].Dataset + } + return cases[i].Name < cases[j].Name + }) + raw, _ := json.Marshal(struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Cases contains the canonically ordered workload declarations bound into the corpus digest. + Cases []ScaleCase `json:"cases"` + }{ + Version: 2, + Cases: cases, + }) + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// runConfigurationIdentity hashes execution-affecting configuration and environment fields for checkpoint compatibility. +func runConfigurationIdentity(cfg config, environment RunEnvironment) string { + payload := struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // SourceCommit identifies the source commit used to build the benchmark executable. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 identifies uncommitted source changes present during the run. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 identifies the benchmark executable used for the run. + BinarySHA256 string `json:"binary_sha256"` + // GOOS supplies the goos input to the anonymous record contract. + GOOS string `json:"goos"` + // GOARCH supplies the goarch input to the anonymous record contract. + GOARCH string `json:"goarch"` + // GoVersion identifies the schema version for go version. + GoVersion string `json:"go_version"` + // Modes records execution-mode order as part of resumable run identity. + Modes []ExecutionMode `json:"modes"` + // Iterations records the number of iterations. + Iterations int `json:"iterations"` + // WarmupIterations records the number of warmup iterations. + WarmupIterations int `json:"warmup_iterations"` + // Round identifies the measurement round. + Round int `json:"round"` + // Block identifies the measurement block used to control carryover effects. + Block int `json:"block"` + // Arm identifies the measurement arm that produced the sample. + Arm string `json:"arm"` + // ArmOrder supplies the arm order input to the anonymous record contract. + ArmOrder int `json:"arm_order"` + // PoolSize sets the database connection-pool size. + PoolSize int `json:"pool_size"` + // Concurrency supplies the concurrency input to the anonymous record contract. + Concurrency []int `json:"concurrency"` + // SessionMemoryCeilingBytes sets the per-session memory ceiling in bytes. + SessionMemoryCeilingBytes int64 `json:"session_memory_ceiling_bytes"` + // PoolMemoryCeilingBytes sets the aggregate pool memory ceiling in bytes. + PoolMemoryCeilingBytes int64 `json:"pool_memory_ceiling_bytes"` + // PostgresReferences records whether independent PostgreSQL references are enabled for the run identity. + PostgresReferences bool `json:"postgres_references"` + // PostgresReferenceArms lists independent PostgreSQL reference arms selected for measurement. + PostgresReferenceArms []string `json:"postgres_reference_arms"` + // PostgresForceShortest selects a forced shortest-path executor for diagnostic runs. + PostgresForceShortest string `json:"postgres_force_shortest"` + // PostgresForceExpansion selects a forced expansion search strategy for diagnostic runs. + PostgresForceExpansion string `json:"postgres_force_expansion"` + // PostgresRepeatableRead indicates whether postgres repeatable read applies. + PostgresRepeatableRead bool `json:"postgres_repeatable_read"` + // PostgresTraversalTelemetry selects the opt-in traversal evidence boundary. + PostgresTraversalTelemetry string `json:"postgres_traversal_telemetry"` + // PostgresExpansionOrientationShadow indicates whether postgres expansion orientation shadow applies. + PostgresExpansionOrientationShadow bool `json:"postgres_expansion_orientation_shadow"` + // PostgresExpansionOrientationTournament indicates whether postgres expansion orientation tournament applies. + PostgresExpansionOrientationTournament bool `json:"postgres_expansion_orientation_tournament"` + // PostgresExpansionOrientationPolicy identifies the postgres expansion orientation policy. + PostgresExpansionOrientationPolicy string `json:"postgres_expansion_orientation_policy"` + // PostgresExpansionSuffixReverseGuard selects the static guarded suffix-reverse statement. + PostgresExpansionSuffixReverseGuard bool `json:"postgres_expansion_suffix_reverse_guard"` + // PostgresExpansionSuffixReverseRetry selects the transaction-local retry candidate. + PostgresExpansionSuffixReverseRetry bool `json:"postgres_expansion_suffix_reverse_retry"` + // PostgresSuffixGuardSuffixLimit binds the tool-only suffix cap override. + PostgresSuffixGuardSuffixLimit int64 `json:"postgres_suffix_guard_suffix_limit"` + // PostgresSuffixGuardStateLimit binds the tool-only reverse-state cap override. + PostgresSuffixGuardStateLimit int64 `json:"postgres_suffix_guard_state_limit"` + // PostgresSuffixRetryOutputRowLimit binds the buffered row cap. + PostgresSuffixRetryOutputRowLimit int64 `json:"postgres_suffix_retry_output_row_limit"` + // PostgresSuffixRetryOutputBytesLimit binds the buffered byte cap. + PostgresSuffixRetryOutputBytesLimit int64 `json:"postgres_suffix_retry_output_bytes_limit"` + // Discovery enables adaptive live-graph discovery instead of the fixed confirmation protocol. + Discovery bool `json:"discovery"` + // TimeoutClasses lists the increasing per-attempt deadlines included in resumable run identity. + TimeoutClasses []time.Duration `json:"timeout_classes"` + // DiscoverySampleFloor sets the minimum live-graph samples required before adaptive discovery may stop. + DiscoverySampleFloor int `json:"discovery_sample_floor"` + }{ + Version: 1, + SourceCommit: environment.SourceCommit, + DirtyDiffSHA256: environment.DirtyDiffSHA256, + BinarySHA256: environment.BinarySHA256, + GOOS: environment.GOOS, + GOARCH: environment.GOARCH, + GoVersion: environment.GoVersion, + Modes: append([]ExecutionMode(nil), cfg.Modes...), + Iterations: cfg.Iterations, + WarmupIterations: cfg.WarmupIterations, + Round: cfg.Round, + Block: cfg.Block, + Arm: cfg.Arm, + ArmOrder: cfg.ArmOrder, + PoolSize: cfg.PoolSize, + Concurrency: append([]int(nil), cfg.Concurrency...), + SessionMemoryCeilingBytes: cfg.SessionMemoryCeilingBytes, + PoolMemoryCeilingBytes: cfg.PoolMemoryCeilingBytes, + PostgresReferences: cfg.PostgresReferences, + PostgresReferenceArms: append([]string(nil), cfg.PostgresReferenceArms...), + PostgresForceShortest: cfg.PostgresForceShortest, + PostgresForceExpansion: cfg.PostgresForceExpansion, + PostgresRepeatableRead: cfg.PostgresRepeatableRead, + PostgresTraversalTelemetry: cfg.PostgresTraversalTelemetry, + PostgresExpansionOrientationShadow: cfg.PostgresExpansionOrientationShadow, + PostgresExpansionOrientationTournament: cfg.PostgresExpansionOrientationTournament, + PostgresExpansionOrientationPolicy: cfg.PostgresExpansionOrientationPolicy, + PostgresExpansionSuffixReverseGuard: cfg.PostgresExpansionSuffixReverseGuard, + PostgresExpansionSuffixReverseRetry: cfg.PostgresExpansionSuffixReverseRetry, + PostgresSuffixGuardSuffixLimit: cfg.PostgresSuffixGuardSuffixLimit, + PostgresSuffixGuardStateLimit: cfg.PostgresSuffixGuardStateLimit, + PostgresSuffixRetryOutputRowLimit: cfg.PostgresSuffixRetryOutputRowLimit, + PostgresSuffixRetryOutputBytesLimit: cfg.PostgresSuffixRetryOutputBytesLimit, + Discovery: cfg.Discovery, + TimeoutClasses: append([]time.Duration(nil), cfg.TimeoutClasses...), + DiscoverySampleFloor: cfg.DiscoverySampleFloor, + } + raw, _ := json.Marshal(payload) + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// readExistingGraphCheckpoint reads a checkpoint, returning an empty checkpoint when the file does not exist. +func readExistingGraphCheckpoint(path, manifestHash, corpusHash, runHash string) ([]CaseResult, error) { + if path == "" { + return nil, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var checkpoint existingGraphCheckpoint + if err := json.Unmarshal(raw, &checkpoint); err != nil { + return nil, fmt.Errorf("decode existing-graph checkpoint: %w", err) + } + if checkpoint.Version != existingGraphCheckpointVersion || checkpoint.ManifestSHA256 != manifestHash || checkpoint.CorpusSHA256 != corpusHash || checkpoint.RunSHA256 != runHash { + return nil, fmt.Errorf("existing-graph checkpoint identity does not match this run") + } + seen := map[string]struct{}{} + runUUID := "" + for _, record := range checkpoint.Records { + if record.WorkloadSHA256 == "" || record.Environment == nil || record.Environment.ArtifactSchemaVersion != 2 || record.Environment.CorpusSHA256 != corpusHash || record.Environment.RunIdentitySHA256 != runHash || record.Environment.RunUUID == "" { + return nil, fmt.Errorf("existing-graph checkpoint record identity does not match this run") + } + if runUUID == "" { + runUUID = record.Environment.RunUUID + } else if record.Environment.RunUUID != runUUID { + return nil, fmt.Errorf("existing-graph checkpoint contains multiple run UUIDs") + } + key := strings.Join([]string{string(record.ExecutionMode), record.Dataset, record.Name}, "/") + if _, found := seen[key]; found { + return nil, fmt.Errorf("existing-graph checkpoint contains duplicate record %s", key) + } + seen[key] = struct{}{} + } + return checkpoint.Records, nil +} + +// writeExistingGraphCheckpoint atomically persists live-graph completion state with restrictive permissions. +func writeExistingGraphCheckpoint(path, manifestHash, corpusHash, runHash string, records []CaseResult) error { + if path == "" { + return nil + } + checkpoint := existingGraphCheckpoint{ + Version: existingGraphCheckpointVersion, + ManifestSHA256: manifestHash, + CorpusSHA256: corpusHash, + RunSHA256: runHash, + Records: records, + } + raw, err := json.MarshalIndent(checkpoint, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + temporary, err := os.CreateTemp(filepath.Dir(path), ".graphbench-checkpoint-*") + if err != nil { + return err + } + temporaryName := temporary.Name() + defer os.Remove(temporaryName) + if _, err := temporary.Write(append(raw, '\n')); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Sync(); err != nil { + _ = temporary.Close() + return err + } + if err := temporary.Close(); err != nil { + return err + } + return os.Rename(temporaryName, path) +} + +// appendExistingGraphProgress appends one progress event as a durable JSON Lines record. +func appendExistingGraphProgress(path string, event ExistingGraphProgress) error { + if path == "" { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer file.Close() + event.At = time.Now().UTC() + return json.NewEncoder(file).Encode(event) +} + +// redactExistingGraphRecord removes raw parameters and Cypher text, pseudonymizes anchor values, and scrubs resolved IDs from diagnostics and plans before a live-run record is persisted. +func redactExistingGraphRecord(record *CaseResult, manifest ExistingGraphAnchorManifest, resolved map[string]graph.ID) { + if record == nil { + return + } + record.Params = nil + redacted := map[string]string{} + for parameter, name := range record.NodeParams { + anchor, found := manifest.Anchors[name] + if !found { + continue + } + seed := anchor.LogicalKey + if seed == "" { + seed = anchor.ContentSHA256 + } + digest := sha256.Sum256([]byte(seed)) + redacted[parameter] = "sha256:" + hex.EncodeToString(digest[:]) + } + record.NodeParams = redacted + record.NodeListParams = nil + record.Cypher = "" + record.ObservedRows = redactObservedRows(record.ObservedRows) + record.SQL = redactResolvedIDs(record.SQL, resolved) + for idx := range record.PostgresPlan { + record.PostgresPlan[idx] = redactResolvedIDs(record.PostgresPlan[idx], resolved) + } + if len(record.PostgresPlanJSON) > 0 { + record.PostgresPlanJSON = redactPlanJSON(record.PostgresPlanJSON, resolved) + } + record.Error = redactDiagnostic(record.Error) + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + reference.ObservedRows = redactObservedRows(reference.ObservedRows) + reference.SQL = redactResolvedIDs(reference.SQL, resolved) + for planIdx := range reference.PostgresPlan { + reference.PostgresPlan[planIdx] = redactResolvedIDs(reference.PostgresPlan[planIdx], resolved) + } + if len(reference.PostgresPlanJSON) > 0 { + reference.PostgresPlanJSON = redactPlanJSON(reference.PostgresPlanJSON, resolved) + } + } + if record.ExistingGraph != nil { + for idx := range record.ExistingGraph.Attempts { + record.ExistingGraph.Attempts[idx].Error = redactDiagnostic(record.ExistingGraph.Attempts[idx].Error) + } + } +} + +// redactObservedRows replaces each normalized observation with a SHA-256 digest for live-graph persistence. +func redactObservedRows(rows []string) []string { + for idx := range rows { + digest := sha256.Sum256([]byte(rows[idx])) + rows[idx] = "sha256:" + hex.EncodeToString(digest[:]) + } + return rows +} + +// redactDiagnostic replaces a nonempty diagnostic with its SHA-256 digest. +func redactDiagnostic(value string) string { + if value == "" { + return "" + } + digest := sha256.Sum256([]byte(value)) + return "sha256:" + hex.EncodeToString(digest[:]) +} + +// redactResolvedIDs replaces resolved physical node IDs and unmapped entity IDs with stable redaction markers. +func redactResolvedIDs(value string, resolved map[string]graph.ID) string { + for _, id := range resolved { + value = regexp.MustCompile(`\b`+regexp.QuoteMeta(fmt.Sprint(id))+`\b`).ReplaceAllString(value, "") + } + value = regexp.MustCompile(`unmapped-(node|edge|relationship):[0-9]+`).ReplaceAllString(value, "unmapped-$1:") + return value +} + +// redactPlanJSON recursively replaces resolved graph IDs in a PostgreSQL JSON plan so live-run artifacts cannot disclose dataset identifiers. +func redactPlanJSON(raw json.RawMessage, resolved map[string]graph.ID) json.RawMessage { + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return nil + } + var redact func(any) any + redact = func(current any) any { + switch typed := current.(type) { + case string: + return redactResolvedIDs(typed, resolved) + case []any: + for idx := range typed { + typed[idx] = redact(typed[idx]) + } + case map[string]any: + for key := range typed { + typed[key] = redact(typed[key]) + } + } + return current + } + encoded, err := json.Marshal(redact(value)) + if err != nil { + return nil + } + return encoded +} + +// validateCompletedWorkloads rejects checkpoint entries that are unknown or bound to stale workload identities. +func validateCompletedWorkloads(completed map[string]string, corpus ScaleCorpus, fixture FixtureMetadata) error { + expectedKeys := map[string]struct{}{} + for _, testCase := range corpus.Cases { + if !testCase.Supports(ModePostgresSQL) { + continue + } + key := existingGraphCaseKey(ModePostgresSQL, testCase) + expectedKeys[key] = struct{}{} + checkpointWorkload, found := completed[key] + if !found { + continue + } + expected := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expected, fixture) + if checkpointWorkload == "" || checkpointWorkload != expected.WorkloadSHA256 { + return fmt.Errorf("existing-graph checkpoint workload identity does not match %s", key) + } + } + for key := range completed { + if _, found := expectedKeys[key]; !found { + return fmt.Errorf("existing-graph checkpoint contains unknown workload %s", key) + } + } + return nil +} + +// idMapForManifest builds an ID map from logical and redacted physical anchor identities. +func idMapForManifest(anchors map[string]graph.ID) opengraph.IDMap { + result := make(opengraph.IDMap, len(anchors)) + for name, id := range anchors { + result[name] = id + } + return result +} + +// scanCheckpointJSONL is deliberately strict: a truncated last line is not a +// completed record and therefore cannot be treated as resumable evidence. +func scanCheckpointJSONL(path string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + var value map[string]any + if err := json.Unmarshal(scanner.Bytes(), &value); err != nil { + return err + } + } + return scanner.Err() +} diff --git a/cmd/graphbench/live_mode_test.go b/cmd/graphbench/live_mode_test.go new file mode 100644 index 00000000..23aa770e --- /dev/null +++ b/cmd/graphbench/live_mode_test.go @@ -0,0 +1,263 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestExistingGraphManifestCorpusSafetyAndRedaction verifies that live-graph mode rejects mutations and strips query, parameter, plan, row, reference, and error disclosures from artifacts. +func TestExistingGraphManifestCorpusSafetyAndRedaction(t *testing.T) { + manifest := ExistingGraphAnchorManifest{ + Version: 1, + Checksum: "manifest", + Anchors: map[string]ExistingGraphAnchor{ + "source": { + LogicalKey: "safe-source", + }, "target": { + LogicalKey: "safe-target", + }, + }, + } + readCase := ScaleCase{ + Name: "read", + Dataset: "live", + Category: "live", + Cypher: `MATCH (n) WHERE n.note = 'create is text' AND id(n) = $source RETURN n`, + NodeParams: map[string]string{"source": "source"}, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + } + require.NoError(t, validateExistingGraphCorpus(ScaleCorpus{ + Cases: []ScaleCase{readCase}, + }, manifest)) + + writeCase := readCase + writeCase.Name = "write" + writeCase.Cypher = "MATCH (n) DELETE n" + require.ErrorContains(t, validateExistingGraphCorpus(ScaleCorpus{ + Cases: []ScaleCase{writeCase}, + }, manifest), "mutation keyword") + writeCase.Cypher = "MATCH (n) RETURN n" + writeCase.WriteScenario = &WriteScenario{} + require.ErrorContains(t, validateExistingGraphCorpus(ScaleCorpus{ + Cases: []ScaleCase{writeCase}, + }, manifest), "write_scenario") + + record := CaseResult{ + Cypher: readCase.Cypher, + Params: map[string]any{"source": 42}, + NodeParams: map[string]string{"source": "source"}, + ObservedRows: []string{"sensitive-property"}, + PostgresPlan: []string{"Index Cond: id = 42"}, + Error: "unmapped-node:77", + PostgresReferences: []PostgresReferenceResult{{ + ObservedRows: []string{"reference-sensitive-property"}, + }}, + ExistingGraph: &ExistingGraphRun{Attempts: []ExistingGraphAttempt{{ + Error: "attempt-sensitive-property 42", + }}}, + } + redactExistingGraphRecord(&record, manifest, map[string]graph.ID{"source": 42}) + require.Empty(t, record.Cypher) + require.Empty(t, record.Params) + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.NodeParams["source"]) + require.NotContains(t, record.NodeParams["source"], "safe-source") + require.NotContains(t, record.ObservedRows[0], "sensitive-property") + require.NotContains(t, record.PostgresPlan[0], "42") + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.Error) + require.NotContains(t, record.Error, "77") + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.PostgresReferences[0].ObservedRows[0]) + require.NotContains(t, record.PostgresReferences[0].ObservedRows[0], "reference-sensitive-property") + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.ExistingGraph.Attempts[0].Error) + require.NotContains(t, record.ExistingGraph.Attempts[0].Error, "attempt-sensitive-property") +} + +// TestExistingGraphManifestRequiresGraphAndLogicalContentIdentity verifies manifest checksums bind graph/content identity and that each anchor chooses exactly one complete logical or physical identity form. +func TestExistingGraphManifestRequiresGraphAndLogicalContentIdentity(t *testing.T) { + path := filepath.Join(t.TempDir(), "anchors.json") + valid := `{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"logical_key":"safe-source"}}}` + require.NoError(t, os.WriteFile(path, []byte(valid), 0o600)) + manifest, err := loadExistingGraphAnchorManifest(path) + require.NoError(t, err) + require.Equal(t, "integration_test", manifest.Graph) + require.Regexp(t, `^[0-9a-f]{64}$`, manifest.Checksum) + + physical := `{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"physical_id":42,"content_sha256":"sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}}}` + require.NoError(t, os.WriteFile(path, []byte(physical), 0o600)) + manifest, err = loadExistingGraphAnchorManifest(path) + require.NoError(t, err) + require.Equal(t, int64(42), *manifest.Anchors["source"].PhysicalID) + + require.NoError(t, os.WriteFile(path, []byte(`{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"physical_id":42}}}`), 0o600)) + _, err = loadExistingGraphAnchorManifest(path) + require.ErrorContains(t, err, "content_sha256") + + require.NoError(t, os.WriteFile(path, []byte(`{"version":1,"graph":"integration_test","content_identity":"sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef","anchors":{"source":{"logical_key":"safe-source","physical_id":42,"content_sha256":"sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"}}}`), 0o600)) + _, err = loadExistingGraphAnchorManifest(path) + require.ErrorContains(t, err, "exactly one") + + require.NoError(t, os.WriteFile(path, []byte(`{"version":1,"graph":"integration_test","anchors":{"source":{"logical_key":"safe-source"}}}`), 0o600)) + _, err = loadExistingGraphAnchorManifest(path) + require.ErrorContains(t, err, "content_identity") +} + +// TestPhysicalExistingGraphAnchorRedactionUsesContentIdentity verifies that artifacts replace a physical anchor ID with an opaque digest derived from its content identity. +func TestPhysicalExistingGraphAnchorRedactionUsesContentIdentity(t *testing.T) { + id := int64(42) + manifest := ExistingGraphAnchorManifest{ + Anchors: map[string]ExistingGraphAnchor{ + "source": { + PhysicalID: &id, + ContentSHA256: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + }, + }, + } + record := CaseResult{ + NodeParams: map[string]string{"source": "source"}, + } + redactExistingGraphRecord(&record, manifest, map[string]graph.ID{"source": graph.ID(id)}) + require.Regexp(t, `^sha256:[0-9a-f]{64}$`, record.NodeParams["source"]) + require.NotContains(t, record.NodeParams["source"], "42") +} + +// TestExistingGraphCheckpointIsIdentityBoundAndResumable verifies round-trip recovery only for matching manifest, corpus, and run identities and rejects duplicate completed records. +func TestExistingGraphCheckpointIsIdentityBoundAndResumable(t *testing.T) { + path := filepath.Join(t.TempDir(), "checkpoint.json") + records := []CaseResult{{ + Dataset: "live", + Name: "case", + WorkloadSHA256: "workload", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: "corpus", + RunIdentitySHA256: "run", + RunUUID: "run-uuid", + }, + }} + require.NoError(t, writeExistingGraphCheckpoint(path, "manifest", "corpus", "run", records)) + loaded, err := readExistingGraphCheckpoint(path, "manifest", "corpus", "run") + require.NoError(t, err) + require.Equal(t, records, loaded) + _, err = readExistingGraphCheckpoint(path, "other", "corpus", "run") + require.ErrorContains(t, err, "identity") + _, err = readExistingGraphCheckpoint(path, "manifest", "corpus", "other-run") + require.ErrorContains(t, err, "identity") + + raw, err := os.ReadFile(path) + require.NoError(t, err) + var checkpoint existingGraphCheckpoint + require.NoError(t, json.Unmarshal(raw, &checkpoint)) + require.Equal(t, existingGraphCheckpointVersion, checkpoint.Version) + + checkpoint.Records = append(checkpoint.Records, checkpoint.Records[0]) + raw, err = json.Marshal(checkpoint) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, raw, 0o600)) + _, err = readExistingGraphCheckpoint(path, "manifest", "corpus", "run") + require.ErrorContains(t, err, "duplicate record") +} + +// TestExistingGraphPlanRedactionPreservesJSONNumbers verifies that plan redaction replaces IDs inside text without corrupting numeric cardinality fields in the JSON document. +func TestExistingGraphPlanRedactionPreservesJSONNumbers(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Plan Rows":42,"Index Cond":"id = 42"}}]`) + redacted := redactPlanJSON(raw, map[string]graph.ID{"source": 42}) + require.JSONEq(t, `[{"Plan":{"Plan Rows":42,"Index Cond":"id = "}}]`, string(redacted)) +} + +// TestExistingGraphProgressIsAppendOnlyJSONL verifies that successive progress events remain two independently parseable JSON Lines records. +func TestExistingGraphProgressIsAppendOnlyJSONL(t *testing.T) { + path := filepath.Join(t.TempDir(), "progress.jsonl") + require.NoError(t, appendExistingGraphProgress(path, ExistingGraphProgress{ + Stage: "case", + CaseKey: "one", + })) + require.NoError(t, appendExistingGraphProgress(path, ExistingGraphProgress{ + Stage: "plan", + CaseKey: "one", + })) + require.NoError(t, scanCheckpointJSONL(path)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, 2, len(splitNonEmptyLines(string(raw)))) +} + +// TestCompleteGateRejectsAdaptiveExistingGraphArtifacts verifies that discovery-selected live-graph measurements cannot enter a complete performance gate. +func TestCompleteGateRejectsAdaptiveExistingGraphArtifacts(t *testing.T) { + records := []CaseResult{{ + ExistingGraph: &ExistingGraphRun{ + Adaptive: true, + }, + }} + require.ErrorContains(t, validatePerformanceArtifactSelections(records, records, false), "adaptive-discovery") +} + +// TestExistingGraphCorpusIdentityIsStable verifies that corpus identity is deterministic and changes when either query text or expected cardinality changes. +func TestExistingGraphCorpusIdentityIsStable(t *testing.T) { + zero := int64(0) + corpus := ScaleCorpus{ + Cases: []ScaleCase{{ + Name: "case", + Dataset: "live", + Category: "live", + Cypher: "RETURN 1", + Expected: ExpectedResult{ + RowCount: &zero, + }, + Params: testutil.Params{}, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }}, + } + require.Equal(t, corpusIdentity(corpus), corpusIdentity(corpus)) + changedQuery := corpus + changedQuery.Cases = append([]ScaleCase(nil), corpus.Cases...) + changedQuery.Cases[0].Cypher = "RETURN 2" + require.NotEqual(t, corpusIdentity(corpus), corpusIdentity(changedQuery)) + + changedExpected := corpus + changedExpected.Cases = append([]ScaleCase(nil), corpus.Cases...) + one := int64(1) + changedExpected.Cases[0].Expected.RowCount = &one + require.NotEqual(t, corpusIdentity(corpus), corpusIdentity(changedExpected)) +} + +// TestExistingGraphCompletedWorkloadsAreFixtureBound verifies that resume records are accepted only for known cases with the same fixture checksum and workload digest. +func TestExistingGraphCompletedWorkloadsAreFixtureBound(t *testing.T) { + corpus := ScaleCorpus{Cases: []ScaleCase{{ + Name: "case", + Dataset: "live", + Cypher: "RETURN 1", + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }}} + fixture := FixtureMetadata{Dataset: "existing_graph", Checksum: "manifest:content:schema:index"} + expected := newCaseResult(corpus.Cases[0], ModePostgresSQL, nil) + attachFixtureMetadata(&expected, fixture) + completed := map[string]string{existingGraphCaseKey(ModePostgresSQL, corpus.Cases[0]): expected.WorkloadSHA256} + require.NoError(t, validateCompletedWorkloads(completed, corpus, fixture)) + + changedFixture := fixture + changedFixture.Checksum = "manifest:other-content:schema:index" + require.ErrorContains(t, validateCompletedWorkloads(completed, corpus, changedFixture), "workload identity") + require.ErrorContains(t, validateCompletedWorkloads(map[string]string{"postgres_sql/other/case": "digest"}, corpus, fixture), "unknown workload") +} + +// splitNonEmptyLines separates platform-independent line endings and discards empty records. +func splitNonEmptyLines(value string) []string { + var lines []string + for _, line := range regexp.MustCompile(`\r?\n`).Split(value, -1) { + if line != "" { + lines = append(lines, line) + } + } + return lines +} diff --git a/cmd/graphbench/main.go b/cmd/graphbench/main.go index bd18d1a3..10f111bf 100644 --- a/cmd/graphbench/main.go +++ b/cmd/graphbench/main.go @@ -21,31 +21,389 @@ import ( "flag" "fmt" "io" + "math" "os" + "slices" + "strconv" "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/testutil" ) +// config contains graphbench command-line selections and safety settings. type config struct { - CorpusRoot string - DatasetDir string - Connection string - PGConnection string + // CorpusRoot locates scale-case and template declarations. + CorpusRoot string + // DatasetDir locates fixture datasets loaded for managed benchmark runs. + DatasetDir string + // Connection contains the backend connection string. + Connection string + // PGConnection contains the PostgreSQL connection string. + PGConnection string + // Neo4jConnection contains the Neo4j connection string. Neo4jConnection string - Modes []ExecutionMode - Iterations int - OutputJSONL string - Summary string - SummaryJSON string - Baseline string + // Modes lists backend execution modes requested for each benchmark round. + Modes []ExecutionMode + // Iterations records the number of iterations. + Iterations int + // WarmupIterations records the number of warmup iterations. + WarmupIterations int + // Round identifies the measurement round. + Round int + // Block identifies the measurement block used to control carryover effects. + Block int + // Arm identifies the measurement arm that produced the sample. + Arm string + // ArmOrder supplies the arm order input to the config contract. + ArmOrder int + // RunUUID supplies an optional stable identity shared by every artifact in one run series. + RunUUID string + // Cases lists exact case names requested by the user. + Cases []string + // Datasets lists exact dataset selectors supplied by the user. + Datasets []string + // Categories lists workload categories used to filter the corpus. + Categories []string + // Tags lists exact tag selectors supplied by the user. + Tags []string + // OutputJSONL selects the benchmark-result JSON Lines destination. + OutputJSONL string + // AppendJSONL selects append-safe JSON Lines output instead of replacing the artifact. + AppendJSONL bool + // Summary selects the Markdown benchmark-summary destination. + Summary string + // SummaryJSON selects the JSON summary destination. + SummaryJSON string + // Baseline identifies the baseline version or result used for comparison. + Baseline string + // DAWGSVersion identifies the schema version for dawgs version. + DAWGSVersion string + // GateBaseline selects the baseline JSON Lines artifact for performance gating. + GateBaseline string + // GateCandidate selects the candidate JSON Lines artifact for performance gating. + GateCandidate string + // GateOutput selects the performance-gate JSON report destination. + GateOutput string + // GateAA selects the host A/A resolution report required by production performance gating. + GateAA string + // GateSeed controls deterministic performance-gate bootstrap resampling. + GateSeed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // Regression sets the largest candidate-to-baseline median ratio accepted by the gate. + Regression float64 + // GateTargets lists exact case names subject to performance gating. + GateTargets []string + // MaterialityRatio sets the relative change required before a difference is material. + MaterialityRatio float64 + // MaterialityAbsolute sets the absolute duration change required before a difference is material. + MaterialityAbsolute time.Duration + // DestructiveLock selects the lock-file path that serializes destructive runs. + DestructiveLock string + // AAArtifacts select one or more benchmark record files used to estimate + // within-arm noise. Repeating the input lets independently appended A/A arms + // remain immutable while the reporter validates them as one logical cohort. + AAArtifacts []string + // AAOutput selects the A/A resolution report destination. + AAOutput string + // ReferenceClosureArtifact selects benchmark records used for production-to-reference closure analysis. + ReferenceClosureArtifact string + // ReferenceClosureOutput selects the reference-closure report destination. + ReferenceClosureOutput string + // ReferenceClosureArm selects the independent reference arm compared with production. + ReferenceClosureArm string + // ReferencePairArtifact selects benchmark records containing the two reference arms to compare. + ReferencePairArtifact string + // ReferencePairOutput selects the paired-reference report destination. + ReferencePairOutput string + // ReferencePairBaseline selects the reference arm treated as the paired baseline. + ReferencePairBaseline string + // ReferencePairCandidate selects the reference arm compared with the paired baseline. + ReferencePairCandidate string + // ReferencePairProtocol selects confirmation or discovery sample requirements for paired references. + ReferencePairProtocol string + // ReferenceTournamentArtifact selects records containing a predeclared three- or five-arm tournament. + ReferenceTournamentArtifact string + // ReferenceTournamentOutput selects the tournament report destination. + ReferenceTournamentOutput string + // ReferenceTournamentArms lists tournament arms with the incumbent first. + ReferenceTournamentArms []string + // ReferenceTournamentProtocol selects confirmation or discovery tournament requirements. + ReferenceTournamentProtocol string + // PoolSize sets the database connection-pool size. + PoolSize int + // Concurrency lists opt-in worker counts for PostgreSQL concurrency measurements. + Concurrency []int + // SessionMemoryCeilingBytes sets the per-session memory ceiling in bytes. + SessionMemoryCeilingBytes int64 + // PoolMemoryCeilingBytes sets the aggregate pool memory ceiling in bytes. + PoolMemoryCeilingBytes int64 + // PostgresReferences enables independent PostgreSQL reference-arm measurement and persistence. + PostgresReferences bool + // PostgresReferenceArms lists independent PostgreSQL reference arms selected for measurement. + PostgresReferenceArms []string + // PostgresForceShortest selects a forced shortest-path executor for diagnostic runs. + PostgresForceShortest string + // PostgresProductionManifest selects a provisional version-2 manifest used + // to measure an exact guarded production statement before evidence closure. + PostgresProductionManifest string + // PostgresRepeatableRead measures the incumbent under the same stable + // snapshot contract required for guarded candidate admission. + PostgresRepeatableRead bool + // PostgresForceExpansion selects a forced expansion search strategy for diagnostic runs. + PostgresForceExpansion string + // PostgresTraversalTelemetry selects off, summary, or an untimed diagnostic replay. + PostgresTraversalTelemetry string + // PostgresExpansionOrientationShadow executes the incumbent while recording the orientation policy's SQL-visible choice. + PostgresExpansionOrientationShadow bool + // PostgresExpansionOrientationTournament executes the guarded selector's + // chosen arm in the same statement. + PostgresExpansionOrientationTournament bool + // PostgresExpansionOrientationPolicy selects an immutable tool-only + // orientation formula. Empty preserves orientation-probe-v1. + PostgresExpansionOrientationPolicy string + // PostgresExpansionSuffixReverseGuard executes the static, full-path-only + // suffix-reverse guard and its exact forward fallback in one statement. + PostgresExpansionSuffixReverseGuard bool + // PostgresExpansionSuffixReverseRetry executes the reverse-only P1 candidate + // with exact forward retry in the same Repeatable Read transaction. + PostgresExpansionSuffixReverseRetry bool + // PostgresExpansionSuffixRouteComponent executes one exact reverse-only + // fixed-suffix statement for the default-off SQL-routing preflight. + PostgresExpansionSuffixRouteComponent bool + // PostgresSuffixRouteComponentClosure records compile, raw-PGX prepared-state, + // and workspace high-water evidence for the fixed-suffix routing preflight. + // It never selects an executor or changes the statement being measured. + PostgresSuffixRouteComponentClosure bool + // PostgresSuffixGuardSuffixLimit overrides the tool-only cap+1 suffix payload limit. + PostgresSuffixGuardSuffixLimit int64 + // PostgresSuffixGuardStateLimit overrides the tool-only cap+1 reverse-state limit. + PostgresSuffixGuardStateLimit int64 + // PostgresSuffixRetryOutputRowLimit overrides the buffered output-row cap. + PostgresSuffixRetryOutputRowLimit int64 + // PostgresSuffixRetryOutputBytesLimit overrides the buffered output-byte cap. + PostgresSuffixRetryOutputBytesLimit int64 + // ConfirmLeft selects the left artifact used for paired confirmation. + ConfirmLeft string + // ConfirmRight selects the right artifact used for paired confirmation. + ConfirmRight string + // ConfirmAA selects the A/A noise report used to classify confirmation deltas. + ConfirmAA string + // ConfirmOutput selects the paired confirmation report destination. + ConfirmOutput string + // ConfirmCases lists exact case names included in paired confirmation. + ConfirmCases []string + // DiagnosticGate marks output as diagnostic and therefore ineligible for a complete release-gate pass. + DiagnosticGate bool + // BundleDir selects the directory that receives portable artifacts and source provenance. + BundleDir string + // BundleEvidence lists named auxiliary artifacts copied into a newly captured bundle. + BundleEvidence []CaptureBundleEvidenceInput + // BundleVerify selects a portable bundle directory for standalone validation. + BundleVerify string + // BundleVerifyOutput selects the standalone bundle-verification JSON destination. + BundleVerifyOutput string + // BundleRequireClean rejects otherwise valid bundles captured from a dirty source tree. + BundleRequireClean bool + // RequireCleanSource refuses a live capture before any database setup when + // tracked or untracked source content is present. + RequireCleanSource bool + // PromotionManifest selects a complete evidence-closure manifest for standalone verification. + PromotionManifest string + // PromotionManifestOutput selects the verification report destination. + PromotionManifestOutput string + // PromotionBindManifest supplies the provisional manifest whose immutable + // identity is attached to one generated evidence report. + PromotionBindManifest string + // PromotionBindRole names the evidence role being bound. + PromotionBindRole string + // PromotionBindInput and PromotionBindOutput select the unbound and bound reports. + PromotionBindInput string + // PromotionBindOutput supplies the promotion bind output input to the config contract. + PromotionBindOutput string + // OperationalGateInput selects the schema-v2 operational evidence document. + OperationalGateInput string + // OperationalGateOutput selects the machine-verifiable operational gate report destination. + OperationalGateOutput string + // BuildCommand supplies the build command input to the config contract. + BuildCommand string + // ExistingGraph selects read-only execution against a pre-existing graph. + ExistingGraph bool + // AnchorManifest selects the live-graph anchor manifest to validate and redact. + AnchorManifest string + // Checkpoint selects the persisted live-graph completion checkpoint. + Checkpoint string + // Resume allows live-graph execution to skip checkpointed workloads with matching identities. + Resume bool + // Progress selects the append-only live-graph progress JSON Lines destination. + Progress string + // Discovery enables adaptive live-graph discovery instead of the fixed confirmation protocol. + Discovery bool + // TimeoutClasses lists increasing per-attempt deadlines for adaptive live-graph discovery. + TimeoutClasses []time.Duration + // DiscoverySampleFloor sets the minimum live-graph samples required before adaptive discovery may stop. + DiscoverySampleFloor int + // ResourceArtifact selects benchmark records evaluated against plan-resource limits. + ResourceArtifact string + // ResourceOutput selects the resource-gate JSON report destination. + ResourceOutput string + // BackendDeltaArtifact selects records used for descriptive PostgreSQL-to-Neo4j comparison. + BackendDeltaArtifact string + // BackendDeltaOutput selects the cross-backend delta report destination. + BackendDeltaOutput string + // ExpandIntoArtifact selects records used to build the fixed-one-hop three-arm study report. + ExpandIntoArtifact string + // ExpandIntoOutput selects the ExpandInto study JSON destination. + ExpandIntoOutput string + // ExpandIntoProtocol selects discovery or confirmation evidence requirements. + ExpandIntoProtocol string + // OrientationShadowArtifact selects true-shadow orientation records. + OrientationShadowArtifact string + // OrientationIncumbentArtifact selects matched exact incumbent records. + OrientationIncumbentArtifact string + // OrientationReverseArtifact selects matched exact forced-reverse records. + OrientationReverseArtifact string + // OrientationAA selects host A/A timing resolution for selector regret. + OrientationAA string + // OrientationOutput selects the selector-regret and probe-overhead report destination. + OrientationOutput string + // OrientationProtocol selects discovery or confirmation evidence requirements. + OrientationProtocol string + // OrientationV2ShadowArtifact selects orientation-probe-v2 shadow records. + OrientationV2ShadowArtifact string + // OrientationV2IncumbentArtifact selects matched exact forward records. + OrientationV2IncumbentArtifact string + // OrientationV2ReverseArtifact selects matched exact reverse records. + OrientationV2ReverseArtifact string + // OrientationV2GuardedArtifact selects actual guarded dual-arm records. + OrientationV2GuardedArtifact string + // OrientationV2AA selects the host A/A timing-resolution report. + OrientationV2AA string + // OrientationV2Freeze binds confirmation to the preregistered discovery identity. + OrientationV2Freeze string + // OrientationV2DiscoveryReport supplies the checksummed training-only report bound by the freeze. + OrientationV2DiscoveryReport string + // OrientationV2FreezeOutput writes the preregistered identity after training-only discovery. + OrientationV2FreezeOutput string + // OrientationV2Output selects the four-arm qualification report destination. + OrientationV2Output string + // OrientationV2Protocol selects discovery or confirmation evidence requirements. + OrientationV2Protocol string + // SuffixGuardIncumbentArtifact selects matched exact-forward feasibility records. + SuffixGuardIncumbentArtifact string + // SuffixGuardReverseArtifact selects matched exact suffix-reverse feasibility records. + SuffixGuardReverseArtifact string + // SuffixGuardGuardedArtifact selects matched production-shaped guard records. + SuffixGuardGuardedArtifact string + // SuffixGuardAA selects matching host A/A timing-resolution evidence. + SuffixGuardAA string + // SuffixGuardOutput selects the training-only feasibility report destination. + SuffixGuardOutput string + // SPI1BaselineArtifact selects exact S4 records for the staged inbound-I1 study. + SPI1BaselineArtifact string + // SPI1CandidateArtifact selects guarded canonical-I1 records for the staged study. + SPI1CandidateArtifact string + // SPI1ResourceReport supplies the candidate artifact's checksummed resource gate. + SPI1ResourceReport string + // SPI1Freeze binds confirmation reporting or holdout capture to training-only discovery. + SPI1Freeze string + // SPI1DiscoveryReport supplies the checksummed training-only report bound by the freeze. + SPI1DiscoveryReport string + // SPI1TrainingBaseline supplies the exact S4 training evidence named by the freeze. + SPI1TrainingBaseline string + // SPI1TrainingCandidate supplies the exact I1 training evidence named by the freeze. + SPI1TrainingCandidate string + // SPI1TrainingResource supplies the exact training resource report named by the freeze. + SPI1TrainingResource string + // SPI1FreezeOutput writes the training-only staged-study freeze manifest. + SPI1FreezeOutput string + // SPI1Output selects the staged S4-to-I1 qualification report destination. + SPI1Output string + // SPI1Protocol selects discovery or confirmation evidence requirements. + SPI1Protocol string + // SPI2BaselineArtifact selects exact S4 distance records for staged SP-I2 qualification. + SPI2BaselineArtifact string + // SPI2CandidateArtifact selects guarded SP-I2 distance records for the staged study. + SPI2CandidateArtifact string + // SPI2ResourceReport supplies the candidate artifact's checksummed resource gate. + SPI2ResourceReport string + // SPI2Freeze binds confirmation reporting or holdout capture to training-only discovery. + SPI2Freeze string + // SPI2DiscoveryReport supplies the checksummed training-only report bound by the freeze. + SPI2DiscoveryReport string + // SPI2TrainingBaseline supplies the exact S4 distance training evidence named by the freeze. + SPI2TrainingBaseline string + // SPI2TrainingCandidate supplies the exact guarded-distance training evidence named by the freeze. + SPI2TrainingCandidate string + // SPI2TrainingResource supplies the exact training resource report named by the freeze. + SPI2TrainingResource string + // SPI2FreezeOutput writes the training-only staged-study freeze manifest. + SPI2FreezeOutput string + // SPI2Output selects the staged S4-to-I2 qualification report destination. + SPI2Output string + // SPI2Protocol selects discovery or confirmation evidence requirements. + SPI2Protocol string + // SPI2Generation explicitly selects the isolated V1 or V2 evidence family. + SPI2Generation string + // SPI2V2DevelopmentTournament enables the fixed open-corpus five-arm + // component schedule. Its artifacts are permanently non-promotional. + SPI2V2DevelopmentTournament bool + // SPI2V2ReadinessComparison enables the fixed open-corpus E0/S4 + // supplemental schedule. Its artifacts are permanently non-promotional. + SPI2V2ReadinessComparison bool + // SPI2V2DevelopmentArtifact selects a raw diagnostic artifact for strict + // schedule and invocation validation. + SPI2V2DevelopmentArtifact string + // SPI2V2DevelopmentStudy selects readiness or tournament validation. + SPI2V2DevelopmentStudy string + // SPI2V2DevelopmentReportArtifact selects the exact five-arm tournament + // artifact consumed by the diagnostic development decision report. + SPI2V2DevelopmentReportArtifact string + // SPI2V2DevelopmentReportOutput writes the diagnostic development report. + SPI2V2DevelopmentReportOutput string + // SPI2V2ComponentCheck enables one exact E1D or E1P semantic/plan check. + SPI2V2ComponentCheck bool + // SPI2V2ComponentAuthorization supplies the exact E1D/E1P authorization + // required before the combined E1DP arm may reach database setup. + SPI2V2ComponentAuthorization string + // SPI2V2ComponentE1DArtifact supplies the exact E1D component-check artifact. + SPI2V2ComponentE1DArtifact string + // SPI2V2ComponentE1PArtifact supplies the exact E1P component-check artifact. + SPI2V2ComponentE1PArtifact string + // SPI2V2ComponentAuthorizationOutput writes the combined-arm authorization. + SPI2V2ComponentAuthorizationOutput string + // SPI2V2SimulationBaselineTrace supplies the frozen clean V1 S4 trace. + SPI2V2SimulationBaselineTrace string + // SPI2V2SimulationCandidateTrace supplies the frozen clean V1 I2 trace. + SPI2V2SimulationCandidateTrace string + // SPI2V2SimulationOutput writes the prospective calibration report. + SPI2V2SimulationOutput string + // P5AdjacencyFeasibilityOutput writes the isolated P5 shadow-adjacency + // physical feasibility report. It never executes a Cypher read path. + P5AdjacencyFeasibilityOutput string } +// parseConfig parses graphbench flags and rejects unsafe or incomplete workflow combinations. func parseConfig(args []string, env func(string) string) (config, error) { flags := flag.NewFlagSet("graphbench", flag.ContinueOnError) flags.SetOutput(io.Discard) var ( - cfg config - rawModes string + cfg config + rawModes string + rawGateTargets string + rawConcurrency string + rawCases string + rawDatasets string + rawCategories string + rawTags string + rawConfirmCases string + rawReferenceArms string + rawTournamentArms string + rawTimeoutClasses string + rawBundleEvidence []string ) flags.StringVar(&cfg.CorpusRoot, "corpus-root", "benchmark/testdata/scale", "scale corpus root") @@ -55,27 +413,1042 @@ func parseConfig(args []string, env func(string) string) (config, error) { flags.StringVar(&cfg.Neo4jConnection, "neo4j-connection", env("NEO4J_CONNECTION_STRING"), "Neo4j connection string") flags.StringVar(&rawModes, "modes", string(ModePostgresSQL), "comma-separated execution modes") flags.IntVar(&cfg.Iterations, "iterations", 3, "timed iterations per case") + flags.IntVar(&cfg.WarmupIterations, "warmup-iterations", 1, "fixed untimed warmup iterations per case") + flags.IntVar(&cfg.Round, "round", 1, "independent benchmark round identifier") + flags.IntVar(&cfg.Block, "block", 1, "matched benchmark block identifier") + flags.StringVar(&cfg.Arm, "arm", "unlabeled", "matched benchmark arm label") + flags.IntVar(&cfg.ArmOrder, "arm-order", 0, "one-based execution order inside the matched block (0 when unpaired)") + flags.StringVar(&cfg.RunUUID, "run-uuid", "", "run-series UUID (generated when empty)") + flags.StringVar(&rawCases, "cases", "", "comma-separated exact case names") + flags.StringVar(&rawDatasets, "datasets", "", "comma-separated exact dataset names") + flags.StringVar(&rawCategories, "categories", "", "comma-separated exact category names") + flags.StringVar(&rawTags, "tags", "", "comma-separated exact case tags") flags.StringVar(&cfg.OutputJSONL, "jsonl-output", "", "JSONL output path (default: stdout)") + flags.BoolVar(&cfg.AppendJSONL, "append-jsonl", false, "append a validated round to an existing JSONL run-series artifact") flags.StringVar(&cfg.Summary, "summary", "", "markdown summary output path") flags.StringVar(&cfg.SummaryJSON, "summary-json", "", "JSON summary output path") flags.StringVar(&cfg.Baseline, "baseline", "", "previous JSONL output for baseline comparison") + flags.StringVar(&cfg.DAWGSVersion, "dawgs-version", "", "DAWGS source version (auto-detected when empty)") + flags.StringVar(&cfg.GateBaseline, "gate-baseline", "", "baseline JSONL artifact for comparison-only mode") + flags.StringVar(&cfg.GateCandidate, "gate-candidate", "", "candidate JSONL artifact for comparison-only mode") + flags.StringVar(&cfg.GateOutput, "gate-output", "", "performance-gate JSON output path (default: stdout)") + flags.StringVar(&cfg.GateAA, "gate-aa", "", "host A/A resolution report required for production performance gating") + flags.Int64Var(&cfg.GateSeed, "seed", 1, "deterministic bootstrap seed") + flags.Float64Var(&cfg.Confidence, "confidence-level", defaultConfidenceLevel, "bootstrap confidence level") + flags.Float64Var(&cfg.Regression, "regression-threshold", minimumTimingNoiseRatio, "minimum allowed comparable-case regression ratio before host A/A noise") + flags.StringVar(&rawGateTargets, "gate-targets", "", "comma-separated PostgreSQL case names expected to improve materially") + flags.Float64Var(&cfg.MaterialityRatio, "materiality-ratio", 0.95, "target median-ratio upper bound") + flags.DurationVar(&cfg.MaterialityAbsolute, "materiality-absolute", 100*time.Microsecond, "target median-saving lower bound") + flags.StringVar(&cfg.DestructiveLock, "destructive-lock", ".coverage/graphbench.lock", "local lock file guarding destructive fixture reloads") + flags.Func("aa-artifact", "JSONL artifact used to calculate baseline A/A measurement resolution (repeat for separately captured arms)", func(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return fmt.Errorf("aa-artifact path must not be empty") + } + cfg.AAArtifacts = append(cfg.AAArtifacts, value) + return nil + }) + flags.StringVar(&cfg.AAOutput, "aa-output", "", "A/A measurement-resolution JSON output path (default: stdout)") + flags.StringVar(&cfg.ReferenceClosureArtifact, "reference-closure-artifact", "", "JSONL artifact containing matched production raw-pgx and PostgreSQL reference samples") + flags.StringVar(&cfg.ReferenceClosureOutput, "reference-closure-output", "", "production/reference closure JSON output path (default: stdout)") + flags.StringVar(&cfg.ReferenceClosureArm, "reference-closure-arm", "s3_unidirectional_trail_cte", "PostgreSQL full-comparator reference arm") + flags.StringVar(&cfg.ReferencePairArtifact, "reference-pair-artifact", "", "JSONL artifact containing two matched PostgreSQL reference arms") + flags.StringVar(&cfg.ReferencePairOutput, "reference-pair-output", "", "matched PostgreSQL reference-pair JSON output path (default: stdout)") + flags.StringVar(&cfg.ReferencePairBaseline, "reference-pair-baseline", "", "baseline PostgreSQL reference arm") + flags.StringVar(&cfg.ReferencePairCandidate, "reference-pair-candidate", "", "candidate PostgreSQL reference arm") + flags.StringVar(&cfg.ReferencePairProtocol, "reference-pair-protocol", referencePairProtocolConfirmation, "reference-pair report protocol (confirmation or discovery)") + flags.StringVar(&cfg.ReferenceTournamentArtifact, "reference-tournament-artifact", "", "JSONL artifact containing a predeclared three- or five-arm PostgreSQL reference tournament") + flags.StringVar(&cfg.ReferenceTournamentOutput, "reference-tournament-output", "", "reference tournament JSON output path (default: stdout)") + flags.StringVar(&rawTournamentArms, "reference-tournament-arms", "", "comma-separated tournament arms with the incumbent first") + flags.StringVar(&cfg.ReferenceTournamentProtocol, "reference-tournament-protocol", referencePairProtocolConfirmation, "reference tournament protocol (confirmation or discovery)") + flags.IntVar(&cfg.PoolSize, "pool-size", 1, "PostgreSQL physical pool size") + flags.StringVar(&rawConcurrency, "concurrency", "", "comma-separated opt-in PostgreSQL concurrency smoke levels") + flags.Int64Var(&cfg.SessionMemoryCeilingBytes, "session-memory-ceiling-bytes", 0, "declared maximum performance workspace bytes per PostgreSQL session") + flags.Int64Var(&cfg.PoolMemoryCeilingBytes, "pool-memory-ceiling-bytes", 0, "declared maximum performance workspace bytes for the complete PostgreSQL pool") + flags.BoolVar(&cfg.PostgresReferences, "postgres-references", false, "capture C1 PostgreSQL component floors and full-query references") + flags.StringVar(&rawReferenceArms, "postgres-reference-arms", "", "comma-separated PostgreSQL reference arms (default: all applicable arms)") + flags.StringVar(&cfg.PostgresForceShortest, "postgres-force-shortest-executor", "", "tool-only forced PostgreSQL shortest executor (includes SP-I2-C-D-V2 and V2 E0/E1/E1D/E1P/E1DP development identities)") + flags.StringVar(&cfg.PostgresProductionManifest, "postgres-production-manifest", "", "provisional version-2 manifest for exact guarded PostgreSQL candidate measurement") + flags.BoolVar(&cfg.PostgresRepeatableRead, "postgres-repeatable-read", false, "measure PostgreSQL under an explicit Repeatable Read transaction") + flags.StringVar(&cfg.PostgresForceExpansion, "postgres-force-expansion-search", "", "tool-only forced PostgreSQL expansion search (supported: EXPANSION-SUFFIX-SEEDED-REVERSE, EXPANSION-ENDPOINT-SEEDED-REVERSE)") + flags.StringVar(&cfg.PostgresTraversalTelemetry, "postgres-traversal-telemetry", postgresTraversalTelemetryOff, "PostgreSQL traversal telemetry level (off, summary, or diagnostic); replays run outside timed samples") + flags.BoolVar(&cfg.PostgresExpansionOrientationShadow, "postgres-expansion-orientation-shadow", false, "tool-only orientation-probe shadow mode; executes only the exact incumbent traversal arm") + flags.BoolVar(&cfg.PostgresExpansionOrientationTournament, "postgres-expansion-orientation-tournament", false, "tool-only guarded orientation-probe mode; executes the selected exact arm") + flags.StringVar(&cfg.PostgresExpansionOrientationPolicy, "postgres-expansion-orientation-policy", "", "tool-only immutable orientation policy (orientation-probe-v1 or orientation-probe-v2; default: v1)") + flags.BoolVar(&cfg.PostgresExpansionSuffixReverseGuard, "postgres-expansion-suffix-reverse-guard", false, "tool-only full-path suffix-reverse guard with exact forward fallback") + flags.BoolVar(&cfg.PostgresExpansionSuffixReverseRetry, "postgres-expansion-suffix-reverse-retry", false, "tool-only reverse-only fixed-suffix candidate with same-transaction exact forward retry") + flags.BoolVar(&cfg.PostgresExpansionSuffixRouteComponent, "postgres-expansion-suffix-route-component", false, "tool-only exact fixed-suffix reverse component with no retry, probe, or cache") + flags.BoolVar(&cfg.PostgresSuffixRouteComponentClosure, "postgres-suffix-route-component-closure", false, "measurement-only fixed-suffix routing closure with compile, raw-PGX prepared-state, and workspace evidence") + flags.Int64Var(&cfg.PostgresSuffixGuardSuffixLimit, "postgres-suffix-guard-suffix-limit", 0, "tool-only suffix payload cap override (0 uses the immutable policy default)") + flags.Int64Var(&cfg.PostgresSuffixGuardStateLimit, "postgres-suffix-guard-state-limit", 0, "tool-only reverse-state cap override (0 uses the immutable policy default)") + flags.Int64Var(&cfg.PostgresSuffixRetryOutputRowLimit, "postgres-suffix-retry-output-row-limit", 0, "tool-only retry candidate output-row cap override (0 uses the immutable policy default)") + flags.Int64Var(&cfg.PostgresSuffixRetryOutputBytesLimit, "postgres-suffix-retry-output-bytes-limit", 0, "tool-only retry candidate output-byte cap override (0 uses the immutable policy default)") + flags.StringVar(&cfg.ConfirmLeft, "confirm-left", "", "left JSONL artifact for paired confirmation mode") + flags.StringVar(&cfg.ConfirmRight, "confirm-right", "", "right JSONL artifact for paired confirmation mode") + flags.StringVar(&cfg.ConfirmAA, "confirm-aa", "", "optional block/reload A/A resolution report") + flags.StringVar(&cfg.ConfirmOutput, "confirm-output", "", "paired confirmation JSON output path (default: stdout)") + flags.StringVar(&rawConfirmCases, "confirm-cases", "", "comma-separated exact primary names for paired confirmation") + flags.BoolVar(&cfg.DiagnosticGate, "diagnostic-gate", false, "allow comparison of matching diagnostic-only subsets") + flags.StringVar(&cfg.BundleDir, "bundle-dir", "", "write a reconstructible capture bundle to this directory") + flags.Func("bundle-evidence", "named auxiliary bundle artifact as name=path (repeatable)", func(value string) error { + rawBundleEvidence = append(rawBundleEvidence, value) + return nil + }) + flags.StringVar(&cfg.BundleVerify, "bundle-verify", "", "standalone verification of a capture bundle directory") + flags.StringVar(&cfg.BundleVerifyOutput, "bundle-verify-output", "", "capture-bundle verification JSON output path (default: stdout)") + flags.BoolVar(&cfg.BundleRequireClean, "bundle-require-clean", false, "require standalone bundle verification to prove a clean source capture") + flags.BoolVar(&cfg.RequireCleanSource, "require-clean-source", false, "refuse a live capture unless the source tree is clean before database setup") + flags.StringVar(&cfg.PromotionManifest, "promotion-manifest", "", "verify a candidate promotion manifest and every bound evidence report") + flags.StringVar(&cfg.PromotionManifestOutput, "promotion-manifest-output", "", "promotion-manifest verification JSON destination (default: stdout)") + flags.StringVar(&cfg.PromotionBindManifest, "promotion-bind-manifest", "", "provisional promotion manifest supplying report identity") + flags.StringVar(&cfg.PromotionBindRole, "promotion-bind-role", "", "promotion evidence role to bind") + flags.StringVar(&cfg.PromotionBindInput, "promotion-bind-input", "", "unbound promotion evidence report") + flags.StringVar(&cfg.PromotionBindOutput, "promotion-bind-output", "", "identity-bound promotion evidence report") + flags.StringVar(&cfg.OperationalGateInput, "operational-gate-input", "", "schema-v2 candidate-bound operational evidence document") + flags.StringVar(&cfg.OperationalGateOutput, "operational-gate-output", "", "operational evidence gate JSON report output path") + flags.StringVar(&cfg.BuildCommand, "build-command", "go build -trimpath ./cmd/graphbench", "reproducible build command recorded in bundles") + flags.BoolVar(&cfg.ExistingGraph, "existing-graph", false, "run non-mutating PostgreSQL cases against an existing graph in read-write sessions without schema, load, clear, vacuum, or persistent writes") + flags.StringVar(&cfg.AnchorManifest, "anchor-manifest", "", "versioned logical-key anchor manifest for existing-graph mode") + flags.StringVar(&cfg.Checkpoint, "checkpoint", "", "atomic existing-graph checkpoint path") + flags.BoolVar(&cfg.Resume, "resume", false, "resume completed records from the matching existing-graph checkpoint") + flags.StringVar(&cfg.Progress, "progress", "", "append-only existing-graph progress JSONL path") + flags.BoolVar(&cfg.Discovery, "discovery", false, "label the run adaptive discovery rather than fixed confirmation") + flags.StringVar(&rawTimeoutClasses, "timeout-classes", "", "comma-separated predeclared per-case timeout classes used by discovery") + flags.IntVar(&cfg.DiscoverySampleFloor, "discovery-sample-floor", 1, "minimum measured samples after adaptive discovery reduction") + flags.StringVar(&cfg.ResourceArtifact, "resource-artifact", "", "JSONL artifact used to calculate the state/resource gate") + flags.StringVar(&cfg.ResourceOutput, "resource-output", "", "state/resource gate JSON output path (default: stdout)") + flags.StringVar(&cfg.BackendDeltaArtifact, "backend-delta-artifact", "", "JSONL artifact used for descriptive matched PostgreSQL/Neo4j deltas") + flags.StringVar(&cfg.BackendDeltaOutput, "backend-delta-output", "", "descriptive backend-delta JSON output path (default: stdout)") + flags.StringVar(&cfg.ExpandIntoArtifact, "expand-into-artifact", "", "JSONL artifact used to build the fixed-one-hop three-arm study report") + flags.StringVar(&cfg.ExpandIntoOutput, "expand-into-output", "", "ExpandInto study JSON output path (default: stdout)") + flags.StringVar(&cfg.ExpandIntoProtocol, "expand-into-protocol", referencePairProtocolDiscovery, "ExpandInto study protocol (discovery or confirmation)") + flags.StringVar(&cfg.OrientationShadowArtifact, "orientation-shadow-artifact", "", "true-shadow orientation JSONL artifact") + flags.StringVar(&cfg.OrientationIncumbentArtifact, "orientation-incumbent-artifact", "", "matched exact incumbent orientation JSONL artifact") + flags.StringVar(&cfg.OrientationReverseArtifact, "orientation-reverse-artifact", "", "matched exact forced-reverse orientation JSONL artifact") + flags.StringVar(&cfg.OrientationAA, "orientation-aa", "", "host A/A report used by orientation selector-regret analysis") + flags.StringVar(&cfg.OrientationOutput, "orientation-output", "", "orientation selector-regret and probe-overhead JSON output path (default: stdout)") + flags.StringVar(&cfg.OrientationProtocol, "orientation-protocol", referencePairProtocolConfirmation, "orientation report protocol (discovery or confirmation)") + flags.StringVar(&cfg.OrientationV2ShadowArtifact, "orientation-v2-shadow-artifact", "", "orientation-probe-v2 shadow JSONL artifact") + flags.StringVar(&cfg.OrientationV2IncumbentArtifact, "orientation-v2-incumbent-artifact", "", "matched exact forward orientation-v2 JSONL artifact") + flags.StringVar(&cfg.OrientationV2ReverseArtifact, "orientation-v2-reverse-artifact", "", "matched exact forced-reverse orientation-v2 JSONL artifact") + flags.StringVar(&cfg.OrientationV2GuardedArtifact, "orientation-v2-guarded-artifact", "", "matched actual guarded orientation-v2 JSONL artifact") + flags.StringVar(&cfg.OrientationV2AA, "orientation-v2-aa", "", "host A/A report used by orientation-v2 qualification") + flags.StringVar(&cfg.OrientationV2Freeze, "orientation-v2-freeze", "", "discovery freeze manifest required by orientation-v2 confirmation") + flags.StringVar(&cfg.OrientationV2DiscoveryReport, "orientation-v2-discovery-report", "", "training-only discovery report bound by the orientation-v2 freeze") + flags.StringVar(&cfg.OrientationV2FreezeOutput, "orientation-v2-freeze-output", "", "write the training-only orientation-v2 discovery freeze manifest") + flags.StringVar(&cfg.OrientationV2Output, "orientation-v2-output", "", "four-arm orientation-v2 qualification JSON output path (default: stdout)") + flags.StringVar(&cfg.OrientationV2Protocol, "orientation-v2-protocol", referencePairProtocolConfirmation, "orientation-v2 report protocol (discovery or confirmation)") + flags.StringVar(&cfg.SuffixGuardIncumbentArtifact, "suffix-guard-incumbent-artifact", "", "six-round exact-forward training artifact for the suffix-reverse stop gate") + flags.StringVar(&cfg.SuffixGuardReverseArtifact, "suffix-guard-reverse-artifact", "", "six-round exact suffix-reverse training artifact") + flags.StringVar(&cfg.SuffixGuardGuardedArtifact, "suffix-guard-guarded-artifact", "", "six-round production-shaped suffix-reverse guard training artifact") + flags.StringVar(&cfg.SuffixGuardAA, "suffix-guard-aa", "", "matching order-balanced incumbent A/A resolution report") + flags.StringVar(&cfg.SuffixGuardOutput, "suffix-guard-output", "", "training-only suffix-reverse feasibility report output path") + flags.StringVar(&cfg.SPI1BaselineArtifact, "sp-i1-baseline-artifact", "", "matched exact S4 JSONL artifact for staged inbound-I1 qualification") + flags.StringVar(&cfg.SPI1CandidateArtifact, "sp-i1-candidate-artifact", "", "matched guarded canonical-I1 JSONL artifact for staged inbound-I1 qualification") + flags.StringVar(&cfg.SPI1ResourceReport, "sp-i1-resource-report", "", "resource-gate report bound to the staged canonical-I1 artifact") + flags.StringVar(&cfg.SPI1Freeze, "sp-i1-freeze", "", "training-only freeze required by SP-I1 confirmation reporting and holdout capture") + flags.StringVar(&cfg.SPI1DiscoveryReport, "sp-i1-discovery-report", "", "training-only discovery report bound by the SP-I1 freeze") + flags.StringVar(&cfg.SPI1TrainingBaseline, "sp-i1-training-baseline-artifact", "", "exact S4 training artifact required to recompute a frozen SP-I1 discovery") + flags.StringVar(&cfg.SPI1TrainingCandidate, "sp-i1-training-candidate-artifact", "", "exact canonical-I1 training artifact required to recompute a frozen SP-I1 discovery") + flags.StringVar(&cfg.SPI1TrainingResource, "sp-i1-training-resource-report", "", "exact training resource report required to recompute a frozen SP-I1 discovery") + flags.StringVar(&cfg.SPI1FreezeOutput, "sp-i1-freeze-output", "", "write the staged SP-I1 training-only freeze manifest") + flags.StringVar(&cfg.SPI1Output, "sp-i1-output", "", "staged S4-to-I1 qualification JSON output path") + flags.StringVar(&cfg.SPI1Protocol, "sp-i1-protocol", referencePairProtocolConfirmation, "staged SP-I1 report protocol (discovery or confirmation)") + flags.StringVar(&cfg.SPI2BaselineArtifact, "sp-i2-baseline-artifact", "", "matched exact S4 distance JSONL artifact for staged SP-I2 qualification") + flags.StringVar(&cfg.SPI2CandidateArtifact, "sp-i2-candidate-artifact", "", "matched guarded SP-I2 distance JSONL artifact") + flags.StringVar(&cfg.SPI2ResourceReport, "sp-i2-resource-report", "", "resource-gate report bound to the staged SP-I2 artifact") + flags.StringVar(&cfg.SPI2Freeze, "sp-i2-freeze", "", "training-only freeze required by SP-I2 confirmation reporting and holdout capture") + flags.StringVar(&cfg.SPI2DiscoveryReport, "sp-i2-discovery-report", "", "training-only discovery report bound by the SP-I2 freeze") + flags.StringVar(&cfg.SPI2TrainingBaseline, "sp-i2-training-baseline-artifact", "", "exact S4 distance training artifact required to recompute frozen SP-I2 discovery") + flags.StringVar(&cfg.SPI2TrainingCandidate, "sp-i2-training-candidate-artifact", "", "exact guarded SP-I2 training artifact required to recompute frozen discovery") + flags.StringVar(&cfg.SPI2TrainingResource, "sp-i2-training-resource-report", "", "exact training resource report required to recompute frozen SP-I2 discovery") + flags.StringVar(&cfg.SPI2FreezeOutput, "sp-i2-freeze-output", "", "write the staged SP-I2 training-only freeze manifest") + flags.StringVar(&cfg.SPI2Output, "sp-i2-output", "", "staged S4-distance-to-I2 qualification JSON output path") + flags.StringVar(&cfg.SPI2Protocol, "sp-i2-protocol", referencePairProtocolConfirmation, "staged SP-I2 report protocol (discovery or confirmation)") + flags.StringVar(&cfg.SPI2Generation, "sp-i2-generation", "", "explicit SP-I2 evidence generation (sp-i2-distance-v1 or sp-i2-distance-v2)") + flags.BoolVar(&cfg.SPI2V2DevelopmentTournament, "sp-i2-v2-development-tournament", false, "run one arm/round of the fixed non-promotional SP-I2 V2 open-corpus component tournament") + flags.BoolVar(&cfg.SPI2V2ReadinessComparison, "sp-i2-v2-readiness-comparison", false, "run one arm/round of the fixed non-promotional SP-I2 V2 E0/S4 readiness comparison") + flags.StringVar(&cfg.SPI2V2DevelopmentArtifact, "sp-i2-v2-development-artifact", "", "validate one complete non-promotional SP-I2 V2 development JSONL artifact") + flags.StringVar(&cfg.SPI2V2DevelopmentStudy, "sp-i2-v2-development-study", "", "development artifact study (readiness or tournament)") + flags.StringVar(&cfg.SPI2V2DevelopmentReportArtifact, "sp-i2-v2-development-report-artifact", "", "exact five-arm tournament JSONL artifact used to create the diagnostic development report") + flags.StringVar(&cfg.SPI2V2DevelopmentReportOutput, "sp-i2-v2-development-report-output", "", "write the diagnostic SP-I2 V2 development report") + flags.BoolVar(&cfg.SPI2V2ComponentCheck, "sp-i2-v2-component-check", false, "capture one exact open-corpus E1D or E1P semantic and plan-invariant check") + flags.StringVar(&cfg.SPI2V2ComponentAuthorization, "sp-i2-v2-component-authorization", "", "checksummed E1D/E1P authorization required to capture E1DP") + flags.StringVar(&cfg.SPI2V2ComponentE1DArtifact, "sp-i2-v2-component-e1d-artifact", "", "exact E1D component-check JSONL artifact") + flags.StringVar(&cfg.SPI2V2ComponentE1PArtifact, "sp-i2-v2-component-e1p-artifact", "", "exact E1P component-check JSONL artifact") + flags.StringVar(&cfg.SPI2V2ComponentAuthorizationOutput, "sp-i2-v2-component-authorization-output", "", "write the checksummed E1DP component authorization") + flags.StringVar(&cfg.SPI2V2SimulationBaselineTrace, "sp-i2-v2-simulation-baseline-trace", "", "frozen clean V1 S4 JSONL trace used for prospective calibration") + flags.StringVar(&cfg.SPI2V2SimulationCandidateTrace, "sp-i2-v2-simulation-candidate-trace", "", "frozen clean V1 I2 JSONL trace used for prospective calibration") + flags.StringVar(&cfg.SPI2V2SimulationOutput, "sp-i2-v2-simulation-output", "", "write the frozen prospective power and coverage report") + flags.StringVar(&cfg.P5AdjacencyFeasibilityOutput, "p5-adjacency-feasibility-output", "", "write the isolated P5 shadow-adjacency feasibility report") if err := flags.Parse(args); err != nil { return config{}, err } + var err error + if cfg.BundleEvidence, err = parseCaptureBundleEvidenceInputs(rawBundleEvidence); err != nil { + return config{}, err + } if cfg.Iterations < 1 { return config{}, fmt.Errorf("iterations must be at least 1") } + if cfg.WarmupIterations < 0 { + return config{}, fmt.Errorf("warmup-iterations must not be negative") + } + if cfg.Round < 1 { + return config{}, fmt.Errorf("round must be at least 1") + } + if cfg.Block < 1 { + return config{}, fmt.Errorf("block must be at least 1") + } + if strings.TrimSpace(cfg.Arm) == "" { + return config{}, fmt.Errorf("arm must not be empty") + } + if cfg.ArmOrder < 0 { + return config{}, fmt.Errorf("arm-order must not be negative") + } + if cfg.PoolSize < 1 { + return config{}, fmt.Errorf("pool-size must be at least 1") + } + if cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryOff && + cfg.PostgresTraversalTelemetry != postgresTraversalTelemetrySummary && + cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic { + return config{}, fmt.Errorf("postgres-traversal-telemetry must be off, summary, or diagnostic") + } + if cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryOff && cfg.PoolSize != 1 { + return config{}, fmt.Errorf("PostgreSQL traversal telemetry requires pool-size 1 to preserve connection identity") + } + if cfg.SessionMemoryCeilingBytes < 0 || cfg.PoolMemoryCeilingBytes < 0 { + return config{}, fmt.Errorf("memory ceilings must not be negative") + } + if cfg.SessionMemoryCeilingBytes > 0 && cfg.PoolMemoryCeilingBytes > 0 && cfg.SessionMemoryCeilingBytes*int64(cfg.PoolSize) > cfg.PoolMemoryCeilingBytes { + return config{}, fmt.Errorf("session memory ceiling times pool size exceeds pool memory ceiling") + } + for _, raw := range strings.Split(rawConcurrency, ",") { + if raw = strings.TrimSpace(raw); raw == "" { + continue + } + level, err := strconv.Atoi(raw) + if err != nil || level < 1 { + return config{}, fmt.Errorf("concurrency levels must be positive integers, got %q", raw) + } + if !slices.Contains(cfg.Concurrency, level) { + cfg.Concurrency = append(cfg.Concurrency, level) + } + } + if (cfg.GateBaseline == "") != (cfg.GateCandidate == "") { + return config{}, fmt.Errorf("gate-baseline and gate-candidate must be supplied together") + } + if cfg.GateAA != "" && cfg.GateBaseline == "" { + return config{}, fmt.Errorf("gate-aa requires gate-baseline and gate-candidate") + } + if (cfg.ConfirmLeft == "") != (cfg.ConfirmRight == "") { + return config{}, fmt.Errorf("confirm-left and confirm-right must be supplied together") + } + if cfg.ConfirmAA != "" && cfg.ConfirmLeft == "" { + return config{}, fmt.Errorf("confirm-aa requires confirm-left and confirm-right") + } + if cfg.ReferenceClosureOutput != "" && cfg.ReferenceClosureArtifact == "" { + return config{}, fmt.Errorf("reference-closure-output requires reference-closure-artifact") + } + if cfg.ReferencePairOutput != "" && cfg.ReferencePairArtifact == "" { + return config{}, fmt.Errorf("reference-pair-output requires reference-pair-artifact") + } + if cfg.ReferencePairArtifact != "" && (cfg.ReferencePairBaseline == "" || cfg.ReferencePairCandidate == "") { + return config{}, fmt.Errorf("reference-pair-artifact requires baseline and candidate arms") + } + if cfg.ReferencePairBaseline != "" && cfg.ReferencePairBaseline == cfg.ReferencePairCandidate { + return config{}, fmt.Errorf("reference-pair baseline and candidate must differ") + } + if cfg.ReferenceTournamentOutput != "" && cfg.ReferenceTournamentArtifact == "" { + return config{}, fmt.Errorf("reference-tournament-output requires reference-tournament-artifact") + } + if cfg.ReferenceTournamentArtifact != "" && len(cfg.ReferenceTournamentArms) == 0 && strings.TrimSpace(rawTournamentArms) == "" { + return config{}, fmt.Errorf("reference-tournament-artifact requires reference-tournament-arms") + } + if cfg.ReferenceTournamentProtocol != referencePairProtocolDiscovery && cfg.ReferenceTournamentProtocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("reference-tournament-protocol must be discovery or confirmation") + } + if cfg.BundleVerifyOutput != "" && cfg.BundleVerify == "" { + return config{}, fmt.Errorf("bundle-verify-output requires bundle-verify") + } + if cfg.PromotionManifestOutput != "" && cfg.PromotionManifest == "" { + return config{}, fmt.Errorf("promotion-manifest-output requires promotion-manifest") + } + promotionBindConfigured := cfg.PromotionBindManifest != "" || cfg.PromotionBindRole != "" || cfg.PromotionBindInput != "" || cfg.PromotionBindOutput != "" + if promotionBindConfigured && (cfg.PromotionBindManifest == "" || cfg.PromotionBindRole == "" || cfg.PromotionBindInput == "" || cfg.PromotionBindOutput == "") { + return config{}, fmt.Errorf("promotion report binding requires manifest, role, input, and output") + } + operationalGateConfigured := cfg.OperationalGateInput != "" || cfg.OperationalGateOutput != "" + if operationalGateConfigured && (cfg.OperationalGateInput == "" || cfg.OperationalGateOutput == "") { + return config{}, fmt.Errorf("operational gate requires operational-gate-input and operational-gate-output") + } + if cfg.BundleRequireClean && cfg.BundleVerify == "" { + return config{}, fmt.Errorf("bundle-require-clean requires bundle-verify") + } + if len(cfg.BundleEvidence) > 0 && cfg.BundleDir == "" { + return config{}, fmt.Errorf("bundle-evidence requires bundle-dir") + } + if cfg.BundleVerify != "" && cfg.BundleDir != "" { + return config{}, fmt.Errorf("bundle-verify and bundle-dir are mutually exclusive") + } + if cfg.PromotionManifest != "" && (cfg.BundleVerify != "" || cfg.BundleDir != "") { + return config{}, fmt.Errorf("promotion-manifest verification is mutually exclusive with bundle operations") + } + if cfg.ExpandIntoOutput != "" && cfg.ExpandIntoArtifact == "" { + return config{}, fmt.Errorf("expand-into-output requires expand-into-artifact") + } + if cfg.ExpandIntoProtocol != referencePairProtocolDiscovery && cfg.ExpandIntoProtocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("expand-into-protocol must be discovery or confirmation") + } + orientationInputs := []string{cfg.OrientationShadowArtifact, cfg.OrientationIncumbentArtifact, cfg.OrientationReverseArtifact, cfg.OrientationAA} + orientationConfigured := false + for _, input := range orientationInputs { + orientationConfigured = orientationConfigured || input != "" + } + if cfg.OrientationOutput != "" { + orientationConfigured = true + } + if orientationConfigured { + for _, input := range orientationInputs { + if input == "" { + return config{}, fmt.Errorf("orientation report requires shadow, incumbent, reverse, and A/A artifacts") + } + } + } + if cfg.OrientationProtocol != referencePairProtocolDiscovery && cfg.OrientationProtocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-protocol must be discovery or confirmation") + } + orientationV2Inputs := []string{ + cfg.OrientationV2ShadowArtifact, cfg.OrientationV2IncumbentArtifact, cfg.OrientationV2ReverseArtifact, + cfg.OrientationV2GuardedArtifact, cfg.OrientationV2AA, + } + orientationV2Configured := cfg.OrientationV2Output != "" + for _, input := range orientationV2Inputs { + orientationV2Configured = orientationV2Configured || input != "" + } + if orientationV2Configured { + for _, input := range orientationV2Inputs { + if input == "" { + return config{}, fmt.Errorf("orientation-v2 report requires shadow, incumbent, reverse, guarded, and A/A artifacts") + } + } + } + if cfg.OrientationV2Protocol != referencePairProtocolDiscovery && cfg.OrientationV2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-v2-protocol must be discovery or confirmation") + } + if orientationV2Configured && cfg.OrientationV2Protocol == referencePairProtocolConfirmation && (cfg.OrientationV2Freeze == "" || cfg.OrientationV2DiscoveryReport == "") { + return config{}, fmt.Errorf("orientation-v2 confirmation requires orientation-v2-freeze and orientation-v2-discovery-report") + } + if orientationV2Configured && cfg.OrientationV2Protocol == referencePairProtocolDiscovery && (cfg.OrientationV2FreezeOutput == "" || cfg.OrientationV2Output == "") { + return config{}, fmt.Errorf("orientation-v2 discovery requires orientation-v2-output and orientation-v2-freeze-output") + } + if cfg.OrientationV2Freeze != "" && cfg.OrientationV2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-v2-freeze is only valid for confirmation") + } + if cfg.OrientationV2DiscoveryReport != "" && cfg.OrientationV2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("orientation-v2-discovery-report is only valid for confirmation") + } + if cfg.OrientationV2FreezeOutput != "" && cfg.OrientationV2Protocol != referencePairProtocolDiscovery { + return config{}, fmt.Errorf("orientation-v2-freeze-output is only valid for discovery") + } + if (cfg.OrientationV2Freeze != "" || cfg.OrientationV2DiscoveryReport != "" || cfg.OrientationV2FreezeOutput != "") && !orientationV2Configured { + return config{}, fmt.Errorf("orientation-v2-freeze requires orientation-v2 report mode") + } + suffixGuardInputs := []string{ + cfg.SuffixGuardIncumbentArtifact, + cfg.SuffixGuardReverseArtifact, + cfg.SuffixGuardGuardedArtifact, + cfg.SuffixGuardAA, + cfg.SuffixGuardOutput, + } + suffixGuardReportConfigured := false + for _, input := range suffixGuardInputs { + suffixGuardReportConfigured = suffixGuardReportConfigured || input != "" + } + if suffixGuardReportConfigured { + for _, input := range suffixGuardInputs { + if input == "" { + return config{}, fmt.Errorf("suffix-guard report requires incumbent, reverse, guarded, A/A, and output artifacts") + } + } + if cfg.OutputJSONL != "" || rawCases != "" || rawDatasets != "" || rawCategories != "" || rawTags != "" { + return config{}, fmt.Errorf("suffix-guard report mode cannot also execute or select benchmark cases") + } + if err := validateDistinctSPI2Paths(map[string]string{ + "incumbent artifact": cfg.SuffixGuardIncumbentArtifact, + "reverse artifact": cfg.SuffixGuardReverseArtifact, + "guarded artifact": cfg.SuffixGuardGuardedArtifact, + "A/A report": cfg.SuffixGuardAA, + "report output": cfg.SuffixGuardOutput, + }); err != nil { + return config{}, fmt.Errorf("suffix-guard report: %w", err) + } + } + spI1ReportInputs := []string{cfg.SPI1BaselineArtifact, cfg.SPI1CandidateArtifact, cfg.SPI1ResourceReport} + spI1TrainingInputs := []string{cfg.SPI1TrainingBaseline, cfg.SPI1TrainingCandidate, cfg.SPI1TrainingResource} + spI1ReportConfigured := cfg.SPI1Output != "" || cfg.SPI1FreezeOutput != "" + for _, input := range spI1ReportInputs { + spI1ReportConfigured = spI1ReportConfigured || input != "" + } + if cfg.SPI1Protocol != referencePairProtocolDiscovery && cfg.SPI1Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("sp-i1-protocol must be discovery or confirmation") + } + if spI1ReportConfigured { + for _, input := range spI1ReportInputs { + if input == "" { + return config{}, fmt.Errorf("SP-I1 report requires baseline, candidate, and resource artifacts") + } + } + if cfg.SPI1Output == "" { + return config{}, fmt.Errorf("SP-I1 report requires sp-i1-output") + } + if cfg.SPI1Protocol == referencePairProtocolDiscovery && cfg.SPI1FreezeOutput == "" { + return config{}, fmt.Errorf("SP-I1 discovery requires sp-i1-freeze-output") + } + if cfg.SPI1Protocol == referencePairProtocolConfirmation && (cfg.SPI1Freeze == "" || cfg.SPI1DiscoveryReport == "") { + return config{}, fmt.Errorf("SP-I1 confirmation requires sp-i1-freeze and sp-i1-discovery-report") + } + } else if (cfg.SPI1Freeze == "") != (cfg.SPI1DiscoveryReport == "") { + return config{}, fmt.Errorf("SP-I1 holdout capture requires both sp-i1-freeze and sp-i1-discovery-report") + } + trainingInputCount := 0 + for _, input := range spI1TrainingInputs { + if input != "" { + trainingInputCount++ + } + } + if cfg.SPI1Freeze != "" && trainingInputCount != len(spI1TrainingInputs) { + return config{}, fmt.Errorf("SP-I1 frozen authorization requires all three exact training evidence artifacts") + } + if cfg.SPI1Freeze == "" && trainingInputCount != 0 { + return config{}, fmt.Errorf("SP-I1 training evidence inputs require a discovery freeze") + } + if !spI1ReportConfigured && cfg.SPI1Freeze != "" && cfg.SPI1Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("SP-I1 holdout capture requires the confirmation protocol") + } + if cfg.SPI1FreezeOutput != "" && cfg.SPI1Protocol != referencePairProtocolDiscovery { + return config{}, fmt.Errorf("sp-i1-freeze-output is only valid for discovery") + } + if spI1ReportConfigured && cfg.SPI1Protocol == referencePairProtocolDiscovery && (cfg.SPI1Freeze != "" || cfg.SPI1DiscoveryReport != "") { + return config{}, fmt.Errorf("SP-I1 discovery creates a freeze and cannot consume confirmation inputs") + } + if spI1ReportConfigured && (cfg.OutputJSONL != "" || rawCases != "" || rawDatasets != "" || rawCategories != "" || rawTags != "") { + return config{}, fmt.Errorf("SP-I1 report mode cannot also execute or select benchmark cases") + } + if spI1ReportConfigured { + if err := validateDistinctSPI1Paths(map[string]string{ + "baseline artifact": cfg.SPI1BaselineArtifact, "candidate artifact": cfg.SPI1CandidateArtifact, + "resource report": cfg.SPI1ResourceReport, "freeze manifest": cfg.SPI1Freeze, + "discovery report": cfg.SPI1DiscoveryReport, "freeze output": cfg.SPI1FreezeOutput, + "training baseline artifact": cfg.SPI1TrainingBaseline, + "training candidate artifact": cfg.SPI1TrainingCandidate, + "training resource report": cfg.SPI1TrainingResource, + "report output": cfg.SPI1Output, + }); err != nil { + return config{}, err + } + } + spI2ReportInputs := []string{cfg.SPI2BaselineArtifact, cfg.SPI2CandidateArtifact, cfg.SPI2ResourceReport} + spI2TrainingInputs := []string{cfg.SPI2TrainingBaseline, cfg.SPI2TrainingCandidate, cfg.SPI2TrainingResource} + spI2ReportConfigured := cfg.SPI2Output != "" || cfg.SPI2FreezeOutput != "" + for _, input := range spI2ReportInputs { + spI2ReportConfigured = spI2ReportConfigured || input != "" + } + spI2ExecutorRequested := cfg.PostgresForceShortest == string(optimize.ShortestPathExecutorI2GuardedDistance) || + isV2GraphBenchExecutor(cfg.PostgresForceShortest) + spI2Requested := spI2ReportConfigured || cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "" || cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ReadinessComparison || cfg.SPI2V2DevelopmentArtifact != "" || cfg.SPI2V2DevelopmentStudy != "" || cfg.SPI2V2DevelopmentReportArtifact != "" || cfg.SPI2V2DevelopmentReportOutput != "" || cfg.SPI2V2ComponentCheck || cfg.SPI2V2ComponentAuthorization != "" || cfg.SPI2V2ComponentE1DArtifact != "" || cfg.SPI2V2ComponentE1PArtifact != "" || cfg.SPI2V2ComponentAuthorizationOutput != "" || cfg.SPI2V2SimulationBaselineTrace != "" || cfg.SPI2V2SimulationCandidateTrace != "" || cfg.SPI2V2SimulationOutput != "" || + spI2TrainingInputCount(spI2TrainingInputs) > 0 || spI2ExecutorRequested || + strings.Contains(rawTags, "sp-i2-distance-v1") || strings.Contains(rawTags, "sp-i2-distance-v2") + if spI2Requested && cfg.SPI2Generation == "" { + return config{}, fmt.Errorf("SP-I2 evidence requires explicit -sp-i2-generation") + } + if cfg.SPI2Generation != "" && cfg.SPI2Generation != spI2GenerationV1 && cfg.SPI2Generation != spI2GenerationV2 { + return config{}, fmt.Errorf("unsupported SP-I2 generation %q", cfg.SPI2Generation) + } + if cfg.SPI2Generation == spI2GenerationV1 { + if cfg.SPI2FreezeOutput != "" { + return config{}, fmt.Errorf("SP-I2 V1 is terminally rejected and cannot create a freeze") + } + if !spI2ReportConfigured && (cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "") { + return config{}, fmt.Errorf("SP-I2 V1 is terminally rejected and cannot authorize holdout capture") + } + if isV2GraphBenchExecutor(cfg.PostgresForceShortest) || strings.Contains(rawTags, "sp-i2-distance-v2") { + return config{}, fmt.Errorf("SP-I2 V1 generation cannot select V2 evidence") + } + } + if cfg.SPI2Generation == spI2GenerationV2 { + if spI2ReportConfigured || cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "" || spI2TrainingInputCount(spI2TrainingInputs) > 0 { + return config{}, fmt.Errorf("SP-I2 V2 evidence cannot use V1 report or freeze flags") + } + if cfg.PostgresForceShortest == string(optimize.ShortestPathExecutorI2GuardedDistance) || + (strings.Contains(rawTags, "sp-i2-distance-v1") && !cfg.SPI2V2DevelopmentTournament && !cfg.SPI2V2ReadinessComparison && !cfg.SPI2V2ComponentCheck) { + return config{}, fmt.Errorf("SP-I2 V2 generation cannot select V1 evidence") + } + if cfg.PostgresForceShortest == string(optimize.ShortestPathExecutorI2GuardedDistanceV2) { + return config{}, fmt.Errorf("SP-I2 V2 is terminally rejected for inadequate prospective power; the formal executor cannot be captured") + } + } + simulationInputs := 0 + for _, path := range []string{cfg.SPI2V2SimulationBaselineTrace, cfg.SPI2V2SimulationCandidateTrace} { + if path != "" { + simulationInputs++ + } + } + if simulationInputs != 0 || cfg.SPI2V2SimulationOutput != "" { + if cfg.SPI2Generation != spI2GenerationV2 || simulationInputs != 2 || cfg.SPI2V2SimulationOutput == "" { + return config{}, fmt.Errorf("SP-I2 V2 simulation requires generation v2, both frozen traces, and an output") + } + if sameCleanPath(cfg.SPI2V2SimulationOutput, cfg.SPI2V2SimulationBaselineTrace) || sameCleanPath(cfg.SPI2V2SimulationOutput, cfg.SPI2V2SimulationCandidateTrace) { + return config{}, fmt.Errorf("SP-I2 V2 simulation output must not overwrite a trace input") + } + if cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ReadinessComparison || cfg.SPI2V2ComponentCheck || + cfg.SPI2V2DevelopmentArtifact != "" || cfg.SPI2V2DevelopmentReportArtifact != "" || cfg.SPI2V2ComponentAuthorizationOutput != "" || + cfg.OutputJSONL != "" || spI2ReportConfigured || cfg.SPI2Freeze != "" { + return config{}, fmt.Errorf("SP-I2 V2 simulation cannot be combined with capture, reporting, or promotion workflows") + } + } + if (cfg.SPI2V2DevelopmentArtifact == "") != (cfg.SPI2V2DevelopmentStudy == "") { + return config{}, fmt.Errorf("SP-I2 V2 development artifact validation requires both artifact and study") + } + if cfg.SPI2V2DevelopmentArtifact != "" { + if cfg.SPI2Generation != spI2GenerationV2 { + return config{}, fmt.Errorf("SP-I2 V2 development artifact validation requires generation %q", spI2GenerationV2) + } + study := spI2V2DevelopmentStudy(cfg.SPI2V2DevelopmentStudy) + if study != spI2V2StudyReadiness && study != spI2V2StudyTournament { + return config{}, fmt.Errorf("SP-I2 V2 development study must be readiness or tournament") + } + if cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ReadinessComparison || spI2ReportConfigured || cfg.SPI2Freeze != "" { + return config{}, fmt.Errorf("SP-I2 V2 development artifact validation cannot be combined with capture or promotional workflows") + } + } + if cfg.SPI2V2DevelopmentReportOutput != "" && cfg.SPI2V2DevelopmentReportArtifact == "" { + return config{}, fmt.Errorf("SP-I2 V2 development report output requires its tournament artifact") + } + if cfg.SPI2V2DevelopmentReportArtifact != "" { + if cfg.SPI2Generation != spI2GenerationV2 { + return config{}, fmt.Errorf("SP-I2 V2 development reporting requires generation %q", spI2GenerationV2) + } + if cfg.SPI2V2DevelopmentArtifact != "" || cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ReadinessComparison || cfg.SPI2V2ComponentCheck || + cfg.SPI2V2ComponentE1DArtifact != "" || cfg.SPI2V2ComponentE1PArtifact != "" || cfg.SPI2V2ComponentAuthorizationOutput != "" || + spI2ReportConfigured || cfg.SPI2Freeze != "" { + return config{}, fmt.Errorf("SP-I2 V2 development reporting cannot be combined with validation, capture, or promotional workflows") + } + } + componentAuthorizationInputs := []string{ + cfg.SPI2V2ComponentE1DArtifact, + cfg.SPI2V2ComponentE1PArtifact, + cfg.SPI2V2ComponentAuthorizationOutput, + } + componentAuthorizationCount := 0 + for _, input := range componentAuthorizationInputs { + if input != "" { + componentAuthorizationCount++ + } + } + if componentAuthorizationCount != 0 && componentAuthorizationCount != len(componentAuthorizationInputs) { + return config{}, fmt.Errorf("SP-I2 V2 component authorization production requires E1D, E1P, and output artifacts") + } + if componentAuthorizationCount != 0 { + if cfg.SPI2Generation != spI2GenerationV2 || cfg.SPI2V2ComponentCheck || cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ReadinessComparison { + return config{}, fmt.Errorf("SP-I2 V2 component authorization production cannot be combined with capture workflows") + } + } + if cfg.SPI2Protocol != referencePairProtocolDiscovery && cfg.SPI2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("sp-i2-protocol must be discovery or confirmation") + } + if spI2ReportConfigured { + for _, input := range spI2ReportInputs { + if input == "" { + return config{}, fmt.Errorf("SP-I2 report requires baseline, candidate, and resource artifacts") + } + } + if cfg.SPI2Output == "" { + return config{}, fmt.Errorf("SP-I2 report requires sp-i2-output") + } + if cfg.SPI2Protocol == referencePairProtocolDiscovery && cfg.SPI2FreezeOutput == "" { + return config{}, fmt.Errorf("SP-I2 discovery requires sp-i2-freeze-output") + } + if cfg.SPI2Protocol == referencePairProtocolConfirmation && (cfg.SPI2Freeze == "" || cfg.SPI2DiscoveryReport == "") { + return config{}, fmt.Errorf("SP-I2 confirmation requires sp-i2-freeze and sp-i2-discovery-report") + } + } else if (cfg.SPI2Freeze == "") != (cfg.SPI2DiscoveryReport == "") { + return config{}, fmt.Errorf("SP-I2 holdout capture requires both sp-i2-freeze and sp-i2-discovery-report") + } + spI2TrainingInputCount := 0 + for _, input := range spI2TrainingInputs { + if input != "" { + spI2TrainingInputCount++ + } + } + if cfg.SPI2Freeze != "" && spI2TrainingInputCount != len(spI2TrainingInputs) { + return config{}, fmt.Errorf("SP-I2 frozen authorization requires all three exact training evidence artifacts") + } + if cfg.SPI2Freeze == "" && spI2TrainingInputCount != 0 { + return config{}, fmt.Errorf("SP-I2 training evidence inputs require a discovery freeze") + } + if !spI2ReportConfigured && cfg.SPI2Freeze != "" && cfg.SPI2Protocol != referencePairProtocolConfirmation { + return config{}, fmt.Errorf("SP-I2 holdout capture requires the confirmation protocol") + } + if cfg.SPI2FreezeOutput != "" && cfg.SPI2Protocol != referencePairProtocolDiscovery { + return config{}, fmt.Errorf("sp-i2-freeze-output is only valid for discovery") + } + if spI2ReportConfigured && cfg.SPI2Protocol == referencePairProtocolDiscovery && (cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "") { + return config{}, fmt.Errorf("SP-I2 discovery creates a freeze and cannot consume confirmation inputs") + } + if spI2ReportConfigured && (cfg.OutputJSONL != "" || rawCases != "" || rawDatasets != "" || rawCategories != "" || rawTags != "") { + return config{}, fmt.Errorf("SP-I2 report mode cannot also execute or select benchmark cases") + } + if spI2ReportConfigured { + if err := validateDistinctSPI2Paths(map[string]string{ + "baseline artifact": cfg.SPI2BaselineArtifact, "candidate artifact": cfg.SPI2CandidateArtifact, + "resource report": cfg.SPI2ResourceReport, "freeze manifest": cfg.SPI2Freeze, + "discovery report": cfg.SPI2DiscoveryReport, "freeze output": cfg.SPI2FreezeOutput, + "training baseline artifact": cfg.SPI2TrainingBaseline, + "training candidate artifact": cfg.SPI2TrainingCandidate, + "training resource report": cfg.SPI2TrainingResource, + "report output": cfg.SPI2Output, + }); err != nil { + return config{}, err + } + } + modeCount := 0 + if cfg.GateBaseline != "" { + modeCount++ + } + if len(cfg.AAArtifacts) != 0 { + modeCount++ + } + if cfg.ConfirmLeft != "" { + modeCount++ + } + if cfg.ReferenceClosureArtifact != "" { + modeCount++ + } + if cfg.ReferencePairArtifact != "" { + modeCount++ + } + if cfg.ReferenceTournamentArtifact != "" { + modeCount++ + } + if cfg.ResourceArtifact != "" { + modeCount++ + } + if cfg.BackendDeltaArtifact != "" { + modeCount++ + } + if cfg.BundleVerify != "" { + modeCount++ + } + if cfg.PromotionManifest != "" { + modeCount++ + } + if promotionBindConfigured { + modeCount++ + } + if operationalGateConfigured { + modeCount++ + } + if cfg.ExpandIntoArtifact != "" { + modeCount++ + } + if orientationConfigured { + modeCount++ + } + if orientationV2Configured { + modeCount++ + } + if suffixGuardReportConfigured { + modeCount++ + } + if spI1ReportConfigured { + modeCount++ + } + if spI2ReportConfigured { + modeCount++ + } + if !spI1ReportConfigured && cfg.SPI1Freeze != "" && modeCount > 0 { + return config{}, fmt.Errorf("SP-I1 holdout authorization cannot be combined with a standalone report mode") + } + if !spI2ReportConfigured && cfg.SPI2Freeze != "" && modeCount > 0 { + return config{}, fmt.Errorf("SP-I2 holdout authorization cannot be combined with a standalone report mode") + } + if modeCount > 1 { + return config{}, fmt.Errorf("performance-gate, A/A, paired-confirmation, reference-closure, reference-pair, reference-tournament, resource-gate, backend-delta, bundle-verify, promotion-manifest, promotion-bind, operational-gate, ExpandInto-report, orientation-report, orientation-v2-report, suffix-guard-report, SP-I1-report, and SP-I2-report modes are mutually exclusive") + } + if modeCount > 0 && cfg.BundleDir != "" { + return config{}, fmt.Errorf("standalone report modes and bundle-dir are mutually exclusive") + } + if len(cfg.AAArtifacts) != 0 && cfg.GateBaseline != "" { + return config{}, fmt.Errorf("aa-artifact and performance-gate mode are mutually exclusive") + } + if cfg.Confidence <= 0 || cfg.Confidence >= 1 || math.IsNaN(cfg.Confidence) || math.IsInf(cfg.Confidence, 0) { + return config{}, fmt.Errorf("confidence-level must be between 0 and 1") + } + if spI1ReportConfigured && (cfg.GateSeed != 1 || cfg.Confidence != defaultConfidenceLevel) { + return config{}, fmt.Errorf("SP-I1 reporting requires frozen seed 1 and confidence %.4f", defaultConfidenceLevel) + } + if spI2ReportConfigured && (cfg.GateSeed != 1 || cfg.Confidence != defaultConfidenceLevel) { + return config{}, fmt.Errorf("SP-I2 reporting requires frozen seed 1 and confidence %.4f", defaultConfidenceLevel) + } + if cfg.Regression < 0 { + return config{}, fmt.Errorf("regression-threshold must not be negative") + } + if cfg.MaterialityRatio <= 0 || cfg.MaterialityRatio >= 1 { + return config{}, fmt.Errorf("materiality-ratio must be between 0 and 1") + } + if cfg.MaterialityAbsolute < 0 { + return config{}, fmt.Errorf("materiality-absolute must not be negative") + } + if cfg.AppendJSONL && cfg.OutputJSONL == "" { + return config{}, fmt.Errorf("append-jsonl requires jsonl-output") + } + if cfg.ResourceOutput != "" && cfg.ResourceArtifact == "" { + return config{}, fmt.Errorf("resource-output requires resource-artifact") + } + if cfg.BackendDeltaOutput != "" && cfg.BackendDeltaArtifact == "" { + return config{}, fmt.Errorf("backend-delta-output requires backend-delta-artifact") + } + if cfg.DiscoverySampleFloor < 1 { + return config{}, fmt.Errorf("discovery-sample-floor must be at least 1") + } + for _, raw := range strings.Split(rawTimeoutClasses, ",") { + if raw = strings.TrimSpace(raw); raw != "" { + timeout, err := time.ParseDuration(raw) + if err != nil || timeout <= 0 { + return config{}, fmt.Errorf("timeout classes must be positive durations, got %q", raw) + } + if len(cfg.TimeoutClasses) > 0 && timeout <= cfg.TimeoutClasses[len(cfg.TimeoutClasses)-1] { + return config{}, fmt.Errorf("timeout classes must be strictly increasing") + } + cfg.TimeoutClasses = append(cfg.TimeoutClasses, timeout) + } + } + for _, target := range strings.Split(rawGateTargets, ",") { + if target = strings.TrimSpace(target); target != "" { + cfg.GateTargets = append(cfg.GateTargets, target) + } + } + if cfg.Cases, err = parseUniqueCSV("case", rawCases); err != nil { + return config{}, err + } + if cfg.Datasets, err = parseUniqueCSV("dataset", rawDatasets); err != nil { + return config{}, err + } + if cfg.Categories, err = parseUniqueCSV("category", rawCategories); err != nil { + return config{}, err + } + if cfg.Tags, err = parseUniqueCSV("tag", rawTags); err != nil { + return config{}, err + } + if cfg.ConfirmCases, err = parseUniqueCSV("confirmation case", rawConfirmCases); err != nil { + return config{}, err + } + if cfg.PostgresReferenceArms, err = parseUniqueCSV("PostgreSQL reference arm", rawReferenceArms); err != nil { + return config{}, err + } + if cfg.ReferenceTournamentArms, err = parseUniqueCSV("reference tournament arm", rawTournamentArms); err != nil { + return config{}, err + } + if cfg.ReferenceTournamentArtifact != "" && len(cfg.ReferenceTournamentArms) != 3 && len(cfg.ReferenceTournamentArms) != 5 { + return config{}, fmt.Errorf("reference tournament requires exactly 3 or 5 arms") + } + for _, arm := range cfg.ReferenceTournamentArms { + if !validPostgresReferenceArm(arm) { + return config{}, fmt.Errorf("unknown PostgreSQL reference tournament arm %q", arm) + } + } + for _, arm := range cfg.PostgresReferenceArms { + if !validPostgresReferenceArm(arm) { + return config{}, fmt.Errorf("unknown PostgreSQL reference arm %q", arm) + } + } + if cfg.ReferenceClosureArtifact != "" && !validPostgresReferenceArm(cfg.ReferenceClosureArm) { + return config{}, fmt.Errorf("unknown PostgreSQL reference closure arm %q", cfg.ReferenceClosureArm) + } + if len(cfg.PostgresReferenceArms) > 0 { + cfg.PostgresReferences = true + } + if cfg.PostgresForceShortest != "" && !validForcedShortestPathExecutor(cfg.PostgresForceShortest) { + return config{}, fmt.Errorf("unsupported PostgreSQL forced shortest executor %q", cfg.PostgresForceShortest) + } + if cfg.PostgresForceExpansion != "" && cfg.PostgresForceExpansion != "EXPANSION-SUFFIX-SEEDED-REVERSE" && cfg.PostgresForceExpansion != "EXPANSION-ENDPOINT-SEEDED-REVERSE" { + return config{}, fmt.Errorf("unsupported PostgreSQL forced expansion search %q", cfg.PostgresForceExpansion) + } + if cfg.PostgresForceShortest != "" && cfg.PostgresForceExpansion != "" { + return config{}, fmt.Errorf("PostgreSQL shortest and expansion search forces are mutually exclusive") + } + orientationMode := cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament + if cfg.PostgresExpansionOrientationShadow && cfg.PostgresExpansionOrientationTournament { + return config{}, fmt.Errorf("PostgreSQL expansion orientation shadow and tournament modes are mutually exclusive") + } + if orientationMode && (cfg.PostgresForceShortest != "" || cfg.PostgresForceExpansion != "") { + return config{}, fmt.Errorf("PostgreSQL expansion orientation and forced traversal selectors are mutually exclusive") + } + suffixMode := cfg.PostgresExpansionSuffixReverseGuard || cfg.PostgresExpansionSuffixReverseRetry || cfg.PostgresExpansionSuffixRouteComponent + if (cfg.PostgresExpansionSuffixReverseGuard && cfg.PostgresExpansionSuffixReverseRetry) || + (cfg.PostgresExpansionSuffixReverseGuard && cfg.PostgresExpansionSuffixRouteComponent) || + (cfg.PostgresExpansionSuffixReverseRetry && cfg.PostgresExpansionSuffixRouteComponent) { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse guard, transaction retry, and direct component modes are mutually exclusive") + } + if suffixMode && (orientationMode || cfg.PostgresForceShortest != "" || cfg.PostgresForceExpansion != "" || cfg.PostgresProductionManifest != "") { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse modes are mutually exclusive with orientation, forced traversal, and production-manifest selectors") + } + if !suffixMode && (cfg.PostgresSuffixGuardSuffixLimit != 0 || cfg.PostgresSuffixGuardStateLimit != 0) { + return config{}, fmt.Errorf("PostgreSQL suffix cap overrides require a suffix-reverse mode") + } + if !cfg.PostgresExpansionSuffixReverseRetry && (cfg.PostgresSuffixRetryOutputRowLimit != 0 || cfg.PostgresSuffixRetryOutputBytesLimit != 0) { + return config{}, fmt.Errorf("PostgreSQL suffix retry output caps require postgres-expansion-suffix-reverse-retry") + } + if cfg.PostgresSuffixGuardSuffixLimit < 0 || cfg.PostgresSuffixGuardStateLimit < 0 || cfg.PostgresSuffixRetryOutputRowLimit < 0 || cfg.PostgresSuffixRetryOutputBytesLimit < 0 { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse cap overrides must not be negative") + } + if suffixMode && !cfg.PostgresRepeatableRead { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse measurements require postgres-repeatable-read") + } + if cfg.PostgresExpansionSuffixReverseGuard && cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse guard measurements require diagnostic traversal telemetry") + } + if cfg.PostgresExpansionSuffixReverseRetry && cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse retry measurements require diagnostic traversal telemetry") + } + if cfg.PostgresExpansionSuffixReverseRetry && cfg.PoolSize != 1 { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse retry measurements require pool-size 1 for exact runtime receipts") + } + if cfg.PostgresExpansionSuffixReverseRetry && (cfg.PostgresReferences || len(cfg.Concurrency) != 0) { + return config{}, fmt.Errorf("PostgreSQL suffix-reverse retry development captures do not support reference or concurrency side measurements") + } + if cfg.PostgresExpansionSuffixRouteComponent && + (cfg.PostgresSuffixGuardSuffixLimit != 0 || cfg.PostgresSuffixGuardStateLimit != 0 || + cfg.PostgresSuffixRetryOutputRowLimit != 0 || cfg.PostgresSuffixRetryOutputBytesLimit != 0) { + return config{}, fmt.Errorf("PostgreSQL suffix-route component does not permit cap overrides") + } + if cfg.PostgresExpansionSuffixRouteComponent && cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic { + return config{}, fmt.Errorf("PostgreSQL suffix-route component measurements require diagnostic traversal telemetry") + } + if cfg.PostgresExpansionSuffixRouteComponent && cfg.PoolSize != 1 { + return config{}, fmt.Errorf("PostgreSQL suffix-route component measurements require pool-size 1 for exact runtime receipts") + } + if cfg.PostgresExpansionSuffixRouteComponent && (cfg.PostgresReferences || len(cfg.Concurrency) != 0) { + return config{}, fmt.Errorf("PostgreSQL suffix-route component captures do not support reference or concurrency side measurements") + } + if cfg.PostgresSuffixRouteComponentClosure && (orientationMode || cfg.PostgresForceShortest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionSuffixReverseGuard || cfg.PostgresExpansionSuffixReverseRetry || cfg.PostgresProductionManifest != "") { + return config{}, fmt.Errorf("PostgreSQL suffix-route closure is mutually exclusive with other selector and suffix-reverse modes") + } + if cfg.PostgresSuffixRouteComponentClosure && !cfg.PostgresRepeatableRead { + return config{}, fmt.Errorf("PostgreSQL suffix-route closure measurements require postgres-repeatable-read") + } + if cfg.PostgresSuffixRouteComponentClosure && cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic { + return config{}, fmt.Errorf("PostgreSQL suffix-route closure measurements require diagnostic traversal telemetry") + } + if cfg.PostgresSuffixRouteComponentClosure && cfg.PoolSize != 1 { + return config{}, fmt.Errorf("PostgreSQL suffix-route closure measurements require pool-size 1") + } + if cfg.PostgresSuffixRouteComponentClosure && (cfg.PostgresReferences || len(cfg.Concurrency) != 0) { + return config{}, fmt.Errorf("PostgreSQL suffix-route closure does not support reference or concurrency side measurements") + } + if cfg.PostgresSuffixRouteComponentClosure && (cfg.SessionMemoryCeilingBytes <= 0 || cfg.PoolMemoryCeilingBytes <= 0) { + return config{}, fmt.Errorf("PostgreSQL suffix-route closure requires positive session and pool memory ceilings") + } + if cfg.PostgresExpansionOrientationPolicy != "" && !orientationMode { + return config{}, fmt.Errorf("PostgreSQL expansion orientation policy requires shadow or tournament mode") + } + if cfg.PostgresExpansionOrientationPolicy != "" && + cfg.PostgresExpansionOrientationPolicy != string(optimize.ExpansionSearchPolicyOrientationProbeV1) && + cfg.PostgresExpansionOrientationPolicy != string(optimize.ExpansionSearchPolicyOrientationProbeV2) { + return config{}, fmt.Errorf("unsupported PostgreSQL expansion orientation policy %q", cfg.PostgresExpansionOrientationPolicy) + } + if (cfg.PostgresExpansionOrientationTournament || cfg.PostgresExpansionOrientationPolicy == string(optimize.ExpansionSearchPolicyOrientationProbeV2)) && !cfg.PostgresRepeatableRead { + return config{}, fmt.Errorf("guarded and orientation-probe-v2 measurements require postgres-repeatable-read") + } + if cfg.PostgresExpansionOrientationPolicy == string(optimize.ExpansionSearchPolicyOrientationProbeV2) && cfg.PostgresTraversalTelemetry == postgresTraversalTelemetryOff { + return config{}, fmt.Errorf("orientation-probe-v2 measurements require PostgreSQL traversal telemetry") + } + if cfg.PostgresProductionManifest != "" && (cfg.PostgresForceShortest != "" || cfg.PostgresForceExpansion != "" || orientationMode) { + return config{}, fmt.Errorf("PostgreSQL production manifest is mutually exclusive with forced and shadow translation modes") + } + if cfg.PostgresProductionManifest != "" && cfg.PostgresRepeatableRead { + return config{}, fmt.Errorf("PostgreSQL production manifest already implies Repeatable Read") + } + if cfg.GateBaseline != "" && !cfg.DiagnosticGate && cfg.GateAA == "" { + return config{}, fmt.Errorf("complete performance gate requires gate-aa host calibration evidence") + } modes, err := parseExecutionModes(rawModes) if err != nil { return config{}, err } cfg.Modes = modes + if cfg.ExistingGraph { + if cfg.AnchorManifest == "" { + return config{}, fmt.Errorf("existing-graph mode requires anchor-manifest") + } + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL { + return config{}, fmt.Errorf("existing-graph mode currently requires only postgres_sql mode") + } + if cfg.Resume && cfg.Checkpoint == "" { + return config{}, fmt.Errorf("resume requires checkpoint") + } + if len(cfg.TimeoutClasses) > 0 && !cfg.Discovery { + return config{}, fmt.Errorf("timeout-classes require discovery mode") + } + } else if cfg.Resume || cfg.AnchorManifest != "" || cfg.Checkpoint != "" || cfg.Progress != "" || cfg.Discovery || len(cfg.TimeoutClasses) > 0 { + return config{}, fmt.Errorf("existing-graph workflow flags require existing-graph mode") + } + if !spI1ReportConfigured && cfg.SPI1Freeze != "" { + if err := validateSPI1HoldoutCaptureConfig(cfg); err != nil { + return config{}, err + } + } + if !spI2ReportConfigured && cfg.SPI2Freeze != "" { + if err := validateSPI2HoldoutCaptureConfig(cfg); err != nil { + return config{}, err + } + } + if cfg.SPI2V2DevelopmentTournament { + if err := validateSPI2V2DevelopmentCaptureConfig(cfg); err != nil { + return config{}, err + } + } + if cfg.SPI2V2ReadinessComparison { + if err := validateSPI2V2ReadinessCaptureConfig(cfg); err != nil { + return config{}, err + } + } + if cfg.SPI2V2ComponentCheck { + if err := validateSPI2V2ComponentCheckCaptureConfig(cfg); err != nil { + return config{}, err + } + } + if cfg.P5AdjacencyFeasibilityOutput != "" { + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL { + return config{}, fmt.Errorf("P5 adjacency feasibility capture requires only postgres_sql mode") + } + if cfg.ExistingGraph { + return config{}, fmt.Errorf("P5 adjacency feasibility capture requires a disposable managed graph") + } + if cfg.PoolSize != 1 { + return config{}, fmt.Errorf("P5 adjacency feasibility capture requires pool-size 1") + } + if cfg.OutputJSONL != "" || cfg.Summary != "" || cfg.SummaryJSON != "" { + return config{}, fmt.Errorf("P5 adjacency feasibility output cannot be combined with benchmark result outputs") + } + if len(cfg.Cases) != 0 || len(cfg.Datasets) != 0 || len(cfg.Categories) != 0 || len(cfg.Tags) != 0 { + return config{}, fmt.Errorf("P5 adjacency feasibility capture does not accept corpus selectors") + } + } return cfg, nil } +// validForcedShortestPathExecutor reports whether graphbench recognizes a +// production executor or a declared tournament identity. +func validForcedShortestPathExecutor(executor string) bool { + switch executor { + case "SP-S0", + "SP-S0-DIRECT", + "SP-S3-U-D", + "SP-S3-U-E+MAT-M0", + "SP-S4-C-D", + "SP-S4-C-WE+MAT-M0", + "SP-I1-C-D", + "SP-I2-C-D", + "SP-I2-C-D-V2", + "SP-I2-C-D-V2-E0", + "SP-I2-C-D-V2-E1", + "SP-I2-C-D-V2-E1D", + "SP-I2-C-D-V2-E1P", + "SP-I2-C-D-V2-E1DP", + "SP-I1-U-E+MAT-M0", + "SP-I1-C-WE+MAT-M0", + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "ASP-A1-DAG", + "ASP-I1-U-DAG+MAT-M0", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL": + return true + default: + return false + } +} + +func spI2TrainingInputCount(inputs []string) int { + count := 0 + for _, input := range inputs { + if input != "" { + count++ + } + } + return count +} + +func isV2GraphBenchExecutor(executor string) bool { + return executor == string(optimize.ShortestPathExecutorI2GuardedDistanceV2) || + executor == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0) || + executor == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1) || + executor == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) || + executor == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P) || + executor == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP) +} + +// parseCaptureBundleEvidenceInputs parses repeatable name=path bundle evidence +// declarations while keeping host paths out of serialized evidence identities. +func parseCaptureBundleEvidenceInputs(rawValues []string) ([]CaptureBundleEvidenceInput, error) { + inputs := make([]CaptureBundleEvidenceInput, 0, len(rawValues)) + seen := map[string]struct{}{} + for _, raw := range rawValues { + name, path, found := strings.Cut(raw, "=") + name = strings.TrimSpace(name) + path = strings.TrimSpace(path) + if !found || !validBundleEvidenceName(name) || path == "" { + return nil, fmt.Errorf("bundle-evidence must be a valid name=path declaration, got %q", raw) + } + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("duplicate bundle-evidence name %q", name) + } + seen[name] = struct{}{} + inputs = append(inputs, CaptureBundleEvidenceInput{ + Name: name, + Path: path, + }) + } + return inputs, nil +} + +// parseUniqueCSV splits comma-separated selectors, rejecting duplicates and empty elements. +func parseUniqueCSV(kind, raw string) ([]string, error) { + var values []string + seen := map[string]struct{}{} + for _, value := range strings.Split(raw, ",") { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, duplicate := seen[value]; duplicate { + return nil, fmt.Errorf("duplicate %s selector %q", kind, value) + } + seen[value] = struct{}{} + values = append(values, value) + } + return values, nil +} + +// selectedCorpusContainsTag selects ed corpus contains tag. +func selectedCorpusContainsTag(corpus ScaleCorpus, tag string) bool { + for _, testCase := range corpus.Cases { + if slices.Contains(testCase.Tags, tag) { + return true + } + } + return false +} + +// parseExecutionModes parses a comma-separated mode list and rejects duplicates or unsupported values. func parseExecutionModes(raw string) ([]ExecutionMode, error) { var ( modes []ExecutionMode @@ -101,28 +1474,471 @@ func parseExecutionModes(raw string) ([]ExecutionMode, error) { return modes, nil } +// fatal logs a formatted fatal error and terminates the command. func fatal(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) os.Exit(1) } +// main runs the graphbench command. func main() { cfg, err := parseConfig(os.Args[1:], os.Getenv) if err != nil { fatal("%v", err) } - - corpus, err := loadScaleCorpus(cfg.CorpusRoot) + if cfg.P5AdjacencyFeasibilityOutput != "" { + pgConnection := cfg.PGConnection + if pgConnection == "" { + pgConnection = cfg.Connection + } + if pgConnection == "" { + fatal("P5 adjacency feasibility capture requires -pg-connection, -connection, PG_CONNECTION_STRING, or CONNECTION_STRING") + } + runLock, err := acquireDestructiveRunLock(cfg.DestructiveLock) + if err != nil { + fatal("acquire destructive run lock: %v", err) + } + defer func() { + if err := runLock.Close(); err != nil { + fatal("release destructive run lock: %v", err) + } + }() + if _, err := runP5AdjacencyFeasibilityCapture(context.Background(), cfg, pgConnection, os.Args); err != nil { + fatal("capture P5 adjacency feasibility: %v", err) + } + return + } + if cfg.SPI2V2SimulationOutput != "" { + report, err := createSPI2PowerSimulationReportV2(cfg.CorpusRoot, cfg.SPI2V2SimulationBaselineTrace, cfg.SPI2V2SimulationCandidateTrace, cfg.SPI2V2SimulationOutput) + if err != nil { + fatal("create SP-I2 V2 power simulation: %v", err) + } + if !report.Passed { + fatal("SP-I2 V2 prospective power or coverage requirement failed; this protocol is terminal") + } + return + } + if cfg.SPI2V2ComponentAuthorizationOutput != "" { + passed, err := createSPI2V2ComponentAuthorization( + cfg.CorpusRoot, + cfg.SPI2V2ComponentE1DArtifact, + cfg.SPI2V2ComponentE1PArtifact, + cfg.SPI2V2ComponentAuthorizationOutput, + ) + if err != nil { + fatal("create SP-I2 V2 component authorization: %v", err) + } + if !passed { + fatal("SP-I2 V2 component authorization failed") + } + return + } + if cfg.SPI2V2DevelopmentReportArtifact != "" { + if _, err := createSPI2V2DevelopmentReport(cfg.CorpusRoot, cfg.SPI2V2DevelopmentReportArtifact, cfg.SPI2V2DevelopmentReportOutput); err != nil { + fatal("create SP-I2 V2 development report: %v", err) + } + return + } + if cfg.SPI2V2DevelopmentArtifact != "" { + if err := validateSPI2V2DevelopmentArtifact(cfg.SPI2V2DevelopmentArtifact, spI2V2DevelopmentStudy(cfg.SPI2V2DevelopmentStudy)); err != nil { + fatal("validate SP-I2 V2 development artifact: %v", err) + } + return + } + if cfg.BundleVerify != "" { + passed, err := createCaptureBundleVerification(cfg.BundleVerify, cfg.BundleVerifyOutput, cfg.BundleRequireClean) + if err != nil { + fatal("verify capture bundle: %v", err) + } + if !passed { + fatal("capture bundle verification failed") + } + return + } + if cfg.PromotionManifest != "" { + passed, err := writePromotionManifestVerification(cfg.PromotionManifest, cfg.PromotionManifestOutput) + if err != nil { + fatal("verify promotion manifest: %v", err) + } + if !passed { + fatal("promotion manifest verification failed") + } + return + } + if cfg.PromotionBindManifest != "" { + if err := bindPromotionEvidenceReport(cfg.PromotionBindManifest, cfg.PromotionBindRole, cfg.PromotionBindInput, cfg.PromotionBindOutput); err != nil { + fatal("bind promotion evidence report: %v", err) + } + return + } + if cfg.OperationalGateInput != "" { + passed, err := createOperationalGateReport(cfg.OperationalGateInput, cfg.OperationalGateOutput) + if err != nil { + fatal("calculate operational gate: %v", err) + } + if !passed { + fatal("operational gate failed") + } + return + } + if cfg.OrientationShadowArtifact != "" { + passed, err := createOrientationSelectorReport( + cfg.OrientationShadowArtifact, + cfg.OrientationIncumbentArtifact, + cfg.OrientationReverseArtifact, + cfg.OrientationAA, + cfg.OrientationOutput, + OrientationSelectorReportOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.OrientationProtocol, + }, + ) + if err != nil { + fatal("calculate orientation selector report: %v", err) + } + if cfg.OrientationProtocol == referencePairProtocolConfirmation && !passed { + fatal("orientation selector qualification failed") + } + return + } + if cfg.OrientationV2ShadowArtifact != "" { + passed, err := createOrientationSelectorV2Report( + cfg.OrientationV2ShadowArtifact, + cfg.OrientationV2IncumbentArtifact, + cfg.OrientationV2ReverseArtifact, + cfg.OrientationV2GuardedArtifact, + cfg.OrientationV2AA, + cfg.OrientationV2Freeze, + cfg.OrientationV2DiscoveryReport, + cfg.OrientationV2FreezeOutput, + cfg.OrientationV2Output, + OrientationSelectorV2ReportOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.OrientationV2Protocol, + }, + ) + if err != nil { + fatal("calculate orientation-v2 selector report: %v", err) + } + if cfg.OrientationV2Protocol == referencePairProtocolConfirmation && !passed { + fatal("orientation-v2 selector qualification failed") + } + return + } + if cfg.SuffixGuardIncumbentArtifact != "" { + passed, err := createSuffixReverseGuardFeasibilityReport( + cfg.SuffixGuardIncumbentArtifact, + cfg.SuffixGuardReverseArtifact, + cfg.SuffixGuardGuardedArtifact, + cfg.SuffixGuardAA, + cfg.SuffixGuardOutput, + SuffixReverseGuardFeasibilityOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + BootstrapCount: defaultBootstrapCount, + }, + ) + if err != nil { + fatal("calculate suffix-reverse guard feasibility: %v", err) + } + if !passed { + fatal("suffix-reverse guard feasibility stop gate failed") + } + return + } + if cfg.SPI1BaselineArtifact != "" { + passed, err := createSPI1QualificationReport( + cfg.SPI1BaselineArtifact, + cfg.SPI1CandidateArtifact, + cfg.SPI1ResourceReport, + cfg.SPI1Freeze, + cfg.SPI1DiscoveryReport, + cfg.SPI1FreezeOutput, + cfg.SPI1Output, + SPI1QualificationOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.SPI1Protocol, + TrainingBaselinePath: cfg.SPI1TrainingBaseline, + TrainingCandidatePath: cfg.SPI1TrainingCandidate, + TrainingResourcePath: cfg.SPI1TrainingResource, + }, + ) + if err != nil { + fatal("calculate staged SP-I1 qualification: %v", err) + } + if cfg.SPI1Protocol == referencePairProtocolConfirmation && !passed { + fatal("staged SP-I1 qualification failed") + } + return + } + if cfg.SPI2BaselineArtifact != "" { + passed, err := createSPI2QualificationReport( + cfg.SPI2BaselineArtifact, + cfg.SPI2CandidateArtifact, + cfg.SPI2ResourceReport, + cfg.SPI2Freeze, + cfg.SPI2DiscoveryReport, + cfg.SPI2FreezeOutput, + cfg.SPI2Output, + SPI2QualificationOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.SPI2Protocol, + TrainingBaselinePath: cfg.SPI2TrainingBaseline, + TrainingCandidatePath: cfg.SPI2TrainingCandidate, + TrainingResourcePath: cfg.SPI2TrainingResource, + }, + ) + if err != nil { + fatal("calculate staged SP-I2 qualification: %v", err) + } + if cfg.SPI2Protocol == referencePairProtocolConfirmation && !passed { + fatal("staged SP-I2 qualification failed") + } + return + } + if cfg.ExpandIntoArtifact != "" { + if err := createExpandIntoStudyReport(cfg.ExpandIntoArtifact, cfg.ExpandIntoOutput, ExpandIntoStudyOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + Protocol: cfg.ExpandIntoProtocol, + MaterialityRatio: cfg.MaterialityRatio, + MaterialityAbsolute: cfg.MaterialityAbsolute, + P95RatioLimit: 1.05, + }); err != nil { + fatal("calculate ExpandInto study: %v", err) + } + return + } + if cfg.GateBaseline != "" { + corpus, err := loadScaleCorpus(cfg.CorpusRoot) + if err != nil { + fatal("load gate corpus declaration: %v", err) + } + selected, _, err := selectRunnableScaleCorpus(corpus, CorpusSelectors{ + Cases: cfg.Cases, + Datasets: cfg.Datasets, + Categories: cfg.Categories, + Tags: cfg.Tags, + }) + if err != nil { + fatal("select gate corpus: %v", err) + } + passed, err := comparePerformanceArtifacts(cfg.GateBaseline, cfg.GateCandidate, cfg.GateOutput, PerfGateOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + RegressionThreshold: cfg.Regression, + DeclaredBackends: selected.DeclaredBackends(), + TargetNames: cfg.GateTargets, + MaterialityRatio: cfg.MaterialityRatio, + MaterialityAbsolute: cfg.MaterialityAbsolute, + DiagnosticMode: cfg.DiagnosticGate, + AAReportPath: cfg.GateAA, + }) + if err != nil { + fatal("compare performance artifacts: %v", err) + } + if !passed { + fatal("performance gate failed") + } + return + } + if len(cfg.AAArtifacts) != 0 { + if err := createAAResolutionReport(cfg.AAArtifacts, cfg.AAOutput, PerfGateOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + }); err != nil { + fatal("calculate A/A measurement resolution: %v", err) + } + return + } + if cfg.ConfirmLeft != "" { + if err := createConfirmationReport(cfg.ConfirmLeft, cfg.ConfirmRight, cfg.ConfirmAA, cfg.ConfirmOutput, ConfirmationOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + CaseNames: cfg.ConfirmCases, + }); err != nil { + fatal("calculate paired confirmation: %v", err) + } + return + } + if cfg.ReferenceClosureArtifact != "" { + passed, err := createReferenceClosureReport(cfg.ReferenceClosureArtifact, cfg.ReferenceClosureOutput, ReferenceClosureOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + ReferenceName: cfg.ReferenceClosureArm, + RatioUpperLimit: 1.10, + AbsoluteResolution: cfg.MaterialityAbsolute, + }) + if err != nil { + fatal("calculate production/reference closure: %v", err) + } + if !passed { + fatal("production/reference closure failed") + } + return + } + if cfg.ReferencePairArtifact != "" { + if err := createReferencePairReport(cfg.ReferencePairArtifact, cfg.ReferencePairOutput, ReferencePairOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + BaselineName: cfg.ReferencePairBaseline, + CandidateName: cfg.ReferencePairCandidate, + Protocol: cfg.ReferencePairProtocol, + }); err != nil { + fatal("calculate matched reference pair: %v", err) + } + return + } + if cfg.ReferenceTournamentArtifact != "" { + passed, err := createReferenceTournamentReport(cfg.ReferenceTournamentArtifact, cfg.ReferenceTournamentOutput, ReferenceTournamentOptions{ + Seed: cfg.GateSeed, + Confidence: cfg.Confidence, + MaterialityRatio: cfg.MaterialityRatio, + MaterialityAbsolute: cfg.MaterialityAbsolute, + P95RatioLimit: 1.05, + Arms: cfg.ReferenceTournamentArms, + Protocol: cfg.ReferenceTournamentProtocol, + }) + if err != nil { + fatal("calculate reference tournament: %v", err) + } + if cfg.ReferenceTournamentProtocol == referencePairProtocolConfirmation && !passed { + fatal("reference tournament qualification failed") + } + return + } + if cfg.ResourceArtifact != "" { + passed, err := createResourceGateReport(cfg.ResourceArtifact, cfg.ResourceOutput) + if err != nil { + fatal("calculate state/resource gate: %v", err) + } + if !passed { + fatal("state/resource gate failed") + } + return + } + if cfg.BackendDeltaArtifact != "" { + if err := createBackendDeltaReport(cfg.BackendDeltaArtifact, cfg.BackendDeltaOutput); err != nil { + fatal("calculate descriptive backend deltas: %v", err) + } + return + } + fullCorpus, err := loadScaleCorpus(cfg.CorpusRoot) if err != nil { fatal("load corpus: %v", err) } + corpus, selection, err := selectRunnableScaleCorpusWithSPI2Protection(fullCorpus, CorpusSelectors{ + Cases: cfg.Cases, + Datasets: cfg.Datasets, + Categories: cfg.Categories, + Tags: cfg.Tags, + }) + if err != nil { + fatal("select corpus: %v", err) + } + if cfg.SPI2Generation == spI2GenerationV2 && selectedCorpusContainsSPI2V2FormalCase(corpus) { + fatal("SP-I2 V2 is terminally rejected for inadequate prospective power; formal corpus execution is forbidden") + } + if selectedCorpusContainsTag(corpus, spI1HoldoutTag) || selectedCorpusContainsSPI1Holdout(corpus) || cfg.SPI1Freeze != "" { + if cfg.SPI1Freeze == "" || cfg.SPI1DiscoveryReport == "" { + fatal("SP-I1 holdout capture requires sp-i1-freeze and sp-i1-discovery-report before database setup") + } + if err := validateSPI1HoldoutCapture( + corpus, cfg.SPI1Freeze, cfg.SPI1DiscoveryReport, + cfg.SPI1TrainingBaseline, cfg.SPI1TrainingCandidate, cfg.SPI1TrainingResource, + ); err != nil { + fatal("authorize SP-I1 holdout capture: %v", err) + } + } + if selectedCorpusContainsTag(corpus, spI2HoldoutTag) || selectedCorpusContainsSPI2Holdout(corpus) || cfg.SPI2Freeze != "" { + if cfg.SPI2Freeze == "" || cfg.SPI2DiscoveryReport == "" { + fatal("SP-I2 holdout capture requires sp-i2-freeze and sp-i2-discovery-report before database setup") + } + if err := validateSPI2HoldoutCapture( + corpus, cfg.SPI2Freeze, cfg.SPI2DiscoveryReport, + cfg.SPI2TrainingBaseline, cfg.SPI2TrainingCandidate, cfg.SPI2TrainingResource, + ); err != nil { + fatal("authorize SP-I2 holdout capture: %v", err) + } + } + if selectedCorpusContainsSPI2V2FormalHoldout(corpus) { + fatal("SP-I2 V2 formal holdout requires a sealed V2 discovery freeze and authorization before database setup") + } + if cfg.RequireCleanSource { + if err := requireCleanSourceCapture(); err != nil { + fatal("refuse capture: %v", err) + } + } + + if !cfg.ExistingGraph { + for _, mode := range cfg.Modes { + var connection string + switch mode { + case ModePostgresSQL: + connection = cfg.PGConnection + case ModeNeo4j: + connection = cfg.Neo4jConnection + default: + continue + } + if connection == "" { + connection = cfg.Connection + } + if connection == "" { + continue + } + } + + runLock, err := acquireDestructiveRunLock(cfg.DestructiveLock) + if err != nil { + fatal("acquire destructive run lock: %v", err) + } + defer func() { + if err := runLock.Close(); err != nil { + fatal("release destructive run lock: %v", err) + } + }() + } var ( - ctx = context.Background() - records []CaseResult + ctx = context.Background() + records []CaseResult + existingManifest ExistingGraphAnchorManifest + startedAt = time.Now() ) + checkpointCorpusHash := corpusIdentity(corpus) + metadata := testutil.ResolveBaselineMetadata(cfg.DAWGSVersion) + environment := resolveRunEnvironment(cfg, os.Args, selection, startedAt, startedAt) + checkpointRunHash := runConfigurationIdentity(cfg, environment) + environment.CorpusSHA256 = checkpointCorpusHash + environment.RunIdentitySHA256 = checkpointRunHash + if cfg.ExistingGraph { + existingManifest, err = loadExistingGraphAnchorManifest(cfg.AnchorManifest) + if err != nil { + fatal("load existing-graph anchor manifest: %v", err) + } + if err := validateExistingGraphCorpus(corpus, existingManifest); err != nil { + fatal("validate existing-graph corpus: %v", err) + } + if cfg.Resume { + records, err = readExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash) + if err != nil { + fatal("resume existing-graph checkpoint: %v", err) + } + for _, record := range records { + if record.Environment != nil && record.Environment.RunUUID != "" { + environment.RunUUID = record.Environment.RunUUID + break + } + } + } + } - for _, mode := range cfg.Modes { + for _, mode := range modesForRound(cfg.Modes, cfg.Round) { switch mode { case ModePostgresSQL: pgConnection := cfg.PGConnection @@ -133,12 +1949,62 @@ func main() { fatal("postgres_sql mode requires -pg-connection, -connection, PG_CONNECTION_STRING, or CONNECTION_STRING") } - runner, err := newPostgresSQLRunner(ctx, cfg.DatasetDir, pgConnection, corpus) + var existingOptions *existingGraphRunnerOptions + if cfg.ExistingGraph { + completed := map[string]string{} + for _, record := range records { + completed[existingGraphCaseKey(record.ExecutionMode, ScaleCase{ + Dataset: record.Dataset, + Name: record.Name, + })] = record.WorkloadSHA256 + } + existingOptions = &existingGraphRunnerOptions{ + Manifest: existingManifest, + ProgressPath: cfg.Progress, + Discovery: cfg.Discovery, + TimeoutClasses: append([]time.Duration(nil), cfg.TimeoutClasses...), + SampleFloor: cfg.DiscoverySampleFloor, + Completed: completed, + OnRecord: func(record CaseResult) error { + setCaseRunMetadata(&record, metadata, environment) + records = append(records, record) + return writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records) + }, + OnComplete: func(postNodes, postEdges int64) error { + for idx := range records { + if records[idx].ExistingGraph != nil { + records[idx].ExistingGraph.PostNodeCount = postNodes + records[idx].ExistingGraph.PostEdgeCount = postEdges + } + } + return writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records) + }, + } + } + runner, err := newPostgresSQLRunnerWithExistingGraph(ctx, cfg.DatasetDir, pgConnection, corpus, cfg.PoolSize, cfg.Round, cfg.Concurrency, cfg.PostgresReferences, cfg.PostgresReferenceArms, cfg.PostgresForceShortest, cfg.PostgresForceExpansion, existingOptions) if err != nil { fatal("open postgres_sql runner: %v", err) } - - nextRecords, err := runner.Run(ctx, cfg.Iterations, corpus) + runner.traversalTelemetry = cfg.PostgresTraversalTelemetry + runner.repeatableRead = cfg.PostgresRepeatableRead + runner.toolOptions.EnableExpansionOrientationShadow = cfg.PostgresExpansionOrientationShadow + runner.toolOptions.EnableExpansionOrientationTournament = cfg.PostgresExpansionOrientationTournament + runner.toolOptions.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicy(cfg.PostgresExpansionOrientationPolicy) + runner.toolOptions.EnableExpansionSuffixReverseGuard = cfg.PostgresExpansionSuffixReverseGuard + runner.toolOptions.EnableExpansionSuffixReverseRetry = cfg.PostgresExpansionSuffixReverseRetry + runner.toolOptions.EnableExpansionSuffixRouteComponent = cfg.PostgresExpansionSuffixRouteComponent + runner.suffixRouteComponentClosure = cfg.PostgresSuffixRouteComponentClosure + runner.sessionMemoryCeilingBytes = cfg.SessionMemoryCeilingBytes + runner.poolMemoryCeilingBytes = cfg.PoolMemoryCeilingBytes + runner.toolOptions.SuffixReverseGuardSuffixRowLimit = cfg.PostgresSuffixGuardSuffixLimit + runner.toolOptions.SuffixReverseGuardStateLimit = cfg.PostgresSuffixGuardStateLimit + runner.toolOptions.SuffixReverseRetryOutputRowLimit = cfg.PostgresSuffixRetryOutputRowLimit + runner.toolOptions.SuffixReverseRetryOutputBytesLimit = cfg.PostgresSuffixRetryOutputBytesLimit + if err := runner.setProductionManifest(cfg.PostgresProductionManifest); err != nil { + _ = runner.Close(ctx) + fatal("configure PostgreSQL production candidate: %v", err) + } + nextRecords, err := runner.Run(ctx, cfg.WarmupIterations, cfg.Iterations, corpus) closeErr := runner.Close(ctx) if err != nil { fatal("run postgres_sql: %v", err) @@ -147,7 +2013,16 @@ func main() { fatal("close postgres_sql: %v", closeErr) } - records = append(records, nextRecords...) + if !cfg.ExistingGraph { + records = append(records, nextRecords...) + } else { + // OnRecord appends each completed record atomically. A resumed run + // may have no new records, while a complete run refreshes the final + // before/after cardinality proof below. + if err := writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records); err != nil { + fatal("finalize existing-graph checkpoint: %v", err) + } + } case ModeNeo4j: neo4jConnection := cfg.Neo4jConnection @@ -163,7 +2038,7 @@ func main() { fatal("open neo4j runner: %v", err) } - nextRecords, err := runner.Run(ctx, cfg.Iterations, corpus) + nextRecords, err := runner.Run(ctx, cfg.WarmupIterations, cfg.Iterations, corpus) closeErr := runner.Close(ctx) if err != nil { fatal("run neo4j: %v", err) @@ -182,14 +2057,43 @@ func main() { } } + if err := validateBackendObservations(records); err != nil { + fatal("validate backend observations: %v", err) + } + + environment.EndedAt = time.Now().UTC() + for idx := range records { + if records[idx].Environment == nil { + setCaseRunMetadata(&records[idx], metadata, environment) + } else if records[idx].Environment.RunUUID == environment.RunUUID { + records[idx].Environment.EndedAt = environment.EndedAt + } + } + if cfg.ExistingGraph { + if err := writeExistingGraphCheckpoint(cfg.Checkpoint, existingManifest.Checksum, checkpointCorpusHash, checkpointRunHash, records); err != nil { + fatal("persist finalized existing-graph checkpoint: %v", err) + } + } + if cfg.Baseline != "" { if err := applyBaseline(cfg.Baseline, records); err != nil { fatal("compare baseline: %v", err) } } - if err := writeJSONLFile(cfg.OutputJSONL, records); err != nil { - fatal("write JSONL: %v", err) + var writeErr error + if cfg.AppendJSONL { + writeErr = appendJSONLFile(cfg.OutputJSONL, records) + } else { + writeErr = writeJSONLFile(cfg.OutputJSONL, records) + } + if writeErr != nil { + fatal("write JSONL: %v", writeErr) + } + if cfg.BundleDir != "" { + if err := writeCaptureBundleWithEvidence(cfg.BundleDir, corpus, records, environment, cfg.BundleEvidence); err != nil { + fatal("write capture bundle: %v", err) + } } summary := buildSummary(records) @@ -204,3 +2108,12 @@ func main() { } } } + +// modesForRound returns execution modes in alternating round order without mutating the configured slice. +func modesForRound(modes []ExecutionMode, round int) []ExecutionMode { + ordered := append([]ExecutionMode(nil), modes...) + if round%2 == 0 { + slices.Reverse(ordered) + } + return ordered +} diff --git a/cmd/graphbench/main_test.go b/cmd/graphbench/main_test.go new file mode 100644 index 00000000..d507d986 --- /dev/null +++ b/cmd/graphbench/main_test.go @@ -0,0 +1,831 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestModesForRoundAlternatesBackendOrderWithoutMutatingConfig verifies odd/even round rotation without modifying the configured backend order. +func TestModesForRoundAlternatesBackendOrderWithoutMutatingConfig(t *testing.T) { + modes := []ExecutionMode{ModePostgresSQL, ModeNeo4j} + + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, modesForRound(modes, 1)) + require.Equal(t, []ExecutionMode{ModeNeo4j, ModePostgresSQL}, modesForRound(modes, 2)) + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, modes) +} + +// TestParseConfigRequiresCompleteGateInputs verifies that baseline gating cannot be enabled without its paired candidate artifact. +func TestParseConfigRequiresCompleteGateInputs(t *testing.T) { + _, err := parseConfig([]string{"-gate-baseline", "baseline.jsonl"}, func(string) string { return "" }) + + require.ErrorContains(t, err, "must be supplied together") +} + +// TestParseConfigDefaultsQualificationConfidence verifies every statistical workflow starts at the frozen 97.5% policy. +func TestParseConfigDefaultsQualificationConfidence(t *testing.T) { + cfg, err := parseConfig(nil, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, defaultConfidenceLevel, cfg.Confidence) + require.Equal(t, minimumTimingNoiseRatio, cfg.Regression) +} + +// TestCleanSourceCaptureGateFailsClosed verifies capture provenance rejects +// both dirty and indeterminate fingerprints before database setup. +func TestCleanSourceCaptureGateFailsClosed(t *testing.T) { + require.NoError(t, validateCleanSourceFingerprint(cleanWorkingTreeSHA256())) + require.ErrorContains(t, validateCleanSourceFingerprint("unknown"), "requires a clean committed source tree") + require.ErrorContains(t, validateCleanSourceFingerprint("deadbeef"), "requires a clean committed source tree") +} + +// TestParseConfigRequiresGateAAForPromotion verifies only explicit diagnostic comparisons may omit host calibration. +func TestParseConfigRequiresGateAAForPromotion(t *testing.T) { + _, err := parseConfig([]string{"-gate-baseline", "baseline.jsonl", "-gate-candidate", "candidate.jsonl"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires gate-aa") + + cfg, err := parseConfig([]string{ + "-gate-baseline", "baseline.jsonl", "-gate-candidate", "candidate.jsonl", "-gate-aa", "aa.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "aa.json", cfg.GateAA) +} + +// TestParseConfigAcceptsNamedBundleEvidence verifies repeatable name=path inputs are retained for capture without conflating their host paths with evidence names. +func TestParseConfigAcceptsNamedBundleEvidence(t *testing.T) { + cfg, err := parseConfig([]string{ + "-bundle-dir", "capture", + "-bundle-evidence", "host-aa=.coverage/aa.json", + "-bundle-evidence", "plan-delta=.coverage/plan-delta.json", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []CaptureBundleEvidenceInput{ + { + Name: "host-aa", + Path: ".coverage/aa.json", + }, + { + Name: "plan-delta", + Path: ".coverage/plan-delta.json", + }, + }, cfg.BundleEvidence) +} + +// TestParseConfigAcceptsStandaloneBundleVerification verifies portable verification can run without a benchmark connection and optionally enforce clean-source provenance. +func TestParseConfigAcceptsStandaloneBundleVerification(t *testing.T) { + cfg, err := parseConfig([]string{ + "-bundle-verify", "capture", + "-bundle-verify-output", "verification.json", + "-bundle-require-clean", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "capture", cfg.BundleVerify) + require.Equal(t, "verification.json", cfg.BundleVerifyOutput) + require.True(t, cfg.BundleRequireClean) +} + +// TestParseConfigAcceptsOnlyStandalonePromotionManifestVerification verifies parse config accepts only standalone promotion manifest verification behavior. +func TestParseConfigAcceptsOnlyStandalonePromotionManifestVerification(t *testing.T) { + cfg, err := parseConfig([]string{ + "-promotion-manifest", "promotion.json", + "-promotion-manifest-output", "verification.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "promotion.json", cfg.PromotionManifest) + require.Equal(t, "verification.json", cfg.PromotionManifestOutput) + + for _, args := range [][]string{ + {"-promotion-manifest-output", "verification.json"}, + {"-promotion-manifest", "promotion.json", "-bundle-verify", "capture"}, + {"-promotion-manifest", "promotion.json", "-resource-artifact", "resources.jsonl"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigRejectsMalformedOrOrphanedBundleFlags verifies capture and verification inputs fail before any artifact or database is touched. +func TestParseConfigRejectsMalformedOrOrphanedBundleFlags(t *testing.T) { + for _, args := range [][]string{ + {"-bundle-evidence", "host-aa=aa.json"}, + {"-bundle-dir", "capture", "-bundle-evidence", "missing-separator"}, + {"-bundle-dir", "capture", "-bundle-evidence", "../escape=aa.json"}, + {"-bundle-dir", "capture", "-bundle-evidence", "host-aa=one.json", "-bundle-evidence", "host-aa=two.json"}, + {"-bundle-verify-output", "verification.json"}, + {"-bundle-require-clean"}, + {"-bundle-verify", "capture", "-bundle-dir", "new-capture"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsExpandIntoStudyProtocols verifies standalone three-arm reports expose the frozen discovery and confirmation evidence contracts. +func TestParseConfigAcceptsExpandIntoStudyProtocols(t *testing.T) { + for _, protocol := range []string{referencePairProtocolDiscovery, referencePairProtocolConfirmation} { + t.Run(protocol, func(t *testing.T) { + cfg, err := parseConfig([]string{ + "-expand-into-artifact", "expand-into.jsonl", + "-expand-into-output", "expand-into.json", + "-expand-into-protocol", protocol, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "expand-into.jsonl", cfg.ExpandIntoArtifact) + require.Equal(t, "expand-into.json", cfg.ExpandIntoOutput) + require.Equal(t, protocol, cfg.ExpandIntoProtocol) + }) + } +} + +// TestParseConfigAcceptsOrientationSelectorReport verifies the matched shadow, +// incumbent, forced-reverse, and A/A artifacts form one standalone workflow. +func TestParseConfigAcceptsOrientationSelectorReport(t *testing.T) { + cfg, err := parseConfig([]string{ + "-orientation-shadow-artifact", "shadow.jsonl", + "-orientation-incumbent-artifact", "incumbent.jsonl", + "-orientation-reverse-artifact", "reverse.jsonl", + "-orientation-aa", "aa.json", + "-orientation-output", "orientation.json", + "-orientation-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "shadow.jsonl", cfg.OrientationShadowArtifact) + require.Equal(t, "incumbent.jsonl", cfg.OrientationIncumbentArtifact) + require.Equal(t, "reverse.jsonl", cfg.OrientationReverseArtifact) + require.Equal(t, "aa.json", cfg.OrientationAA) + require.Equal(t, "orientation.json", cfg.OrientationOutput) + require.Equal(t, referencePairProtocolConfirmation, cfg.OrientationProtocol) +} + +// TestParseConfigAcceptsOrientationSelectorV2Report verifies parse config accepts orientation selector v2 report behavior. +func TestParseConfigAcceptsOrientationSelectorV2Report(t *testing.T) { + cfg, err := parseConfig([]string{ + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", + "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", + "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", + "-orientation-v2-freeze", "orientation-v2-freeze.json", + "-orientation-v2-discovery-report", "orientation-v2-discovery.json", + "-orientation-v2-output", "orientation-v2.json", + "-orientation-v2-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "shadow-v2.jsonl", cfg.OrientationV2ShadowArtifact) + require.Equal(t, "incumbent.jsonl", cfg.OrientationV2IncumbentArtifact) + require.Equal(t, "reverse.jsonl", cfg.OrientationV2ReverseArtifact) + require.Equal(t, "guarded-v2.jsonl", cfg.OrientationV2GuardedArtifact) + require.Equal(t, "aa.json", cfg.OrientationV2AA) + require.Equal(t, "orientation-v2-freeze.json", cfg.OrientationV2Freeze) + require.Equal(t, "orientation-v2-discovery.json", cfg.OrientationV2DiscoveryReport) + require.Equal(t, "orientation-v2.json", cfg.OrientationV2Output) +} + +// TestParseConfigAcceptsOrientationSelectorV2DiscoveryFreeze verifies parse config accepts orientation selector v2 discovery freeze behavior. +func TestParseConfigAcceptsOrientationSelectorV2DiscoveryFreeze(t *testing.T) { + cfg, err := parseConfig([]string{ + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", + "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", + "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", + "-orientation-v2-output", "orientation-v2-discovery.json", + "-orientation-v2-freeze-output", "orientation-v2-freeze.json", + "-orientation-v2-protocol", referencePairProtocolDiscovery, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, referencePairProtocolDiscovery, cfg.OrientationV2Protocol) + require.Equal(t, "orientation-v2-discovery.json", cfg.OrientationV2Output) + require.Equal(t, "orientation-v2-freeze.json", cfg.OrientationV2FreezeOutput) +} + +// TestParseConfigRejectsIncompleteOrMixedOrientationSelectorV2Report verifies parse config rejects incomplete or mixed orientation selector v2 report behavior. +func TestParseConfigRejectsIncompleteOrMixedOrientationSelectorV2Report(t *testing.T) { + complete := []string{ + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", + "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", + "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", + "-orientation-v2-freeze", "orientation-v2-freeze.json", + "-orientation-v2-discovery-report", "orientation-v2-discovery.json", + } + for _, args := range [][]string{ + {"-orientation-v2-shadow-artifact", "shadow-v2.jsonl"}, + { + "-orientation-v2-shadow-artifact", "shadow-v2.jsonl", "-orientation-v2-incumbent-artifact", "incumbent.jsonl", + "-orientation-v2-reverse-artifact", "reverse.jsonl", "-orientation-v2-guarded-artifact", "guarded-v2.jsonl", + "-orientation-v2-aa", "aa.json", "-orientation-v2-output", "report.json", + }, + append(append([]string(nil), complete...), "-orientation-v2-protocol", "exploratory"), + append(append([]string(nil), complete...), "-orientation-shadow-artifact", "shadow-v1.jsonl", "-orientation-incumbent-artifact", "incumbent.jsonl", "-orientation-reverse-artifact", "reverse.jsonl", "-orientation-aa", "aa.json"), + append(append([]string(nil), complete...), "-expand-into-artifact", "expand.jsonl"), + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsSPI1StagedDiscoveryAndConfirmation verifies parse config accepts spi1 staged discovery and confirmation behavior. +func TestParseConfigAcceptsSPI1StagedDiscoveryAndConfirmation(t *testing.T) { + discovery, err := parseConfig([]string{ + "-sp-i1-baseline-artifact", "s4-training.jsonl", + "-sp-i1-candidate-artifact", "i1-training.jsonl", + "-sp-i1-resource-report", "i1-training-resource.json", + "-sp-i1-output", "sp-i1-discovery.json", + "-sp-i1-freeze-output", "sp-i1-freeze.json", + "-sp-i1-protocol", referencePairProtocolDiscovery, + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "s4-training.jsonl", discovery.SPI1BaselineArtifact) + require.Equal(t, "i1-training.jsonl", discovery.SPI1CandidateArtifact) + require.Equal(t, "i1-training-resource.json", discovery.SPI1ResourceReport) + require.Equal(t, "sp-i1-freeze.json", discovery.SPI1FreezeOutput) + + confirmation, err := parseConfig([]string{ + "-sp-i1-baseline-artifact", "s4-confirmation.jsonl", + "-sp-i1-candidate-artifact", "i1-confirmation.jsonl", + "-sp-i1-resource-report", "i1-confirmation-resource.json", + "-sp-i1-output", "sp-i1-confirmation.json", + "-sp-i1-freeze", "sp-i1-freeze.json", + "-sp-i1-discovery-report", "sp-i1-discovery.json", + "-sp-i1-training-baseline-artifact", "s4-training.jsonl", + "-sp-i1-training-candidate-artifact", "i1-training.jsonl", + "-sp-i1-training-resource-report", "i1-training-resource.json", + "-sp-i1-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "sp-i1-freeze.json", confirmation.SPI1Freeze) + require.Equal(t, "sp-i1-discovery.json", confirmation.SPI1DiscoveryReport) +} + +func TestParseConfigRejectsSPI2V1DiscoveryAndAcceptsVerification(t *testing.T) { + _, err := parseConfig([]string{ + "-sp-i2-baseline-artifact", "s4-distance-training.jsonl", + "-sp-i2-candidate-artifact", "i2-training.jsonl", + "-sp-i2-resource-report", "i2-training-resource.json", + "-sp-i2-output", "sp-i2-discovery.json", + "-sp-i2-freeze-output", "sp-i2-freeze.json", + "-sp-i2-protocol", referencePairProtocolDiscovery, + "-sp-i2-generation", spI2GenerationV1, + }, func(string) string { return "" }) + require.ErrorContains(t, err, "terminally rejected") + + confirmation, err := parseConfig([]string{ + "-sp-i2-baseline-artifact", "s4-distance-confirmation.jsonl", + "-sp-i2-candidate-artifact", "i2-confirmation.jsonl", + "-sp-i2-resource-report", "i2-confirmation-resource.json", + "-sp-i2-output", "sp-i2-confirmation.json", + "-sp-i2-freeze", "sp-i2-freeze.json", + "-sp-i2-discovery-report", "sp-i2-discovery.json", + "-sp-i2-training-baseline-artifact", "s4-distance-training.jsonl", + "-sp-i2-training-candidate-artifact", "i2-training.jsonl", + "-sp-i2-training-resource-report", "i2-training-resource.json", + "-sp-i2-protocol", referencePairProtocolConfirmation, + "-sp-i2-generation", spI2GenerationV1, + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "sp-i2-freeze.json", confirmation.SPI2Freeze) + require.Equal(t, "sp-i2-discovery.json", confirmation.SPI2DiscoveryReport) +} + +func TestV2GraphBenchExecutorIncludesEveryDevelopmentArm(t *testing.T) { + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorI2GuardedDistanceV2, + optimize.ShortestPathExecutorI2GuardedDistanceV2E0, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP, + } { + require.True(t, isV2GraphBenchExecutor(string(executor)), executor) + } + require.False(t, isV2GraphBenchExecutor(string(optimize.ShortestPathExecutorI2GuardedDistance))) +} + +func TestParseConfigAcceptsSPI2V2DevelopmentTournamentPosition(t *testing.T) { + executor := string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0) + cfg, err := parseConfig([]string{ + "-modes", string(ModePostgresSQL), + "-iterations", "100", + "-warmup-iterations", "25", + "-round", "1", + "-block", "1", + "-arm", executor, + "-arm-order", "1", + "-run-uuid", "development-series", + "-tags", spI2TrainingTag, + "-jsonl-output", "development.jsonl", + "-postgres-force-shortest-executor", executor, + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", postgresTraversalTelemetryDiagnostic, + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-development-tournament", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.SPI2V2DevelopmentTournament) +} + +func TestParseConfigAcceptsSPI2V2ReadinessPosition(t *testing.T) { + executor := string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0) + cfg, err := parseConfig([]string{ + "-modes", string(ModePostgresSQL), + "-iterations", "100", + "-warmup-iterations", "25", + "-round", "2", + "-block", "2", + "-arm", executor, + "-arm-order", "1", + "-run-uuid", "readiness-series", + "-tags", spI2TrainingTag, + "-jsonl-output", "readiness.jsonl", + "-append-jsonl", + "-postgres-force-shortest-executor", executor, + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", postgresTraversalTelemetryDiagnostic, + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-readiness-comparison", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.SPI2V2ReadinessComparison) +} + +func TestParseConfigRejectsV1CorpusForOrdinaryV2Capture(t *testing.T) { + _, err := parseConfig([]string{ + "-modes", string(ModePostgresSQL), + "-tags", spI2TrainingTag, + "-postgres-force-shortest-executor", string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0), + "-sp-i2-generation", spI2GenerationV2, + }, func(string) string { return "" }) + require.ErrorContains(t, err, "cannot select V1 evidence") +} + +func TestParseConfigAcceptsSPI2V2DevelopmentArtifactValidation(t *testing.T) { + cfg, err := parseConfig([]string{ + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-development-artifact", "development.jsonl", + "-sp-i2-v2-development-study", string(spI2V2StudyTournament), + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "development.jsonl", cfg.SPI2V2DevelopmentArtifact) + + for _, args := range [][]string{ + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-development-artifact", "development.jsonl"}, + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-development-study", string(spI2V2StudyReadiness)}, + {"-sp-i2-generation", spI2GenerationV1, "-sp-i2-v2-development-artifact", "development.jsonl", "-sp-i2-v2-development-study", string(spI2V2StudyReadiness)}, + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-development-artifact", "development.jsonl", "-sp-i2-v2-development-study", "other"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigAcceptsSPI2V2DevelopmentReportProduction(t *testing.T) { + cfg, err := parseConfig([]string{ + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-development-report-artifact", "tournament.jsonl", + "-sp-i2-v2-development-report-output", "report.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "tournament.jsonl", cfg.SPI2V2DevelopmentReportArtifact) + require.Equal(t, "report.json", cfg.SPI2V2DevelopmentReportOutput) + + invalid := [][]string{ + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-development-report-output", "report.json"}, + {"-sp-i2-generation", spI2GenerationV1, "-sp-i2-v2-development-report-artifact", "tournament.jsonl"}, + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-development-report-artifact", "tournament.jsonl", "-sp-i2-v2-development-artifact", "tournament.jsonl", "-sp-i2-v2-development-study", string(spI2V2StudyTournament)}, + } + for _, args := range invalid { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err) + } +} + +func TestParseConfigAcceptsIsolatedSPI2V2PowerSimulation(t *testing.T) { + cfg, err := parseConfig([]string{ + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-simulation-baseline-trace", "s4.jsonl", + "-sp-i2-v2-simulation-candidate-trace", "i2.jsonl", + "-sp-i2-v2-simulation-output", "simulation.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "simulation.json", cfg.SPI2V2SimulationOutput) + + for _, args := range [][]string{ + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-simulation-output", "simulation.json"}, + {"-sp-i2-generation", spI2GenerationV1, "-sp-i2-v2-simulation-baseline-trace", "s4.jsonl", "-sp-i2-v2-simulation-candidate-trace", "i2.jsonl", "-sp-i2-v2-simulation-output", "simulation.json"}, + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-simulation-baseline-trace", "s4.jsonl", "-sp-i2-v2-simulation-candidate-trace", "i2.jsonl", "-sp-i2-v2-simulation-output", "s4.jsonl"}, + {"-sp-i2-generation", spI2GenerationV2, "-sp-i2-v2-simulation-baseline-trace", "s4.jsonl", "-sp-i2-v2-simulation-candidate-trace", "i2.jsonl", "-sp-i2-v2-simulation-output", "simulation.json", "-jsonl-output", "capture.jsonl"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigRejectsTerminalSPI2V2FormalExecutor(t *testing.T) { + _, err := parseConfig([]string{ + "-sp-i2-generation", spI2GenerationV2, + "-postgres-force-shortest-executor", string(optimize.ShortestPathExecutorI2GuardedDistanceV2), + }, func(string) string { return "" }) + require.ErrorContains(t, err, "terminally rejected") +} + +func TestParseConfigAcceptsSPI2V2ComponentCheck(t *testing.T) { + executor := string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P) + cfg, err := parseConfig([]string{ + "-modes", string(ModePostgresSQL), + "-iterations", "1", + "-warmup-iterations", "1", + "-round", "1", + "-block", "1", + "-arm", executor, + "-arm-order", "1", + "-run-uuid", "component-check", + "-tags", spI2TrainingTag, + "-jsonl-output", "component.jsonl", + "-postgres-force-shortest-executor", executor, + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", postgresTraversalTelemetryDiagnostic, + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-component-check", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.SPI2V2ComponentCheck) +} + +func TestParseConfigAcceptsSPI2V2ComponentAuthorizationProduction(t *testing.T) { + cfg, err := parseConfig([]string{ + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-component-e1d-artifact", "e1d.jsonl", + "-sp-i2-v2-component-e1p-artifact", "e1p.jsonl", + "-sp-i2-v2-component-authorization-output", "authorization.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "authorization.json", cfg.SPI2V2ComponentAuthorizationOutput) + + _, err = parseConfig([]string{ + "-sp-i2-generation", spI2GenerationV2, + "-sp-i2-v2-component-e1d-artifact", "e1d.jsonl", + "-sp-i2-v2-component-authorization-output", "authorization.json", + }, func(string) string { return "" }) + require.Error(t, err) +} + +func TestParseConfigRejectsIncompleteOrMixedSPI2StagedWorkflow(t *testing.T) { + discovery := []string{ + "-sp-i2-baseline-artifact", "s4.jsonl", + "-sp-i2-candidate-artifact", "i2.jsonl", + "-sp-i2-resource-report", "resource.json", + "-sp-i2-output", "report.json", + "-sp-i2-freeze-output", "freeze.json", + "-sp-i2-protocol", referencePairProtocolDiscovery, + } + for _, args := range [][]string{ + {"-sp-i2-baseline-artifact", "s4.jsonl"}, + {"-sp-i2-freeze", "freeze.json"}, + append(append([]string(nil), discovery...), "-sp-i2-freeze", "old-freeze.json", "-sp-i2-discovery-report", "old-report.json"), + append(append([]string(nil), discovery...), "-sp-i2-protocol", "exploratory"), + append(append([]string(nil), discovery...), "-sp-i2-output", "s4.jsonl"), + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsSPI1HoldoutCaptureAuthorization verifies parse config accepts spi1 holdout capture authorization behavior. +func TestParseConfigAcceptsSPI1HoldoutCaptureAuthorization(t *testing.T) { + cfg, err := parseConfig([]string{ + "-sp-i1-freeze", "sp-i1-freeze.json", + "-sp-i1-discovery-report", "sp-i1-discovery.json", + "-sp-i1-training-baseline-artifact", "s4-training.jsonl", + "-sp-i1-training-candidate-artifact", "i1-training.jsonl", + "-sp-i1-training-resource-report", "i1-training-resource.json", + "-tags", "sp-i1-inbound-v1-training,sp-i1-inbound-v1-holdout", + "-iterations", "50", + "-warmup-iterations", "20", + "-round", "1", + "-block", "1", + "-arm", "sp-i1-s4", + "-arm-order", "1", + "-run-uuid", "sp-i1-confirmation-run", + "-postgres-force-shortest-executor", "SP-S4-C-WE+MAT-M0", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", postgresTraversalTelemetryDiagnostic, + "-jsonl-output", "sp-i1-s4-confirmation.jsonl", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "sp-i1-freeze.json", cfg.SPI1Freeze) + require.Empty(t, cfg.SPI1BaselineArtifact) +} + +// TestParseConfigRejectsIncompleteOrMixedSPI1StagedWorkflow verifies parse config rejects incomplete or mixed spi1 staged workflow behavior. +func TestParseConfigRejectsIncompleteOrMixedSPI1StagedWorkflow(t *testing.T) { + discovery := []string{ + "-sp-i1-baseline-artifact", "s4.jsonl", + "-sp-i1-candidate-artifact", "i1.jsonl", + "-sp-i1-resource-report", "resource.json", + "-sp-i1-output", "report.json", + "-sp-i1-freeze-output", "freeze.json", + "-sp-i1-protocol", referencePairProtocolDiscovery, + } + for _, args := range [][]string{ + {"-sp-i1-baseline-artifact", "s4.jsonl"}, + {"-sp-i1-freeze", "freeze.json"}, + append(append([]string(nil), discovery...), "-sp-i1-freeze", "old-freeze.json", "-sp-i1-discovery-report", "old-report.json"), + append(append([]string(nil), discovery...), "-sp-i1-protocol", "exploratory"), + append(append([]string(nil), discovery...), "-resource-artifact", "other.jsonl"), + append(append([]string(nil), discovery...), "-sp-i1-output", "s4.jsonl"), + append(append([]string(nil), discovery...), "-seed", "2"), + append(append([]string(nil), discovery...), "-confidence-level", "0.95"), + {"-sp-i1-freeze", "freeze.json", "-sp-i1-discovery-report", "discovery.json", "-resource-artifact", "candidate.jsonl"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsProductionManifestAndRejectsToolMixing verifies parse config accepts production manifest and rejects tool mixing behavior. +func TestParseConfigAcceptsProductionManifestAndRejectsToolMixing(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-production-manifest", "provisional.json"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "provisional.json", cfg.PostgresProductionManifest) + + _, err = parseConfig([]string{ + "-postgres-production-manifest", "provisional.json", + "-postgres-force-shortest-executor", "ASP-I1-U-DAG+MAT-M0", + }, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") + + cfg, err = parseConfig([]string{"-postgres-repeatable-read"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresRepeatableRead) + _, err = parseConfig([]string{"-postgres-production-manifest", "provisional.json", "-postgres-repeatable-read"}, func(string) string { return "" }) + require.ErrorContains(t, err, "already implies Repeatable Read") +} + +// TestParseConfigRejectsIncompleteOrientationSelectorReport verifies the +// report cannot silently omit an exact comparator, A/A floor, or standalone +// workflow boundary. +func TestParseConfigRejectsIncompleteOrientationSelectorReport(t *testing.T) { + complete := []string{ + "-orientation-shadow-artifact", "shadow.jsonl", + "-orientation-incumbent-artifact", "incumbent.jsonl", + "-orientation-reverse-artifact", "reverse.jsonl", + "-orientation-aa", "aa.json", + } + for _, args := range [][]string{ + {"-orientation-shadow-artifact", "shadow.jsonl"}, + append(append([]string(nil), complete...), "-orientation-protocol", "exploratory"), + append(append([]string(nil), complete...), "-expand-into-artifact", "expand.jsonl"), + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigRejectsInvalidExpandIntoStudyMode verifies report output, protocol, and standalone-mode exclusivity fail closed. +func TestParseConfigRejectsInvalidExpandIntoStudyMode(t *testing.T) { + for _, args := range [][]string{ + {"-expand-into-output", "expand-into.json"}, + {"-expand-into-artifact", "expand-into.jsonl", "-expand-into-protocol", "exploratory"}, + {"-expand-into-artifact", "expand-into.jsonl", "-bundle-verify", "capture"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsPoolAndConcurrencySmokeLevels verifies numeric pool parsing and stable deduplication of requested concurrency levels. +func TestParseConfigAcceptsPoolAndConcurrencySmokeLevels(t *testing.T) { + cfg, err := parseConfig([]string{"-pool-size", "4", "-concurrency", "1,4,8,4"}, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, 4, cfg.PoolSize) + require.Equal(t, []int{1, 4, 8}, cfg.Concurrency) +} + +// TestParseConfigAcceptsReferencePairDiscoveryProtocol verifies that the discovery protocol flag selects the corresponding reference-pair workflow. +func TestParseConfigAcceptsReferencePairDiscoveryProtocol(t *testing.T) { + cfg, err := parseConfig([]string{"-reference-pair-protocol", "discovery"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, referencePairProtocolDiscovery, cfg.ReferencePairProtocol) +} + +// TestParseConfigAcceptsReferenceTournament verifies a predeclared arm order +// is preserved because the first arm defines the incumbent. +func TestParseConfigAcceptsReferenceTournament(t *testing.T) { + arms := "expand_into_pair_join,expand_into_lower_degree_scan,expand_into_pair_cache" + cfg, err := parseConfig([]string{ + "-reference-tournament-artifact", "tournament.jsonl", + "-reference-tournament-output", "tournament.json", + "-reference-tournament-arms", arms, + "-reference-tournament-protocol", referencePairProtocolConfirmation, + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"}, cfg.ReferenceTournamentArms) + require.Equal(t, referencePairProtocolConfirmation, cfg.ReferenceTournamentProtocol) +} + +// TestParseConfigRejectsInvalidReferenceTournament verifies parse config rejects invalid reference tournament behavior. +func TestParseConfigRejectsInvalidReferenceTournament(t *testing.T) { + for _, args := range [][]string{ + {"-reference-tournament-output", "tournament.json"}, + {"-reference-tournament-artifact", "tournament.jsonl"}, + {"-reference-tournament-artifact", "tournament.jsonl", "-reference-tournament-arms", "expand_into_pair_join,expand_into_pair_cache"}, + {"-reference-tournament-artifact", "tournament.jsonl", "-reference-tournament-arms", "expand_into_pair_join,unknown,expand_into_pair_cache"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigRejectsPoolMemoryBelowPerSessionBudget verifies that the pool ceiling must cover the per-session budget for every configured connection. +func TestParseConfigRejectsPoolMemoryBelowPerSessionBudget(t *testing.T) { + _, err := parseConfig([]string{ + "-pool-size", "4", + "-session-memory-ceiling-bytes", "100", + "-pool-memory-ceiling-bytes", "399", + }, func(string) string { return "" }) + + require.ErrorContains(t, err, "session memory ceiling times pool size") +} + +// TestParseConfigAcceptsDiagnosticSelectorsAndRunMetadata verifies parsing of case filters, warmups, arm identity, and block metadata used to reproduce diagnostic runs. +func TestParseConfigAcceptsDiagnosticSelectorsAndRunMetadata(t *testing.T) { + cfg, err := parseConfig([]string{ + "-cases", "case-a,case-b", "-datasets", "fixture", "-categories", "lookup", "-tags", "primary,control", + "-warmup-iterations", "20", "-arm", "candidate", "-arm-order", "2", "-block", "7", "-run-uuid", "run-1", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []string{"case-a", "case-b"}, cfg.Cases) + require.Equal(t, 20, cfg.WarmupIterations) + require.Equal(t, "candidate", cfg.Arm) + require.Equal(t, 7, cfg.Block) +} + +// TestParseConfigRejectsDuplicateExactSelectors verifies that repeated exact case names are rejected before corpus selection. +func TestParseConfigRejectsDuplicateExactSelectors(t *testing.T) { + _, err := parseConfig([]string{"-cases", "case-a,case-a"}, func(string) string { return "" }) + require.ErrorContains(t, err, "duplicate case selector") +} + +// TestParseConfigAcceptsOnlyQualifiedForcedShortestExecutor verifies the supported shortest-executor allowlist and rejects an incomplete strategy name. +func TestParseConfigAcceptsOnlyQualifiedForcedShortestExecutor(t *testing.T) { + for _, executor := range []string{ + "SP-S0", + "SP-S0-DIRECT", + "SP-S3-U-D", + "SP-S3-U-E+MAT-M0", + "SP-S4-C-D", + "SP-S4-C-WE+MAT-M0", + "SP-I1-C-D", + "SP-I1-U-E+MAT-M0", + "SP-I1-C-WE+MAT-M0", + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "ASP-A1-DAG", + "ASP-I1-U-DAG+MAT-M0", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL", + } { + t.Run(executor, func(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-force-shortest-executor", executor}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, executor, cfg.PostgresForceShortest) + }) + } + + _, err := parseConfig([]string{"-postgres-force-shortest-executor", "SP-S1"}, func(string) string { return "" }) + require.ErrorContains(t, err, "unsupported PostgreSQL forced shortest executor") +} + +// TestParseConfigExistingGraphWorkflow verifies that a fully specified live-graph discovery run retains checkpoint, resume, progress, timeout, and sampling settings. +func TestParseConfigExistingGraphWorkflow(t *testing.T) { + cfg, err := parseConfig([]string{ + "-existing-graph", "-anchor-manifest", "anchors.json", "-checkpoint", "checkpoint.json", + "-resume", "-progress", "progress.jsonl", "-discovery", "-timeout-classes", "100ms,1s", + "-discovery-sample-floor", "2", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.ExistingGraph) + require.True(t, cfg.Resume) + require.True(t, cfg.Discovery) + require.Equal(t, []time.Duration{100 * time.Millisecond, time.Second}, cfg.TimeoutClasses) + require.Equal(t, 2, cfg.DiscoverySampleFloor) +} + +// TestParseConfigRejectsUnsafeExistingGraphCombinations verifies that live-graph mode requires an anchor manifest and disallows mismatched backends or orphaned resume/discovery flags. +func TestParseConfigRejectsUnsafeExistingGraphCombinations(t *testing.T) { + for _, args := range [][]string{ + {"-existing-graph"}, + {"-existing-graph", "-anchor-manifest", "anchors.json", "-modes", "postgres_sql,neo4j"}, + {"-existing-graph", "-anchor-manifest", "anchors.json", "-resume"}, + {"-existing-graph", "-anchor-manifest", "anchors.json", "-timeout-classes", "1s"}, + {"-anchor-manifest", "anchors.json"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigAcceptsOnlyQualifiedForcedExpansionSearch verifies the expansion-strategy allowlist and prevents simultaneous forced expansion and shortest-path strategies. +func TestParseConfigAcceptsOnlyQualifiedForcedExpansionSearch(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", cfg.PostgresForceExpansion) + cfg, err = parseConfig([]string{"-postgres-force-expansion-search", "EXPANSION-ENDPOINT-SEEDED-REVERSE"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "EXPANSION-ENDPOINT-SEEDED-REVERSE", cfg.PostgresForceExpansion) + + _, err = parseConfig([]string{"-postgres-force-expansion-search", "unknown-strategy"}, func(string) string { return "" }) + require.ErrorContains(t, err, "unsupported PostgreSQL forced expansion search") + + _, err = parseConfig([]string{ + "-postgres-force-shortest-executor", "SP-S3-U-D", + "-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE", + }, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") +} + +// TestParseConfigRequiresOutputForJSONLAppend verifies that append mode names a destination and is retained once that destination is present. +func TestParseConfigRequiresOutputForJSONLAppend(t *testing.T) { + _, err := parseConfig([]string{"-append-jsonl"}, func(string) string { return "" }) + require.ErrorContains(t, err, "append-jsonl requires jsonl-output") + + cfg, err := parseConfig([]string{"-append-jsonl", "-jsonl-output", "rounds.jsonl"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.AppendJSONL) +} + +// TestParseConfigAcceptsMultipleAAArtifacts verifies independently captured +// A/A arms can be passed to the reporter without an unvalidated external merge. +func TestParseConfigAcceptsMultipleAAArtifacts(t *testing.T) { + cfg, err := parseConfig([]string{ + "-aa-artifact", "aa-a.jsonl", + "-aa-artifact", "aa-b.jsonl", + "-aa-output", "aa.json", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, []string{"aa-a.jsonl", "aa-b.jsonl"}, cfg.AAArtifacts) +} + +// TestParseConfigAcceptsReferenceClosureMode verifies reference-closure artifact parsing, confidence propagation, required output pairing, and exclusion of incompatible A/A mode. +func TestParseConfigAcceptsReferenceClosureMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-reference-closure-artifact", "reference.jsonl", + "-reference-closure-output", "report.json", + "-reference-closure-arm", "s3_unidirectional_trail_cte", + "-confidence-level", "0.975", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "reference.jsonl", cfg.ReferenceClosureArtifact) + require.Equal(t, 0.975, cfg.Confidence) + + _, err = parseConfig([]string{"-reference-closure-output", "report.json"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires reference-closure-artifact") + _, err = parseConfig([]string{"-reference-closure-artifact", "reference.jsonl", "-aa-artifact", "aa.jsonl"}, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") +} + +// TestParseConfigAcceptsReferencePairMode verifies that pair-report configuration retains its artifact and explicit baseline/candidate arm names. +func TestParseConfigAcceptsReferencePairMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-reference-pair-artifact", "pair.jsonl", + "-reference-pair-baseline", "s3", + "-reference-pair-candidate", "s1", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "pair.jsonl", cfg.ReferencePairArtifact) + require.Equal(t, "s3", cfg.ReferencePairBaseline) + require.Equal(t, "s1", cfg.ReferencePairCandidate) +} diff --git a/cmd/graphbench/measure.go b/cmd/graphbench/measure.go index 7aaa7a93..ec28819b 100644 --- a/cmd/graphbench/measure.go +++ b/cmd/graphbench/measure.go @@ -18,12 +18,100 @@ package main import ( "context" + "crypto/sha256" + "encoding/json" + "errors" "fmt" + "math" + "slices" + "sort" "time" "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" ) +// stableObservationSHA256 binds an ordered set of normalized public rows to a +// compact evidence value. Callers must sort rows before fingerprinting them. +func stableObservationSHA256(rows []string) (string, error) { + encoded, err := json.Marshal(rows) + if err != nil { + return "", fmt.Errorf("encode stable observations: %w", err) + } + sum := sha256.Sum256(encoded) + return fmt.Sprintf("%x", sum), nil +} + +// errScaleWriteRollback signals the intentional rollback used to isolate a measured write. +var errScaleWriteRollback = errors.New("scale write rollback") + +// resolvedWriteScenario contains a write scenario after symbolic fixture parameters are resolved. +type resolvedWriteScenario struct { + // SelectionCypher contains the write-selection Cypher statement. + SelectionCypher string + // SelectionParams contains resolved parameters for the write-selection query. + SelectionParams map[string]any + // AffectedEntity identifies the entity class counted after a write. + AffectedEntity string + // ExpectedMatched sets the required number of matched entities. + ExpectedMatched int64 + // ExpectedAffected sets the required number of affected entities. + ExpectedAffected int64 + // PostState supplies the post state input to the resolvedWriteScenario contract. + PostState []resolvedStateQuery +} + +// resolvedStateQuery contains a post-write state query after fixture parameters are resolved. +type resolvedStateQuery struct { + // Name labels the post-write state assertion in diagnostics and results. + Name string + // Cypher contains the Cypher statement under test. + Cypher string + // Params supplies literal query parameters. + Params map[string]any + // Expected supplies the expected input to the resolvedStateQuery contract. + Expected ExpectedResult +} + +// writeMeasurement captures a write's matched and affected counts, duration, and post-state observations. +type writeMeasurement struct { + // Matched records entities matched by the write selection. + Matched int64 + // Affected records entities changed by the measured write. + Affected int64 + // Duration records elapsed time for this observation. + Duration time.Duration + // PostState contains the observed results of post-write validation queries. + PostState []StateQueryResult +} + +// timedReadAttestation is the runtime receipt captured outside a measured +// query's latency boundary for that exact invocation. +type timedReadAttestation struct { + // InvocationID identifies the invocation id. + InvocationID string + // RequestedIdentity identifies the requested identity. + RequestedIdentity string + // RuntimeIdentity identifies the runtime identity. + RuntimeIdentity string + // RuntimeBranch supplies the runtime branch input to the timedReadAttestation contract. + RuntimeBranch string + // FallbackExecuted supplies the fallback executed input to the timedReadAttestation contract. + FallbackExecuted *bool + // Events supplies the events input to the timedReadAttestation contract. + Events []RuntimeReceiptEvent +} + +// timedReadAttestor arms and reads invocation-local runtime evidence. Begin +// and Complete execute outside the duration measurement. +type timedReadAttestor interface { + // Begin supplies the begin input to the timedReadAttestor contract. + Begin(context.Context, int) error + // Complete supplies the complete input to the timedReadAttestor contract. + Complete(context.Context, int) (timedReadAttestation, error) +} + +// countCypherRows executes a Cypher query and returns the number of result rows. func countCypherRows(tx graph.Transaction, cypher string, params map[string]any) (int64, error) { result := tx.Query(cypher, params) defer result.Close() @@ -33,39 +121,811 @@ func countCypherRows(tx graph.Transaction, cypher string, params map[string]any) rowCount++ } - return rowCount, result.Error() + if err := result.Error(); err != nil { + return 0, err + } + + return rowCount, nil +} + +// countRawRows executes a raw backend query and returns the number of result rows. +func countRawRows(tx graph.Transaction, sql string, params map[string]any) (int64, error) { + result := tx.Raw(sql, params) + defer result.Close() + + var rowCount int64 + for result.Next() { + rowCount++ + } + + if err := result.Error(); err != nil { + return 0, err + } + + return rowCount, nil +} + +// stableNodeObservation serializes a node using fixture-stable identity, kinds, and properties. +type stableNodeObservation struct { + // Identity contains the stable fixture identity emitted in observations. + Identity string `json:"identity"` + // Kinds lists stable node kinds in deterministic observation order. + Kinds []string `json:"kinds,omitempty"` + // Properties contains normalized property values. + Properties map[string]any `json:"properties,omitempty"` +} + +// stableRelationshipObservation serializes a relationship using stable endpoints, kind, identity, and properties. +type stableRelationshipObservation struct { + // Identity contains the stable fixture identity emitted in observations. + Identity string `json:"identity,omitempty"` + // Start contains the stable identity of the relationship's start node. + Start string `json:"start"` + // End contains the stable identity of the relationship's end node. + End string `json:"end"` + // Kind names the relationship kind preserved in the stable observation. + Kind string `json:"kind"` + // Properties contains normalized property values. + Properties map[string]any `json:"properties,omitempty"` +} + +// stablePathObservation serializes an ordered path as stable node and relationship observations. +type stablePathObservation struct { + // Nodes contains the stable node sequence. + Nodes []stableNodeObservation `json:"nodes"` + // Relationships contains the ordered stable relationship sequence in the path. + Relationships []stableRelationshipObservation `json:"relationships"` +} + +// reverseIDMap inverts fixture node-key mappings for stable result serialization. +func reverseIDMap(idMap opengraph.IDMap) map[graph.ID]string { + reversed := make(map[graph.ID]string, len(idMap)) + for name, id := range idMap { + reversed[id] = name + } + return reversed +} + +// stableIdentity maps a database identifier to its fixture key, falling back to its decimal representation. +func stableIdentity(id graph.ID, reversed map[graph.ID]string) string { + if name, found := reversed[id]; found { + return name + } + return fmt.Sprintf("unmapped-node:%d", id) +} + +// stableProperties returns properties with database identifiers replaced by stable fixture keys. +func stableProperties(properties *graph.Properties) map[string]any { + if properties == nil { + return nil + } + return properties.Map +} + +// stableNode converts a backend node to a fixture-stable serialized observation. +func stableNode(node *graph.Node, reversed map[graph.ID]string) stableNodeObservation { + kinds := node.Kinds.Strings() + sort.Strings(kinds) + return stableNodeObservation{ + Identity: stableIdentity(node.ID, reversed), + Kinds: kinds, + Properties: stableProperties(node.Properties), + } +} + +// stableRelationship converts a backend relationship to stable endpoints, kind, identity, and properties. +func stableRelationship(relationship *graph.Relationship, reversed map[graph.ID]string) stableRelationshipObservation { + kind := "" + if relationship.Kind != nil { + kind = relationship.Kind.String() + } + identity := "" + if relationship.Properties != nil { + if logicalKey, err := relationship.Properties.Get("logical_key").String(); err == nil { + identity = logicalKey + } + } + return stableRelationshipObservation{ + Identity: identity, + Start: stableIdentity(relationship.StartID, reversed), + End: stableIdentity(relationship.EndID, reversed), + Kind: kind, + Properties: stableProperties(relationship.Properties), + } +} + +// stablePath converts a backend path to stable ordered node and relationship observations. +func stablePath(path graph.Path, reversed map[graph.ID]string) (stablePathObservation, error) { + if len(path.Nodes) == 0 { + return stablePathObservation{}, fmt.Errorf("path has no nodes") + } + nodesByID := make(map[graph.ID]*graph.Node, len(path.Nodes)) + for _, node := range path.Nodes { + if node == nil { + return stablePathObservation{}, fmt.Errorf("path has a nil node") + } + nodesByID[node.ID] = node + } + + // Neo4j exposes the distinct node collection for cyclic paths while + // PostgreSQL exposes one node per traversal position. Reconstruct the public + // walk from the ordered relationships so cycles and self-loops normalize to + // the same repeated-node sequence on both backends. + orderedNodes := make([]*graph.Node, 1, len(path.Edges)+1) + orderedNodes[0] = path.Nodes[0] + currentID := path.Nodes[0].ID + for idx, relationship := range path.Edges { + if relationship == nil { + return stablePathObservation{}, fmt.Errorf("path relationship %d is nil", idx) + } + nextID := relationship.EndID + switch { + case relationship.StartID == currentID: + case relationship.EndID == currentID: + nextID = relationship.StartID + default: + return stablePathObservation{}, fmt.Errorf("path relationship %d is not contiguous with node ID %d", idx, currentID) + } + next, found := nodesByID[nextID] + if !found { + return stablePathObservation{}, fmt.Errorf("path relationship %d references missing node ID %d", idx, nextID) + } + orderedNodes = append(orderedNodes, next) + currentID = nextID + } + + observation := stablePathObservation{ + Nodes: make([]stableNodeObservation, len(orderedNodes)), + Relationships: make([]stableRelationshipObservation, len(path.Edges)), + } + for idx, node := range orderedNodes { + observation.Nodes[idx] = stableNode(node, reversed) + } + seenRelationships := make(map[graph.ID]struct{}, len(path.Edges)) + for idx, relationship := range path.Edges { + if _, duplicate := seenRelationships[relationship.ID]; duplicate { + return stablePathObservation{}, fmt.Errorf("path reuses relationship ID %d", relationship.ID) + } + seenRelationships[relationship.ID] = struct{}{} + observation.Relationships[idx] = stableRelationship(relationship, reversed) + } + return observation, nil +} + +// stableRowValues normalizes result values to stable scalar IDs or canonical path JSON. +func stableRowValues(values []any, mapper graph.ValueMapper, reversed map[graph.ID]string, scalarNodeIDs bool, pathValues bool) ([]any, error) { + stable := make([]any, len(values)) + for idx, value := range values { + switch typed := value.(type) { + case *graph.Node: + stable[idx] = stableNode(typed, reversed) + case graph.Node: + stable[idx] = stableNode(&typed, reversed) + case *graph.Relationship: + stable[idx] = stableRelationship(typed, reversed) + case graph.Relationship: + stable[idx] = stableRelationship(&typed, reversed) + case graph.Path: + path, err := stablePath(typed, reversed) + if err != nil { + return nil, err + } + stable[idx] = path + case *graph.Path: + path, err := stablePath(*typed, reversed) + if err != nil { + return nil, err + } + stable[idx] = path + default: + var relationship graph.Relationship + if mapper.Map(value, &relationship) { + stable[idx] = stableRelationship(&relationship, reversed) + continue + } + + var node graph.Node + if mapper.Map(value, &node) { + stable[idx] = stableNode(&node, reversed) + continue + } + + // The PostgreSQL path mapper accepts a map without path fields as an + // empty path, so only attempt this mapping when the result contract + // says the row contains paths. + if pathValues { + var path graph.Path + if mapper.Map(value, &path) { + observation, err := stablePath(path, reversed) + if err != nil { + return nil, err + } + stable[idx] = observation + continue + } + } + + if scalarNodeIDs { + if id, ok := scaleInt64(value); ok { + stable[idx] = stableIdentity(graph.ID(id), reversed) + continue + } + } + stable[idx] = value + } + } + return stable, nil +} + +// expectedPathRows serializes expected paths to the same canonical representation as observed paths. +func expectedPathRows(rows []ExpectedPath) ([]string, error) { + encoded := make([]string, len(rows)) + for idx, row := range rows { + value, err := json.Marshal(row) + if err != nil { + return nil, err + } + encoded[idx] = string(value) + } + + sort.Strings(encoded) + return encoded, nil +} + +// observedPathRows extracts and sorts canonical path observations from normalized rows. +func observedPathRows(rows []string) ([]string, error) { + encoded := make([]string, len(rows)) + for idx, row := range rows { + var values []json.RawMessage + if err := json.Unmarshal([]byte(row), &values); err != nil { + return nil, err + } + if len(values) != 1 { + return nil, fmt.Errorf("expected one path column, got %d", len(values)) + } + var path stablePathObservation + if err := json.Unmarshal(values[0], &path); err != nil { + return nil, err + } + signature := ExpectedPath{ + Nodes: make([]string, len(path.Nodes)), + RelationshipKinds: make([]string, len(path.Relationships)), + } + includeRelationshipKeys := false + for _, relationship := range path.Relationships { + includeRelationshipKeys = includeRelationshipKeys || relationship.Identity != "" + } + if includeRelationshipKeys { + signature.RelationshipKeys = make([]string, len(path.Relationships)) + } + for nodeIdx, node := range path.Nodes { + signature.Nodes[nodeIdx] = node.Identity + } + for relationshipIdx, relationship := range path.Relationships { + signature.RelationshipKinds[relationshipIdx] = relationship.Kind + if includeRelationshipKeys { + signature.RelationshipKeys[relationshipIdx] = relationship.Identity + } + } + value, err := json.Marshal(signature) + if err != nil { + return nil, err + } + encoded[idx] = string(value) + } + sort.Strings(encoded) + return encoded, nil +} + +// observeCypherRows executes Cypher and returns row count plus normalized observations. +func observeCypherRows(tx graph.Transaction, cypher string, params map[string]any, idMap opengraph.IDMap, scalarNodeIDs bool, pathValues bool) (int64, []string, error) { + result := tx.Query(cypher, params) + return observeResultRows(result, idMap, scalarNodeIDs, pathValues) +} + +// observeRawRows executes raw SQL and returns row count plus normalized observations. +func observeRawRows(tx graph.Transaction, sql string, params map[string]any, idMap opengraph.IDMap, scalarNodeIDs bool, pathValues bool) (int64, []string, error) { + result := tx.Raw(sql, params) + return observeResultRows(result, idMap, scalarNodeIDs, pathValues) +} + +// observeResultRows drains a result iterator into a count and sorted stable observations. +func observeResultRows(result graph.Result, idMap opengraph.IDMap, scalarNodeIDs bool, pathValues bool) (int64, []string, error) { + defer result.Close() + + var ( + rowCount int64 + rows []string + ) + for result.Next() { + rowCount++ + stableValues, err := stableRowValues(result.Values(), result.Mapper(), reverseIDMap(idMap), scalarNodeIDs, pathValues) + if err != nil { + return 0, nil, fmt.Errorf("stabilize observed row %d: %w", rowCount, err) + } + encoded, err := json.Marshal(stableValues) + if err != nil { + return 0, nil, fmt.Errorf("encode observed row %d: %w", rowCount, err) + } + rows = append(rows, string(encoded)) + } + if err := result.Error(); err != nil { + return 0, nil, err + } + + // Cypher does not promise row order without ORDER BY. Comparing sorted row + // encodings preserves multiplicity while avoiding a false mismatch when an + // otherwise identical plan returns rows in another order. + sort.Strings(rows) + return rowCount, rows, nil +} + +// validateExpectedObservations compares normalized rows with explicit scalar, ID-row, or path expectations. +func validateExpectedObservations(expected ExpectedResult, observed []string) error { + if len(expected.IDRows) > 0 { + expectedRows := make([]string, len(expected.IDRows)) + for idx, row := range expected.IDRows { + encoded, err := json.Marshal(row) + if err != nil { + return err + } + expectedRows[idx] = string(encoded) + } + sort.Strings(expectedRows) + if !slices.Equal(expectedRows, observed) { + return fmt.Errorf("stable ID rows differ: expected=%v observed=%v", expectedRows, observed) + } + } + if len(expected.PathRows) > 0 { + expectedRows, err := expectedPathRows(expected.PathRows) + if err != nil { + return err + } + observedRows, err := observedPathRows(observed) + if err != nil { + return err + } + if !slices.Equal(expectedRows, observedRows) { + return fmt.Errorf("stable path rows differ: expected=%v observed=%v", expectedRows, observedRows) + } + } + if expected.ScalarInt != nil { + expectedRow := fmt.Sprintf("[%d]", *expected.ScalarInt) + if len(observed) != 1 || observed[0] != expectedRow { + return fmt.Errorf("scalar result differs: expected=%s observed=%v", expectedRow, observed) + } + } + return nil +} + +// observeCypher runs a Cypher query in a read transaction and returns stable observations. +func observeCypher(tx graph.Transaction, cypher string, params map[string]any) (StateQueryResult, error) { + result := tx.Query(cypher, params) + defer result.Close() + + var observation StateQueryResult + for result.Next() { + observation.RowCount++ + if observation.RowCount == 1 && len(result.Values()) > 0 { + if scalar, ok := scaleInt64(result.Values()[0]); ok { + observation.ScalarInt = &scalar + } + } + } + + if err := result.Error(); err != nil { + return StateQueryResult{}, err + } + + return observation, nil +} + +// resultContainsNodeIDs reports whether the expected result kind requires stable node-identifier mapping. +func resultContainsNodeIDs(expected ExpectedResult) bool { + return expected.ResultKind == "id_set" || expected.ResultKind == "id_rows" +} + +// resultContainsPaths reports whether expected observations require canonical path normalization. +func resultContainsPaths(expected ExpectedResult) bool { + return expected.ResultKind == "path_set" +} + +// measureCypher executes cypher and records its timing observations. +func measureCypher(ctx context.Context, db graph.Database, cypher string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, iterations int) (int64, []string, DurationStats, error) { + return measureCypherWithWarmups(ctx, db, cypher, params, expected, idMap, 0, iterations) +} + +// measureCypherWithWarmups executes cypher with warmups and records its timing observations. +func measureCypherWithWarmups(ctx context.Context, db graph.Database, cypher string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int) (int64, []string, DurationStats, error) { + return measureReadWithWarmups(ctx, db, cypher, params, expected, idMap, warmupIterations, iterations, false) +} + +// measureCypherWithWarmupsOptions derives execution options for measure cypher with warmups. +func measureCypherWithWarmupsOptions(ctx context.Context, db graph.Database, cypher string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, cypher, params, expected, idMap, warmupIterations, iterations, false, nil, options...) } -func measureCypher(ctx context.Context, db graph.Database, cypher string, params map[string]any, iterations int) (int64, DurationStats, error) { +// measureRawSQLWithWarmups executes raw SQL with warmups and records its timing observations. +func measureRawSQLWithWarmups(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int) (int64, []string, DurationStats, error) { + return measureReadWithWarmups(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true) +} + +// measureRawSQLWithWarmupsOptions derives execution options for measure raw sql with warmups. +func measureRawSQLWithWarmupsOptions(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true, nil, options...) +} + +// measureRawSQLWithWarmupsAndAttestation preserves the ordinary raw-SQL +// measurement boundary while binding each timed sample to an exact runtime +// receipt armed immediately before and read immediately after execution. +func measureRawSQLWithWarmupsAndAttestation(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, attestor timedReadAttestor) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true, attestor) +} + +// measureRawSQLWithWarmupsAndAttestationOptions measures a raw production +// statement under explicit graph transaction options while keeping receipt +// arming and reading outside the timed transaction. +func measureRawSQLWithWarmupsAndAttestationOptions(ctx context.Context, db graph.Database, sql string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, attestor timedReadAttestor, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, sql, params, expected, idMap, warmupIterations, iterations, true, attestor, options...) +} + +// measureReadWithWarmups executes read with warmups and records its timing observations. +func measureReadWithWarmups(ctx context.Context, db graph.Database, query string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, raw bool) (int64, []string, DurationStats, error) { + return measureReadWithWarmupsAndAttestation(ctx, db, query, params, expected, idMap, warmupIterations, iterations, raw, nil) +} + +// measureReadWithWarmupsAndAttestation supports benchmark evidence processing for measure read with warmups and attestation. +func measureReadWithWarmupsAndAttestation(ctx context.Context, db graph.Database, query string, params map[string]any, expected ExpectedResult, idMap opengraph.IDMap, warmupIterations, iterations int, raw bool, attestor timedReadAttestor, options ...graph.TransactionOption) (int64, []string, DurationStats, error) { if iterations < 1 { - return 0, DurationStats{}, fmt.Errorf("iterations must be at least 1") + return 0, nil, DurationStats{}, fmt.Errorf("iterations must be at least 1") + } + if warmupIterations < 0 { + return 0, nil, DurationStats{}, fmt.Errorf("warmup iterations must not be negative") } - var warmupRows int64 + coldStart := time.Now() + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + _, err := countReadRows(tx, query, params, raw) + return err + }, options...); err != nil { + return 0, nil, DurationStats{}, err + } + coldDuration := time.Since(coldStart) + for range warmupIterations { + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + _, err := countReadRows(tx, query, params, raw) + return err + }, options...); err != nil { + return 0, nil, DurationStats{}, err + } + } + + var ( + warmupRows int64 + preflightObserved []string + stabilizeNodeIDs = resultContainsNodeIDs(expected) + stabilizePaths = resultContainsPaths(expected) + stabilization timedReadAttestation + ) + if attestor != nil { + if err := attestor.Begin(ctx, 0); err != nil { + return 0, nil, DurationStats{}, fmt.Errorf("arm excluded runtime receipt stabilization: %w", err) + } + } if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { var err error - warmupRows, err = countCypherRows(tx, cypher, params) + warmupRows, preflightObserved, err = observeReadRows(tx, query, params, idMap, stabilizeNodeIDs, stabilizePaths, raw) return err - }); err != nil { - return 0, DurationStats{}, err + }, options...); err != nil { + if attestor != nil { + _, _ = attestor.Complete(context.WithoutCancel(ctx), 0) + } + return 0, nil, DurationStats{}, err + } + if attestor != nil { + var err error + if stabilization, err = attestor.Complete(ctx, 0); err != nil { + return 0, nil, DurationStats{}, fmt.Errorf("read excluded runtime receipt stabilization: %w", err) + } } durations := make([]time.Duration, iterations) + attestations := make([]timedReadAttestation, iterations) for idx := range iterations { + if attestor != nil { + if err := attestor.Begin(ctx, idx+1); err != nil { + return 0, nil, DurationStats{}, fmt.Errorf("arm timed runtime attestation %d: %w", idx+1, err) + } + } start := time.Now() if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { - _, err := countCypherRows(tx, cypher, params) + _, err := countReadRows(tx, query, params, raw) return err - }); err != nil { - return 0, DurationStats{}, err + }, options...); err != nil { + if attestor != nil { + _, _ = attestor.Complete(context.WithoutCancel(ctx), idx+1) + } + return 0, nil, DurationStats{}, err } durations[idx] = time.Since(start) + if attestor != nil { + attestation, err := attestor.Complete(ctx, idx+1) + if err != nil { + return 0, nil, DurationStats{}, fmt.Errorf("read timed runtime attestation %d: %w", idx+1, err) + } + attestations[idx] = attestation + } + } + + var ( + postflightRows int64 + postflightObserved []string + ) + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + postflightRows, postflightObserved, err = observeReadRows(tx, query, params, idMap, stabilizeNodeIDs, stabilizePaths, raw) + return err + }, options...); err != nil { + return 0, nil, DurationStats{}, err + } + if postflightRows != warmupRows { + return 0, nil, DurationStats{}, fmt.Errorf("postflight row count changed: preflight=%d postflight=%d", warmupRows, postflightRows) + } + if !slices.Equal(preflightObserved, postflightObserved) { + return 0, nil, DurationStats{}, fmt.Errorf("postflight result changed despite stable row count") + } + if err := validateExpectedObservations(expected, preflightObserved); err != nil { + return 0, nil, DurationStats{}, err } stats, err := computeDurationStats(durations) if err != nil { - return 0, DurationStats{}, err + return 0, nil, DurationStats{}, err + } + stats.WarmupIterations = warmupIterations + if attestor != nil { + stats.ReceiptStabilization = &RuntimeStabilizationReceipt{ + InvocationID: stabilization.InvocationID, + RequestedIdentity: stabilization.RequestedIdentity, + RuntimeIdentity: stabilization.RuntimeIdentity, + RuntimeBranch: stabilization.RuntimeBranch, + FallbackExecuted: stabilization.FallbackExecuted, + Events: append([]RuntimeReceiptEvent(nil), stabilization.Events...), + } + for idx := range attestations { + stats.Samples[idx].RuntimeInvocationID = attestations[idx].InvocationID + stats.Samples[idx].RequestedIdentity = attestations[idx].RequestedIdentity + stats.Samples[idx].RuntimeIdentity = attestations[idx].RuntimeIdentity + stats.Samples[idx].RuntimeBranch = attestations[idx].RuntimeBranch + stats.Samples[idx].FallbackExecuted = attestations[idx].FallbackExecuted + stats.Samples[idx].RuntimeAttestation = "timed_invocation" + stats.Samples[idx].RuntimeReceiptEvents = append([]RuntimeReceiptEvent(nil), attestations[idx].Events...) + } + } + + stats.Samples = append([]LatencySample{{ + Round: 1, + Iteration: 0, + Classification: "cold", + Duration: coldDuration, + }}, stats.Samples...) + + return warmupRows, preflightObserved, stats, nil +} + +// countReadRows dispatches to raw SQL or Cypher row counting according to raw. +func countReadRows(tx graph.Transaction, query string, params map[string]any, raw bool) (int64, error) { + if raw { + return countRawRows(tx, query, params) + } + return countCypherRows(tx, query, params) +} + +// observeReadRows dispatches a read observation to raw SQL or Cypher execution. +func observeReadRows(tx graph.Transaction, query string, params map[string]any, idMap opengraph.IDMap, scalarNodeIDs, pathValues, raw bool) (int64, []string, error) { + if raw { + return observeRawRows(tx, query, params, idMap, scalarNodeIDs, pathValues) + } + return observeCypherRows(tx, query, params, idMap, scalarNodeIDs, pathValues) +} + +// measureWriteCypher executes write cypher and records its timing observations. +func measureWriteCypher( + ctx context.Context, + db graph.Database, + cypher string, + params map[string]any, + scenario resolvedWriteScenario, + iterations int, +) (writeMeasurement, DurationStats, error) { + return measureWriteCypherWithWarmups(ctx, db, cypher, params, scenario, 0, iterations) +} + +// measureWriteCypherWithWarmups executes write cypher with warmups and records its timing observations. +func measureWriteCypherWithWarmups( + ctx context.Context, + db graph.Database, + cypher string, + params map[string]any, + scenario resolvedWriteScenario, + warmupIterations int, + iterations int, +) (writeMeasurement, DurationStats, error) { + if iterations < 1 { + return writeMeasurement{}, DurationStats{}, fmt.Errorf("iterations must be at least 1") + } + if warmupIterations < 0 { + return writeMeasurement{}, DurationStats{}, fmt.Errorf("warmup iterations must not be negative") + } + + // The first untimed execution remains the cold diagnostic. Additional + // configured warmups are also untimed and must preserve its semantics. + warmup, err := measureWriteIteration(ctx, db, cypher, params, scenario) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + for idx := 0; idx < warmupIterations; idx++ { + next, err := measureWriteIteration(ctx, db, cypher, params, scenario) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + if next.Matched != warmup.Matched || next.Affected != warmup.Affected { + return writeMeasurement{}, DurationStats{}, fmt.Errorf("warm-up iteration %d changed cardinality", idx+1) + } + } + + durations := make([]time.Duration, iterations) + for idx := range iterations { + measurement, err := measureWriteIteration(ctx, db, cypher, params, scenario) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + if measurement.Matched != warmup.Matched || measurement.Affected != warmup.Affected { + return writeMeasurement{}, DurationStats{}, fmt.Errorf( + "write iteration %d changed cardinality: matched=%d affected=%d, warm-up matched=%d affected=%d", + idx+1, + measurement.Matched, + measurement.Affected, + warmup.Matched, + warmup.Affected, + ) + } + durations[idx] = measurement.Duration + } + + stats, err := computeDurationStats(durations) + if err != nil { + return writeMeasurement{}, DurationStats{}, err + } + stats.WarmupIterations = warmupIterations + + stats.Samples = append([]LatencySample{{ + Round: 1, + Iteration: 0, + Classification: "cold", + Duration: warmup.Duration, + }}, stats.Samples...) + + return warmup, stats, nil +} + +// measureWriteIteration executes write iteration and records its timing observations. +func measureWriteIteration( + ctx context.Context, + db graph.Database, + cypher string, + params map[string]any, + scenario resolvedWriteScenario, +) (writeMeasurement, error) { + var measurement writeMeasurement + + err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + matched, err := countCypherRows(tx, scenario.SelectionCypher, scenario.SelectionParams) + if err != nil { + return fmt.Errorf("count matched rows: %w", err) + } + measurement.Matched = matched + if matched != scenario.ExpectedMatched { + return fmt.Errorf("expected %d matched rows, got %d", scenario.ExpectedMatched, matched) + } + + before, err := countAffectedEntities(tx, scenario.AffectedEntity) + if err != nil { + return err + } + + start := time.Now() + if _, err := countCypherRows(tx, cypher, params); err != nil { + return fmt.Errorf("execute mutation: %w", err) + } + measurement.Duration = time.Since(start) + + after, err := countAffectedEntities(tx, scenario.AffectedEntity) + if err != nil { + return err + } + measurement.Affected = before - after + if measurement.Affected != scenario.ExpectedAffected { + return fmt.Errorf("expected %d affected %ss, got %d", scenario.ExpectedAffected, scenario.AffectedEntity, measurement.Affected) + } + + for _, stateQuery := range scenario.PostState { + observation, err := observeCypher(tx, stateQuery.Cypher, stateQuery.Params) + if err != nil { + return fmt.Errorf("post-state %q: %w", stateQuery.Name, err) + } + observation.Name = stateQuery.Name + if err := checkStateExpectation(observation, stateQuery.Expected); err != nil { + return fmt.Errorf("post-state %q: %w", stateQuery.Name, err) + } + measurement.PostState = append(measurement.PostState, observation) + } + + return errScaleWriteRollback + }) + if errors.Is(err, errScaleWriteRollback) { + return measurement, nil + } + if err != nil { + return writeMeasurement{}, err + } + + return writeMeasurement{}, fmt.Errorf("write scenario committed instead of rolling back") +} + +// countAffectedEntities returns the transaction-visible node or relationship count selected by entity. +func countAffectedEntities(tx graph.Transaction, entity string) (int64, error) { + switch entity { + case "node": + return tx.Nodes().Count() + case "relationship": + return tx.Relationships().Count() + default: + return 0, fmt.Errorf("unsupported affected entity %q", entity) + } +} + +// checkStateExpectation validates a post-write observation against its declared row-count and scalar expectations. +func checkStateExpectation(observation StateQueryResult, expected ExpectedResult) error { + if expected.RowCount != nil && observation.RowCount != *expected.RowCount { + return fmt.Errorf("expected %d rows, got %d", *expected.RowCount, observation.RowCount) + } + if expected.ScalarInt != nil { + if observation.ScalarInt == nil { + return fmt.Errorf("expected scalar integer %d, got no integer scalar", *expected.ScalarInt) + } + if *observation.ScalarInt != *expected.ScalarInt { + return fmt.Errorf("expected scalar integer %d, got %d", *expected.ScalarInt, *observation.ScalarInt) + } + } + + return nil +} + +// scaleInt64 converts supported integral numeric representations to int64 without unsigned overflow. +func scaleInt64(value any) (int64, bool) { + switch typedValue := value.(type) { + case int: + return int64(typedValue), true + case int32: + return int64(typedValue), true + case int64: + return typedValue, true + case uint: + if uint64(typedValue) <= math.MaxInt64 { + return int64(typedValue), true + } + case uint32: + return int64(typedValue), true + case uint64: + if typedValue <= math.MaxInt64 { + return int64(typedValue), true + } + case float64: + if math.Trunc(typedValue) == typedValue { + return int64(typedValue), true + } } - return warmupRows, stats, nil + return 0, false } diff --git a/cmd/graphbench/measure_test.go b/cmd/graphbench/measure_test.go new file mode 100644 index 00000000..3c1edd09 --- /dev/null +++ b/cmd/graphbench/measure_test.go @@ -0,0 +1,449 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "errors" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/stretchr/testify/require" +) + +// TestStableRowValuesReverseMapsNodeIDs verifies that scalar and node IDs become fixture keys while node-kind metadata is preserved. +func TestStableRowValuesReverseMapsNodeIDs(t *testing.T) { + values, err := stableRowValues( + []any{int64(101), graph.NewNode(102, nil, graph.StringKind("Group"))}, + graph.NewValueMapper(), + reverseIDMap(opengraph.IDMap{"start": 101, "end": 102}), + true, + false, + ) + require.NoError(t, err) + require.Equal(t, "start", values[0]) + require.Equal(t, stableNodeObservation{ + Identity: "end", + Kinds: []string{"Group"}, + }, values[1]) +} + +// TestResultContainsNodeIDs verifies that only ID-set and ID-row expectations request physical-to-logical ID normalization. +func TestResultContainsNodeIDs(t *testing.T) { + require.True(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "id_set"})) + require.True(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "id_rows"})) + require.False(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "scalar"})) + require.False(t, resultContainsNodeIDs(ExpectedResult{ResultKind: "path_set"})) +} + +// TestStableRowValuesMapsNativePathValues verifies that driver-native paths normalize into logical node identities and directed relationship observations. +func TestStableRowValuesMapsNativePathValues(t *testing.T) { + start := graph.NewNode(1, nil, graph.StringKind("Start")) + end := graph.NewNode(2, nil, graph.StringKind("End")) + edge := graph.NewRelationship(3, 1, 2, nil, graph.StringKind("Edge")) + mapper := graph.NewValueMapper(func(value, target any) bool { + path, sourceOK := value.(string) + mapped, targetOK := target.(*graph.Path) + if sourceOK && targetOK && path == "native-path" { + *mapped = graph.Path{ + Nodes: []*graph.Node{start, end}, + Edges: []*graph.Relationship{edge}, + } + return true + } + return false + }) + + values, err := stableRowValues( + []any{"native-path"}, + mapper, + reverseIDMap(opengraph.IDMap{"start": 1, "end": 2}), + false, + true, + ) + + require.NoError(t, err) + require.Equal(t, stablePathObservation{ + Nodes: []stableNodeObservation{ + { + Identity: "start", + Kinds: []string{"Start"}, + }, + { + Identity: "end", + Kinds: []string{"End"}, + }, + }, + Relationships: []stableRelationshipObservation{{ + Start: "start", + End: "end", + Kind: "Edge", + }}, + }, values[0]) +} + +// TestStablePathReconstructsRepeatedCycleAndSelfLoopNodes verifies a backend +// path that supplies distinct nodes still produces the complete ordered Cypher +// walk, including repeated occurrences at cycles and self-loops. +func TestStablePathReconstructsRepeatedCycleAndSelfLoopNodes(t *testing.T) { + root := graph.NewNode(1, nil) + cycle := graph.NewNode(2, nil) + terminal := graph.NewNode(3, nil) + path := graph.Path{ + Nodes: []*graph.Node{root, cycle, terminal}, + Edges: []*graph.Relationship{ + graph.NewRelationship(10, 1, 2, nil, graph.StringKind("Expand")), + graph.NewRelationship(11, 2, 1, nil, graph.StringKind("Expand")), + graph.NewRelationship(12, 1, 1, nil, graph.StringKind("Expand")), + graph.NewRelationship(13, 1, 3, nil, graph.StringKind("Complete")), + }, + } + + observed, err := stablePath(path, reverseIDMap(opengraph.IDMap{ + "root": 1, "cycle": 2, "terminal": 3, + })) + + require.NoError(t, err) + require.Equal(t, []string{"root", "cycle", "root", "root", "terminal"}, []string{ + observed.Nodes[0].Identity, + observed.Nodes[1].Identity, + observed.Nodes[2].Identity, + observed.Nodes[3].Identity, + observed.Nodes[4].Identity, + }) +} + +// TestStablePathReconstructsInboundTraversal verifies relationship storage +// direction does not reverse the public path walk. +func TestStablePathReconstructsInboundTraversal(t *testing.T) { + root := graph.NewNode(1, nil) + terminal := graph.NewNode(2, nil) + observed, err := stablePath(graph.Path{ + Nodes: []*graph.Node{root, terminal}, + Edges: []*graph.Relationship{ + graph.NewRelationship(10, 2, 1, nil, graph.StringKind("Expand")), + }, + }, reverseIDMap(opengraph.IDMap{"root": 1, "terminal": 2})) + + require.NoError(t, err) + require.Equal(t, "root", observed.Nodes[0].Identity) + require.Equal(t, "terminal", observed.Nodes[1].Identity) +} + +// TestStablePathRejectsNoncontiguousRelationships verifies malformed backend +// path values cannot manufacture a stable observation. +func TestStablePathRejectsNoncontiguousRelationships(t *testing.T) { + _, err := stablePath(graph.Path{ + Nodes: []*graph.Node{graph.NewNode(1, nil), graph.NewNode(2, nil), graph.NewNode(3, nil)}, + Edges: []*graph.Relationship{ + graph.NewRelationship(10, 2, 3, nil, graph.StringKind("Expand")), + }, + }, nil) + + require.ErrorContains(t, err, "is not contiguous") +} + +// TestStableRowValuesRejectsRelationshipReuseWithinPath verifies that observation normalization rejects a trail containing the same physical relationship twice. +func TestStableRowValuesRejectsRelationshipReuseWithinPath(t *testing.T) { + start := graph.NewNode(1, nil) + end := graph.NewNode(2, nil) + relationship := graph.NewRelationship(10, 1, 2, nil, graph.StringKind("Edge")) + _, err := stableRowValues([]any{graph.Path{ + Nodes: []*graph.Node{start, end, start}, + Edges: []*graph.Relationship{relationship, relationship}, + }}, graph.NewValueMapper(), reverseIDMap(opengraph.IDMap{"start": 1, "end": 2}), false, true) + require.ErrorContains(t, err, "reuses relationship ID 10") +} + +// TestStableRelationshipUsesLogicalFixtureKeyAsCrossBackendIdentity verifies that a relationship's logical_key property, rather than its backend ID, identifies it across engines. +func TestStableRelationshipUsesLogicalFixtureKeyAsCrossBackendIdentity(t *testing.T) { + properties := graph.NewProperties().Set("logical_key", "branch-0001-level-02") + relationship := graph.NewRelationship(99, 1, 2, properties, graph.StringKind("MemberOf")) + + stable := stableRelationship(relationship, map[graph.ID]string{1: "start", 2: "end"}) + require.Equal(t, "branch-0001-level-02", stable.Identity) + require.Equal(t, "start", stable.Start) + require.Equal(t, "end", stable.End) +} + +// TestObserveCypherReturnsZeroValueOnResultError verifies that an iterator failure cannot leak a partially populated state observation. +func TestObserveCypherReturnsZeroValueOnResultError(t *testing.T) { + tx := &scaleWriteTestTransaction{ + database: &scaleWriteTestDatabase{}, + } + + observation, err := observeCypher(tx, "unexpected", nil) + + require.ErrorContains(t, err, "unexpected query") + require.Equal(t, StateQueryResult{}, observation) +} + +// TestMeasureWriteCypherRollsBackWarmupAndEveryIteration verifies matched/affected/post-state measurements, cold-versus-warm classification, and rollback after every sampled mutation. +func TestMeasureWriteCypherRollsBackWarmupAndEveryIteration(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 1, + } + postStateCount := int64(2) + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + PostState: []resolvedStateQuery{{ + Name: "surviving relationships", + Cypher: "relationship count", + Expected: ExpectedResult{ScalarInt: &postStateCount}, + }}, + } + + measurement, stats, err := measureWriteCypher(context.Background(), database, "delete", nil, scenario, 2) + + require.NoError(t, err) + require.Equal(t, int64(1), measurement.Matched) + require.Equal(t, int64(1), measurement.Affected) + require.Equal(t, int64(2), *measurement.PostState[0].ScalarInt) + require.Equal(t, 2, stats.Iterations) + require.Len(t, stats.Samples, 3) + require.Equal(t, "cold", stats.Samples[0].Classification) + require.Equal(t, "warm", stats.Samples[1].Classification) + require.Equal(t, 3, database.writeTransactions) + require.Equal(t, int64(3), database.relationships, "every write transaction must roll back") +} + +// TestMeasureWriteCypherRecordsConfiguredUntimedWarmups verifies that configured warmups execute transactions and update metadata without entering the timing sample set. +func TestMeasureWriteCypherRecordsConfiguredUntimedWarmups(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 1, + } + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + } + + _, stats, err := measureWriteCypherWithWarmups(context.Background(), database, "delete", nil, scenario, 2, 1) + require.NoError(t, err) + require.Equal(t, 2, stats.WarmupIterations) + require.Len(t, stats.Samples, 2, "configured warmups must not become samples") + require.Equal(t, 4, database.writeTransactions, "cold + two warmups + one timed transaction") +} + +// TestMeasureWriteCypherRejectsOverBroadMutation verifies that deleting more relationships than declared fails validation and leaves the fixture unchanged. +func TestMeasureWriteCypherRejectsOverBroadMutation(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 2, + } + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + PostState: []resolvedStateQuery{{ + Name: "survivors", + Cypher: "relationship count", + Expected: ExpectedResult{RowCount: int64Pointer(1)}, + }}, + } + + _, _, err := measureWriteCypher(context.Background(), database, "delete", nil, scenario, 1) + require.ErrorContains(t, err, "expected 1 affected relationships, got 2") + require.Equal(t, int64(3), database.relationships) +} + +// TestMeasureWriteCypherRejectsUnderBroadMutation verifies that deleting fewer relationships than declared fails validation and leaves the fixture unchanged. +func TestMeasureWriteCypherRejectsUnderBroadMutation(t *testing.T) { + database := &scaleWriteTestDatabase{ + nodes: 2, + relationships: 3, + deleteCount: 0, + } + scenario := resolvedWriteScenario{ + SelectionCypher: "selection", + AffectedEntity: "relationship", + ExpectedMatched: 1, + ExpectedAffected: 1, + PostState: []resolvedStateQuery{{ + Name: "survivors", + Cypher: "relationship count", + Expected: ExpectedResult{RowCount: int64Pointer(1)}, + }}, + } + + _, _, err := measureWriteCypher(context.Background(), database, "delete", nil, scenario, 1) + require.ErrorContains(t, err, "expected 1 affected relationships, got 0") + require.Equal(t, int64(3), database.relationships) +} + +// int64Pointer returns a pointer to the supplied integer for optional expectations. +func int64Pointer(value int64) *int64 { + return &value +} + +// scaleWriteTestDatabase models mutable entity counts and rollback boundaries for write measurements. +type scaleWriteTestDatabase struct { + // Database supplies methods outside the transaction interaction under test. + graph.Database + + // nodes is the mutable node cardinality visible to count queries. + nodes int64 + + // relationships is the mutable relationship cardinality restored on rollback. + relationships int64 + + // deleteCount controls how many relationships the synthetic mutation removes. + deleteCount int64 + + // writeTransactions counts cold, warmup, and measured transaction attempts. + writeTransactions int +} + +// WriteTransaction runs the delegate and restores entity counts when its sentinel error requests rollback. +func (s *scaleWriteTestDatabase) WriteTransaction(_ context.Context, delegate graph.TransactionDelegate, _ ...graph.TransactionOption) error { + s.writeTransactions++ + originalNodes := s.nodes + originalRelationships := s.relationships + err := delegate(&scaleWriteTestTransaction{database: s}) + if err != nil { + s.nodes = originalNodes + s.relationships = originalRelationships + } + + return err +} + +// scaleWriteTestTransaction interprets the synthetic selection, deletion, and post-state query names used by write measurements. +type scaleWriteTestTransaction struct { + // Transaction supplies operations outside the query and count surfaces under test. + graph.Transaction + + // database owns the mutable cardinalities affected by synthetic queries. + database *scaleWriteTestDatabase +} + +// Query maps synthetic query names to selection rows, cardinality mutation, post-state counts, or a terminal error. +func (s *scaleWriteTestTransaction) Query(cypher string, _ map[string]any) graph.Result { + switch cypher { + case "selection": + return &scaleWriteTestResult{rows: [][]any{{int64(1)}}} + case "delete": + s.database.relationships -= s.database.deleteCount + return &scaleWriteTestResult{} + case "relationship count": + return &scaleWriteTestResult{rows: [][]any{{s.database.relationships}}} + default: + return &scaleWriteTestResult{err: errors.New("unexpected query")} + } +} + +// Nodes returns the current node-cardinality snapshot used to compute affected entities. +func (s *scaleWriteTestTransaction) Nodes() graph.NodeQuery { + return &scaleWriteTestNodeQuery{count: s.database.nodes} +} + +// Relationships returns the current relationship-cardinality snapshot used to compute affected entities. +func (s *scaleWriteTestTransaction) Relationships() graph.RelationshipQuery { + return &scaleWriteTestRelationshipQuery{count: s.database.relationships} +} + +// scaleWriteTestNodeQuery exposes a fixed node cardinality through the graph query interface. +type scaleWriteTestNodeQuery struct { + // NodeQuery supplies query methods other than Count. + graph.NodeQuery + + // count is the node cardinality returned to mutation accounting. + count int64 +} + +// Count returns the node snapshot without a query failure. +func (s *scaleWriteTestNodeQuery) Count() (int64, error) { + return s.count, nil +} + +// scaleWriteTestRelationshipQuery exposes a fixed relationship cardinality through the graph query interface. +type scaleWriteTestRelationshipQuery struct { + // RelationshipQuery supplies query methods other than Count. + graph.RelationshipQuery + + // count is the relationship cardinality returned to mutation accounting. + count int64 +} + +// Count returns the relationship snapshot without a query failure. +func (s *scaleWriteTestRelationshipQuery) Count() (int64, error) { + return s.count, nil +} + +// scaleWriteTestResult iterates configured rows and errors for write-measurement tests. +type scaleWriteTestResult struct { + // rows contains the synthetic values exposed by iteration. + rows [][]any + + // idx is the one-based cursor position after a successful Next call. + idx int + + // err is returned after iteration completes. + err error +} + +// Next advances the one-based cursor while synthetic rows remain. +func (s *scaleWriteTestResult) Next() bool { + if s.idx >= len(s.rows) { + return false + } + s.idx++ + return true +} + +// Keys returns no column names because write-measurement observations consume values positionally. +func (s *scaleWriteTestResult) Keys() []string { + return nil +} + +// Values returns the current synthetic row or nil before and after valid iteration. +func (s *scaleWriteTestResult) Values() []any { + if s.idx == 0 || s.idx > len(s.rows) { + return nil + } + + return s.rows[s.idx-1] +} + +// Mapper returns the zero mapper because the synthetic rows contain primitive counts only. +func (s *scaleWriteTestResult) Mapper() graph.ValueMapper { + return graph.ValueMapper{} +} + +// Scan satisfies graph.Result; these tests consume rows through Values. +func (s *scaleWriteTestResult) Scan(...any) error { + return nil +} + +// Error returns the configured terminal iterator error. +func (s *scaleWriteTestResult) Error() error { + return s.err +} + +// Close satisfies graph.Result; this fake owns no resource. +func (s *scaleWriteTestResult) Close() {} diff --git a/cmd/graphbench/neo4j.go b/cmd/graphbench/neo4j.go index 429fa8f1..0767a9e9 100644 --- a/cmd/graphbench/neo4j.go +++ b/cmd/graphbench/neo4j.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "net/url" + "strconv" "strings" neo4jcore "github.com/neo4j/neo4j-go-driver/v5/neo4j" @@ -30,13 +31,19 @@ import ( "github.com/specterops/dawgs/util/size" ) +// neo4jRunner owns the Neo4j driver and database used to execute benchmark cases. type neo4jRunner struct { - datasetDir string - db graph.Database - planDriver neo4jcore.DriverWithContext + // datasetDir locates fixture and corpus files on disk. + datasetDir string + // db provides graph transactions for fixture preparation and query execution. + db graph.Database + // planDriver supplies the Neo4j driver used only for untimed PROFILE or EXPLAIN capture. + planDriver neo4jcore.DriverWithContext + // databaseName selects the Neo4j database targeted by the benchmark session. databaseName string } +// newNeo4jRunner opens a Neo4j driver and selects the optional database encoded in the URI. func newNeo4jRunner(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus) (*neo4jRunner, error) { db, err := dawgs.Open(ctx, dawgsneo4j.DriverName, dawgs.Config{ GraphQueryMemoryLimit: size.Gibibyte, @@ -71,6 +78,7 @@ func newNeo4jRunner(ctx context.Context, datasetDir, connection string, corpus S }, nil } +// Close releases both Neo4j drivers owned by the benchmark runner. func (s *neo4jRunner) Close(ctx context.Context) error { var closeErr error if s.planDriver != nil { @@ -85,13 +93,18 @@ func (s *neo4jRunner) Close(ctx context.Context) error { return closeErr } -func (s *neo4jRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { +// Run reloads each fixture dataset and measures every corpus case supported by Neo4j. +func (s *neo4jRunner) Run(ctx context.Context, warmupIterations, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { var ( records []CaseResult casesByDataset = scaleCasesByDataset(corpus) ) for _, datasetName := range scaleCorpusDatasets(corpus) { + fixture, err := fixtureMetadata(s.datasetDir, datasetName) + if err != nil { + return nil, err + } if err := clearGraph(ctx, s.db); err != nil { return nil, fmt.Errorf("clear graph for %s: %w", datasetName, err) } @@ -106,7 +119,8 @@ func (s *neo4jRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpu continue } - record := s.runCase(ctx, iterations, testCase, idMap) + record := s.runCase(ctx, warmupIterations, iterations, testCase, idMap) + attachFixtureMetadata(&record, fixture) records = append(records, record) } } @@ -114,7 +128,8 @@ func (s *neo4jRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpu return records, nil } -func (s *neo4jRunner) runCase(ctx context.Context, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { +// runCase resolves fixture parameters, measures the selected Neo4j read or write workload, and records correctness and timing status in one CaseResult. +func (s *neo4jRunner) runCase(ctx context.Context, warmupIterations, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { params, err := resolveCaseParams(testCase, idMap) record := newCaseResult(testCase, ModeNeo4j, params) if err != nil { @@ -123,18 +138,42 @@ func (s *neo4jRunner) runCase(ctx context.Context, iterations int, testCase Scal return record } - rowCount, stats, err := measureCypher(ctx, s.db, testCase.Cypher, params, iterations) - if err != nil { - record.Status = StatusError - record.Error = err.Error() - return record - } + if testCase.WriteScenario == nil { + rowCount, observedRows, stats, err := measureCypherWithWarmups(ctx, s.db, testCase.Cypher, params, testCase.Expected, idMap, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + record.RowCount = rowCount + record.ObservedRows = observedRows + record.Stats = stats + labelLatencySamples(&record.Stats, ModeNeo4j, testCase) + applyRowExpectation(&record) + } else { + scenario, err := resolveWriteScenario(testCase, idMap) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + measurement, stats, err := measureWriteCypherWithWarmups(ctx, s.db, testCase.Cypher, params, scenario, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } - record.RowCount = rowCount - record.Stats = stats - applyRowExpectation(&record) + record.MatchedCount = &measurement.Matched + record.AffectedCount = &measurement.Affected + record.PostState = measurement.PostState + record.Stats = stats + labelLatencySamples(&record.Stats, ModeNeo4j, testCase) + } - plan, operators, err := s.explain(ctx, testCase.Cypher, params) + plan, operators, err := s.explain(ctx, testCase.Cypher, params, testCase.WriteScenario != nil) if err != nil { if record.Status == StatusOK { record.Status = StatusError @@ -148,9 +187,14 @@ func (s *neo4jRunner) runCase(ctx context.Context, iterations int, testCase Scal return record } -func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params map[string]any) (plan *Neo4jPlanNode, operators []string, err error) { +// explain submits native Neo4j PROFILE for reads and EXPLAIN for writes after the timed block. +func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params map[string]any, write bool) (plan *Neo4jPlanNode, operators []string, err error) { + accessMode := neo4jcore.AccessModeRead + if write { + accessMode = neo4jcore.AccessModeWrite + } session := s.planDriver.NewSession(ctx, neo4jcore.SessionConfig{ - AccessMode: neo4jcore.AccessModeRead, + AccessMode: accessMode, DatabaseName: s.databaseName, }) defer func() { @@ -159,7 +203,7 @@ func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params ma } }() - result, err := session.Run(ctx, "EXPLAIN "+cypherWithoutTerminator(cypherQuery), params) + result, err := session.Run(ctx, neo4jPlanCaptureStatement(cypherQuery, write), params) if err != nil { return nil, nil, err } @@ -168,21 +212,63 @@ func (s *neo4jRunner) explain(ctx context.Context, cypherQuery string, params ma if err != nil { return nil, nil, err } - if summary.Plan() == nil { + if write { + explainPlan := summary.Plan() + if explainPlan == nil { + return nil, nil, nil + } + + metadata := neo4jProfileMetadata(explainPlan.Arguments(), neo4jServerAgent(summary), false) + planNode := convertNeo4jPlan(explainPlan) + planNode.ProfileMetadata = &metadata + + return &planNode, neo4jOperators(planNode), nil + } + + profile := summary.Profile() + if profile == nil { return nil, nil, nil } - planNode := convertNeo4jPlan(summary.Plan()) + metadata := neo4jProfileMetadata(profile.Arguments(), neo4jServerAgent(summary), true) + planNode := convertNeo4jProfiledPlan(profile, metadata.internalTraversalOpaque()) + planNode.ProfileMetadata = &metadata + return &planNode, neo4jOperators(planNode), nil } +// neo4jPlanCaptureStatement selects PROFILE only for read-only cases and retains non-executing EXPLAIN for writes. +func neo4jPlanCaptureStatement(cypherQuery string, write bool) string { + command := "PROFILE" + if write { + command = "EXPLAIN" + } + + return command + " " + cypherWithoutTerminator(cypherQuery) +} + +// neo4jServerAgent supports benchmark evidence processing for neo4j server agent. +func neo4jServerAgent(summary neo4jcore.ResultSummary) string { + if server := summary.Server(); server != nil { + return server.Agent() + } + + return "" +} + +// neo4jPlanDriverConfig contains a Neo4j server URI and optional target database parsed from a connection string. type neo4jPlanDriverConfig struct { - Target string - Username string - Password string + // Target contains the Neo4j server URI without a database path. + Target string + // Username contains the Neo4j username decoded from the connection URI. + Username string + // Password contains the Neo4j password decoded from the connection URI. + Password string + // DatabaseName selects the Neo4j database targeted by the session. DatabaseName string } +// parseNeo4jPlanDriverConfig parses a Neo4j connection string while preserving its server URI and database path. func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { connectionURL, err := url.Parse(connStr) if err != nil { @@ -218,6 +304,7 @@ func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { }, nil } +// neo4jDatabaseName returns the optional single-segment database name encoded in a Neo4j URI path. func neo4jDatabaseName(connectionURL *url.URL) (string, error) { databasePath := strings.Trim(connectionURL.EscapedPath(), "/") if databasePath == "" { @@ -238,6 +325,7 @@ func neo4jDatabaseName(connectionURL *url.URL) (string, error) { return databaseName, nil } +// openNeo4jPlanDriver parses the benchmark connection settings and returns a context-aware driver together with the selected database name. func openNeo4jPlanDriver(connStr string) (neo4jcore.DriverWithContext, string, error) { cfg, err := parseNeo4jPlanDriverConfig(connStr) if err != nil { @@ -252,18 +340,71 @@ func openNeo4jPlanDriver(connStr string) (neo4jcore.DriverWithContext, string, e return driver, cfg.DatabaseName, nil } +// Neo4jProfileMetadata identifies the planner, runtime, and server used for a captured plan. +type Neo4jProfileMetadata struct { + // CaptureMode identifies the capture mode. + CaptureMode string `json:"capture_mode"` + // Profiled indicates whether profiled applies. + Profiled bool `json:"profiled"` + // Planner supplies the planner input to the Neo4jProfileMetadata contract. + Planner string `json:"planner,omitempty"` + // PlannerImplementation supplies the planner implementation input to the Neo4jProfileMetadata contract. + PlannerImplementation string `json:"planner_implementation,omitempty"` + // PlannerVersion identifies the schema version for planner version. + PlannerVersion string `json:"planner_version,omitempty"` + // Runtime supplies the runtime input to the Neo4jProfileMetadata contract. + Runtime string `json:"runtime,omitempty"` + // RuntimeImplementation supplies the runtime implementation input to the Neo4jProfileMetadata contract. + RuntimeImplementation string `json:"runtime_implementation,omitempty"` + // RuntimeVersion identifies the schema version for runtime version. + RuntimeVersion string `json:"runtime_version,omitempty"` + // CypherVersion identifies the schema version for cypher version. + CypherVersion string `json:"cypher_version,omitempty"` + // ServerAgent supplies the server agent input to the Neo4jProfileMetadata contract. + ServerAgent string `json:"server_agent,omitempty"` +} + +// Neo4jPlanNode models the recursive operator tree returned by Neo4j PROFILE or EXPLAIN. type Neo4jPlanNode struct { - Operator string `json:"operator"` - Arguments map[string]string `json:"arguments,omitempty"` - Identifiers []string `json:"identifiers,omitempty"` - Children []Neo4jPlanNode `json:"children,omitempty"` + // Operator identifies the backend plan operator at this node. + Operator string `json:"operator"` + // Arguments maps backend plan argument names to stable string representations. + Arguments map[string]string `json:"arguments,omitempty"` + // Identifiers lists variables or identifiers referenced by the Neo4j plan node. + Identifiers []string `json:"identifiers,omitempty"` + // EstimatedRows records planner-estimated output rows when Neo4j supplies them. + EstimatedRows *float64 `json:"estimated_rows,omitempty"` + // ActualRows records rows emitted by an executed PROFILE operator. + ActualRows *int64 `json:"actual_rows,omitempty"` + // Loops records operator loops when Neo4j exposes them as a plan argument. + Loops *int64 `json:"loops,omitempty"` + // DBHits records data-store accesses reported for an executed PROFILE operator. + DBHits *int64 `json:"db_hits,omitempty"` + // PageCacheHits records page-cache hits reported for an executed PROFILE operator. + PageCacheHits *int64 `json:"page_cache_hits,omitempty"` + // PageCacheMisses records page-cache misses reported for an executed PROFILE operator. + PageCacheMisses *int64 `json:"page_cache_misses,omitempty"` + // PageCacheHitRatio supplies the page cache hit ratio input to the Neo4jPlanNode contract. + PageCacheHitRatio *float64 `json:"page_cache_hit_ratio,omitempty"` + // TimeNS records operator time in nanoseconds when exposed by the Neo4j server. + TimeNS *int64 `json:"time_ns,omitempty"` + // InternalTraversalWork marks Neo4j 4.4 SP/ASP relationship work as opaque. + InternalTraversalWork string `json:"internal_traversal_work,omitempty"` + // ProfileMetadata records root planner/runtime and capture metadata. + ProfileMetadata *Neo4jProfileMetadata `json:"profile_metadata,omitempty"` + // Children contains child Neo4j plan operators in backend order. + Children []Neo4jPlanNode `json:"children,omitempty"` } +// convertNeo4jPlan recursively converts a Neo4j plan into the stable serialized plan-node schema. func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { + arguments := plan.Arguments() node := Neo4jPlanNode{ - Operator: plan.Operator(), - Arguments: stringifyArguments(plan.Arguments()), - Identifiers: append([]string(nil), plan.Identifiers()...), + Operator: normalizeNeo4jOperator(plan.Operator()), + Arguments: stringifyArguments(arguments), + Identifiers: append([]string(nil), plan.Identifiers()...), + EstimatedRows: neo4jFloatArgument(arguments, "EstimatedRows", "estimatedRows"), + Loops: neo4jIntArgument(arguments, "Loops", "loops"), } for _, child := range plan.Children() { @@ -273,6 +414,117 @@ func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { return node } +// convertNeo4jProfiledPlan recursively converts executed PROFILE data while preserving child order. +func convertNeo4jProfiledPlan(plan neo4jcore.ProfiledPlan, opaqueInternalTraversal bool) Neo4jPlanNode { + arguments := plan.Arguments() + operator := normalizeNeo4jOperator(plan.Operator()) + node := Neo4jPlanNode{ + Operator: operator, + Arguments: stringifyArguments(arguments), + Identifiers: append([]string(nil), plan.Identifiers()...), + EstimatedRows: neo4jFloatArgument(arguments, "EstimatedRows", "estimatedRows"), + ActualRows: neo4jInt64Pointer(plan.Records()), + Loops: neo4jIntArgument(arguments, "Loops", "loops"), + DBHits: neo4jInt64Pointer(plan.DbHits()), + PageCacheHits: neo4jInt64Pointer(plan.PageCacheHits()), + PageCacheMisses: neo4jInt64Pointer(plan.PageCacheMisses()), + PageCacheHitRatio: neo4jFloat64Pointer(plan.PageCacheHitRatio()), + TimeNS: neo4jInt64Pointer(plan.Time()), + } + if opaqueInternalTraversal && strings.Contains(strings.ToLower(neo4jOperatorBase(operator)), "shortestpath") { + node.InternalTraversalWork = "opaque" + } + + for _, child := range plan.Children() { + node.Children = append(node.Children, convertNeo4jProfiledPlan(child, opaqueInternalTraversal)) + } + + return node +} + +// neo4jProfileMetadata derives metadata describing neo4j profile. +func neo4jProfileMetadata(arguments map[string]any, serverAgent string, profiled bool) Neo4jProfileMetadata { + captureMode := "EXPLAIN" + if profiled { + captureMode = "PROFILE" + } + + return Neo4jProfileMetadata{ + CaptureMode: captureMode, + Profiled: profiled, + Planner: neo4jStringArgument(arguments, "planner"), + PlannerImplementation: neo4jStringArgument(arguments, "planner-impl"), + PlannerVersion: neo4jStringArgument(arguments, "planner-version"), + Runtime: neo4jStringArgument(arguments, "runtime"), + RuntimeImplementation: neo4jStringArgument(arguments, "runtime-impl"), + RuntimeVersion: neo4jStringArgument(arguments, "runtime-version"), + CypherVersion: neo4jStringArgument(arguments, "version"), + ServerAgent: serverAgent, + } +} + +// internalTraversalOpaque supports benchmark evidence processing for internal traversal opaque. +func (s Neo4jProfileMetadata) internalTraversalOpaque() bool { + return strings.HasPrefix(s.PlannerVersion, "4.4") || + strings.HasPrefix(s.RuntimeVersion, "4.4") || + strings.Contains(s.CypherVersion, "4.4") || + strings.Contains(s.ServerAgent, "/4.4") +} + +// neo4jStringArgument supports benchmark evidence processing for neo4j string argument. +func neo4jStringArgument(arguments map[string]any, name string) string { + if value, ok := arguments[name]; ok { + return fmt.Sprint(value) + } + + return "" +} + +// neo4jFloatArgument supports benchmark evidence processing for neo4j float argument. +func neo4jFloatArgument(arguments map[string]any, names ...string) *float64 { + for _, name := range names { + value, ok := arguments[name] + if !ok { + continue + } + + parsed, err := strconv.ParseFloat(fmt.Sprint(value), 64) + if err == nil { + return neo4jFloat64Pointer(parsed) + } + } + + return nil +} + +// neo4jIntArgument supports benchmark evidence processing for neo4j int argument. +func neo4jIntArgument(arguments map[string]any, names ...string) *int64 { + for _, name := range names { + value, ok := arguments[name] + if !ok { + continue + } + + parsed, err := strconv.ParseInt(fmt.Sprint(value), 10, 64) + if err == nil { + return neo4jInt64Pointer(parsed) + } + } + + return nil +} + +// neo4jInt64Pointer returns an addressable representation of neo4j int64. +func neo4jInt64Pointer(value int64) *int64 { + return &value +} + +// neo4jFloat64Pointer returns an addressable representation of neo4j float64. +func neo4jFloat64Pointer(value float64) *float64 { + return &value +} + +// stringifyArguments converts plan arguments to stable strings in a fresh map. func stringifyArguments(arguments map[string]any) map[string]string { if len(arguments) == 0 { return nil @@ -286,6 +538,7 @@ func stringifyArguments(arguments map[string]any) map[string]string { return values } +// neo4jOperators flattens a Neo4j plan tree in traversal order with exactly one backend suffix. func neo4jOperators(root Neo4jPlanNode) []string { var ( operators []string @@ -293,7 +546,7 @@ func neo4jOperators(root Neo4jPlanNode) []string { ) walk = func(node Neo4jPlanNode) { - operators = append(operators, node.Operator+"@neo4j") + operators = append(operators, normalizeNeo4jOperator(node.Operator)) for _, child := range node.Children { walk(child) } @@ -303,6 +556,27 @@ func neo4jOperators(root Neo4jPlanNode) []string { return operators } +// normalizeNeo4jOperator normalizes neo4j operator. +func normalizeNeo4jOperator(operator string) string { + base := neo4jOperatorBase(operator) + if base == "" { + return "" + } + + return base + "@neo4j" +} + +// neo4jOperatorBase supports benchmark evidence processing for neo4j operator base. +func neo4jOperatorBase(operator string) string { + operator = strings.TrimSpace(operator) + for strings.HasSuffix(operator, "@neo4j") { + operator = strings.TrimSpace(strings.TrimSuffix(operator, "@neo4j")) + } + + return operator +} + +// cypherWithoutTerminator trims surrounding whitespace and one trailing Cypher semicolon. func cypherWithoutTerminator(cypherQuery string) string { return strings.TrimSuffix(strings.TrimSpace(cypherQuery), ";") } diff --git a/cmd/graphbench/neo4j_test.go b/cmd/graphbench/neo4j_test.go index a01058c9..036f7da5 100644 --- a/cmd/graphbench/neo4j_test.go +++ b/cmd/graphbench/neo4j_test.go @@ -20,9 +20,11 @@ import ( "net/url" "testing" + neo4jcore "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/stretchr/testify/require" ) +// TestParseNeo4jPlanDriverConfig verifies parse neo4j plan driver config behavior. func TestParseNeo4jPlanDriverConfig(t *testing.T) { cfg, err := parseNeo4jPlanDriverConfig("neo4j://neo4j:secret@example.com:7687/neo4jdb?x=1") @@ -33,6 +35,7 @@ func TestParseNeo4jPlanDriverConfig(t *testing.T) { require.Equal(t, "neo4jdb", cfg.DatabaseName) } +// TestNeo4jDatabaseNameRejectsNestedPath verifies neo4j database name rejects nested path behavior. func TestNeo4jDatabaseNameRejectsNestedPath(t *testing.T) { for _, connStr := range []string{ "neo4j://neo4j:secret@example.com:7687/a/b", @@ -46,13 +49,181 @@ func TestNeo4jDatabaseNameRejectsNestedPath(t *testing.T) { } } +// TestNeo4jOperatorsAnnotatesOperators verifies neo4j operators annotates operators behavior. func TestNeo4jOperatorsAnnotatesOperators(t *testing.T) { operators := neo4jOperators(Neo4jPlanNode{ - Operator: "ProduceResults", + Operator: "ProduceResults@neo4j@neo4j", Children: []Neo4jPlanNode{{ - Operator: "AllNodesScan", + Operator: "AllNodesScan@neo4j", }}, }) require.Equal(t, []string{"ProduceResults@neo4j", "AllNodesScan@neo4j"}, operators) } + +// TestNeo4jPlanCaptureStatementProfilesReadsAndExplainsWrites verifies neo4j plan capture statement profiles reads and explains writes behavior. +func TestNeo4jPlanCaptureStatementProfilesReadsAndExplainsWrites(t *testing.T) { + require.Equal(t, "PROFILE MATCH (n) RETURN n", neo4jPlanCaptureStatement(" MATCH (n) RETURN n; ", false)) + require.Equal(t, "EXPLAIN CREATE (n)", neo4jPlanCaptureStatement("CREATE (n);", true)) +} + +// TestConvertNeo4jPlanPreservesEndpointChildOrder verifies convert neo4j plan preserves endpoint child order behavior. +func TestConvertNeo4jPlanPreservesEndpointChildOrder(t *testing.T) { + plan := stubNeo4jPlan{ + operator: "CartesianProduct@neo4j@neo4j", + arguments: map[string]any{"EstimatedRows": 2.5, "Loops": int64(3)}, + children: []neo4jcore.Plan{ + stubNeo4jPlan{ + operator: "NodeIndexSeek", + identifiers: []string{"start"}, + }, + stubNeo4jPlan{ + operator: "NodeIndexSeek", + identifiers: []string{"end"}, + }, + }, + } + + converted := convertNeo4jPlan(plan) + + require.Equal(t, "CartesianProduct@neo4j", converted.Operator) + require.Equal(t, 2.5, *converted.EstimatedRows) + require.Equal(t, int64(3), *converted.Loops) + require.Equal(t, []string{"start"}, converted.Children[0].Identifiers) + require.Equal(t, []string{"end"}, converted.Children[1].Identifiers) +} + +// TestConvertNeo4jProfiledPlanCapturesMetricsMetadataAndOpaqueShortestPath verifies convert neo4j profiled plan captures metrics metadata and opaque shortest path behavior. +func TestConvertNeo4jProfiledPlanCapturesMetricsMetadataAndOpaqueShortestPath(t *testing.T) { + profile := stubNeo4jProfiledPlan{ + operator: "ProduceResults@neo4j", + arguments: map[string]any{ + "EstimatedRows": 1.5, + "planner": "COST", + "planner-impl": "IDP", + "planner-version": "4.4", + "runtime": "INTERPRETED", + "runtime-impl": "INTERPRETED", + "runtime-version": "4.4", + "version": "CYPHER 4.4", + }, + dbHits: 11, + records: 7, + pageCacheHits: 13, + pageCacheMisses: 2, + pageCacheHitRatio: 0.86, + timeNS: 101, + children: []neo4jcore.ProfiledPlan{ + stubNeo4jProfiledPlan{ + operator: "ShortestPath@neo4j@neo4j", + dbHits: 1, + records: 1, + }, + stubNeo4jProfiledPlan{ + operator: "NodeIndexSeek", + identifiers: []string{"end"}, + dbHits: 3, + records: 1, + }, + }, + } + metadata := neo4jProfileMetadata(profile.Arguments(), "Neo4j/4.4.44", true) + + converted := convertNeo4jProfiledPlan(profile, metadata.internalTraversalOpaque()) + converted.ProfileMetadata = &metadata + + require.Equal(t, "ProduceResults@neo4j", converted.Operator) + require.Equal(t, 1.5, *converted.EstimatedRows) + require.Equal(t, int64(7), *converted.ActualRows) + require.Equal(t, int64(11), *converted.DBHits) + require.Equal(t, int64(13), *converted.PageCacheHits) + require.Equal(t, int64(2), *converted.PageCacheMisses) + require.Equal(t, 0.86, *converted.PageCacheHitRatio) + require.Equal(t, int64(101), *converted.TimeNS) + require.Equal(t, "PROFILE", converted.ProfileMetadata.CaptureMode) + require.True(t, converted.ProfileMetadata.Profiled) + require.Equal(t, "4.4", converted.ProfileMetadata.PlannerVersion) + require.Equal(t, "4.4", converted.ProfileMetadata.RuntimeVersion) + require.Equal(t, "ShortestPath@neo4j", converted.Children[0].Operator) + require.Equal(t, "opaque", converted.Children[0].InternalTraversalWork) + require.Empty(t, converted.Children[1].InternalTraversalWork) + require.Equal(t, []string{"end"}, converted.Children[1].Identifiers) +} + +// stubNeo4jPlan groups state that must remain consistent while processing stub neo4j plan. +type stubNeo4jPlan struct { + // operator retains the operator while stubNeo4jPlan is assembled or evaluated. + operator string + // arguments retains the arguments while stubNeo4jPlan is assembled or evaluated. + arguments map[string]any + // identifiers retains the identifiers while stubNeo4jPlan is assembled or evaluated. + identifiers []string + // children retains the children while stubNeo4jPlan is assembled or evaluated. + children []neo4jcore.Plan +} + +// Operator prepares or inspects test evidence for operator. +func (s stubNeo4jPlan) Operator() string { return s.operator } + +// Arguments prepares or inspects test evidence for arguments. +func (s stubNeo4jPlan) Arguments() map[string]any { return s.arguments } + +// Identifiers prepares or inspects test evidence for identifiers. +func (s stubNeo4jPlan) Identifiers() []string { return s.identifiers } + +// Children prepares or inspects test evidence for children. +func (s stubNeo4jPlan) Children() []neo4jcore.Plan { return s.children } + +// stubNeo4jProfiledPlan groups state that must remain consistent while processing stub neo4j profiled plan. +type stubNeo4jProfiledPlan struct { + // operator retains the operator while stubNeo4jProfiledPlan is assembled or evaluated. + operator string + // arguments retains the arguments while stubNeo4jProfiledPlan is assembled or evaluated. + arguments map[string]any + // identifiers retains the identifiers while stubNeo4jProfiledPlan is assembled or evaluated. + identifiers []string + // dbHits retains the db hits while stubNeo4jProfiledPlan is assembled or evaluated. + dbHits int64 + // records retains the records while stubNeo4jProfiledPlan is assembled or evaluated. + records int64 + // children retains the children while stubNeo4jProfiledPlan is assembled or evaluated. + children []neo4jcore.ProfiledPlan + // pageCacheMisses retains the page cache misses while stubNeo4jProfiledPlan is assembled or evaluated. + pageCacheMisses int64 + // pageCacheHits retains the page cache hits while stubNeo4jProfiledPlan is assembled or evaluated. + pageCacheHits int64 + // pageCacheHitRatio retains the page cache hit ratio while stubNeo4jProfiledPlan is assembled or evaluated. + pageCacheHitRatio float64 + // timeNS retains the time ns while stubNeo4jProfiledPlan is assembled or evaluated. + timeNS int64 +} + +// Operator prepares or inspects test evidence for operator. +func (s stubNeo4jProfiledPlan) Operator() string { return s.operator } + +// Arguments prepares or inspects test evidence for arguments. +func (s stubNeo4jProfiledPlan) Arguments() map[string]any { return s.arguments } + +// Identifiers prepares or inspects test evidence for identifiers. +func (s stubNeo4jProfiledPlan) Identifiers() []string { return s.identifiers } + +// DbHits prepares or inspects test evidence for db hits. +func (s stubNeo4jProfiledPlan) DbHits() int64 { return s.dbHits } + +// Records prepares or inspects test evidence for records. +func (s stubNeo4jProfiledPlan) Records() int64 { return s.records } + +// Children prepares or inspects test evidence for children. +func (s stubNeo4jProfiledPlan) Children() []neo4jcore.ProfiledPlan { return s.children } + +// PageCacheMisses prepares or inspects test evidence for page cache misses. +func (s stubNeo4jProfiledPlan) PageCacheMisses() int64 { return s.pageCacheMisses } + +// PageCacheHits prepares or inspects test evidence for page cache hits. +func (s stubNeo4jProfiledPlan) PageCacheHits() int64 { return s.pageCacheHits } + +// PageCacheHitRatio derives the statistical value used to evaluate page cache hit ratio. +func (s stubNeo4jProfiledPlan) PageCacheHitRatio() float64 { return s.pageCacheHitRatio } + +// Time prepares or inspects test evidence for time. +func (s stubNeo4jProfiledPlan) Time() int64 { return s.timeNS } diff --git a/cmd/graphbench/operational_gate.go b/cmd/graphbench/operational_gate.go new file mode 100644 index 00000000..84b70321 --- /dev/null +++ b/cmd/graphbench/operational_gate.go @@ -0,0 +1,1499 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "reflect" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + pgdriver "github.com/specterops/dawgs/drivers/pg" +) + +const ( + // operationalGateVersion identifies the serialized operational-evidence schema. + operationalGateVersion = 2 + + // OperationalScenarioCandidateMatrix exercises an admitted candidate under + // one pool-size, concurrency, and PostgreSQL plan-cache-mode cell. + OperationalScenarioCandidateMatrix OperationalEvidenceScenario = "candidate_matrix" + // OperationalScenarioLowWorkMem proves admitted execution under constrained work_mem. + OperationalScenarioLowWorkMem OperationalEvidenceScenario = "low_work_mem" + // OperationalScenarioCancellation proves bounded cancellation and same-session recovery. + OperationalScenarioCancellation OperationalEvidenceScenario = "cancellation_replay" + // OperationalScenarioConcurrentWriter proves Repeatable Read stability across a concurrent commit. + OperationalScenarioConcurrentWriter OperationalEvidenceScenario = "repeatable_read_concurrent_writer" + // OperationalScenarioSessionIsolation proves invocation-local state does not cross sessions. + OperationalScenarioSessionIsolation OperationalEvidenceScenario = "session_isolation" + // OperationalScenarioForcedOverflow proves the candidate's exact overflow fallback receipt chain. + OperationalScenarioForcedOverflow OperationalEvidenceScenario = "forced_overflow_fallback" +) + +var ( + defaultOperationalPoolSizes = []int{1, 2, 8} + defaultOperationalConcurrency = []int{1, 8, 16} + defaultOperationalPlanCacheModes = []string{"auto", "force_custom_plan", "force_generic_plan"} +) + +// OperationalEvidenceScenario identifies the independently validated operational proof in a record. +type OperationalEvidenceScenario string + +// OperationalGateRequirements freezes the generic operational contract while +// allowing a policy identity to differ from the candidate arm it dispatches. +type OperationalGateRequirements struct { + // CandidateRuntimeIdentity is the admitted executor recorded at the timed invocation boundary. + CandidateRuntimeIdentity string `json:"candidate_runtime_identity"` + // FallbackRuntimeIdentity is the first exact fallback required in an overflow receipt chain. + FallbackRuntimeIdentity string `json:"fallback_runtime_identity"` + // PoolSizes contains every required connection-pool size. + PoolSizes []int `json:"pool_sizes"` + // ConcurrencyLevels contains every required concurrent worker count. + ConcurrencyLevels []int `json:"concurrency_levels"` + // PlanCacheModes contains every required PostgreSQL plan_cache_mode. + PlanCacheModes []string `json:"plan_cache_modes"` + // LowWorkMemMaximumBytes is the largest work_mem setting accepted as constrained evidence. + LowWorkMemMaximumBytes int64 `json:"low_work_mem_maximum_bytes"` + // CancellationMaximum is the exclusive cancellation-latency ceiling. + CancellationMaximum time.Duration `json:"cancellation_maximum"` + // RequireCleanSource rejects operational evidence captured from a dirty source tree. + RequireCleanSource bool `json:"require_clean_source"` + // CandidateSQLFingerprint repeats the independently frozen manifest identity + // anchor for every non-overflow scenario. It cannot introduce a new digest. + CandidateSQLFingerprint string `json:"candidate_sql_fingerprint"` +} + +// defaultOperationalGateRequirements returns the promotion-grade operational matrix. +func defaultOperationalGateRequirements(candidateRuntimeIdentity, fallbackRuntimeIdentity string) OperationalGateRequirements { + return OperationalGateRequirements{ + CandidateRuntimeIdentity: candidateRuntimeIdentity, + FallbackRuntimeIdentity: fallbackRuntimeIdentity, + PoolSizes: append([]int(nil), defaultOperationalPoolSizes...), + ConcurrencyLevels: append([]int(nil), defaultOperationalConcurrency...), + PlanCacheModes: append([]string(nil), defaultOperationalPlanCacheModes...), + LowWorkMemMaximumBytes: 64 * 1024, + CancellationMaximum: 250 * time.Millisecond, + RequireCleanSource: true, + } +} + +// OperationalCancellationEvidence records the expected timeout and the +// successful replay performed after rollback on the same PostgreSQL backend. +type OperationalCancellationEvidence struct { + SQLState string `json:"sql_state"` + Latency time.Duration `json:"latency"` + TransactionRolledBack bool `json:"transaction_rolled_back"` + CancelledBackendPID uint32 `json:"cancelled_backend_pid"` + ReplayBackendPID uint32 `json:"replay_backend_pid"` + ReplaySucceeded bool `json:"replay_succeeded"` + ReplayCandidateReceipt LatencySample `json:"replay_candidate_receipt"` +} + +// OperationalSnapshotEvidence records a reader snapshot before and after a +// distinct concurrent writer commits. +type OperationalSnapshotEvidence struct { + ReaderBackendPID uint32 `json:"reader_backend_pid"` + WriterBackendPID uint32 `json:"writer_backend_pid"` + ReaderIsolation string `json:"reader_isolation"` + WriterAffectedRows int64 `json:"writer_affected_rows"` + WriterCommitted bool `json:"writer_committed"` + ObservationBeforeSHA256 string `json:"observation_before_sha256"` + ObservationAfterSHA256 string `json:"observation_after_sha256"` + PostCommitObservationSHA256 string `json:"post_commit_observation_sha256"` +} + +// OperationalSessionIsolationEvidence records two invocation-local sessions +// and the rows each session could observe from the other's invocation. +type OperationalSessionIsolationEvidence struct { + SessionABackendPID uint32 `json:"session_a_backend_pid"` + SessionBBackendPID uint32 `json:"session_b_backend_pid"` + SessionAInvocationID string `json:"session_a_invocation_id"` + SessionBInvocationID string `json:"session_b_invocation_id"` + SessionAOwnRows int64 `json:"session_a_own_rows"` + SessionBOwnRows int64 `json:"session_b_own_rows"` + SessionAObservedBRows int64 `json:"session_a_observed_b_rows"` + SessionBObservedARows int64 `json:"session_b_observed_a_rows"` + SessionACandidateReceipt LatencySample `json:"session_a_candidate_receipt"` + SessionBCandidateReceipt LatencySample `json:"session_b_candidate_receipt"` +} + +// OperationalEvidenceRecord binds one proof to the exact promotion identity, +// source archive, benchmark environment, case, and runtime receipts. +type OperationalEvidenceRecord struct { + ID string `json:"id"` + Scenario OperationalEvidenceScenario `json:"scenario"` + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + SourceSHA256 string `json:"source_sha256"` + Concurrency int `json:"concurrency,omitempty"` + Result CaseResult `json:"result"` + Cancellation *OperationalCancellationEvidence `json:"cancellation,omitempty"` + Snapshot *OperationalSnapshotEvidence `json:"snapshot,omitempty"` + SessionIsolation *OperationalSessionIsolationEvidence `json:"session_isolation,omitempty"` +} + +// OperationalGateInput is the strict, portable source document consumed by +// the operational report generator. +type OperationalGateInput struct { + Version int `json:"version"` + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + Requirements OperationalGateRequirements `json:"requirements"` + Records []OperationalEvidenceRecord `json:"records"` +} + +// OperationalMatrixCell identifies one required candidate execution cell. +type OperationalMatrixCell struct { + PoolSize int `json:"pool_size"` + Concurrency int `json:"concurrency"` + PlanCacheMode string `json:"plan_cache_mode"` +} + +// OperationalGateCoverage reports independently machine-checkable coverage of +// the matrix and each non-matrix operational proof. +type OperationalGateCoverage struct { + RequiredMatrixCells int `json:"required_matrix_cells"` + ObservedMatrixCells int `json:"observed_matrix_cells"` + MissingMatrixCells []OperationalMatrixCell `json:"missing_matrix_cells,omitempty"` + LowWorkMem bool `json:"low_work_mem"` + CancellationReplay bool `json:"cancellation_replay"` + RepeatableReadWriter bool `json:"repeatable_read_concurrent_writer"` + SessionIsolation bool `json:"session_isolation"` + ForcedOverflowFallback bool `json:"forced_overflow_fallback"` +} + +// OperationalGateRecord reports validation of one source evidence record. +type OperationalGateRecord struct { + ID string `json:"id"` + Scenario OperationalEvidenceScenario `json:"scenario"` + Dataset string `json:"dataset,omitempty"` + Name string `json:"name,omitempty"` + PoolSize int `json:"pool_size,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + PlanCacheMode string `json:"plan_cache_mode,omitempty"` + WorkMemBytes int64 `json:"work_mem_bytes,omitempty"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +// OperationalGateReport is the promotion-manifest operational evidence role. +// PromotionIdentity is deliberately repeated verbatim for manifest closure. +type OperationalGateReport struct { + Version int `json:"version"` + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + Requirements OperationalGateRequirements `json:"requirements"` + // Input retains the complete source evidence so final promotion + // verification can independently rebuild every gate decision. + Input OperationalGateInput `json:"input"` + // InputSHA256 binds Input's canonical JSON representation. It detects raw + // evidence changes even when a forged summary is left untouched. + InputSHA256 string `json:"input_sha256"` + Passed bool `json:"passed"` + Coverage OperationalGateCoverage `json:"coverage"` + Records []OperationalGateRecord `json:"records"` + Reasons []string `json:"reasons,omitempty"` +} + +// buildOperationalGateReport validates operational evidence without relying +// on filenames, CLI arguments, or human interpretation of integration logs. +func buildOperationalGateReport(identity PromotionEvidenceIdentity, requirements OperationalGateRequirements, records []OperationalEvidenceRecord) OperationalGateReport { + input, inputSHA256, inputErr := canonicalOperationalGateInput(identity, requirements, records) + if inputErr == nil { + identity = input.PromotionIdentity + requirements = input.Requirements + records = input.Records + } + report := OperationalGateReport{ + Version: operationalGateVersion, + PromotionIdentity: cloneOperationalPromotionIdentity(identity), + Requirements: cloneOperationalRequirements(requirements), + Input: input, + InputSHA256: inputSHA256, + Passed: true, + } + if inputErr != nil { + report.Reasons = append(report.Reasons, "operational input cannot be canonically embedded: "+inputErr.Error()) + } + report.Reasons = append(report.Reasons, validateOperationalIdentity(identity)...) + report.Reasons = append(report.Reasons, validateOperationalRequirements(identity, requirements)...) + if len(records) == 0 { + report.Reasons = append(report.Reasons, "operational evidence is empty") + } + + validMatrix := map[OperationalMatrixCell]struct{}{} + validScenarios := map[OperationalEvidenceScenario]bool{} + seenMatrix := map[OperationalMatrixCell]int{} + seenScenarios := map[OperationalEvidenceScenario]int{} + seenIDs := map[string]struct{}{} + operationalWorkload := "" + operationalCandidateSQL := "" + operationalTranslationTarget := "" + var databaseIdentity *PostgresEnvironment + + for _, record := range records { + decision := OperationalGateRecord{ + ID: record.ID, + Scenario: record.Scenario, + Dataset: record.Result.Dataset, + Name: record.Result.Name, + Concurrency: record.Concurrency, + } + if record.Result.Environment != nil { + decision.PoolSize = record.Result.Environment.PoolSize + } + if record.Result.PostgresEnvironment != nil { + decision.PlanCacheMode = normalizedPlanCacheMode(record.Result.PostgresEnvironment.PlanCacheMode) + if workMemBytes, err := parsePostgresMemoryBytes(record.Result.PostgresEnvironment.WorkMem); err != nil { + decision.Reasons = append(decision.Reasons, "invalid PostgreSQL work_mem: "+err.Error()) + } else { + decision.WorkMemBytes = workMemBytes + } + } + + if strings.TrimSpace(record.ID) == "" { + decision.Reasons = append(decision.Reasons, "record id is missing") + } else if _, duplicate := seenIDs[record.ID]; duplicate { + decision.Reasons = append(decision.Reasons, "record id is duplicated") + } else { + seenIDs[record.ID] = struct{}{} + } + decision.Reasons = append(decision.Reasons, validateOperationalRecordBinding(identity, requirements, record)...) + workload, workloadErr := operationalWorkloadBinding(record.Result) + if workloadErr != nil { + decision.Reasons = append(decision.Reasons, "operational workload binding: "+workloadErr.Error()) + } else if operationalWorkload == "" { + operationalWorkload = workload + } else if workload != operationalWorkload { + decision.Reasons = append(decision.Reasons, "operational scenarios do not use one exact authorized workload") + } + translationTarget, translationErr := operationalTranslationTargetBinding(record.Result) + if translationErr != nil { + decision.Reasons = append(decision.Reasons, "operational translation binding: "+translationErr.Error()) + } else if operationalTranslationTarget == "" { + operationalTranslationTarget = translationTarget + } else if translationTarget != operationalTranslationTarget { + decision.Reasons = append(decision.Reasons, "operational scenarios do not use one exact authorized translation target") + } + if record.Scenario != OperationalScenarioForcedOverflow { + if operationalCandidateSQL == "" { + operationalCandidateSQL = record.Result.SQLFingerprint + } else if record.Result.SQLFingerprint != operationalCandidateSQL { + decision.Reasons = append(decision.Reasons, "non-overflow operational scenarios do not use one exact candidate SQL") + } + } + if record.Result.PostgresEnvironment != nil { + if databaseIdentity == nil { + copy := *record.Result.PostgresEnvironment + databaseIdentity = © + } else if !sameOperationalDatabase(databaseIdentity, record.Result.PostgresEnvironment) { + decision.Reasons = append(decision.Reasons, "PostgreSQL database identity differs across operational records") + } + } + + switch record.Scenario { + case OperationalScenarioCandidateMatrix: + decision.Reasons = append(decision.Reasons, validateOperationalMatrixCell(record, requirements)...) + decision.Reasons = append(decision.Reasons, validateOperationalConcurrencyBlock(record)...) + poolSize := 0 + if record.Result.Environment != nil { + poolSize = record.Result.Environment.PoolSize + } + decision.Reasons = append(decision.Reasons, validateOperationalCandidateResult(record.Result, identity, requirements, poolSize == 1)...) + cell := OperationalMatrixCell{PoolSize: decision.PoolSize, Concurrency: decision.Concurrency, PlanCacheMode: decision.PlanCacheMode} + seenMatrix[cell]++ + if seenMatrix[cell] > 1 { + decision.Reasons = append(decision.Reasons, "candidate matrix cell is duplicated") + } + case OperationalScenarioLowWorkMem: + seenScenarios[record.Scenario]++ + if seenScenarios[record.Scenario] > 1 { + decision.Reasons = append(decision.Reasons, "operational scenario is duplicated") + } + decision.Reasons = append(decision.Reasons, validateOperationalCandidateResult(record.Result, identity, requirements, true)...) + if decision.WorkMemBytes <= 0 || decision.WorkMemBytes > requirements.LowWorkMemMaximumBytes { + decision.Reasons = append(decision.Reasons, fmt.Sprintf("work_mem exceeds constrained ceiling %d bytes", requirements.LowWorkMemMaximumBytes)) + } + case OperationalScenarioCancellation: + seenScenarios[record.Scenario]++ + if seenScenarios[record.Scenario] > 1 { + decision.Reasons = append(decision.Reasons, "operational scenario is duplicated") + } + decision.Reasons = append(decision.Reasons, validateOperationalCandidateResult(record.Result, identity, requirements, true)...) + decision.Reasons = append(decision.Reasons, validateOperationalCancellation(record.Cancellation, record.Result, requirements)...) + case OperationalScenarioConcurrentWriter: + seenScenarios[record.Scenario]++ + if seenScenarios[record.Scenario] > 1 { + decision.Reasons = append(decision.Reasons, "operational scenario is duplicated") + } + decision.Reasons = append(decision.Reasons, validateOperationalCandidateResult(record.Result, identity, requirements, true)...) + decision.Reasons = append(decision.Reasons, validateOperationalSnapshot(record.Snapshot)...) + case OperationalScenarioSessionIsolation: + seenScenarios[record.Scenario]++ + if seenScenarios[record.Scenario] > 1 { + decision.Reasons = append(decision.Reasons, "operational scenario is duplicated") + } + decision.Reasons = append(decision.Reasons, validateOperationalCandidateResult(record.Result, identity, requirements, true)...) + decision.Reasons = append(decision.Reasons, validateOperationalSessionIsolation(record.SessionIsolation, record.Result, requirements)...) + case OperationalScenarioForcedOverflow: + seenScenarios[record.Scenario]++ + if seenScenarios[record.Scenario] > 1 { + decision.Reasons = append(decision.Reasons, "operational scenario is duplicated") + } + decision.Reasons = append(decision.Reasons, validateOperationalFallbackResult(record.Result, identity, requirements)...) + default: + decision.Reasons = append(decision.Reasons, fmt.Sprintf("unsupported operational scenario %q", record.Scenario)) + } + + decision.Passed = len(decision.Reasons) == 0 + if !decision.Passed { + report.Passed = false + } else { + validScenarios[record.Scenario] = true + if record.Scenario == OperationalScenarioCandidateMatrix { + validMatrix[OperationalMatrixCell{ + PoolSize: decision.PoolSize, + Concurrency: decision.Concurrency, + PlanCacheMode: decision.PlanCacheMode, + }] = struct{}{} + } + } + report.Records = append(report.Records, decision) + } + if len(records) != len(defaultOperationalPoolSizes)*len(defaultOperationalConcurrency)*len(defaultOperationalPlanCacheModes)+5 { + report.Reasons = append(report.Reasons, "operational evidence must contain exactly 32 records") + } + + for _, poolSize := range requirements.PoolSizes { + for _, concurrency := range requirements.ConcurrencyLevels { + for _, mode := range requirements.PlanCacheModes { + cell := OperationalMatrixCell{PoolSize: poolSize, Concurrency: concurrency, PlanCacheMode: normalizedPlanCacheMode(mode)} + report.Coverage.RequiredMatrixCells++ + if _, found := validMatrix[cell]; found { + report.Coverage.ObservedMatrixCells++ + } else { + report.Coverage.MissingMatrixCells = append(report.Coverage.MissingMatrixCells, cell) + report.Reasons = append(report.Reasons, fmt.Sprintf("candidate matrix is missing pool_size=%d concurrency=%d plan_cache_mode=%s", poolSize, concurrency, cell.PlanCacheMode)) + } + } + } + } + report.Coverage.LowWorkMem = validScenarios[OperationalScenarioLowWorkMem] + report.Coverage.CancellationReplay = validScenarios[OperationalScenarioCancellation] + report.Coverage.RepeatableReadWriter = validScenarios[OperationalScenarioConcurrentWriter] + report.Coverage.SessionIsolation = validScenarios[OperationalScenarioSessionIsolation] + report.Coverage.ForcedOverflowFallback = validScenarios[OperationalScenarioForcedOverflow] + for _, required := range []struct { + scenario OperationalEvidenceScenario + present bool + }{ + {scenario: OperationalScenarioLowWorkMem, present: report.Coverage.LowWorkMem}, + {scenario: OperationalScenarioCancellation, present: report.Coverage.CancellationReplay}, + {scenario: OperationalScenarioConcurrentWriter, present: report.Coverage.RepeatableReadWriter}, + {scenario: OperationalScenarioSessionIsolation, present: report.Coverage.SessionIsolation}, + {scenario: OperationalScenarioForcedOverflow, present: report.Coverage.ForcedOverflowFallback}, + } { + if !required.present { + report.Reasons = append(report.Reasons, fmt.Sprintf("required operational scenario %s is missing valid evidence", required.scenario)) + } + } + if len(report.Reasons) > 0 { + report.Passed = false + } + return report +} + +// canonicalOperationalGateInput creates an immutable, JSON-round-tripped copy +// of the exact evidence evaluated by the report and returns the digest used by +// final promotion verification. Evaluating the copy also ensures producer and +// verifier observe the same JSON number and timestamp representations. +func canonicalOperationalGateInput(identity PromotionEvidenceIdentity, requirements OperationalGateRequirements, records []OperationalEvidenceRecord) (OperationalGateInput, string, error) { + source := OperationalGateInput{ + Version: operationalGateVersion, + PromotionIdentity: cloneOperationalPromotionIdentity(identity), + Requirements: cloneOperationalRequirements(requirements), + Records: records, + } + raw, err := json.Marshal(source) + if err != nil { + return source, "", fmt.Errorf("encode: %w", err) + } + input, err := decodeOperationalGateInput(bytes.NewReader(raw)) + if err != nil { + return source, "", err + } + digest, err := operationalGateInputSHA256(input) + if err != nil { + return input, "", err + } + return input, digest, nil +} + +// operationalGateInputSHA256 hashes the canonical JSON representation of the +// complete embedded operational evidence document. +func operationalGateInputSHA256(input OperationalGateInput) (string, error) { + raw, err := json.Marshal(input) + if err != nil { + return "", fmt.Errorf("encode canonical operational input: %w", err) + } + digest := sha256.Sum256(raw) + return fmt.Sprintf("%x", digest), nil +} + +// validateRecomputedOperationalGateReport is the final-promotion trust +// boundary. It verifies the embedded source evidence, independently rebuilds +// the report, and compares every substantive decision with the serialized +// summary. A checksum-consistent but fabricated passing summary therefore +// cannot authorize promotion. +func validateRecomputedOperationalGateReport(report OperationalGateReport, expectedIdentity PromotionEvidenceIdentity) error { + if report.Version != operationalGateVersion { + return fmt.Errorf("operational report version must be %d", operationalGateVersion) + } + if !reflect.DeepEqual(report.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("operational report promotion identity does not match manifest") + } + if report.Input.Version != operationalGateVersion { + return fmt.Errorf("embedded operational input version must be %d", operationalGateVersion) + } + if !reflect.DeepEqual(report.Input.PromotionIdentity, expectedIdentity) || + !reflect.DeepEqual(report.Input.PromotionIdentity, report.PromotionIdentity) { + return fmt.Errorf("embedded operational input promotion identity does not match report and manifest") + } + if !reflect.DeepEqual(report.Input.Requirements, report.Requirements) { + return fmt.Errorf("embedded operational input requirements do not match report") + } + if !lowercaseSHA256(report.InputSHA256) { + return fmt.Errorf("operational report input_sha256 is not a canonical SHA-256 digest") + } + inputSHA256, err := operationalGateInputSHA256(report.Input) + if err != nil { + return err + } + if inputSHA256 != report.InputSHA256 { + return fmt.Errorf("operational report embedded input SHA-256 does not match") + } + + recomputed := buildOperationalGateReport(report.Input.PromotionIdentity, report.Input.Requirements, report.Input.Records) + if recomputed.InputSHA256 != report.InputSHA256 || !reflect.DeepEqual(recomputed.Input, report.Input) { + return fmt.Errorf("operational report embedded input is not canonical") + } + if !reflect.DeepEqual(report.Coverage, recomputed.Coverage) { + return fmt.Errorf("operational report coverage differs from recomputed input") + } + if !reflect.DeepEqual(report.Records, recomputed.Records) { + return fmt.Errorf("operational report record decisions differ from recomputed input") + } + if report.Passed != recomputed.Passed || !reflect.DeepEqual(report.Reasons, recomputed.Reasons) { + return fmt.Errorf("operational report passing disposition differs from recomputed input") + } + if !recomputed.Passed { + return fmt.Errorf("recomputed operational input did not pass: %s", strings.Join(recomputed.Reasons, "; ")) + } + return nil +} + +// loadOperationalGateInput strictly decodes one operational evidence +// document. Unknown fields and concatenated JSON are rejected so misspelled +// proof fields cannot be silently treated as absent evidence. +func loadOperationalGateInput(path string) (OperationalGateInput, error) { + var input OperationalGateInput + if strings.TrimSpace(path) == "" { + return input, fmt.Errorf("operational gate requires an explicit input path") + } + file, err := os.Open(path) + if err != nil { + return input, fmt.Errorf("read operational gate input: %w", err) + } + defer file.Close() + + return decodeOperationalGateInput(file) +} + +func decodeOperationalGateInput(reader io.Reader) (OperationalGateInput, error) { + var input OperationalGateInput + raw, err := io.ReadAll(reader) + if err != nil { + return OperationalGateInput{}, fmt.Errorf("read operational gate input: %w", err) + } + if err := rejectDuplicateJSONObjectKeys(raw); err != nil { + return OperationalGateInput{}, fmt.Errorf("decode operational gate input: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&input); err != nil { + return OperationalGateInput{}, fmt.Errorf("decode operational gate input: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return OperationalGateInput{}, fmt.Errorf("operational gate input contains trailing JSON data") + } + return OperationalGateInput{}, fmt.Errorf("decode trailing operational gate input: %w", err) + } + if input.Version != operationalGateVersion { + return OperationalGateInput{}, fmt.Errorf("operational gate input version must be %d, got %d", operationalGateVersion, input.Version) + } + return input, nil +} + +// createOperationalGateReport loads a strict evidence document, evaluates it, +// and writes the passing or failing machine-verifiable report. +func createOperationalGateReport(inputPath, outputPath string) (bool, error) { + input, err := loadOperationalGateInput(inputPath) + if err != nil { + return false, err + } + if err := validateOperationalGatePaths(inputPath, outputPath); err != nil { + return false, err + } + report := buildOperationalGateReport(input.PromotionIdentity, input.Requirements, input.Records) + if err := writeOperationalGateReport(outputPath, report); err != nil { + return false, err + } + return report.Passed, nil +} + +// validateOperationalGatePaths prevents report creation from replacing or +// aliasing the immutable source evidence document. +func validateOperationalGatePaths(inputPath, outputPath string) error { + if strings.TrimSpace(outputPath) == "" { + return nil + } + inputAbsolute, err := filepath.Abs(filepath.Clean(inputPath)) + if err != nil { + return fmt.Errorf("resolve operational gate input: %w", err) + } + if evaluated, err := filepath.EvalSymlinks(inputAbsolute); err == nil { + inputAbsolute = evaluated + } + outputAbsolute, err := filepath.Abs(filepath.Clean(outputPath)) + if err != nil { + return fmt.Errorf("resolve operational gate output: %w", err) + } + if evaluated, err := filepath.EvalSymlinks(outputAbsolute); err == nil { + outputAbsolute = evaluated + } else if evaluatedParent, parentErr := filepath.EvalSymlinks(filepath.Dir(outputAbsolute)); parentErr == nil { + outputAbsolute = filepath.Join(evaluatedParent, filepath.Base(outputAbsolute)) + } + if inputAbsolute == outputAbsolute { + return fmt.Errorf("operational gate input and output must use distinct paths") + } + inputInfo, inputErr := os.Stat(inputPath) + outputInfo, outputErr := os.Stat(outputPath) + if inputErr == nil && outputErr == nil && os.SameFile(inputInfo, outputInfo) { + return fmt.Errorf("operational gate input and output must not alias the same file") + } + if outputErr != nil && !os.IsNotExist(outputErr) { + return fmt.Errorf("inspect operational gate output: %w", outputErr) + } + return nil +} + +// writeOperationalGateReport writes a manifest-consumable operational report. +func writeOperationalGateReport(path string, report OperationalGateReport) (err error) { + if strings.TrimSpace(path) == "" { + return fmt.Errorf("operational gate requires an explicit report output path") + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +// validateOperationalIdentity rejects incomplete promotion bindings before any +// operational observation can be considered. +func validateOperationalIdentity(identity PromotionEvidenceIdentity) []string { + var reasons []string + for name, value := range map[string]string{ + "candidate": identity.Candidate, "selector_version": identity.SelectorVersion, + "execution_boundary": identity.ExecutionBoundary, "source_commit": identity.SourceCommit, + "fallback_executor": identity.FallbackExecutor, + } { + if strings.TrimSpace(value) == "" { + reasons = append(reasons, "promotion identity "+name+" is missing") + } + } + for name, value := range map[string]string{ + "source_sha256": identity.SourceSHA256, "binary_sha256": identity.BinarySHA256, "corpus_sha256": identity.CorpusSHA256, + } { + if !lowercaseSHA256(value) { + reasons = append(reasons, "promotion identity "+name+" is not a canonical SHA-256 digest") + } + } + if len(identity.Caps) == 0 { + reasons = append(reasons, "promotion identity caps are missing") + } + if len(identity.Buckets) == 0 { + reasons = append(reasons, "promotion identity buckets are missing") + } + sort.Strings(reasons) + return reasons +} + +// validateOperationalRequirements validates the report's frozen matrix declaration. +func validateOperationalRequirements(identity PromotionEvidenceIdentity, requirements OperationalGateRequirements) []string { + var reasons []string + expectedCandidate, supported := operationalCandidateRuntimeIdentity(identity.Candidate) + if !supported { + reasons = append(reasons, "promotion candidate has no registered operational runtime mapping") + } else if requirements.CandidateRuntimeIdentity != expectedCandidate { + reasons = append(reasons, "candidate runtime identity differs from the registered promotion candidate mapping") + } else if strings.TrimSpace(requirements.CandidateRuntimeIdentity) == "" { + reasons = append(reasons, "candidate runtime identity is missing") + } + if strings.TrimSpace(requirements.FallbackRuntimeIdentity) == "" { + reasons = append(reasons, "fallback runtime identity is missing") + } else if requirements.FallbackRuntimeIdentity != identity.FallbackExecutor { + reasons = append(reasons, "fallback runtime identity differs from promotion fallback executor") + } + if requirements.CancellationMaximum <= 0 || requirements.CancellationMaximum > 250*time.Millisecond { + reasons = append(reasons, "cancellation maximum must be positive and no greater than 250ms") + } + if requirements.LowWorkMemMaximumBytes <= 0 || requirements.LowWorkMemMaximumBytes > 64*1024 { + reasons = append(reasons, "low work_mem ceiling must be positive and no greater than 64kB") + } + if !requirements.RequireCleanSource { + reasons = append(reasons, "operational evidence must require a clean source tree") + } + if !lowercaseSHA256(identity.OperationalCandidateSQLSHA256) { + reasons = append(reasons, "promotion identity operational candidate SQL SHA-256 must be a canonical digest") + } + if !lowercaseSHA256(requirements.CandidateSQLFingerprint) { + reasons = append(reasons, "operational candidate SQL fingerprint must be a canonical SHA-256 digest") + } else if requirements.CandidateSQLFingerprint != identity.OperationalCandidateSQLSHA256 { + reasons = append(reasons, "operational candidate SQL fingerprint differs from the promotion identity anchor") + } + if !slices.Equal(requirements.PoolSizes, defaultOperationalPoolSizes) { + reasons = append(reasons, "operational pool-size matrix must be exactly 1,2,8") + } + if !slices.Equal(requirements.ConcurrencyLevels, defaultOperationalConcurrency) { + reasons = append(reasons, "operational concurrency matrix must be exactly 1,8,16") + } + if len(requirements.PlanCacheModes) != len(defaultOperationalPlanCacheModes) { + reasons = append(reasons, "operational plan-cache matrix is incomplete") + } else if !slices.Equal(requirements.PlanCacheModes, defaultOperationalPlanCacheModes) { + reasons = append(reasons, "operational plan-cache matrix must be exactly auto,force_custom_plan,force_generic_plan") + } + sort.Strings(reasons) + return reasons +} + +// validateOperationalRecordBinding enforces source, binary, corpus, and exact +// promotion identity on every independently captured record. +func validateOperationalRecordBinding(identity PromotionEvidenceIdentity, requirements OperationalGateRequirements, record OperationalEvidenceRecord) []string { + var reasons []string + if !reflect.DeepEqual(record.PromotionIdentity, identity) { + reasons = append(reasons, "record promotion identity does not match report") + } + if record.SourceSHA256 != identity.SourceSHA256 { + reasons = append(reasons, "record source archive does not match promotion identity") + } + result := record.Result + if result.ExecutionMode != ModePostgresSQL { + reasons = append(reasons, "operational record is not PostgreSQL SQL execution") + } + if result.Status != StatusOK { + reasons = append(reasons, "operational record status is not ok") + } + if strings.TrimSpace(result.Source) == "" || strings.TrimSpace(result.Dataset) == "" || strings.TrimSpace(result.Name) == "" || + strings.TrimSpace(result.Category) == "" || !lowercaseSHA256(result.WorkloadSHA256) { + reasons = append(reasons, "operational record lacks a bound workload identity") + } + reasons = append(reasons, validateOperationalAuthorizedWorkload(identity, requirements, record)...) + if !result.StableObservation { + reasons = append(reasons, "operational record lacks a stable observation") + } + if result.Environment == nil { + reasons = append(reasons, "run environment is missing") + } else { + if result.Environment.ArtifactSchemaVersion != 2 { + reasons = append(reasons, "operational evidence requires artifact schema v2") + } + if result.Environment.SourceCommit != identity.SourceCommit { + reasons = append(reasons, "run source commit does not match promotion identity") + } + if result.Environment.BinarySHA256 != identity.BinarySHA256 { + reasons = append(reasons, "run binary does not match promotion identity") + } + if result.Environment.CorpusSHA256 != identity.CorpusSHA256 { + reasons = append(reasons, "run corpus does not match promotion identity") + } + if requirements.RequireCleanSource && result.Environment.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + reasons = append(reasons, "operational evidence was captured from a dirty source tree") + } + } + if result.Fixture == nil || result.Fixture.Dataset != result.Dataset || !lowercaseSHA256(result.Fixture.Checksum) || + strings.TrimSpace(result.Fixture.Configuration) == "" || !result.Fixture.PhysicalValidated || + result.Fixture.NodeCount <= 0 || result.Fixture.EdgeCount <= 0 || + result.Fixture.PhysicalNodeCount != int64(result.Fixture.NodeCount) || + result.Fixture.PhysicalEdgeCount != int64(result.Fixture.EdgeCount) { + reasons = append(reasons, "operational record lacks one physically validated fixture identity") + } + if result.PostgresEnvironment == nil { + reasons = append(reasons, "PostgreSQL environment is missing") + } else if !strings.EqualFold(strings.TrimSpace(result.PostgresEnvironment.TransactionIsolation), "repeatable read") { + reasons = append(reasons, "operational evidence requires Repeatable Read") + } + return reasons +} + +// operationalCandidateRuntimeIdentity freezes the exact executor arm that an +// operational matrix must exercise for every promotable policy. A policy may +// emit a different identity from the executor it admits, but callers cannot +// choose that mapping in their evidence document. +func operationalCandidateRuntimeIdentity(candidate string) (string, bool) { + switch candidate { + case string(optimize.ShortestPathExecutorASPI1DAG), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorI2GuardedDistance): + return candidate, true + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), + string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return string(optimize.ExpansionSearchSuffixSeededReverse), true + default: + return "", false + } +} + +// validateOperationalAuthorizedWorkload binds every operational scenario to +// one exact manifest query bucket and to the SQL and workload shape actually +// measured by GraphBench. +func validateOperationalAuthorizedWorkload(identity PromotionEvidenceIdentity, requirements OperationalGateRequirements, record OperationalEvidenceRecord) []string { + var reasons []string + result := record.Result + cypherQuery := strings.TrimSpace(result.Cypher) + if cypherQuery == "" { + return []string{"operational record has no Cypher query to authorize"} + } + querySHA256 := pgdriver.TraversalPolicyQuerySHA256(cypherQuery) + var matches []PromotionBucket + for _, bucket := range identity.Buckets { + if slices.Contains(bucket.QuerySHA256, querySHA256) { + matches = append(matches, bucket) + } + } + if len(matches) != 1 { + return []string{fmt.Sprintf("operational query must match exactly one promotion bucket, matched %d", len(matches))} + } + bucket := matches[0] + shape := result.Shape + if !isOrientationProbePolicy(identity.Candidate) && + (shape.MinDepth == nil || shape.MaxDepth == nil || *shape.MinDepth != bucket.MinimumDepth || *shape.MaxDepth != bucket.MaximumDepth || + shape.Direction != bucket.Direction || shape.RelationshipKindCount != bucket.RelationshipKindCount || + len(shape.EdgeKinds) != shape.RelationshipKindCount || (len(shape.EdgeKinds) == 0) != bucket.UntypedRelationship) { + reasons = append(reasons, "operational workload shape differs from its authorized promotion bucket") + } + if !slices.Contains(bucket.QualificationSplit, shape.QualificationSplit) { + reasons = append(reasons, "operational workload split is not authorized by its promotion bucket") + } + if result.TraversalTelemetry == nil || result.TraversalTelemetry.Summary.ObservationMode != bucket.ObservationMode { + reasons = append(reasons, "operational observation mode differs from its authorized promotion bucket") + } + if strings.TrimSpace(result.SQL) == "" || !lowercaseSHA256(result.SQLFingerprint) || result.SQLFingerprint != sqlFingerprint(result.SQL) { + reasons = append(reasons, "operational SQL fingerprint is missing or does not bind the measured SQL") + } + if record.Scenario != OperationalScenarioForcedOverflow && result.SQLFingerprint != identity.OperationalCandidateSQLSHA256 { + reasons = append(reasons, "operational SQL fingerprint differs from the independently frozen production candidate SQL") + } + reasons = append(reasons, validateOperationalTranslationBinding(identity, bucket, record)...) + return reasons +} + +// validateOperationalTranslationBinding ties rendered SQL to the exact +// production-canary target carried by GraphBench's EXPLAIN translation. This +// prevents a self-consistent fingerprint over unrelated SQL from satisfying +// an authorized Cypher query. +func validateOperationalTranslationBinding(identity PromotionEvidenceIdentity, bucket PromotionBucket, record OperationalEvidenceRecord) []string { + result := record.Result + if result.Optimization == nil { + return []string{"operational record lacks optimization target evidence"} + } + outcome, ok := singleTraversalOutcome(result.Optimization.TargetOutcomes) + if !ok { + return []string{"operational record must contain one exact traversal optimization target"} + } + if outcome.TargetKind != "traversal" { + return []string{"operational optimization target is not a traversal"} + } + if isOrientationProbePolicy(identity.Candidate) { + return validateOperationalOrientationTarget(identity, bucket, record, outcome) + } + return validateOperationalShortestTarget(identity, bucket, record, outcome) +} + +func validateOperationalShortestTarget(identity PromotionEvidenceIdentity, bucket PromotionBucket, record OperationalEvidenceRecord, outcome translate.TargetLoweringOutcome) []string { + if outcome.Family != "SP" && outcome.Family != "ASP" { + return []string{"operational optimization target is not an authorized SP/ASP traversal"} + } + if outcome.MinimumDepth == nil || outcome.MaximumDepth == nil || *outcome.MinimumDepth != int64(bucket.MinimumDepth) || *outcome.MaximumDepth != int64(bucket.MaximumDepth) || + outcome.Direction != bucket.Direction || outcome.ObservationMode != bucket.ObservationMode || + outcome.RelationshipKindCount != bucket.RelationshipKindCount || outcome.UntypedRelationship != bucket.UntypedRelationship { + return []string{"operational optimization target differs from its authorized promotion bucket"} + } + if outcome.Candidate != identity.Candidate || outcome.Selected != identity.Candidate || outcome.Applied != identity.Candidate || + outcome.Fallback != identity.FallbackExecutor || outcome.SelectorVersion != identity.SelectorVersion || + outcome.ExecutionBoundary != identity.ExecutionBoundary || outcome.SelectionMode != "production_canary" || + outcome.EmittedPolicy != operationalCandidatePolicy(identity.Candidate) || + len(outcome.EmittedCandidates) != 2 || !slices.Contains(outcome.EmittedCandidates, identity.Candidate) || + !slices.Contains(outcome.EmittedCandidates, identity.FallbackExecutor) || + !slices.Contains(outcome.PlannedCandidates, identity.Candidate) || !slices.Contains(outcome.PlannedCandidates, identity.FallbackExecutor) || + outcome.Eligible == nil || !*outcome.Eligible || outcome.StaticallyEligible == nil || !*outcome.StaticallyEligible { + return []string{"operational optimization target does not prove the exact production candidate policy"} + } + if reasons := validateOperationalTargetCaps(identity, outcome, record.Scenario == OperationalScenarioForcedOverflow); len(reasons) != 0 { + return reasons + } + return nil +} + +func validateOperationalOrientationTarget(identity PromotionEvidenceIdentity, bucket PromotionBucket, record OperationalEvidenceRecord, outcome translate.TargetLoweringOutcome) []string { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + if outcome.Family != "fixed_suffix_expansion" || outcome.MinimumDepth == nil || outcome.MaximumDepth == nil || + *outcome.MinimumDepth != int64(bucket.MinimumDepth) || *outcome.MaximumDepth != int64(bucket.MaximumDepth) || + outcome.ObservationMode != bucket.ObservationMode || bucket.Direction != "outbound" || + bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship || !operationalEligibilityFact(outcome, "qualified_fixed_suffix_topology") { + return []string{"operational orientation target differs from its authorized promotion bucket"} + } + if outcome.Candidate != reverse || outcome.Selected != forward || outcome.Applied != forward || outcome.Fallback != forward || + outcome.EmittedPolicy != identity.Candidate || outcome.SelectorVersion != identity.SelectorVersion || + outcome.ExecutionBoundary != identity.ExecutionBoundary || outcome.SelectionMode != "production_canary" || + len(outcome.EmittedCandidates) != 2 || !slices.Contains(outcome.EmittedCandidates, reverse) || !slices.Contains(outcome.EmittedCandidates, forward) || + !slices.Contains(outcome.PlannedCandidates, reverse) || !slices.Contains(outcome.PlannedCandidates, forward) || + outcome.Eligible == nil || !*outcome.Eligible || outcome.StaticallyEligible == nil || !*outcome.StaticallyEligible { + return []string{"operational orientation target does not prove the exact production policy"} + } + forcedOverflow := record.Scenario == OperationalScenarioForcedOverflow + if outcome.ProbeCaps == nil || outcome.Admission == nil || !outcome.Admission.RequiresCompleteProbes || string(outcome.Admission.FallbackStrategy) != forward { + return []string{"operational orientation target lacks its complete bounded admission"} + } + actualCaps := map[string]int64{ + "root_row_limit": outcome.ProbeCaps.RootRowLimit, + "reverse_seed_row_limit": outcome.ProbeCaps.ReverseSeedRowLimit, + "directional_degree_row_limit": outcome.ProbeCaps.DirectionalDegreeRowLimit, + "state_limit": outcome.Admission.StateLimit, + } + for name, expected := range identity.Caps { + actual, found := actualCaps[name] + if !found || (!forcedOverflow && actual != expected) || (forcedOverflow && (actual <= 0 || actual > expected)) { + return []string{"operational orientation target cap differs from promotion identity: " + name} + } + } + if len(actualCaps) != len(identity.Caps) || outcome.StateLimit != outcome.Admission.StateLimit { + return []string{"operational orientation target contains an unauthorized cap contract"} + } + return nil +} + +func operationalEligibilityFact(outcome translate.TargetLoweringOutcome, name string) bool { + for _, fact := range outcome.EligibilityFacts { + if fact.Name == name { + return fact.Eligible + } + } + return false +} + +func operationalCandidatePolicy(candidate string) string { + switch candidate { + case string(optimize.ShortestPathExecutorASPI1DAG): + return optimize.ShortestPathPolicyASPI1GuardedV1 + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return optimize.ShortestPathPolicyI1CanonicalGuardedV1 + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return optimize.ShortestPathPolicyI2DistanceGuardedV1 + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return candidate + default: + return "" + } +} + +// validateOperationalTargetCaps accepts the manifest caps verbatim for every +// normal scenario. Forced overflow may lower positive caps to make overflow +// deterministic, but it may not change the target, policy, or add dimensions. +func validateOperationalTargetCaps(identity PromotionEvidenceIdentity, outcome translate.TargetLoweringOutcome, forcedOverflow bool) []string { + actual := map[string]int64{ + "state_limit": outcome.StateLimit, "frontier_limit": outcome.FrontierLimit, + "predecessor_limit": outcome.PredecessorLimit, "enumeration_limit": outcome.EnumerationLimit, + "output_bytes_limit": outcome.OutputBytesLimit, + } + for name, value := range actual { + expected, required := identity.Caps[name] + if !required { + if value != 0 { + return []string{"operational optimization target contains an unauthorized cap " + name} + } + continue + } + if forcedOverflow { + if value <= 0 || value > expected { + return []string{"forced-overflow optimization cap is not a positive bounded variant of " + name} + } + } else if value != expected { + return []string{"operational optimization target cap differs from promotion identity: " + name} + } + } + return nil +} + +// operationalTranslationTargetBinding excludes only the cap values that the +// forced-overflow scenario is explicitly allowed to reduce. +func operationalTranslationTargetBinding(result CaseResult) (string, error) { + if result.Optimization == nil { + return "", fmt.Errorf("optimization target evidence is missing") + } + outcome, ok := singleTraversalOutcome(result.Optimization.TargetOutcomes) + if !ok { + return "", fmt.Errorf("one exact traversal optimization target is required") + } + outcome.StateLimit = 0 + outcome.FrontierLimit = 0 + outcome.PredecessorLimit = 0 + outcome.EnumerationLimit = 0 + outcome.OutputBytesLimit = 0 + if outcome.ProbeCaps != nil { + probeCaps := *outcome.ProbeCaps + probeCaps.RootRowLimit = 0 + probeCaps.ReverseSeedRowLimit = 0 + probeCaps.DirectionalDegreeRowLimit = 0 + probeCaps.SurvivalRowLimit = 0 + outcome.ProbeCaps = &probeCaps + } + if outcome.Admission != nil { + admission := *outcome.Admission + admission.StateLimit = 0 + outcome.Admission = &admission + } + raw, err := json.Marshal(outcome) + if err != nil { + return "", fmt.Errorf("encode optimization target: %w", err) + } + return sqlFingerprint(string(raw)), nil +} + +// operationalWorkloadBinding hashes every logical and resolved workload input +// that must remain identical across scenarios. Rendered SQL is deliberately +// excluded because forced-overflow evidence may change only guarded cap +// literals; validateOperationalTranslationBinding independently proves that +// both SQL variants describe the same authorized translation target. +func operationalWorkloadBinding(result CaseResult) (string, error) { + var fixture any + if result.Fixture != nil { + fixture = struct { + Dataset string `json:"dataset"` + Checksum string `json:"checksum"` + NodeCount int `json:"node_count"` + EdgeCount int `json:"edge_count"` + PhysicalNodeCount int64 `json:"physical_node_count"` + PhysicalEdgeCount int64 `json:"physical_edge_count"` + Configuration string `json:"configuration"` + Shortest *ShortestFixtureExpectations `json:"shortest"` + FixedSuffixExpansion *FixedSuffixExpansionFixtureExpectations `json:"fixed_suffix_expansion"` + EndpointSeededExpansion *EndpointSeededExpansionFixtureExpectations `json:"endpoint_seeded_expansion"` + }{ + Dataset: result.Fixture.Dataset, Checksum: result.Fixture.Checksum, + NodeCount: result.Fixture.NodeCount, EdgeCount: result.Fixture.EdgeCount, + PhysicalNodeCount: result.Fixture.PhysicalNodeCount, PhysicalEdgeCount: result.Fixture.PhysicalEdgeCount, + Configuration: result.Fixture.Configuration, Shortest: result.Fixture.Shortest, + FixedSuffixExpansion: result.Fixture.FixedSuffixExpansion, + EndpointSeededExpansion: result.Fixture.EndpointSeededExpansion, + } + } + payload := struct { + Version int `json:"version"` + Source string `json:"source"` + Dataset string `json:"dataset"` + Name string `json:"name"` + WorkloadSHA256 string `json:"workload_sha256"` + QuerySHA256 string `json:"query_sha256"` + Params map[string]any `json:"params"` + NodeParams map[string]string `json:"node_params"` + NodeListParams map[string][]string `json:"node_list_params"` + Fixture any `json:"fixture"` + }{ + Version: 1, Source: result.Source, Dataset: result.Dataset, Name: result.Name, + WorkloadSHA256: result.WorkloadSHA256, + QuerySHA256: pgdriver.TraversalPolicyQuerySHA256(result.Cypher), + Params: result.Params, NodeParams: result.NodeParams, NodeListParams: result.NodeListParams, + Fixture: fixture, + } + raw, err := json.Marshal(payload) + if err != nil { + return "", fmt.Errorf("encode canonical workload: %w", err) + } + return sqlFingerprint(string(raw)), nil +} + +// validateOperationalMatrixCell rejects records outside the frozen Cartesian +// matrix instead of silently treating them as harmless extra evidence. +func validateOperationalMatrixCell(record OperationalEvidenceRecord, requirements OperationalGateRequirements) []string { + if record.Result.Environment == nil || record.Result.PostgresEnvironment == nil { + return nil + } + poolSize := record.Result.Environment.PoolSize + mode := normalizedPlanCacheMode(record.Result.PostgresEnvironment.PlanCacheMode) + if !slices.Contains(requirements.PoolSizes, poolSize) || + !slices.Contains(requirements.ConcurrencyLevels, record.Concurrency) || + !slices.Contains(requirements.PlanCacheModes, mode) { + return []string{fmt.Sprintf("candidate matrix record is outside the required matrix: pool_size=%d concurrency=%d plan_cache_mode=%s", poolSize, record.Concurrency, mode)} + } + return nil +} + +// validateOperationalConcurrencyBlock proves the declared matrix cell was +// actually executed and drained successfully rather than merely labeled. +func validateOperationalConcurrencyBlock(record OperationalEvidenceRecord) []string { + var reasons []string + if record.Result.Environment == nil || record.Concurrency <= 0 { + return []string{"candidate matrix lacks positive pool and concurrency settings"} + } + var matches []ConcurrencyBlock + for _, block := range record.Result.Concurrency { + if block.PoolSize == record.Result.Environment.PoolSize && block.Concurrency == record.Concurrency { + matches = append(matches, block) + } + } + if len(matches) != 1 { + return []string{fmt.Sprintf("candidate matrix requires exactly one matching concurrency block, found %d", len(matches))} + } + block := matches[0] + iterations := record.Result.Stats.Iterations + expectedOperations := record.Concurrency * iterations + if iterations <= 0 || block.Operations != expectedOperations || len(block.Samples) != block.Operations { + reasons = append(reasons, "concurrency block lacks a complete successful operation set") + } + if block.Wall <= 0 || block.QPS <= 0 { + reasons = append(reasons, "concurrency block lacks positive wall time or throughput") + } + workers := make(map[int]struct{}, record.Concurrency) + workerIterations := make(map[[2]int]struct{}, expectedOperations) + connections := make(map[string]struct{}) + coldConnections := make(map[string]int) + for _, sample := range block.Samples { + connectionID := strings.TrimSpace(sample.ConnectionID) + if connectionID == "" || sample.Total <= 0 || sample.ExecuteDrain <= 0 || sample.Total < sample.ExecuteDrain { + reasons = append(reasons, "concurrency sample lacks connection and execution evidence") + break + } + if pid, err := strconv.ParseUint(connectionID, 10, 32); err != nil || pid == 0 { + reasons = append(reasons, "concurrency sample connection is not a PostgreSQL backend PID") + break + } + if sample.Worker < 1 || sample.Worker > record.Concurrency { + reasons = append(reasons, "concurrency sample identifies a worker outside the declared range") + break + } + if sample.Iteration < 1 || sample.Iteration > iterations { + reasons = append(reasons, "concurrency sample identifies an iteration outside the measured range") + break + } + key := [2]int{sample.Worker, sample.Iteration} + if _, duplicate := workerIterations[key]; duplicate { + reasons = append(reasons, "concurrency block duplicates a worker iteration") + break + } + workerIterations[key] = struct{}{} + if sample.Classification != "cold-session" && sample.Classification != "warm-session" { + reasons = append(reasons, "concurrency sample has a non-producer session classification") + break + } + if sample.Classification == "cold-session" { + coldConnections[connectionID]++ + } + connections[connectionID] = struct{}{} + workers[sample.Worker] = struct{}{} + } + if len(workers) != record.Concurrency { + reasons = append(reasons, fmt.Sprintf("concurrency block exercised %d of %d declared workers", len(workers), record.Concurrency)) + } + if len(workerIterations) != expectedOperations { + reasons = append(reasons, fmt.Sprintf("concurrency block completed %d of %d worker iterations", len(workerIterations), expectedOperations)) + } + if len(connections) == 0 || len(connections) > record.Result.Environment.PoolSize || len(connections) > record.Concurrency { + reasons = append(reasons, "concurrency block connection usage exceeds its pool or worker bounds") + } + for connectionID := range connections { + if coldConnections[connectionID] != 1 { + reasons = append(reasons, "concurrency sample session classification contradicts connection reuse") + break + } + } + return reasons +} + +// validateOperationalCandidateResult validates admitted execution. Single-pool +// and exceptional records require exact per-invocation receipts. Larger-pool +// matrix records instead retain GraphBench's honest replay attribution and are +// proven by the independently validated concurrency block. +func validateOperationalCandidateResult(result CaseResult, identity PromotionEvidenceIdentity, requirements OperationalGateRequirements, requireTimedReceipts bool) []string { + var reasons []string + if result.TraversalTelemetry == nil { + return []string{"candidate traversal telemetry is missing"} + } + if err := ValidateTraversalExecutionTelemetry(result.TraversalTelemetry); err != nil { + reasons = append(reasons, "candidate traversal telemetry: "+err.Error()) + } + reasons = append(reasons, validateOperationalSPI2Attribution(result, identity)...) + summary := result.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable { + reasons = append(reasons, "candidate runtime outcome is unavailable") + } + if summary.RequestedIdentity != requirements.CandidateRuntimeIdentity || summary.RuntimeIdentity != requirements.CandidateRuntimeIdentity || summary.AppliedIdentity != requirements.CandidateRuntimeIdentity { + reasons = append(reasons, "candidate summary does not identify admitted candidate execution") + } + if summary.SelectorVersion != identity.SelectorVersion || summary.ExecutionBoundary != identity.ExecutionBoundary { + reasons = append(reasons, "candidate summary selector or execution boundary differs from promotion identity") + } + if identity.Candidate != requirements.CandidateRuntimeIdentity && summary.EmittedIdentity != identity.Candidate { + reasons = append(reasons, "candidate summary emitted policy differs from promotion identity") + } + if summary.FallbackExecuted == nil || *summary.FallbackExecuted { + reasons = append(reasons, "candidate summary executed or omitted fallback outcome") + } + if summary.Overflow == nil || *summary.Overflow { + reasons = append(reasons, "candidate summary overflow outcome is not false") + } + if strings.TrimSpace(summary.RuntimeBranch) == "" || summary.RuntimeBranch == "mixed" || summary.RuntimeBranch == "runtime_outcome_unavailable" { + reasons = append(reasons, "candidate runtime branch is unavailable or mixed") + } + warm := operationalWarmSamples(result) + if len(warm) == 0 { + reasons = append(reasons, "candidate record has no warm samples") + } + for _, sample := range warm { + if requireTimedReceipts { + reasons = append(reasons, validateOperationalCandidateSample(sample, result, requirements)...) + } else { + reasons = append(reasons, validateOperationalPooledCandidateSample(sample, result, requirements)...) + } + } + return reasons +} + +// validateOperationalPooledCandidateSample accepts only the producer's honest +// pool>1 replay metadata. A submitted timed receipt would falsely imply one +// session-local attestor covered a measurement that may use many sessions. +func validateOperationalPooledCandidateSample(sample LatencySample, result CaseResult, requirements OperationalGateRequirements) []string { + var reasons []string + if sample.RequestedIdentity != requirements.CandidateRuntimeIdentity || sample.RuntimeIdentity != requirements.CandidateRuntimeIdentity || + sample.FallbackExecuted == nil || *sample.FallbackExecuted { + reasons = append(reasons, "pooled candidate sample does not match admitted replay outcome") + } + if sample.RuntimeAttestation != "same_case_invocation_local_replay" || sample.RuntimeInvocationID != "" || + len(sample.RuntimeReceiptEvents) != 0 || sample.ConnectionID != "" { + reasons = append(reasons, "pooled candidate sample must retain non-attested GraphBench replay metadata") + } + if sample.Dataset != result.Dataset || sample.Case != result.Name || sample.Backend != ModePostgresSQL { + reasons = append(reasons, "pooled candidate sample workload identity differs from its record") + } + return reasons +} + +// validateOperationalFallbackResult validates exact forced-overflow selection, +// including nested fallback chains whose terminal executor follows the manifest fallback. +func validateOperationalFallbackResult(result CaseResult, identity PromotionEvidenceIdentity, requirements OperationalGateRequirements) []string { + var reasons []string + if result.TraversalTelemetry == nil { + return []string{"overflow traversal telemetry is missing"} + } + if err := ValidateTraversalExecutionTelemetry(result.TraversalTelemetry); err != nil { + reasons = append(reasons, "overflow traversal telemetry: "+err.Error()) + } + reasons = append(reasons, validateOperationalSPI2Attribution(result, identity)...) + summary := result.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable { + reasons = append(reasons, "overflow runtime outcome is unavailable") + } + if summary.RequestedIdentity != requirements.CandidateRuntimeIdentity || summary.RuntimeIdentity != requirements.FallbackRuntimeIdentity || summary.AppliedIdentity != requirements.FallbackRuntimeIdentity || summary.FallbackIdentity != requirements.FallbackRuntimeIdentity { + reasons = append(reasons, "overflow summary does not identify the exact configured fallback") + } + if summary.SelectorVersion != identity.SelectorVersion || summary.ExecutionBoundary != identity.ExecutionBoundary { + reasons = append(reasons, "overflow summary selector or execution boundary differs from promotion identity") + } + if identity.Candidate != requirements.CandidateRuntimeIdentity && summary.EmittedIdentity != identity.Candidate { + reasons = append(reasons, "overflow summary emitted policy differs from promotion identity") + } + if summary.FallbackExecuted == nil || !*summary.FallbackExecuted || summary.Overflow == nil || !*summary.Overflow { + reasons = append(reasons, "overflow summary lacks true overflow and fallback outcomes") + } + warm := operationalWarmSamples(result) + if len(warm) == 0 { + reasons = append(reasons, "overflow record has no warm timed samples") + } + for _, sample := range warm { + if sample.RequestedIdentity != requirements.CandidateRuntimeIdentity || sample.FallbackExecuted == nil || !*sample.FallbackExecuted || sample.RuntimeAttestation != "timed_invocation" || strings.TrimSpace(sample.RuntimeInvocationID) == "" { + reasons = append(reasons, "overflow warm sample lacks candidate request and fallback attribution") + continue + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + reasons = append(reasons, "overflow warm sample receipt chain: "+err.Error()) + continue + } + if reason := validateOperationalEventInvocation(sample); reason != "" { + reasons = append(reasons, reason) + } + if !receiptChainContainsIdentity(sample.RuntimeReceiptEvents, requirements.FallbackRuntimeIdentity, true) { + reasons = append(reasons, "overflow receipt chain does not contain the exact configured fallback") + } + } + return reasons +} + +// validateOperationalSPI2Attribution requires each SP-I2 operational record +// to carry the same exact diagnostic proof used by resource qualification. +// Summary-only identity claims cannot establish that the inactive statement +// arm stayed uninitialized or that the selected arm produced the public rows. +func validateOperationalSPI2Attribution(result CaseResult, identity PromotionEvidenceIdentity) []string { + if identity.Candidate != string(optimize.ShortestPathExecutorI2GuardedDistance) { + return nil + } + + telemetry := result.TraversalTelemetry + if telemetry == nil { + return []string{"SP-I2 operational attribution telemetry is missing"} + } + var reasons []string + if telemetry.Level != TraversalTelemetryLevelDiagnostic || telemetry.Diagnostic == nil { + reasons = append(reasons, "SP-I2 operational records require an untimed diagnostic replay") + } else if telemetry.Diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete { + reasons = append(reasons, "SP-I2 operational records require complete diagnostic counters") + } + + contract, _ := guardedInlineResourceContractForArchitecture(string(optimize.ShortestPathExecutorI2GuardedDistance)) + gateCase := &ResourceGateCase{} + appendGuardedInlineResourceBindingReasons(gateCase, result, contract) + appendInlineDistanceAttributionReasons(gateCase, telemetry) + for _, reason := range gateCase.Reasons { + reasons = append(reasons, "SP-I2 operational attribution: "+reason) + } + return reasons +} + +// validateOperationalCandidateSample validates one candidate receipt independently of its enclosing scenario. +func validateOperationalCandidateSample(sample LatencySample, result CaseResult, requirements OperationalGateRequirements) []string { + var reasons []string + if sample.RequestedIdentity != requirements.CandidateRuntimeIdentity || sample.RuntimeIdentity != requirements.CandidateRuntimeIdentity || sample.FallbackExecuted == nil || *sample.FallbackExecuted { + reasons = append(reasons, "candidate warm sample does not identify singular admitted execution") + } + if sample.RuntimeAttestation != "timed_invocation" || strings.TrimSpace(sample.RuntimeInvocationID) == "" || strings.TrimSpace(sample.ConnectionID) == "" { + reasons = append(reasons, "candidate warm sample lacks timed invocation and connection attribution") + } + if sample.Dataset != result.Dataset || sample.Case != result.Name || sample.Backend != ModePostgresSQL { + reasons = append(reasons, "candidate warm sample workload identity differs from its record") + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + reasons = append(reasons, "candidate warm sample receipt chain: "+err.Error()) + } else if reason := validateOperationalEventInvocation(sample); reason != "" { + reasons = append(reasons, reason) + } + return reasons +} + +// validateOperationalCancellation validates timeout, rollback, same-PID reuse, +// and the candidate receipt emitted by the successful replay. +func validateOperationalCancellation(evidence *OperationalCancellationEvidence, result CaseResult, requirements OperationalGateRequirements) []string { + if evidence == nil { + return []string{"cancellation evidence is missing"} + } + var reasons []string + if evidence.SQLState != "57014" { + reasons = append(reasons, "cancellation did not report PostgreSQL SQLSTATE 57014") + } + if evidence.Latency <= 0 || evidence.Latency >= requirements.CancellationMaximum { + reasons = append(reasons, fmt.Sprintf("cancellation latency must be positive and below %s", requirements.CancellationMaximum)) + } + if !evidence.TransactionRolledBack { + reasons = append(reasons, "cancelled transaction was not rolled back") + } + if evidence.CancelledBackendPID == 0 || evidence.CancelledBackendPID != evidence.ReplayBackendPID { + reasons = append(reasons, "post-rollback replay did not reuse the cancelled backend PID") + } + if !evidence.ReplaySucceeded { + reasons = append(reasons, "post-rollback replay did not succeed") + } + if evidence.ReplayCandidateReceipt.ConnectionID != strconv.FormatUint(uint64(evidence.ReplayBackendPID), 10) { + reasons = append(reasons, "post-rollback replay receipt is not bound to the reused backend PID") + } + reasons = append(reasons, validateOperationalCandidateSample(evidence.ReplayCandidateReceipt, result, requirements)...) + return reasons +} + +// validateOperationalSnapshot validates a stable Repeatable Read observation while a distinct writer commits. +func validateOperationalSnapshot(evidence *OperationalSnapshotEvidence) []string { + if evidence == nil { + return []string{"concurrent-writer snapshot evidence is missing"} + } + var reasons []string + if !strings.EqualFold(strings.TrimSpace(evidence.ReaderIsolation), "repeatable read") { + reasons = append(reasons, "concurrent-writer reader did not use Repeatable Read") + } + if evidence.ReaderBackendPID == 0 || evidence.WriterBackendPID == 0 || evidence.ReaderBackendPID == evidence.WriterBackendPID { + reasons = append(reasons, "concurrent writer was not a distinct PostgreSQL backend") + } + if !evidence.WriterCommitted { + reasons = append(reasons, "concurrent writer did not commit") + } + if evidence.WriterAffectedRows <= 0 { + reasons = append(reasons, "concurrent writer did not affect any rows") + } + if !lowercaseSHA256(evidence.ObservationBeforeSHA256) || evidence.ObservationBeforeSHA256 != evidence.ObservationAfterSHA256 { + reasons = append(reasons, "reader observation changed across the concurrent commit") + } + if !lowercaseSHA256(evidence.PostCommitObservationSHA256) || evidence.PostCommitObservationSHA256 == evidence.ObservationBeforeSHA256 { + reasons = append(reasons, "post-transaction observation does not prove the concurrent writer changed visible state") + } + return reasons +} + +// validateOperationalSessionIsolation validates independent invocation IDs, +// distinct sessions, own-row visibility, and zero cross-session visibility. +func validateOperationalSessionIsolation(evidence *OperationalSessionIsolationEvidence, result CaseResult, requirements OperationalGateRequirements) []string { + if evidence == nil { + return []string{"session-isolation evidence is missing"} + } + var reasons []string + if evidence.SessionABackendPID == 0 || evidence.SessionBBackendPID == 0 || evidence.SessionABackendPID == evidence.SessionBBackendPID { + reasons = append(reasons, "session-isolation evidence does not use distinct PostgreSQL backends") + } + if strings.TrimSpace(evidence.SessionAInvocationID) == "" || strings.TrimSpace(evidence.SessionBInvocationID) == "" || evidence.SessionAInvocationID == evidence.SessionBInvocationID { + reasons = append(reasons, "session-isolation evidence lacks distinct invocation IDs") + } + if evidence.SessionAOwnRows <= 0 || evidence.SessionBOwnRows <= 0 || evidence.SessionAObservedBRows != 0 || evidence.SessionBObservedARows != 0 { + reasons = append(reasons, "session-local evidence contains missing own rows or cross-session rows") + } + for _, receipt := range []struct { + name string + pid uint32 + invocation string + sample LatencySample + }{ + {name: "session A", pid: evidence.SessionABackendPID, invocation: evidence.SessionAInvocationID, sample: evidence.SessionACandidateReceipt}, + {name: "session B", pid: evidence.SessionBBackendPID, invocation: evidence.SessionBInvocationID, sample: evidence.SessionBCandidateReceipt}, + } { + if receipt.sample.ConnectionID != strconv.FormatUint(uint64(receipt.pid), 10) || receipt.sample.RuntimeInvocationID != receipt.invocation { + reasons = append(reasons, receipt.name+" receipt does not match its backend and invocation") + } + reasons = append(reasons, validateOperationalCandidateSample(receipt.sample, result, requirements)...) + } + return reasons +} + +// operationalWarmSamples returns only timed warm samples used for runtime attribution. +func operationalWarmSamples(result CaseResult) []LatencySample { + var samples []LatencySample + for _, sample := range result.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + samples = append(samples, sample) + } + } + return samples +} + +// validateOperationalEventInvocation binds every receipt event to the timed invocation. +func validateOperationalEventInvocation(sample LatencySample) string { + for _, event := range sample.RuntimeReceiptEvents { + if event.InvocationID != sample.RuntimeInvocationID { + return "runtime receipt event is not bound to its timed invocation" + } + } + return "" +} + +// receiptChainContainsIdentity reports whether a fallback identity appears in an ordered receipt chain. +func receiptChainContainsIdentity(events []RuntimeReceiptEvent, identity string, fallback bool) bool { + for _, event := range events { + if event.RuntimeIdentity == identity && event.FallbackExecuted == fallback { + return true + } + } + return false +} + +// normalizedPlanCacheMode returns the canonical PostgreSQL mode spelling. +func normalizedPlanCacheMode(value string) string { + return strings.ToLower(strings.TrimSpace(value)) +} + +// parsePostgresMemoryBytes parses the integral PostgreSQL memory-setting forms +// emitted by current_setting, including the server-minimum 64kB work_mem. +func parsePostgresMemoryBytes(value string) (int64, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return 0, fmt.Errorf("setting is empty") + } + index := 0 + for index < len(trimmed) && trimmed[index] >= '0' && trimmed[index] <= '9' { + index++ + } + if index == 0 { + return 0, fmt.Errorf("setting %q has no integral value", value) + } + amount, err := strconv.ParseInt(trimmed[:index], 10, 64) + if err != nil || amount <= 0 { + return 0, fmt.Errorf("setting %q has an invalid value", value) + } + unit := strings.ToLower(strings.TrimSpace(trimmed[index:])) + multiplier := int64(1) + switch unit { + case "", "b": + case "kb": + multiplier = 1024 + case "mb": + multiplier = 1024 * 1024 + case "gb": + multiplier = 1024 * 1024 * 1024 + default: + return 0, fmt.Errorf("setting %q has an unsupported unit", value) + } + if amount > (1<<63-1)/multiplier { + return 0, fmt.Errorf("setting %q overflows bytes", value) + } + return amount * multiplier, nil +} + +// sameOperationalDatabase compares server and schema identity while allowing +// plan_cache_mode and work_mem to vary across required matrix cells. +func sameOperationalDatabase(left, right *PostgresEnvironment) bool { + return left.Version == right.Version && left.Database == right.Database && + left.TempFileLimit == right.TempFileLimit && left.GraphPartitionCount == right.GraphPartitionCount && + left.PostmasterStartedAt.Equal(right.PostmasterStartedAt) && left.DatabaseOID == right.DatabaseOID && + left.Autovacuum == right.Autovacuum && left.SchemaFingerprint == right.SchemaFingerprint && + left.IndexFingerprint == right.IndexFingerprint +} + +// cloneOperationalPromotionIdentity prevents callers from mutating a completed report through shared maps or slices. +func cloneOperationalPromotionIdentity(identity PromotionEvidenceIdentity) PromotionEvidenceIdentity { + identity.Caps = clonePromotionCaps(identity.Caps) + identity.Buckets = clonePromotionBuckets(identity.Buckets) + return identity +} + +// cloneOperationalRequirements prevents callers from mutating a completed report through shared slices. +func cloneOperationalRequirements(requirements OperationalGateRequirements) OperationalGateRequirements { + requirements.PoolSizes = append([]int(nil), requirements.PoolSizes...) + requirements.ConcurrencyLevels = append([]int(nil), requirements.ConcurrencyLevels...) + requirements.PlanCacheModes = append([]string(nil), requirements.PlanCacheModes...) + return requirements +} diff --git a/cmd/graphbench/operational_gate_test.go b/cmd/graphbench/operational_gate_test.go new file mode 100644 index 00000000..0046d5fb --- /dev/null +++ b/cmd/graphbench/operational_gate_test.go @@ -0,0 +1,1588 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + pgdriver "github.com/specterops/dawgs/drivers/pg" + "github.com/stretchr/testify/require" +) + +const ( + operationalTestCandidate = "SP-I2-C-D" + operationalTestFallback = "SP-S4-C-D" + operationalTestTerminal = "SP-S3-U-E+MAT-M0" + operationalTestCypher = "MATCH p = shortestPath((r)<-[:Traverse*1..32]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)" + operationalTestSQL = "select 1::int8 as distance" + operationalTestOrientationCypher = "MATCH (r)-[:Expand*0..16]->()-[:EnterSuffix]->()-[:ContinueSuffix]->()-[:CompleteSuffix]->(e) WHERE id(r) = $root_id RETURN id(e)" + + operationalMainHelper = "GRAPHBENCH_OPERATIONAL_GATE_MAIN_HELPER" + operationalMainHelperInput = "GRAPHBENCH_OPERATIONAL_GATE_MAIN_INPUT" + operationalMainHelperOutput = "GRAPHBENCH_OPERATIONAL_GATE_MAIN_OUTPUT" +) + +func passingPromotionOperationalReport(t *testing.T, identity PromotionEvidenceIdentity) OperationalGateReport { + t.Helper() + requirements := defaultOperationalGateRequirements(identity.Candidate, identity.FallbackExecutor) + if runtimeIdentity, supported := operationalCandidateRuntimeIdentity(identity.Candidate); supported { + requirements.CandidateRuntimeIdentity = runtimeIdentity + } + requirements.CandidateSQLFingerprint = identity.OperationalCandidateSQLSHA256 + records := operationalTestEvidence(identity) + for index := range records { + operationalTestBindRecordToIdentity(&records[index], identity) + } + return buildOperationalGateReport(identity, requirements, records) +} + +func operationalTestRequirements() OperationalGateRequirements { + requirements := defaultOperationalGateRequirements(operationalTestCandidate, operationalTestFallback) + requirements.CandidateSQLFingerprint = operationalTestPromotionIdentity().OperationalCandidateSQLSHA256 + return requirements +} + +// TestParseConfigAcceptsOperationalGateMode verifies the strict input/output +// pair selects a standalone operational-report mode. +func TestParseConfigAcceptsOperationalGateMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-operational-gate-input", "operational-input.json", + "-operational-gate-output", "operational-report.json", + }, func(string) string { return "" }) + + require.NoError(t, err) + require.Equal(t, "operational-input.json", cfg.OperationalGateInput) + require.Equal(t, "operational-report.json", cfg.OperationalGateOutput) +} + +// TestParseConfigRejectsIncompleteOrMixedOperationalGate verifies neither +// half of the file contract nor another standalone report mode can be ignored. +func TestParseConfigRejectsIncompleteOrMixedOperationalGate(t *testing.T) { + complete := []string{ + "-operational-gate-input", "operational-input.json", + "-operational-gate-output", "operational-report.json", + } + for _, test := range []struct { + args []string + reason string + }{ + {args: []string{"-operational-gate-input", "operational-input.json"}, reason: "requires operational-gate-input and operational-gate-output"}, + {args: []string{"-operational-gate-output", "operational-report.json"}, reason: "requires operational-gate-input and operational-gate-output"}, + {args: append(append([]string(nil), complete...), "-resource-artifact", "resource.jsonl"), reason: "mutually exclusive"}, + {args: append(append([]string(nil), complete...), "-promotion-manifest", "promotion.json"), reason: "mutually exclusive"}, + } { + _, err := parseConfig(test.args, func(string) string { return "" }) + require.ErrorContains(t, err, test.reason, test.args) + } +} + +// TestOperationalGateMainFailsClosed runs the real main dispatch in a child +// test process. The failing report must be persisted before main exits one. +func TestOperationalGateMainFailsClosed(t *testing.T) { + if os.Getenv(operationalMainHelper) == "1" { + os.Args = []string{ + "graphbench", + "-operational-gate-input", os.Getenv(operationalMainHelperInput), + "-operational-gate-output", os.Getenv(operationalMainHelperOutput), + } + main() + return + } + + identity := operationalTestPromotionIdentity() + records := operationalTestEvidence(identity) + records[0].Result.Environment.BinarySHA256 = strings.Repeat("f", 64) + input := OperationalGateInput{ + Version: operationalGateVersion, + PromotionIdentity: identity, + Requirements: operationalTestRequirements(), + Records: records, + } + directory := t.TempDir() + inputPath := filepath.Join(directory, "input.json") + outputPath := filepath.Join(directory, "report.json") + operationalTestWriteJSON(t, inputPath, input) + + command := exec.Command(os.Args[0], "-test.run=^TestOperationalGateMainFailsClosed$") + command.Env = append(os.Environ(), + operationalMainHelper+"=1", + operationalMainHelperInput+"="+inputPath, + operationalMainHelperOutput+"="+outputPath, + ) + output, err := command.CombinedOutput() + var exitError *exec.ExitError + require.ErrorAs(t, err, &exitError, string(output)) + require.Equal(t, 1, exitError.ExitCode(), string(output)) + require.Contains(t, string(output), "operational gate failed") + + raw, err := os.ReadFile(outputPath) + require.NoError(t, err) + var report OperationalGateReport + require.NoError(t, json.Unmarshal(raw, &report)) + require.False(t, report.Passed) + require.True(t, operationalTestReportContains(report, "run binary does not match promotion identity")) +} + +// TestOperationalGateAcceptsCompleteCandidateBoundEvidence verifies the full +// promotion matrix and every independent operational proof serialize as a +// manifest-consumable passing report. +func TestOperationalGateAcceptsCompleteCandidateBoundEvidence(t *testing.T) { + identity := operationalTestPromotionIdentity() + requirements := operationalTestRequirements() + report := buildOperationalGateReport(identity, requirements, operationalTestEvidence(identity)) + + require.True(t, report.Passed, "global=%v records=%v", report.Reasons, report.Records) + require.Empty(t, report.Reasons) + require.Equal(t, 27, report.Coverage.RequiredMatrixCells) + require.Equal(t, 27, report.Coverage.ObservedMatrixCells) + require.Empty(t, report.Coverage.MissingMatrixCells) + require.True(t, report.Coverage.LowWorkMem) + require.True(t, report.Coverage.CancellationReplay) + require.True(t, report.Coverage.RepeatableReadWriter) + require.True(t, report.Coverage.SessionIsolation) + require.True(t, report.Coverage.ForcedOverflowFallback) + require.Len(t, report.Records, 32) + require.Equal(t, operationalGateVersion, report.Input.Version) + require.Equal(t, identity, report.Input.PromotionIdentity) + require.Equal(t, requirements, report.Input.Requirements) + require.Len(t, report.Input.Records, 32) + require.True(t, lowercaseSHA256(report.InputSHA256)) + require.NoError(t, validateRecomputedOperationalGateReport(report, identity)) + for _, record := range report.Records { + require.True(t, record.Passed, "%s: %v", record.ID, record.Reasons) + } + + path := filepath.Join(t.TempDir(), "operational.json") + require.NoError(t, writeOperationalGateReport(path, report)) + raw, err := os.ReadFile(path) + require.NoError(t, err) + var document map[string]any + require.NoError(t, json.Unmarshal(raw, &document)) + require.Equal(t, true, document["passed"]) + require.Contains(t, document, "promotion_identity") + encodedIdentity, err := json.Marshal(document["promotion_identity"]) + require.NoError(t, err) + var decoded PromotionEvidenceIdentity + require.NoError(t, json.Unmarshal(encodedIdentity, &decoded)) + require.Equal(t, identity, decoded) +} + +// TestOperationalGateRejectsSPI2PlanAttributionTampering proves every SP-I2 +// operational record must independently bind its typed counters, exact named +// plan branches, inactive executor arm, and public output cardinality. +func TestOperationalGateRejectsSPI2PlanAttributionTampering(t *testing.T) { + identity := operationalTestPromotionIdentity() + tests := []struct { + name string + mutate func([]OperationalEvidenceRecord) + reason string + }{ + { + name: "summary-only pooled candidate", + mutate: func(records []OperationalEvidenceRecord) { + records[9].Result.TraversalTelemetry.Level = TraversalTelemetryLevelSummary + records[9].Result.TraversalTelemetry.Diagnostic = nil + }, + reason: "SP-I2 operational records require an untimed diagnostic replay", + }, + { + name: "missing candidate marker", + mutate: func(records []OperationalEvidenceRecord) { + delete(records[0].Result.TraversalTelemetry.Diagnostic.PlanReplay.Counters, "sp_i2_candidate_marker_rows") + }, + reason: "missing exact plan counter sp_i2_candidate_marker_rows", + }, + { + name: "dual markers", + mutate: func(records []OperationalEvidenceRecord) { + telemetry := records[0].Result.TraversalTelemetry + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_fallback_marker_rows"] = 1 + value := int64(1) + telemetry.Diagnostic.Counters.InlineShortestDistance.FallbackMarkerRows = &value + }, + reason: "must attribute exactly one candidate or fallback marker", + }, + { + name: "candidate initializes fallback executor", + mutate: func(records []OperationalEvidenceRecord) { + telemetry := records[0].Result.TraversalTelemetry + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_fallback_executor_loops"] = 1 + value := int64(1) + telemetry.Diagnostic.Counters.InlineShortestDistance.FallbackExecutorLoops = &value + }, + reason: "candidate selection did not suppress the fallback executor and output arm", + }, + { + name: "candidate claims admission at cap plus one", + mutate: func(records []OperationalEvidenceRecord) { + telemetry := records[0].Result.TraversalTelemetry + value := telemetry.Summary.Caps["state_rows"] + 1 + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_distance_rows"] = value + telemetry.Diagnostic.Counters.InlineShortestDistance.StateRows = &value + telemetry.Diagnostic.Counters.InlineShortestDistance.FrontierRows = &value + }, + reason: "candidate selection exceeds its state or conservative frontier cap", + }, + { + name: "typed plan drift", + mutate: func(records []OperationalEvidenceRecord) { + value := int64(3) + records[0].Result.TraversalTelemetry.Diagnostic.Counters.InlineShortestDistance.StateRows = &value + }, + reason: "typed counter does not match plan counter sp_i2_distance_rows", + }, + { + name: "candidate target drift", + mutate: func(records []OperationalEvidenceRecord) { + records[0].Result.TraversalTelemetry.Diagnostic.PlanReplay.Counters["sp_i2_target_rows"] = 0 + }, + reason: "candidate branch does not agree with its target receipt", + }, + { + name: "public output drift", + mutate: func(records []OperationalEvidenceRecord) { + records[0].Result.RowCount = 2 + }, + reason: "typed output does not match the exact public observation", + }, + { + name: "fallback initializes candidate executor", + mutate: func(records []OperationalEvidenceRecord) { + record := operationalTestScenario(records, OperationalScenarioForcedOverflow) + telemetry := record.Result.TraversalTelemetry + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_candidate_executor_loops"] = 1 + value := int64(1) + telemetry.Diagnostic.Counters.InlineShortestDistance.CandidateExecutorLoops = &value + }, + reason: "fallback selection did not suppress the candidate executor and output arm", + }, + { + name: "fallback lacks cap plus one sentinel", + mutate: func(records []OperationalEvidenceRecord) { + record := operationalTestScenario(records, OperationalScenarioForcedOverflow) + telemetry := record.Result.TraversalTelemetry + value := telemetry.Summary.Caps["state_rows"] + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_distance_rows"] = value + telemetry.Diagnostic.Counters.InlineShortestDistance.StateRows = &value + telemetry.Diagnostic.Counters.InlineShortestDistance.FrontierRows = &value + }, + reason: "fallback selection lacks an exact state or conservative frontier cap+1 sentinel", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + records := operationalTestEvidence(identity) + test.mutate(records) + report := buildOperationalGateReport(identity, operationalTestRequirements(), records) + require.False(t, report.Passed) + require.True(t, operationalTestReportContains(report, test.reason), "global=%v records=%v", report.Reasons, report.Records) + }) + } +} + +// TestValidateRecomputedOperationalGateReportRejectsTampering proves final +// promotion derives its decision from the embedded raw evidence. Each raw +// mutation refreshes the input digest to model an attacker who also rewrites +// that shallow checksum; the unchanged passing summary must still be rejected. +func TestValidateRecomputedOperationalGateReportRejectsTampering(t *testing.T) { + identity := operationalTestPromotionIdentity() + passing := buildOperationalGateReport(identity, operationalTestRequirements(), operationalTestEvidence(identity)) + require.True(t, passing.Passed, "global=%v records=%v", passing.Reasons, passing.Records) + + tests := []struct { + name string + mutate func(*OperationalGateReport) + }{ + {name: "sql", mutate: func(report *OperationalGateReport) { + report.Input.Records[0].Result.SQL = "select 2::int8 as distance" + report.Input.Records[0].Result.SQLFingerprint = sqlFingerprint(report.Input.Records[0].Result.SQL) + }}, + {name: "optimization", mutate: func(report *OperationalGateReport) { + report.Input.Records[0].Result.Optimization.TargetOutcomes[0].Applied = operationalTestFallback + }}, + {name: "receipt", mutate: func(report *OperationalGateReport) { + report.Input.Records[0].Result.Stats.Samples[0].RuntimeReceiptEvents[0].RuntimeIdentity = operationalTestFallback + }}, + {name: "cancellation", mutate: func(report *OperationalGateReport) { + operationalTestScenario(report.Input.Records, OperationalScenarioCancellation).Cancellation.ReplaySucceeded = false + }}, + {name: "snapshot", mutate: func(report *OperationalGateReport) { + operationalTestScenario(report.Input.Records, OperationalScenarioConcurrentWriter).Snapshot.ObservationAfterSHA256 = strings.Repeat("0", 64) + }}, + {name: "session isolation", mutate: func(report *OperationalGateReport) { + operationalTestScenario(report.Input.Records, OperationalScenarioSessionIsolation).SessionIsolation.SessionAObservedBRows = 1 + }}, + {name: "raw source binding", mutate: func(report *OperationalGateReport) { + report.Input.Records[0].SourceSHA256 = strings.Repeat("0", 64) + }}, + {name: "embedded identity", mutate: func(report *OperationalGateReport) { + report.Input.PromotionIdentity.BinarySHA256 = strings.Repeat("0", 64) + }}, + {name: "embedded requirements", mutate: func(report *OperationalGateReport) { + report.Input.Requirements.CancellationMaximum = time.Second + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + report := operationalTestCloneReport(t, passing) + test.mutate(&report) + var err error + report.InputSHA256, err = operationalGateInputSHA256(report.Input) + require.NoError(t, err) + raw, err := json.Marshal(report) + require.NoError(t, err) + require.Error(t, validatePromotionOperationalReport(raw, identity)) + }) + } +} + +// TestValidateRecomputedOperationalGateReportRejectsSummaryForgery verifies +// raw evidence cannot be paired with edited coverage, decisions, or a passing +// disposition, even when the embedded-input digest remains valid. +func TestValidateRecomputedOperationalGateReportRejectsSummaryForgery(t *testing.T) { + identity := operationalTestPromotionIdentity() + passing := buildOperationalGateReport(identity, operationalTestRequirements(), operationalTestEvidence(identity)) + tests := []struct { + name string + mutate func(*OperationalGateReport) + reason string + }{ + {name: "coverage", mutate: func(report *OperationalGateReport) { report.Coverage.ObservedMatrixCells-- }, reason: "coverage differs"}, + {name: "record decision", mutate: func(report *OperationalGateReport) { report.Records[0].Passed = false }, reason: "record decisions differ"}, + {name: "passed", mutate: func(report *OperationalGateReport) { report.Passed = false }, reason: "passing disposition differs"}, + {name: "reasons", mutate: func(report *OperationalGateReport) { report.Reasons = []string{"forged"} }, reason: "passing disposition differs"}, + {name: "input digest", mutate: func(report *OperationalGateReport) { report.InputSHA256 = strings.Repeat("0", 64) }, reason: "SHA-256 does not match"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + report := operationalTestCloneReport(t, passing) + test.mutate(&report) + raw, err := json.Marshal(report) + require.NoError(t, err) + require.ErrorContains(t, validatePromotionOperationalReport(raw, identity), test.reason) + }) + } +} + +// TestValidatePromotionOperationalReportRejectsAmbiguousEmbeddedEvidence +// keeps strict decoding at the final manifest boundary, including nested raw +// input fields and concatenated JSON documents. +func TestValidatePromotionOperationalReportRejectsAmbiguousEmbeddedEvidence(t *testing.T) { + identity := operationalTestPromotionIdentity() + report := buildOperationalGateReport(identity, operationalTestRequirements(), operationalTestEvidence(identity)) + raw, err := json.Marshal(report) + require.NoError(t, err) + + var document map[string]any + require.NoError(t, json.Unmarshal(raw, &document)) + document["input"].(map[string]any)["unexpected_raw_proof"] = true + unknown, err := json.Marshal(document) + require.NoError(t, err) + require.ErrorContains(t, validatePromotionOperationalReport(unknown, identity), "unknown field") + require.ErrorContains(t, validatePromotionOperationalReport(append(raw, []byte(`{}`)...), identity), "trailing JSON data") +} + +// TestOperationalGateAcceptsProducerPhysicalSizeVariation verifies relation +// allocation diagnostics may vary between independent captures without +// changing the canonical logical fixture binding. +func TestOperationalGateAcceptsProducerPhysicalSizeVariation(t *testing.T) { + identity := operationalTestPromotionIdentity() + records := operationalTestEvidence(identity) + for index := range records { + records[index].Result.Fixture.NodeRelationBytes = int64(4096 + index*8192) + records[index].Result.Fixture.EdgeRelationBytes = int64(8192 + index*16384) + } + report := buildOperationalGateReport(identity, operationalTestRequirements(), records) + require.True(t, report.Passed, "global=%v records=%v", report.Reasons, report.Records) +} + +// TestOperationalGateAuthorizesOrientationByTarget verifies a real +// fixed-suffix outer shape is not mistaken for the single variable-expansion +// target authorized by an orientation bucket. +func TestOperationalGateAuthorizesOrientationByTarget(t *testing.T) { + query := "MATCH (r)-[:Expand*0..16]->()-[:EnterSuffix]->()-[:ContinueSuffix]->()-[:CompleteSuffix]->(e) WHERE id(r) = $root_id RETURN id(e)" + digest := strings.Repeat("a", 64) + policy := string(optimize.ExpansionSearchPolicyOrientationProbeV2) + identity := PromotionEvidenceIdentity{ + Candidate: policy, SelectorVersion: policy, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: strings.Repeat("b", 64), CorpusSHA256: strings.Repeat("c", 64), + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "fixed-suffix", QuerySHA256: []string{pgdriver.TraversalPolicyQuerySHA256(query)}, + Direction: "outbound", ObservationMode: "endpoint_ids", MinimumDepth: 0, MaximumDepth: 16, + RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + record := operationalTestRecord(operationalTestPromotionIdentity(), "orientation", OperationalScenarioCandidateMatrix, "auto", "4MB", 1) + minimumDepth, maximumDepth := 0, 16 + record.Result.Cypher = query + record.Result.Shape = WorkloadShape{ + QualificationSplit: "training", EdgeKinds: []string{"Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"}, + MinDepth: &minimumDepth, MaxDepth: &maximumDepth, + } + record.Result.TraversalTelemetry.Summary.ObservationMode = "endpoint_ids" + eligible := true + minimumTargetDepth, maximumTargetDepth := int64(0), int64(16) + record.Result.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Lowering: optimize.LoweringExpansionSearchStrategy, TargetKind: "traversal", Family: "fixed_suffix_expansion", + Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), Selected: string(optimize.ExpansionSearchStepwiseForward), + Applied: string(optimize.ExpansionSearchStepwiseForward), Fallback: string(optimize.ExpansionSearchStepwiseForward), + PlannedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedPolicy: policy, SelectorVersion: policy, ExecutionBoundary: "guarded_dual_arm", SelectionMode: "production_canary", + ObservationMode: "endpoint_ids", MinimumDepth: &minimumTargetDepth, MaximumDepth: &maximumTargetDepth, + Eligible: &eligible, StaticallyEligible: &eligible, + EligibilityFacts: []translate.TargetEligibilityFact{{Name: "qualified_fixed_suffix_topology", Eligible: true}}, + ProbeCaps: &optimize.ExpansionSearchProbeCaps{ + RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: optimize.ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, + Admission: &optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchOrientationStateLimit, RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, + StateLimit: optimize.ExpansionSearchOrientationStateLimit, + }}} + requirements := defaultOperationalGateRequirements(string(optimize.ExpansionSearchSuffixSeededReverse), string(optimize.ExpansionSearchStepwiseForward)) + requirements.CandidateSQLFingerprint = record.Result.SQLFingerprint + identity.OperationalCandidateSQLSHA256 = record.Result.SQLFingerprint + require.Empty(t, validateOperationalAuthorizedWorkload(identity, requirements, record)) + + record.Result.Optimization.TargetOutcomes[0].EligibilityFacts[0].Eligible = false + require.Contains(t, validateOperationalAuthorizedWorkload(identity, requirements, record), "operational orientation target differs from its authorized promotion bucket") +} + +// TestOperationalGateRejectsUnregisteredGuardPolicy verifies a caller cannot +// invent the relationship between a policy identity and a runtime arm. +func TestOperationalGateRejectsUnregisteredGuardPolicy(t *testing.T) { + identity := operationalTestPromotionIdentity() + identity.Candidate = "suffix-reverse-guard-v1" + records := operationalTestEvidence(identity) + for index := range records { + records[index].Result.TraversalTelemetry.Summary.EmittedIdentity = identity.Candidate + } + requirements := operationalTestRequirements() + + report := buildOperationalGateReport(identity, requirements, records) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "promotion candidate has no registered operational runtime mapping") +} + +// TestOperationalCandidateRuntimeIdentityFreezesPolicyMapping verifies policy +// identities cannot choose an arbitrary exact traversal arm. +func TestOperationalCandidateRuntimeIdentityFreezesPolicyMapping(t *testing.T) { + for _, policy := range []string{ + string(optimize.ExpansionSearchPolicyOrientationProbeV1), + string(optimize.ExpansionSearchPolicyOrientationProbeV2), + } { + actual, supported := operationalCandidateRuntimeIdentity(policy) + require.True(t, supported) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), actual) + } + actual, supported := operationalCandidateRuntimeIdentity("suffix-reverse-guard-v1") + require.False(t, supported) + require.Empty(t, actual) +} + +// TestCreateOperationalGateReport verifies the file boundary preserves the +// exact source document identity and writes failed gates as useful evidence. +func TestCreateOperationalGateReport(t *testing.T) { + identity := operationalTestPromotionIdentity() + input := OperationalGateInput{ + Version: operationalGateVersion, + PromotionIdentity: identity, + Requirements: operationalTestRequirements(), + Records: operationalTestEvidence(identity), + } + inputPath := filepath.Join(t.TempDir(), "operational-input.json") + outputPath := filepath.Join(t.TempDir(), "operational-report.json") + operationalTestWriteJSON(t, inputPath, input) + + passed, err := createOperationalGateReport(inputPath, outputPath) + require.NoError(t, err) + require.True(t, passed) + output, err := os.ReadFile(outputPath) + require.NoError(t, err) + var report OperationalGateReport + require.NoError(t, json.Unmarshal(output, &report)) + require.True(t, report.Passed, report.Reasons) + require.Equal(t, identity, report.PromotionIdentity) + + input.Records[0].Result.Environment.BinarySHA256 = strings.Repeat("f", 64) + operationalTestWriteJSON(t, inputPath, input) + passed, err = createOperationalGateReport(inputPath, outputPath) + require.NoError(t, err) + require.False(t, passed) + output, err = os.ReadFile(outputPath) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(output, &report)) + require.False(t, report.Passed) + require.True(t, operationalTestReportContains(report, "run binary does not match promotion identity")) +} + +// TestCreateOperationalGateReportPreservesInput rejects an output path that +// would replace the immutable evidence document. +func TestCreateOperationalGateReportPreservesInput(t *testing.T) { + identity := operationalTestPromotionIdentity() + input := OperationalGateInput{ + Version: operationalGateVersion, + PromotionIdentity: identity, + Requirements: operationalTestRequirements(), + Records: operationalTestEvidence(identity), + } + path := filepath.Join(t.TempDir(), "operational-input.json") + operationalTestWriteJSON(t, path, input) + + _, err := createOperationalGateReport(path, path) + require.ErrorContains(t, err, "distinct paths") + loaded, err := loadOperationalGateInput(path) + require.NoError(t, err) + require.Equal(t, identity, loaded.PromotionIdentity) +} + +// TestLoadOperationalGateInputRejectsAmbiguousJSON verifies the loader fails +// closed on schema drift and concatenated documents. +func TestLoadOperationalGateInputRejectsAmbiguousJSON(t *testing.T) { + identity := operationalTestPromotionIdentity() + valid := OperationalGateInput{ + Version: operationalGateVersion, + PromotionIdentity: identity, + Requirements: operationalTestRequirements(), + Records: operationalTestEvidence(identity), + } + validJSON, err := json.Marshal(valid) + require.NoError(t, err) + + for _, test := range []struct { + name string + input string + reason string + }{ + {name: "unknown field", input: strings.TrimSuffix(string(validJSON), "}") + `,"unexpected":true}`, reason: "unknown field"}, + {name: "duplicate field", input: strings.Replace(string(validJSON), `"version":2`, `"version":2,"version":2`, 1), reason: "duplicate JSON object key"}, + {name: "nested duplicate field", input: strings.Replace(string(validJSON), `"candidate_runtime_identity":"`+operationalTestCandidate+`"`, `"candidate_runtime_identity":"`+operationalTestCandidate+`","candidate_runtime_identity":"`+operationalTestCandidate+`"`, 1), reason: "duplicate JSON object key"}, + {name: "trailing document", input: string(validJSON) + `{}`, reason: "trailing JSON data"}, + {name: "unsupported version", input: strings.Replace(string(validJSON), `"version":2`, `"version":1`, 1), reason: "input version must be 2"}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "input.json") + require.NoError(t, os.WriteFile(path, []byte(test.input), 0o600)) + _, err := loadOperationalGateInput(path) + require.ErrorContains(t, err, test.reason) + }) + } + _, err = loadOperationalGateInput("") + require.ErrorContains(t, err, "explicit input path") +} + +// TestOperationalGateFailsClosedOnMissingOrContradictoryEvidence verifies each +// class of operational proof is substantive rather than a presence-only flag. +func TestOperationalGateFailsClosedOnMissingOrContradictoryEvidence(t *testing.T) { + tests := []struct { + name string + mutate func([]OperationalEvidenceRecord) []OperationalEvidenceRecord + reason string + }{ + { + name: "matrix cell", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + return records[1:] + }, + reason: "candidate matrix is missing pool_size=1 concurrency=1 plan_cache_mode=auto", + }, + { + name: "duplicate matrix cell", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + duplicate := records[0] + duplicate.ID = "duplicate-matrix" + return append(records, duplicate) + }, + reason: "candidate matrix cell is duplicated", + }, + { + name: "duplicate exceptional scenario", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := *operationalTestScenario(records, OperationalScenarioLowWorkMem) + record.ID = "duplicate-low-memory" + return append(records, record) + }, + reason: "operational scenario is duplicated", + }, + { + name: "promotion identity", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].PromotionIdentity.BinarySHA256 = strings.Repeat("f", 64) + return records + }, + reason: "record promotion identity does not match report", + }, + { + name: "source archive", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].SourceSHA256 = strings.Repeat("f", 64) + return records + }, + reason: "record source archive does not match promotion identity", + }, + { + name: "binary", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Environment.BinarySHA256 = strings.Repeat("f", 64) + return records + }, + reason: "run binary does not match promotion identity", + }, + { + name: "dirty source", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Environment.DirtyDiffSHA256 = strings.Repeat("d", 64) + return records + }, + reason: "operational evidence was captured from a dirty source tree", + }, + { + name: "candidate receipt", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + sample := &records[0].Result.Stats.Samples[0] + sample.RuntimeReceiptEvents = nil + return records + }, + reason: "event chain is missing", + }, + { + name: "fabricated pooled receipt", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := &records[9] + record.Result.Stats.Samples = []LatencySample{operationalTestCandidateSample(record.Result, "forged-pooled", "101")} + return records + }, + reason: "pooled candidate sample must retain non-attested GraphBench replay metadata", + }, + { + name: "matrix declaration", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Concurrency = 7 + records[0].Result.Concurrency[0].Concurrency = 7 + return records + }, + reason: "candidate matrix record is outside the required matrix", + }, + { + name: "matrix workers", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := &records[3] + record.Result.Concurrency[0].Samples[1].Worker = 1 + return records + }, + reason: "concurrency block duplicates a worker iteration", + }, + { + name: "stable observation", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.StableObservation = false + return records + }, + reason: "operational record lacks a stable observation", + }, + { + name: "resolved endpoint substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioCancellation) + record.Result.Params["end_id"] = int64(999) + return records + }, + reason: "operational scenarios do not use one exact authorized workload", + }, + { + name: "symbolic endpoint substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioSessionIsolation) + record.Result.NodeParams["end_id"] = "easy-end" + return records + }, + reason: "operational scenarios do not use one exact authorized workload", + }, + { + name: "symbolic endpoint list substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioConcurrentWriter) + record.Result.NodeListParams["targets"] = []string{"easy-end"} + return records + }, + reason: "operational scenarios do not use one exact authorized workload", + }, + { + name: "unauthorized query", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Cypher += " LIMIT 1" + return records + }, + reason: "operational query must match exactly one promotion bucket, matched 0", + }, + { + name: "sql fingerprint", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.SQL += " where false" + return records + }, + reason: "operational SQL fingerprint is missing or does not bind the measured SQL", + }, + { + name: "self-consistent substituted SQL", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := &records[0] + record.Result.SQL = "select 0::int8 as distance" + record.Result.SQLFingerprint = sqlFingerprint(record.Result.SQL) + return records + }, + reason: "non-overflow operational scenarios do not use one exact candidate SQL", + }, + { + name: "global self-consistent substituted SQL", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + for index := range records { + records[index].Result.SQL = "select 0::int8 as distance" + records[index].Result.SQLFingerprint = sqlFingerprint(records[index].Result.SQL) + } + return records + }, + reason: "operational SQL fingerprint differs from the independently frozen production candidate SQL", + }, + { + name: "missing optimization target", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Optimization = nil + return records + }, + reason: "operational record lacks optimization target evidence", + }, + { + name: "optimization selector substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Optimization.TargetOutcomes[0].SelectorVersion = "unregistered-selector" + return records + }, + reason: "operational optimization target does not prove the exact production candidate policy", + }, + { + name: "optimization bucket substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Optimization.TargetOutcomes[0].Direction = "outbound" + return records + }, + reason: "operational optimization target differs from its authorized promotion bucket", + }, + { + name: "optimization cap substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Optimization.TargetOutcomes[0].StateLimit-- + return records + }, + reason: "operational optimization target cap differs from promotion identity: state_limit", + }, + { + name: "authorized shape", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + *records[0].Result.Shape.MaxDepth = 31 + return records + }, + reason: "operational workload shape differs from its authorized promotion bucket", + }, + { + name: "physical fixture", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Fixture.PhysicalValidated = false + return records + }, + reason: "operational record lacks one physically validated fixture identity", + }, + { + name: "forged physical cardinality", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[0].Result.Fixture.PhysicalEdgeCount++ + return records + }, + reason: "operational record lacks one physically validated fixture identity", + }, + { + name: "fixture configuration substitution", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioLowWorkMem) + record.Result.Fixture.Configuration = "easier-fixture" + return records + }, + reason: "operational scenarios do not use one exact authorized workload", + }, + { + name: "cross-scenario workload", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioCancellation) + record.Result.WorkloadSHA256 = strings.Repeat("5", 64) + return records + }, + reason: "operational scenarios do not use one exact authorized workload", + }, + { + name: "low work mem", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioLowWorkMem) + record.Result.PostgresEnvironment.WorkMem = "65kB" + return records + }, + reason: "work_mem exceeds constrained ceiling 65536 bytes", + }, + { + name: "cancellation latency", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioCancellation) + record.Cancellation.Latency = 250 * time.Millisecond + return records + }, + reason: "cancellation latency must be positive and below 250ms", + }, + { + name: "rollback reuse", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioCancellation) + record.Cancellation.ReplayBackendPID++ + return records + }, + reason: "post-rollback replay did not reuse the cancelled backend PID", + }, + { + name: "repeatable read writer", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioConcurrentWriter) + record.Snapshot.ObservationAfterSHA256 = strings.Repeat("f", 64) + return records + }, + reason: "reader observation changed across the concurrent commit", + }, + { + name: "concurrent writer effect", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioConcurrentWriter) + record.Snapshot.WriterAffectedRows = 0 + return records + }, + reason: "concurrent writer did not affect any rows", + }, + { + name: "post transaction visibility", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioConcurrentWriter) + record.Snapshot.PostCommitObservationSHA256 = record.Snapshot.ObservationBeforeSHA256 + return records + }, + reason: "post-transaction observation does not prove the concurrent writer changed visible state", + }, + { + name: "session isolation", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioSessionIsolation) + record.SessionIsolation.SessionAObservedBRows = 1 + return records + }, + reason: "session-local evidence contains missing own rows or cross-session rows", + }, + { + name: "forced overflow receipt", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioForcedOverflow) + events := record.Result.Stats.Samples[0].RuntimeReceiptEvents + events[0].RuntimeIdentity = "wrong-fallback" + return records + }, + reason: "overflow receipt chain does not contain the exact configured fallback", + }, + { + name: "forced overflow cap expansion", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + record := operationalTestScenario(records, OperationalScenarioForcedOverflow) + record.Result.Optimization.TargetOutcomes[0].StateLimit = record.PromotionIdentity.Caps["state_limit"] + 1 + return records + }, + reason: "forced-overflow optimization cap is not a positive bounded variant of state_limit", + }, + { + name: "database identity", + mutate: func(records []OperationalEvidenceRecord) []OperationalEvidenceRecord { + records[len(records)-1].Result.PostgresEnvironment.DatabaseOID++ + return records + }, + reason: "PostgreSQL database identity differs across operational records", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + identity := operationalTestPromotionIdentity() + records := test.mutate(operationalTestEvidence(identity)) + report := buildOperationalGateReport(identity, operationalTestRequirements(), records) + require.False(t, report.Passed) + require.True(t, operationalTestReportContains(report, test.reason), "missing %q in report: global=%v records=%v", test.reason, report.Reasons, report.Records) + }) + } +} + +// TestOperationalGateRequirementsAreImmutable verifies callers cannot weaken +// the prescribed matrix, work_mem, cancellation, or fallback contract. +func TestOperationalGateRequirementsAreImmutable(t *testing.T) { + tests := []struct { + name string + mutate func(*OperationalGateRequirements) + reason string + }{ + {name: "pool", mutate: func(value *OperationalGateRequirements) { value.PoolSizes = []int{1} }, reason: "operational pool-size matrix must be exactly 1,2,8"}, + {name: "concurrency", mutate: func(value *OperationalGateRequirements) { value.ConcurrencyLevels = []int{1} }, reason: "operational concurrency matrix must be exactly 1,8,16"}, + {name: "cache", mutate: func(value *OperationalGateRequirements) { value.PlanCacheModes[2] = "off" }, reason: "operational plan-cache matrix must be exactly auto,force_custom_plan,force_generic_plan"}, + {name: "memory", mutate: func(value *OperationalGateRequirements) { value.LowWorkMemMaximumBytes = 128 * 1024 }, reason: "low work_mem ceiling must be positive and no greater than 64kB"}, + {name: "cancellation", mutate: func(value *OperationalGateRequirements) { value.CancellationMaximum = time.Second }, reason: "cancellation maximum must be positive and no greater than 250ms"}, + {name: "clean source", mutate: func(value *OperationalGateRequirements) { value.RequireCleanSource = false }, reason: "operational evidence must require a clean source tree"}, + {name: "candidate SQL", mutate: func(value *OperationalGateRequirements) { value.CandidateSQLFingerprint = "" }, reason: "operational candidate SQL fingerprint must be a canonical SHA-256 digest"}, + {name: "fallback", mutate: func(value *OperationalGateRequirements) { value.FallbackRuntimeIdentity = "other" }, reason: "fallback runtime identity differs from promotion fallback executor"}, + {name: "candidate mapping", mutate: func(value *OperationalGateRequirements) { value.CandidateRuntimeIdentity = operationalTestFallback }, reason: "candidate runtime identity differs from the registered promotion candidate mapping"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + identity := operationalTestPromotionIdentity() + requirements := operationalTestRequirements() + test.mutate(&requirements) + report := buildOperationalGateReport(identity, requirements, operationalTestEvidence(identity)) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, test.reason) + }) + } +} + +// TestOperationalGateRequiresManifestSQLAnchor verifies the operational input +// cannot choose a self-consistent candidate SQL independently of promotion. +func TestOperationalGateRequiresManifestSQLAnchor(t *testing.T) { + identity := operationalTestPromotionIdentity() + requirements := operationalTestRequirements() + requirements.CandidateSQLFingerprint = strings.Repeat("d", 64) + report := buildOperationalGateReport(identity, requirements, operationalTestEvidence(identity)) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "operational candidate SQL fingerprint differs from the promotion identity anchor") + + identity.OperationalCandidateSQLSHA256 = "" + requirements.CandidateSQLFingerprint = "" + report = buildOperationalGateReport(identity, requirements, operationalTestEvidence(identity)) + require.False(t, report.Passed) + require.Contains(t, report.Reasons, "promotion identity operational candidate SQL SHA-256 must be a canonical digest") +} + +// TestParsePostgresMemoryBytes verifies canonical PostgreSQL work_mem forms and invalid input. +func TestParsePostgresMemoryBytes(t *testing.T) { + for value, expected := range map[string]int64{ + "64kB": 64 * 1024, + "4MB": 4 * 1024 * 1024, + "1GB": 1024 * 1024 * 1024, + "512": 512, + } { + actual, err := parsePostgresMemoryBytes(value) + require.NoError(t, err) + require.Equal(t, expected, actual) + } + for _, value := range []string{"", "zero", "0MB", "1TB"} { + _, err := parsePostgresMemoryBytes(value) + require.Error(t, err) + } +} + +func operationalTestPromotionIdentity() PromotionEvidenceIdentity { + digest := strings.Repeat("a", 64) + return PromotionEvidenceIdentity{ + Candidate: operationalTestCandidate, + SelectorVersion: "sp-static-v8-hidden-fanin", + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: operationalTestFallback, + SourceCommit: "deadbeef", + SourceSHA256: digest, + BinarySHA256: strings.Repeat("b", 64), + CorpusSHA256: strings.Repeat("c", 64), + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), + Caps: map[string]int64{"state_limit": 4096, "frontier_limit": 1024}, + Buckets: []PromotionBucket{{ + Name: "hidden-fanin", + QuerySHA256: []string{pgdriver.TraversalPolicyQuerySHA256(operationalTestCypher)}, + Direction: "inbound", + ObservationMode: "distance", + MinimumDepth: 1, + MaximumDepth: 32, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } +} + +func operationalTestEvidence(identity PromotionEvidenceIdentity) []OperationalEvidenceRecord { + var records []OperationalEvidenceRecord + for _, poolSize := range defaultOperationalPoolSizes { + for _, concurrency := range defaultOperationalConcurrency { + for _, mode := range defaultOperationalPlanCacheModes { + id := fmt.Sprintf("matrix-%s-p%d-c%d", mode, poolSize, concurrency) + record := operationalTestRecord(identity, id, OperationalScenarioCandidateMatrix, mode, "4MB", poolSize) + record.Concurrency = concurrency + if poolSize != 1 { + record.Result.Stats.Samples = []LatencySample{operationalTestPooledCandidateSample(record.Result)} + } + samples := make([]ConcurrencySample, 0, concurrency) + connectionCount := poolSize + if concurrency < connectionCount { + connectionCount = concurrency + } + for worker := 1; worker <= concurrency; worker++ { + connection := (worker-1)%connectionCount + 1 + classification := "warm-session" + if worker <= connectionCount { + classification = "cold-session" + } + samples = append(samples, ConcurrencySample{ + Worker: worker, Iteration: 1, ConnectionID: fmt.Sprint(100 + connection), Classification: classification, + ExecuteDrain: 500 * time.Microsecond, Total: time.Millisecond, + }) + } + record.Result.Concurrency = []ConcurrencyBlock{{ + Concurrency: concurrency, + PoolSize: poolSize, + Operations: concurrency, + Wall: time.Millisecond, + QPS: float64(concurrency) * 1000, + Samples: samples, + }} + records = append(records, record) + } + } + } + + lowMemory := operationalTestRecord(identity, "low-memory", OperationalScenarioLowWorkMem, "force_generic_plan", "64kB", 1) + records = append(records, lowMemory) + + cancellation := operationalTestRecord(identity, "cancellation", OperationalScenarioCancellation, "auto", "4MB", 1) + replay := operationalTestCandidateSample(cancellation.Result, "cancel-replay", "301") + cancellation.Cancellation = &OperationalCancellationEvidence{ + SQLState: "57014", + Latency: 5 * time.Millisecond, + TransactionRolledBack: true, + CancelledBackendPID: 301, + ReplayBackendPID: 301, + ReplaySucceeded: true, + ReplayCandidateReceipt: replay, + } + records = append(records, cancellation) + + snapshot := operationalTestRecord(identity, "snapshot", OperationalScenarioConcurrentWriter, "auto", "4MB", 2) + snapshot.Snapshot = &OperationalSnapshotEvidence{ + ReaderBackendPID: 401, + WriterBackendPID: 402, + ReaderIsolation: "repeatable read", + WriterAffectedRows: 1, + WriterCommitted: true, + ObservationBeforeSHA256: strings.Repeat("e", 64), + ObservationAfterSHA256: strings.Repeat("e", 64), + PostCommitObservationSHA256: strings.Repeat("f", 64), + } + records = append(records, snapshot) + + sessions := operationalTestRecord(identity, "sessions", OperationalScenarioSessionIsolation, "auto", "4MB", 2) + sessionA := operationalTestCandidateSample(sessions.Result, "session-a", "501") + sessionB := operationalTestCandidateSample(sessions.Result, "session-b", "502") + sessions.SessionIsolation = &OperationalSessionIsolationEvidence{ + SessionABackendPID: 501, + SessionBBackendPID: 502, + SessionAInvocationID: "session-a", + SessionBInvocationID: "session-b", + SessionAOwnRows: 1, + SessionBOwnRows: 1, + SessionACandidateReceipt: sessionA, + SessionBCandidateReceipt: sessionB, + } + records = append(records, sessions) + + overflow := operationalTestRecord(identity, "overflow", OperationalScenarioForcedOverflow, "force_custom_plan", "4MB", 1) + overflow.Result.Optimization.TargetOutcomes[0].StateLimit = 1 + overflow.Result.Optimization.TargetOutcomes[0].FrontierLimit = 1 + overflow.Result.SQL = "select 1::int8 as distance /* forced overflow caps=1,1 */" + overflow.Result.SQLFingerprint = sqlFingerprint(overflow.Result.SQL) + overflow.Result.TraversalTelemetry = operationalTestTelemetry(identity, true) + overflow.Result.Stats.Samples = []LatencySample{operationalTestFallbackSample(overflow.Result, "overflow-invocation", "601")} + records = append(records, overflow) + return records +} + +func operationalTestBindRecordToIdentity(record *OperationalEvidenceRecord, identity PromotionEvidenceIdentity) { + bucket := identity.Buckets[0] + query := operationalTestCypher + switch identity.Candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + query = "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p" + case string(optimize.ShortestPathExecutorI2GuardedDistance): + query = "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)" + case string(optimize.ShortestPathExecutorASPI1DAG): + query = "MATCH p = allShortestPaths((s)-[:Traverse*1..64]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), string(optimize.ExpansionSearchPolicyOrientationProbeV2): + query = operationalTestOrientationCypher + } + record.PromotionIdentity = cloneOperationalPromotionIdentity(identity) + record.SourceSHA256 = identity.SourceSHA256 + record.Result.Environment.SourceCommit = identity.SourceCommit + record.Result.Environment.BinarySHA256 = identity.BinarySHA256 + record.Result.Environment.CorpusSHA256 = identity.CorpusSHA256 + record.Result.Cypher = query + record.Result.SQL = operationalTestSQL + record.Result.SQLFingerprint = sqlFingerprint(operationalTestSQL) + record.Result.Shape.Direction = bucket.Direction + record.Result.Shape.RelationshipKindCount = bucket.RelationshipKindCount + record.Result.Shape.EdgeKinds = []string{"Traverse"} + record.Result.Shape.MinDepth = operationalTestInt(bucket.MinimumDepth) + record.Result.Shape.MaxDepth = operationalTestInt(bucket.MaximumDepth) + record.Result.Shape.QualificationSplit = bucket.QualificationSplit[0] + + runtimeIdentity, _ := operationalCandidateRuntimeIdentity(identity.Candidate) + fallback := record.Scenario == OperationalScenarioForcedOverflow + record.Result.TraversalTelemetry = operationalTestIdentityTelemetry(identity, runtimeIdentity, fallback, bucket.ObservationMode) + record.Result.Optimization = operationalTestIdentityOptimization(identity, bucket, fallback) + if fallback { + record.Result.SQL = operationalTestSQL + " /* forced overflow */" + record.Result.SQLFingerprint = sqlFingerprint(record.Result.SQL) + record.Result.Stats.Samples = []LatencySample{operationalTestIdentitySample(record.Result, runtimeIdentity, identity.FallbackExecutor, true, "overflow-invocation", "601")} + } else if record.Scenario != OperationalScenarioCandidateMatrix || record.Result.Environment.PoolSize == 1 { + record.Result.Stats.Samples = []LatencySample{operationalTestIdentitySample(record.Result, runtimeIdentity, runtimeIdentity, false, record.ID+"-invocation", "101")} + } else { + sample := operationalTestPooledCandidateSample(record.Result) + sample.RequestedIdentity = runtimeIdentity + sample.RuntimeIdentity = runtimeIdentity + record.Result.Stats.Samples = []LatencySample{sample} + } + if record.Cancellation != nil { + record.Cancellation.ReplayCandidateReceipt = operationalTestIdentitySample(record.Result, runtimeIdentity, runtimeIdentity, false, "cancel-replay", "301") + } + if record.SessionIsolation != nil { + record.SessionIsolation.SessionACandidateReceipt = operationalTestIdentitySample(record.Result, runtimeIdentity, runtimeIdentity, false, "session-a", "501") + record.SessionIsolation.SessionBCandidateReceipt = operationalTestIdentitySample(record.Result, runtimeIdentity, runtimeIdentity, false, "session-b", "502") + } +} + +func operationalTestIdentityOptimization(identity PromotionEvidenceIdentity, bucket PromotionBucket, forcedOverflow bool) *translate.OptimizationSummary { + runtimeIdentity, _ := operationalCandidateRuntimeIdentity(identity.Candidate) + eligible := true + minimumDepth, maximumDepth := int64(bucket.MinimumDepth), int64(bucket.MaximumDepth) + outcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, TargetKind: "traversal", Family: "SP", + Candidate: identity.Candidate, Selected: identity.Candidate, Applied: identity.Candidate, + Fallback: identity.FallbackExecutor, EmittedPolicy: operationalCandidatePolicy(identity.Candidate), + PlannedCandidates: []string{identity.FallbackExecutor, identity.Candidate}, + EmittedCandidates: []string{identity.Candidate, identity.FallbackExecutor}, + ExecutionBoundary: identity.ExecutionBoundary, SelectorVersion: identity.SelectorVersion, + SelectionMode: "production_canary", ObservationMode: bucket.ObservationMode, Direction: bucket.Direction, + RelationshipKindCount: bucket.RelationshipKindCount, UntypedRelationship: bucket.UntypedRelationship, + Eligible: &eligible, StaticallyEligible: &eligible, MinimumDepth: &minimumDepth, MaximumDepth: &maximumDepth, + } + if identity.Candidate == string(optimize.ShortestPathExecutorASPI1DAG) { + outcome.Family = "ASP" + } + for name, value := range identity.Caps { + if forcedOverflow { + value = 1 + } + switch name { + case "state_limit": + outcome.StateLimit = value + case "frontier_limit": + outcome.FrontierLimit = value + case "predecessor_limit": + outcome.PredecessorLimit = value + case "enumeration_limit": + outcome.EnumerationLimit = value + case "output_bytes_limit": + outcome.OutputBytesLimit = value + } + } + if isOrientationProbePolicy(identity.Candidate) { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + outcome.Lowering = optimize.LoweringExpansionSearchStrategy + outcome.Family = "fixed_suffix_expansion" + outcome.Candidate, outcome.Selected, outcome.Applied, outcome.Fallback = reverse, forward, forward, forward + outcome.EmittedPolicy = identity.Candidate + outcome.PlannedCandidates = []string{forward, reverse} + outcome.EmittedCandidates = []string{forward, reverse} + outcome.Direction = "" + outcome.EligibilityFacts = []translate.TargetEligibilityFact{{Name: "qualified_fixed_suffix_topology", Eligible: true}} + capValue := func(name string) int64 { + value := identity.Caps[name] + if forcedOverflow { + return 1 + } + return value + } + outcome.ProbeCaps = &optimize.ExpansionSearchProbeCaps{ + RootRowLimit: capValue("root_row_limit"), ReverseSeedRowLimit: capValue("reverse_seed_row_limit"), + DirectionalDegreeRowLimit: capValue("directional_degree_row_limit"), + } + outcome.Admission = &optimize.ExpansionSearchAdmission{ + StateLimit: capValue("state_limit"), RequiresCompleteProbes: true, FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + } + outcome.StateLimit = outcome.Admission.StateLimit + _ = runtimeIdentity + } + return &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}} +} + +func operationalTestIdentityTelemetry(identity PromotionEvidenceIdentity, runtimeIdentity string, fallback bool, observationMode string) *TraversalExecutionTelemetry { + telemetry := operationalTestTelemetry(identity, fallback && identity.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance)) + summary := &telemetry.Summary + summary.RequestedIdentity = runtimeIdentity + summary.PlannedIdentities = []string{runtimeIdentity, identity.FallbackExecutor} + summary.EmittedIdentity = operationalCandidatePolicy(identity.Candidate) + summary.RuntimeIdentity = runtimeIdentity + summary.AppliedIdentity = runtimeIdentity + summary.SelectorVersion = identity.SelectorVersion + summary.ExecutionBoundary = identity.ExecutionBoundary + summary.ObservationMode = observationMode + summary.RuntimeBranch = "selected_candidate" + if identity.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) { + summary.RuntimeBranch = "inline_canonical_distance" + } + summary.FallbackIdentity = "" + summary.FallbackExecuted = operationalTestBool(false) + summary.Overflow = operationalTestBool(false) + if fallback { + summary.RuntimeIdentity = identity.FallbackExecutor + summary.AppliedIdentity = identity.FallbackExecutor + summary.RuntimeBranch = "exact_configured_fallback" + if identity.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) { + summary.RuntimeBranch = "exact_s4_distance_fallback" + } + summary.FallbackIdentity = identity.FallbackExecutor + summary.Provenance["fallback_identity"] = "test" + summary.FallbackExecuted = operationalTestBool(true) + summary.Overflow = operationalTestBool(true) + } + if identity.Candidate != string(optimize.ShortestPathExecutorI2GuardedDistance) { + telemetry.Level = TraversalTelemetryLevelSummary + telemetry.Diagnostic = nil + } + return telemetry +} + +func operationalTestIdentitySample(result CaseResult, requested, runtime string, fallback bool, invocation, connection string) LatencySample { + sample := operationalTestCandidateSample(result, invocation, connection) + sample.RequestedIdentity = requested + sample.RuntimeIdentity = runtime + sample.RuntimeBranch = "selected_candidate" + sample.FallbackExecuted = operationalTestBool(fallback) + sample.RuntimeReceiptEvents = []RuntimeReceiptEvent{{InvocationID: invocation, Ordinal: 1, RuntimeIdentity: runtime, RuntimeBranch: sample.RuntimeBranch, FallbackExecuted: fallback}} + if fallback { + sample.RuntimeBranch = "exact_configured_fallback" + sample.RuntimeReceiptEvents[0].RuntimeBranch = sample.RuntimeBranch + } + return sample +} + +func operationalTestInt(value int) *int { return &value } + +func operationalTestRecord(identity PromotionEvidenceIdentity, id string, scenario OperationalEvidenceScenario, planCacheMode, workMem string, poolSize int) OperationalEvidenceRecord { + minimumDepth, maximumDepth := 1, 32 + sql := operationalTestSQL + result := CaseResult{ + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: identity.CorpusSHA256, + SourceCommit: identity.SourceCommit, + DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: identity.BinarySHA256, + PoolSize: poolSize, + }, + PostgresEnvironment: &PostgresEnvironment{ + Version: "PostgreSQL 17", + Database: "operational", + PlanCacheMode: planCacheMode, + TransactionIsolation: "repeatable read", + WorkMem: workMem, + TempFileLimit: "-1", + GraphPartitionCount: 1, + PostmasterStartedAt: time.Unix(1_700_000_000, 0).UTC(), + DatabaseOID: 42, + Autovacuum: "on", + SchemaFingerprint: strings.Repeat("1", 64), + IndexFingerprint: strings.Repeat("2", 64), + }, + Fixture: &FixtureMetadata{ + Dataset: "operational-dataset", Checksum: strings.Repeat("4", 64), + NodeCount: 2, EdgeCount: 1, PhysicalValidated: true, PhysicalNodeCount: 2, PhysicalEdgeCount: 1, + Configuration: "generated-shortest-operational-v1", + }, + Source: "generated_sp_i2_distance_v1.json", + Dataset: "operational-dataset", + Name: "operational-case", + WorkloadSHA256: strings.Repeat("3", 64), + Category: "generated_shortest_path_v2", + Shape: WorkloadShape{ + QualificationSplit: "training", RootPredicate: "bound_id", TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, Direction: "inbound", RelationshipKindCount: 1, + MinDepth: &minimumDepth, MaxDepth: &maximumDepth, + }, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Cypher: operationalTestCypher, + Params: map[string]any{"root_id": int64(101), "end_id": int64(202)}, + NodeParams: map[string]string{"root_id": "root", "end_id": "end"}, + NodeListParams: map[string][]string{"targets": {"end"}}, + SQL: sql, + SQLFingerprint: sqlFingerprint(sql), + StableObservation: true, + RowCount: 1, + ObservedRows: []string{"[1]"}, + TraversalTelemetry: operationalTestTelemetry(identity, false), + Optimization: operationalTestOptimization(identity), + } + result.Stats = DurationStats{ + Iterations: 1, + Median: time.Millisecond, + Samples: []LatencySample{operationalTestCandidateSample(result, id+"-invocation", "101")}, + } + return OperationalEvidenceRecord{ + ID: id, + Scenario: scenario, + PromotionIdentity: cloneOperationalPromotionIdentity(identity), + SourceSHA256: identity.SourceSHA256, + Result: result, + } +} + +func operationalTestOptimization(identity PromotionEvidenceIdentity) *translate.OptimizationSummary { + minimumDepth, maximumDepth := int64(1), int64(32) + eligible := true + return &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Lowering: optimize.LoweringShortestPathExecutor, TargetKind: "traversal", Family: "SP", + Candidate: identity.Candidate, Selected: identity.Candidate, Applied: identity.Candidate, + Fallback: identity.FallbackExecutor, EmittedPolicy: optimize.ShortestPathPolicyI2DistanceGuardedV1, + PlannedCandidates: []string{identity.FallbackExecutor, identity.Candidate}, + EmittedCandidates: []string{identity.Candidate, identity.FallbackExecutor}, + ExecutionBoundary: identity.ExecutionBoundary, SelectorVersion: identity.SelectorVersion, + SelectionMode: "production_canary", ObservationMode: "distance", Direction: "inbound", + RelationshipKindCount: 1, Eligible: &eligible, StaticallyEligible: &eligible, + MinimumDepth: &minimumDepth, MaximumDepth: &maximumDepth, + StateLimit: identity.Caps["state_limit"], FrontierLimit: identity.Caps["frontier_limit"], + }}} +} + +func operationalTestTelemetry(identity PromotionEvidenceIdentity, fallback bool) *TraversalExecutionTelemetry { + runtimeIdentity := operationalTestCandidate + runtimeBranch := "inline_canonical_distance" + applied := operationalTestCandidate + overflow := false + provenance := map[string]string{ + "requested_identity": "test", + "planned_identities": "test", + "emitted_identity": "test", + "runtime_identity": "test", + "applied_identity": "test", + "selector_version": "test", + "scheduler_version": "test", + "runtime_branch": "test", + "runtime_outcome_available": "test", + "observation_mode": "test", + "overflow": "test", + "fallback_executed": "test", + "caps.state_rows": "test", + } + fallbackIdentity := "" + if fallback { + runtimeIdentity = operationalTestFallback + runtimeBranch = "exact_s4_distance_fallback" + applied = operationalTestFallback + overflow = true + fallbackIdentity = operationalTestFallback + provenance["fallback_identity"] = "test" + } + available := true + stateLimit := identity.Caps["state_limit"] + frontierLimit := identity.Caps["frontier_limit"] + stateRows := int64(2) + outputRows := int64(1) + candidateMarkerRows := int64(1) + fallbackMarkerRows := int64(0) + candidateBranchRows := int64(1) + fallbackBranchRows := int64(0) + candidateExecutorLoops := int64(1) + fallbackExecutorLoops := int64(0) + if fallback { + stateLimit = 1 + frontierLimit = 1 + candidateMarkerRows = 0 + fallbackMarkerRows = 1 + candidateBranchRows = 0 + fallbackBranchRows = 1 + candidateExecutorLoops = 0 + fallbackExecutorLoops = 1 + } + provenance["caps.state_rows"] = "test" + provenance["caps.frontier_rows"] = "test" + planCounters := map[string]int64{ + "sp_i2_distance_rows": stateRows, + "sp_i2_target_rows": candidateBranchRows, + "sp_i2_output_rows": outputRows, + "sp_i2_candidate_marker_rows": candidateMarkerRows, + "sp_i2_fallback_marker_rows": fallbackMarkerRows, + "sp_i2_candidate_branch_rows": candidateBranchRows, + "sp_i2_fallback_branch_rows": fallbackBranchRows, + "sp_i2_candidate_executor_loops": candidateExecutorLoops, + "sp_i2_fallback_executor_loops": fallbackExecutorLoops, + } + diagnosticProvenance := map[string]string{} + planProvenance := map[string]string{} + for _, name := range []string{ + "state_rows", "frontier_rows", "output_rows", "candidate_marker_rows", "fallback_marker_rows", + "candidate_branch_rows", "fallback_branch_rows", "candidate_executor_loops", "fallback_executor_loops", + } { + diagnosticProvenance["inline_shortest_distance."+name] = "test" + } + for name := range planCounters { + planProvenance["counters."+name] = "test" + } + timedSample := false + return &TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: TraversalTelemetryLevelDiagnostic, + Summary: TraversalExecutionSummary{ + RequestedIdentity: operationalTestCandidate, + PlannedIdentities: []string{operationalTestCandidate, operationalTestFallback}, + EmittedIdentity: "sp-i2-distance-guarded-v1", + RuntimeIdentity: runtimeIdentity, + AppliedIdentity: applied, + SelectorVersion: identity.SelectorVersion, + SchedulerVersion: "single_ended_level", + ExecutionBoundary: identity.ExecutionBoundary, + ObservationMode: "distance", + Caps: map[string]int64{"state_rows": stateLimit, "frontier_rows": frontierLimit}, + RuntimeOutcomeAvailable: &available, + RuntimeBranch: runtimeBranch, + Overflow: operationalTestBool(overflow), + FallbackExecuted: operationalTestBool(fallback), + FallbackIdentity: fallbackIdentity, + Provenance: provenance, + }, + Diagnostic: &TraversalExecutionDiagnostic{ + InvocationID: "operational-diagnostic", + ConnectionID: "operational-diagnostic-connection", + TimedSample: &timedSample, + RequiredFamilies: []TraversalTelemetryFamily{TraversalTelemetryFamilySP}, + Counters: TraversalDiagnosticCounters{InlineShortestDistance: &InlineDistanceTraversalCounters{ + StateRows: &stateRows, FrontierRows: &stateRows, OutputRows: &outputRows, + CandidateMarkerRows: &candidateMarkerRows, FallbackMarkerRows: &fallbackMarkerRows, + CandidateBranchRows: &candidateBranchRows, FallbackBranchRows: &fallbackBranchRows, + CandidateExecutorLoops: &candidateExecutorLoops, FallbackExecutorLoops: &fallbackExecutorLoops, + }}, + CounterStatus: TraversalTelemetryCounterStatusComplete, + PlanReplay: &TraversalPlanReplayEvidence{ + Source: "postgres_explain_analyze_timing_off", Counters: planCounters, Provenance: planProvenance, + }, + Provenance: diagnosticProvenance, + }, + } +} + +func operationalTestCandidateSample(result CaseResult, invocation, connection string) LatencySample { + fallback := false + return LatencySample{ + Round: 1, Iteration: 1, Case: result.Name, Dataset: result.Dataset, Backend: ModePostgresSQL, + ConnectionID: connection, Classification: "warm", Duration: time.Millisecond, + RequestedIdentity: operationalTestCandidate, RuntimeIdentity: operationalTestCandidate, + RuntimeBranch: "inline_canonical_distance", FallbackExecuted: &fallback, + RuntimeAttestation: "timed_invocation", RuntimeInvocationID: invocation, + RuntimeReceiptEvents: []RuntimeReceiptEvent{{ + InvocationID: invocation, Ordinal: 1, RuntimeIdentity: operationalTestCandidate, + RuntimeBranch: "inline_canonical_distance", FallbackExecuted: false, + }}, + } +} + +func operationalTestPooledCandidateSample(result CaseResult) LatencySample { + fallback := false + return LatencySample{ + Round: 1, Iteration: 1, Case: result.Name, Dataset: result.Dataset, Backend: ModePostgresSQL, + Classification: "warm", Duration: time.Millisecond, + RequestedIdentity: operationalTestCandidate, RuntimeIdentity: operationalTestCandidate, + RuntimeBranch: "inline_canonical_distance", FallbackExecuted: &fallback, + RuntimeAttestation: "same_case_invocation_local_replay", + } +} + +func operationalTestFallbackSample(result CaseResult, invocation, connection string) LatencySample { + fallback := true + return LatencySample{ + Round: 1, Iteration: 1, Case: result.Name, Dataset: result.Dataset, Backend: ModePostgresSQL, + ConnectionID: connection, Classification: "warm", Duration: time.Millisecond, + RequestedIdentity: operationalTestCandidate, RuntimeIdentity: operationalTestTerminal, + RuntimeBranch: "exact_relationship_trail_fallback", FallbackExecuted: &fallback, + RuntimeAttestation: "timed_invocation", RuntimeInvocationID: invocation, + RuntimeReceiptEvents: []RuntimeReceiptEvent{ + {InvocationID: invocation, Ordinal: 1, RuntimeIdentity: operationalTestFallback, RuntimeBranch: "exact_s4_distance_fallback", FallbackExecuted: true}, + {InvocationID: invocation, Ordinal: 2, RuntimeIdentity: operationalTestTerminal, RuntimeBranch: "exact_relationship_trail_fallback", FallbackExecuted: true}, + }, + } +} + +func operationalTestBool(value bool) *bool { + return &value +} + +func operationalTestScenario(records []OperationalEvidenceRecord, scenario OperationalEvidenceScenario) *OperationalEvidenceRecord { + for index := range records { + if records[index].Scenario == scenario { + return &records[index] + } + } + panic("scenario not found: " + string(scenario)) +} + +func operationalTestReportContains(report OperationalGateReport, reason string) bool { + for _, actual := range report.Reasons { + if strings.Contains(actual, reason) { + return true + } + } + for _, record := range report.Records { + for _, actual := range record.Reasons { + if strings.Contains(actual, reason) { + return true + } + } + } + return false +} + +func operationalTestWriteJSON(t *testing.T, path string, value any) { + t.Helper() + raw, err := json.Marshal(value) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, raw, 0o600)) +} + +func operationalTestCloneReport(t *testing.T, report OperationalGateReport) OperationalGateReport { + t.Helper() + raw, err := json.Marshal(report) + require.NoError(t, err) + var clone OperationalGateReport + require.NoError(t, json.Unmarshal(raw, &clone)) + return clone +} diff --git a/cmd/graphbench/orientation_policy.go b/cmd/graphbench/orientation_policy.go new file mode 100644 index 00000000..2c669d30 --- /dev/null +++ b/cmd/graphbench/orientation_policy.go @@ -0,0 +1,36 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + +// isOrientationProbePolicy recognizes immutable orientation selector +// identities without treating an unqualified version as production eligible. +func isOrientationProbePolicy(identity string) bool { + switch optimize.ExpansionSearchPolicy(identity) { + case optimize.ExpansionSearchPolicyOrientationProbeV1, + optimize.ExpansionSearchPolicyOrientationProbeV2: + return true + default: + return false + } +} + +// isSuffixReverseGuardPolicy recognizes the reverse-first fixed-suffix guard +// without folding it into the topology-scored orientation policy lineage. +func isSuffixReverseGuardPolicy(identity string) bool { + return optimize.ExpansionSearchPolicy(identity) == optimize.ExpansionSearchPolicySuffixReverseGuardV1 +} + +func isSuffixReverseRetryPolicy(identity string) bool { + return optimize.ExpansionSearchPolicy(identity) == optimize.ExpansionSearchPolicySuffixReverseRetryV1 +} + +// isGuardedExpansionPolicy recognizes same-statement ordinary-expansion +// policies that must expose exactly one candidate or fallback runtime branch. +func isGuardedExpansionPolicy(identity string) bool { + return isOrientationProbePolicy(identity) || isSuffixReverseGuardPolicy(identity) +} diff --git a/cmd/graphbench/orientation_policy_test.go b/cmd/graphbench/orientation_policy_test.go new file mode 100644 index 00000000..a40b1de1 --- /dev/null +++ b/cmd/graphbench/orientation_policy_test.go @@ -0,0 +1,30 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestOrientationProbePolicyRecognitionIsVersionExplicit verifies orientation probe policy recognition is version explicit behavior. +func TestOrientationProbePolicyRecognitionIsVersionExplicit(t *testing.T) { + require.True(t, isOrientationProbePolicy("orientation-probe-v1")) + require.True(t, isOrientationProbePolicy("orientation-probe-v2")) + require.False(t, isOrientationProbePolicy("orientation-probe-v3")) + require.False(t, isOrientationProbePolicy("ORIENTATION-PROBE-V2")) + require.False(t, isOrientationProbePolicy("")) +} + +// TestSuffixReverseGuardPolicyIsNotAnOrientationGeneration verifies the new +// admission-only policy cannot consume orientation-v2 report or manifest paths. +func TestSuffixReverseGuardPolicyIsNotAnOrientationGeneration(t *testing.T) { + require.True(t, isSuffixReverseGuardPolicy("suffix-reverse-guard-v1")) + require.True(t, isGuardedExpansionPolicy("suffix-reverse-guard-v1")) + require.False(t, isOrientationProbePolicy("suffix-reverse-guard-v1")) + require.False(t, isSuffixReverseGuardPolicy("orientation-probe-v2")) +} diff --git a/cmd/graphbench/orientation_selector_report.go b/cmd/graphbench/orientation_selector_report.go new file mode 100644 index 00000000..e715b0d9 --- /dev/null +++ b/cmd/graphbench/orientation_selector_report.go @@ -0,0 +1,658 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// orientationSelectorReportVersion reserves the stable protocol value used to recognize orientation selector report version across artifacts and executions. +const orientationSelectorReportVersion = 1 + +// OrientationSelectorReportOptions configures the matched shadow/incumbent/ +// reverse comparison and its frozen qualification protocol. +type OrientationSelectorReportOptions struct { + // Seed makes randomized statistical procedures reproducible. + Seed int64 + // Confidence sets the requested statistical confidence level. + Confidence float64 + // BootstrapCount records the number of bootstrap count. + BootstrapCount int + // Protocol identifies the protocol. + Protocol string +} + +// OrientationLatencyGate records one frozen relative-or-absolute latency +// rule. A case passes when either the ratio upper bound or absolute upper gap +// stays within its declared limit. +type OrientationLatencyGate struct { + // BaselineIdentity identifies the baseline identity. + BaselineIdentity string `json:"baseline_identity"` + // ObservedIdentity identifies the observed identity. + ObservedIdentity string `json:"observed_identity"` + // BaselineSamples supplies the baseline samples input to the OrientationLatencyGate contract. + BaselineSamples int `json:"baseline_samples"` + // ObservedSamples supplies the observed samples input to the OrientationLatencyGate contract. + ObservedSamples int `json:"observed_samples"` + // Ratio supplies the ratio input to the OrientationLatencyGate contract. + Ratio RatioInterval `json:"median_ratio"` + // AbsoluteChange supplies the absolute change input to the OrientationLatencyGate contract. + AbsoluteChange DurationInterval `json:"median_absolute_change"` + // RatioUpperLimit supplies the ratio upper limit input to the OrientationLatencyGate contract. + RatioUpperLimit float64 `json:"ratio_upper_limit"` + // AbsoluteFloor supplies the absolute floor input to the OrientationLatencyGate contract. + AbsoluteFloor time.Duration `json:"absolute_floor"` + // AbsoluteGapUpper supplies the absolute gap upper input to the OrientationLatencyGate contract. + AbsoluteGapUpper time.Duration `json:"absolute_gap_upper"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` +} + +// OrientationSelectorCase reports shadow attribution, exact-arm regret, and +// probe-only overhead for one topology bucket. +type OrientationSelectorCase struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset"` + // Name identifies the name. + Name string `json:"name"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // QualificationRole supplies the qualification role input to the OrientationSelectorCase contract. + QualificationRole string `json:"qualification_role"` + // ThresholdTuningEligible indicates whether threshold tuning eligible applies. + ThresholdTuningEligible bool `json:"threshold_tuning_eligible"` + // QualificationEligible indicates whether qualification eligible applies. + QualificationEligible bool `json:"qualification_eligible"` + // Rounds records the number of rounds. + Rounds int `json:"matched_rounds"` + // WouldSelectIdentity identifies the would select identity. + WouldSelectIdentity string `json:"would_select_identity"` + // FastestExactIdentity identifies the fastest exact identity. + FastestExactIdentity string `json:"fastest_exact_identity"` + // ExactObservationsMatched indicates whether exact observations matched applies. + ExactObservationsMatched bool `json:"exact_observations_matched"` + // SelectorRegret supplies the selector regret input to the OrientationSelectorCase contract. + SelectorRegret OrientationLatencyGate `json:"selector_regret"` + // ProbeOverhead supplies the probe overhead input to the OrientationSelectorCase contract. + ProbeOverhead OrientationLatencyGate `json:"probe_overhead"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` +} + +// OrientationSelectorReport validates that shadow selection is attributable, +// low-regret, and cheap while the incumbent remains the only shadow execution +// arm. Diagnostic and legacy records never contribute to qualification. +type OrientationSelectorReport struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Policy identifies the policy. + Policy string `json:"policy"` + // Protocol identifies the protocol. + Protocol string `json:"protocol"` + // Seed makes randomized statistical procedures reproducible. + Seed int64 `json:"seed"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // ShadowArtifactSHA256 binds the referenced shadow artifact content by SHA-256 digest. + ShadowArtifactSHA256 string `json:"shadow_artifact_sha256,omitempty"` + // IncumbentArtifactSHA256 binds the referenced incumbent artifact content by SHA-256 digest. + IncumbentArtifactSHA256 string `json:"incumbent_artifact_sha256,omitempty"` + // ReverseArtifactSHA256 binds the referenced reverse artifact content by SHA-256 digest. + ReverseArtifactSHA256 string `json:"reverse_artifact_sha256,omitempty"` + // AAReportSHA256 binds the referenced aa report content by SHA-256 digest. + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + // SelectorRegretRatioLimit supplies the selector regret ratio limit input to the OrientationSelectorReport contract. + SelectorRegretRatioLimit float64 `json:"selector_regret_ratio_upper_limit"` + // ProbeOverheadRatioLimit supplies the probe overhead ratio limit input to the OrientationSelectorReport contract. + ProbeOverheadRatioLimit float64 `json:"probe_overhead_ratio_upper_limit"` + // ProbeOverheadAbsoluteLimit supplies the probe overhead absolute limit input to the OrientationSelectorReport contract. + ProbeOverheadAbsoluteLimit time.Duration `json:"probe_overhead_absolute_limit"` + // EvidencePassed indicates whether evidence passed applies. + EvidencePassed bool `json:"evidence_passed"` + // TrainingCases supplies the training cases input to the OrientationSelectorReport contract. + TrainingCases int `json:"training_cases"` + // HoldoutCases supplies the holdout cases input to the OrientationSelectorReport contract. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed indicates whether qualification passed applies. + QualificationPassed bool `json:"qualification_passed"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []OrientationSelectorCase `json:"cases"` +} + +// orientationSelectorSeries accumulates matched observations used to evaluate orientation selector. +type orientationSelectorSeries struct { + // shadow retains the shadow while orientationSelectorSeries is assembled or evaluated. + shadow roundSamples + // incumbent retains the incumbent while orientationSelectorSeries is assembled or evaluated. + incumbent roundSamples + // reverse retains the reverse while orientationSelectorSeries is assembled or evaluated. + reverse roundSamples + // wouldSelect retains the would select while orientationSelectorSeries is assembled or evaluated. + wouldSelect string +} + +// buildOrientationSelectorReport compares a true-shadow artifact with matched +// exact incumbent and forced-reverse artifacts. The shadow's public result and +// runtime identity must remain incumbent even when would_select names reverse. +func buildOrientationSelectorReport( + shadowRecords, incumbentRecords, reverseRecords []CaseResult, + aa *AAResolutionReport, + options OrientationSelectorReportOptions, +) (OrientationSelectorReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return OrientationSelectorReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return OrientationSelectorReport{}, fmt.Errorf("bootstrap count must be positive") + } + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolConfirmation + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 20, 10, 20, 50 + if protocol == referencePairProtocolDiscovery { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 5, 5, 20, 10 + } else if protocol != referencePairProtocolConfirmation { + return OrientationSelectorReport{}, fmt.Errorf("unsupported orientation selector protocol %q", protocol) + } + + if err := validateAAResolutionEvidence(aa, incumbentRecords, options.Confidence); err != nil { + return OrientationSelectorReport{}, fmt.Errorf("incumbent A/A evidence: %w", err) + } + incumbentHost, err := artifactHostFingerprint(incumbentRecords) + if err != nil { + return OrientationSelectorReport{}, err + } + for name, records := range map[string][]CaseResult{"shadow": shadowRecords, "reverse": reverseRecords} { + host, err := artifactHostFingerprint(records) + if err != nil { + return OrientationSelectorReport{}, fmt.Errorf("%s artifact host: %w", name, err) + } + if host != incumbentHost { + return OrientationSelectorReport{}, fmt.Errorf("%s artifact host does not match incumbent host", name) + } + } + + series, keys, err := collectOrientationSelectorSeries(shadowRecords, incumbentRecords, reverseRecords) + if err != nil { + return OrientationSelectorReport{}, err + } + report := OrientationSelectorReport{ + Version: orientationSelectorReportVersion, + Policy: string(optimize.ExpansionSearchPolicyOrientationProbeV1), + Protocol: protocol, + Seed: options.Seed, + Confidence: options.Confidence, + SelectorRegretRatioLimit: 1.10, + ProbeOverheadRatioLimit: 1.10, + ProbeOverheadAbsoluteLimit: 100 * time.Microsecond, + EvidencePassed: true, + } + trainingPassed, holdoutPassed := true, true + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for index, key := range keys { + current := series[key] + shadow, incumbent := matchedRounds(current.shadow, current.incumbent) + incumbent, reverse := matchedRounds(incumbent, current.reverse) + shadow, incumbent = matchedRounds(shadow, incumbent) + if len(shadow) < minimumRounds || len(shadow) > maximumRounds { + return OrientationSelectorReport{}, fmt.Errorf("%s/%s requires %d-%d matched orientation rounds, got %d", key.dataset, key.name, minimumRounds, maximumRounds, len(shadow)) + } + for _, round := range sortedRounds(shadow) { + if len(shadow[round]) < minimumSamples || len(incumbent[round]) < minimumSamples || len(reverse[round]) < minimumSamples { + return OrientationSelectorReport{}, fmt.Errorf("%s/%s round %d requires %d samples per orientation arm", key.dataset, key.name, round, minimumSamples) + } + } + if err := validateOrientationArmOrder(shadowRecords, incumbentRecords, reverseRecords, key, sortedRounds(shadow), minimumWarmups); err != nil { + return OrientationSelectorReport{}, err + } + + split, err := qualificationSplit(key, shadowRecords, incumbentRecords, reverseRecords) + if err != nil { + return OrientationSelectorReport{}, err + } + role, tuningEligible, qualificationEligible := orientationQualificationRole(split, protocol) + fastestIdentity, fastest := fastestOrientationExactArm(incumbent, reverse) + selected := incumbent + if current.wouldSelect == string(optimize.ExpansionSearchSuffixSeededReverse) { + selected = reverse + } + seed := options.Seed + int64(index)*7919 + _, selectorFloorAbsolute, err := aaTimingFloor(aa, key, false, 0) + if err != nil { + return OrientationSelectorReport{}, err + } + selectorRegret := orientationLatencyGate( + fastestIdentity, + current.wouldSelect, + fastest, + selected, + 1.10, + selectorFloorAbsolute, + seed, + gateOptions, + ) + probeOverhead := orientationLatencyGate( + string(optimize.ExpansionSearchStepwiseForward), + string(optimize.ExpansionSearchPolicyOrientationProbeV1), + incumbent, + shadow, + 1.10, + report.ProbeOverheadAbsoluteLimit, + seed+3, + gateOptions, + ) + entry := OrientationSelectorCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + QualificationRole: role, + ThresholdTuningEligible: tuningEligible, + QualificationEligible: qualificationEligible, + Rounds: len(shadow), + WouldSelectIdentity: current.wouldSelect, + FastestExactIdentity: fastestIdentity, + ExactObservationsMatched: true, + SelectorRegret: selectorRegret, + ProbeOverhead: probeOverhead, + Passed: selectorRegret.Passed && probeOverhead.Passed, + } + if !selectorRegret.Passed { + entry.Reasons = append(entry.Reasons, "selector regret exceeds the 1.10/A/A floor") + } + if !probeOverhead.Passed { + entry.Reasons = append(entry.Reasons, "shadow probe overhead exceeds 10% and 100us") + } + if !entry.Passed { + report.EvidencePassed = false + } + if qualificationEligible { + switch split { + case "training": + report.TrainingCases++ + trainingPassed = trainingPassed && entry.Passed + case "holdout": + report.HoldoutCases++ + holdoutPassed = holdoutPassed && entry.Passed + } + } + report.Cases = append(report.Cases, entry) + } + report.TrainingPassed = protocol == referencePairProtocolConfirmation && report.TrainingCases > 0 && trainingPassed + report.HoldoutPassed = protocol == referencePairProtocolConfirmation && report.HoldoutCases > 0 && holdoutPassed + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +// collectOrientationSelectorSeries collects orientation selector series. +func collectOrientationSelectorSeries( + shadowRecords, incumbentRecords, reverseRecords []CaseResult, +) (map[performanceKey]*orientationSelectorSeries, []performanceKey, error) { + series := map[performanceKey]*orientationSelectorSeries{} + for _, record := range shadowRecords { + if record.ExecutionMode != ModePostgresSQL || record.TraversalTelemetry == nil || record.TraversalTelemetry.Summary.WouldSelectIdentity == "" { + continue + } + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + if series[key] == nil { + series[key] = &orientationSelectorSeries{ + shadow: roundSamples{}, + incumbent: roundSamples{}, + reverse: roundSamples{}, + } + } + if err := validateOrientationRecord(record, "shadow"); err != nil { + return nil, nil, err + } + wouldSelect := record.TraversalTelemetry.Summary.WouldSelectIdentity + if series[key].wouldSelect != "" && series[key].wouldSelect != wouldSelect { + return nil, nil, fmt.Errorf("%s/%s changes shadow would_select identity across rounds", key.dataset, key.name) + } + series[key].wouldSelect = wouldSelect + appendOrientationWarmSamples(series[key].shadow, record) + } + if len(series) == 0 { + return nil, nil, fmt.Errorf("shadow artifact has no attributable orientation shadow records") + } + + for arm, records := range map[string][]CaseResult{"incumbent": incumbentRecords, "reverse": reverseRecords} { + for _, record := range records { + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + current := series[key] + if current == nil { + continue + } + if err := validateOrientationRecord(record, arm); err != nil { + return nil, nil, err + } + if arm == "incumbent" { + appendOrientationWarmSamples(current.incumbent, record) + } else { + appendOrientationWarmSamples(current.reverse, record) + } + } + } + + keys := make([]performanceKey, 0, len(series)) + for key, current := range series { + if len(current.incumbent) == 0 || len(current.reverse) == 0 { + return nil, nil, fmt.Errorf("%s/%s lacks matched incumbent or forced-reverse records", key.dataset, key.name) + } + if err := validateOrientationExactObservations(key, shadowRecords, incumbentRecords, reverseRecords); err != nil { + return nil, nil, err + } + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + return series, keys, nil +} + +// validateOrientationRecord validates orientation record. +func validateOrientationRecord(record CaseResult, arm string) error { + if record.Status != StatusOK || record.Environment == nil || record.TraversalTelemetry == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing record", record.Dataset, record.Name, arm) + } + summary := record.TraversalTelemetry.Summary + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + switch arm { + case "shadow": + if summary.EmittedIdentity != string(optimize.ExpansionSearchPolicyOrientationProbeV1) || + summary.SelectorVersion != string(optimize.ExpansionSearchPolicyOrientationProbeV1) || + summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || summary.RuntimeBranch != "shadow_incumbent" || + (summary.WouldSelectIdentity != forward && summary.WouldSelectIdentity != reverse) || + summary.FallbackExecuted == nil || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s shadow telemetry does not prove incumbent-only orientation shadow execution", record.Dataset, record.Name) + } + case "incumbent": + if summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s incumbent artifact did not execute the exact forward arm", record.Dataset, record.Name) + } + case "reverse": + if summary.RuntimeIdentity != reverse || summary.AppliedIdentity != reverse || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s reverse artifact did not execute the exact forced reverse arm", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown orientation arm %q", arm) + } + return nil +} + +// appendOrientationWarmSamples appends orientation warm samples. +func appendOrientationWarmSamples(series roundSamples, record CaseResult) { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[sample.Round] = append(series[sample.Round], sample.Duration) + } + } +} + +// validateOrientationExactObservations validates orientation exact observations. +func validateOrientationExactObservations(key performanceKey, artifacts ...[]CaseResult) error { + workload := "" + var observed []string + rowCount := int64(-1) + binary := "" + for _, records := range artifacts { + matched := false + armSQL := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + matched = true + if !record.StableObservation || record.WorkloadSHA256 == "" || record.SQLFingerprint == "" || record.Environment == nil || record.Environment.BinarySHA256 == "" { + return fmt.Errorf("%s/%s lacks stable observation or executable/SQL identity", key.dataset, key.name) + } + if workload != "" && workload != record.WorkloadSHA256 { + return fmt.Errorf("%s/%s workload identity differs across orientation arms", key.dataset, key.name) + } + workload = record.WorkloadSHA256 + if rowCount >= 0 && (rowCount != record.RowCount || !slices.Equal(observed, record.ObservedRows)) { + return fmt.Errorf("%s/%s exact observations differ across orientation arms", key.dataset, key.name) + } + rowCount, observed = record.RowCount, append([]string(nil), record.ObservedRows...) + if binary != "" && binary != record.Environment.BinarySHA256 { + return fmt.Errorf("%s/%s executable identity differs across orientation arms", key.dataset, key.name) + } + binary = record.Environment.BinarySHA256 + if armSQL != "" && armSQL != record.SQLFingerprint { + return fmt.Errorf("%s/%s SQL fingerprint changes within an orientation arm", key.dataset, key.name) + } + armSQL = record.SQLFingerprint + } + if !matched { + return fmt.Errorf("%s/%s is missing from one orientation artifact", key.dataset, key.name) + } + } + return nil +} + +// validateOrientationArmOrder validates orientation arm order. +func validateOrientationArmOrder( + shadowRecords, incumbentRecords, reverseRecords []CaseResult, + key performanceKey, + rounds []int, + minimumWarmups int, +) error { + armRecords := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // records retains the records while anonymous record is assembled or evaluated. + records []CaseResult + }{ + { + name: "shadow", + records: shadowRecords, + }, + { + name: "incumbent", + records: incumbentRecords, + }, + { + name: "reverse", + records: reverseRecords, + }, + } + evidence := make([]map[int]pairedRoundEvidence, len(armRecords)) + positionCounts := make([][4]int, len(armRecords)) + for index, arm := range armRecords { + current, err := collectPairedRoundEvidence(arm.records, key) + if err != nil { + return err + } + evidence[index] = current + } + for _, round := range rounds { + seenPositions := map[int]struct{}{} + block, runUUID := 0, "" + for index, arm := range armRecords { + current, found := evidence[index][round] + if !found || current.Warmups < minimumWarmups || current.Arm == "" || current.Arm == "unlabeled" { + return fmt.Errorf("%s/%s round %d lacks %s arm identity or %d warmups", key.dataset, key.name, round, arm.name, minimumWarmups) + } + if current.ArmOrder < 1 || current.ArmOrder > 3 { + return fmt.Errorf("%s/%s round %d has invalid three-arm order", key.dataset, key.name, round) + } + if _, duplicate := seenPositions[current.ArmOrder]; duplicate { + return fmt.Errorf("%s/%s round %d has duplicate three-arm order", key.dataset, key.name, round) + } + seenPositions[current.ArmOrder] = struct{}{} + positionCounts[index][current.ArmOrder]++ + if block == 0 { + block, runUUID = current.Block, current.RunUUID + } else if current.Block != block || current.RunUUID != runUUID { + return fmt.Errorf("%s/%s round %d has mismatched three-arm block or run UUID", key.dataset, key.name, round) + } + } + if block < 1 || runUUID == "" { + return fmt.Errorf("%s/%s round %d has missing three-arm block or run UUID", key.dataset, key.name, round) + } + } + for index, counts := range positionCounts { + minimum, maximum := counts[1], counts[1] + for position := 2; position <= 3; position++ { + minimum = min(minimum, counts[position]) + maximum = max(maximum, counts[position]) + } + if maximum-minimum > 1 { + return fmt.Errorf("%s/%s %s arm order is not position-balanced", key.dataset, key.name, armRecords[index].name) + } + } + return nil +} + +// orientationQualificationRole classifies orientation qualification role for downstream policy decisions. +func orientationQualificationRole(split, protocol string) (role string, tuningEligible, qualificationEligible bool) { + switch split { + case "training": + return "selector_training", true, protocol == referencePairProtocolConfirmation + case "holdout": + return "frozen_evaluation", false, protocol == referencePairProtocolConfirmation + case "diagnostic": + return "diagnostic_only", false, false + default: + return "legacy_diagnostic", false, false + } +} + +// fastestOrientationExactArm supports benchmark evidence processing for fastest orientation exact arm. +func fastestOrientationExactArm(incumbent, reverse roundSamples) (string, roundSamples) { + if roundMedianEstimate(reverse) < roundMedianEstimate(incumbent) { + return string(optimize.ExpansionSearchSuffixSeededReverse), reverse + } + return string(optimize.ExpansionSearchStepwiseForward), incumbent +} + +// roundMedianEstimate derives the statistical value used to evaluate round median estimate. +func roundMedianEstimate(samples roundSamples) float64 { + rounds := sortedRounds(samples) + medians := make([]float64, 0, len(rounds)) + for _, round := range rounds { + medians = append(medians, durationQuantile(samples[round], 0.5)) + } + return quantile(medians, 0.5) +} + +// orientationLatencyGate supports benchmark evidence processing for orientation latency gate. +func orientationLatencyGate( + baselineIdentity, observedIdentity string, + baseline, observed roundSamples, + ratioLimit float64, + absoluteFloor time.Duration, + seed int64, + options PerfGateOptions, +) OrientationLatencyGate { + ratio := bootstrapRoundMedianRatio(baseline, observed, seed, options) + change := negateDurationInterval(bootstrapRoundMedianSaving(baseline, observed, seed+1, options)) + absoluteGapUpper := max(time.Duration(0), change.Upper) + return OrientationLatencyGate{ + BaselineIdentity: baselineIdentity, + ObservedIdentity: observedIdentity, + BaselineSamples: sampleCount(baseline), + ObservedSamples: sampleCount(observed), + Ratio: ratio, + AbsoluteChange: change, + RatioUpperLimit: ratioLimit, + AbsoluteFloor: absoluteFloor, + AbsoluteGapUpper: absoluteGapUpper, + Passed: ratio.Upper <= ratioLimit || absoluteGapUpper <= absoluteFloor, + } +} + +// createOrientationSelectorReport loads the three exact arm artifacts and A/A +// calibration, builds the report, and writes an indented JSON document. +func createOrientationSelectorReport( + shadowPath, incumbentPath, reversePath, aaPath, outputPath string, + options OrientationSelectorReportOptions, +) (bool, error) { + shadow, err := readJSONLFile(shadowPath) + if err != nil { + return false, fmt.Errorf("read orientation shadow artifact: %w", err) + } + incumbent, err := readJSONLFile(incumbentPath) + if err != nil { + return false, fmt.Errorf("read orientation incumbent artifact: %w", err) + } + reverse, err := readJSONLFile(reversePath) + if err != nil { + return false, fmt.Errorf("read orientation reverse artifact: %w", err) + } + aa, aaSHA, err := loadAAResolutionReport(aaPath) + if err != nil { + return false, fmt.Errorf("read orientation A/A report: %w", err) + } + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, aa, options) + if err != nil { + return false, err + } + report.ShadowArtifactSHA256, err = fileSHA256(shadowPath) + if err != nil { + return false, err + } + report.IncumbentArtifactSHA256, err = fileSHA256(incumbentPath) + if err != nil { + return false, err + } + report.ReverseArtifactSHA256, err = fileSHA256(reversePath) + if err != nil { + return false, err + } + report.AAReportSHA256 = aaSHA + return report.QualificationPassed, writeOrientationSelectorReport(outputPath, report) +} + +// writeOrientationSelectorReport writes orientation selector report. +func writeOrientationSelectorReport(path string, report OrientationSelectorReport) (err error) { + output := os.Stdout + if path != "" { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/orientation_selector_report_test.go b/cmd/graphbench/orientation_selector_report_test.go new file mode 100644 index 00000000..c742e30e --- /dev/null +++ b/cmd/graphbench/orientation_selector_report_test.go @@ -0,0 +1,364 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestOrientationSelectorReportPassesMatchedLowRegretLowOverheadEvidence verifies orientation selector report passes matched low regret low overhead evidence behavior. +func TestOrientationSelectorReportPassesMatchedLowRegretLowOverheadEvidence(t *testing.T) { + trainingShadow, trainingIncumbent, trainingReverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond+50*time.Microsecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + holdoutShadow, holdoutIncumbent, holdoutReverse := orientationSelectorRecords( + "holdout", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond+50*time.Microsecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + renameOrientationRecords("training-fixed-suffix", trainingShadow, trainingIncumbent, trainingReverse) + renameOrientationRecords("holdout-fixed-suffix", holdoutShadow, holdoutIncumbent, holdoutReverse) + shadow := append(trainingShadow, holdoutShadow...) + incumbent := append(trainingIncumbent, holdoutIncumbent...) + reverse := append(trainingReverse, holdoutReverse...) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 7, + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + + require.NoError(t, err) + require.True(t, report.EvidencePassed) + require.Equal(t, 1, report.TrainingCases) + require.Equal(t, 1, report.HoldoutCases) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.Equal(t, 1.10, report.SelectorRegretRatioLimit) + require.Equal(t, 1.10, report.ProbeOverheadRatioLimit) + require.Equal(t, 100*time.Microsecond, report.ProbeOverheadAbsoluteLimit) + require.Len(t, report.Cases, 2) + entry := report.Cases[1] + if entry.QualificationSplit != "training" { + entry = report.Cases[0] + } + require.Equal(t, "training", entry.QualificationSplit) + require.Equal(t, "selector_training", entry.QualificationRole) + require.True(t, entry.ThresholdTuningEligible) + require.True(t, entry.QualificationEligible) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), entry.WouldSelectIdentity) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), entry.FastestExactIdentity) + require.True(t, entry.SelectorRegret.Passed) + require.True(t, entry.ProbeOverhead.Passed) + require.True(t, entry.ExactObservationsMatched) +} + +// TestOrientationSelectorReportFailsRegretWhenShadowChoosesSlowArm verifies orientation selector report fails regret when shadow chooses slow arm behavior. +func TestOrientationSelectorReportFailsRegretWhenShadowChoosesSlowArm(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond+25*time.Microsecond, + 10*time.Millisecond, + time.Millisecond, + ) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 11, + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + + require.NoError(t, err) + require.False(t, report.EvidencePassed) + require.False(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.False(t, report.Cases[0].SelectorRegret.Passed) + require.True(t, report.Cases[0].ProbeOverhead.Passed) + require.Contains(t, report.Cases[0].Reasons, "selector regret exceeds the 1.10/A/A floor") +} + +// TestOrientationSelectorReportAllowsAbsoluteProbeFloor verifies orientation selector report allows absolute probe floor behavior. +func TestOrientationSelectorReportAllowsAbsoluteProbeFloor(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchStepwiseForward), + 250*time.Microsecond, + 200*time.Microsecond, + 300*time.Microsecond, + ) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 13, + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + + require.NoError(t, err) + probe := report.Cases[0].ProbeOverhead + require.Greater(t, probe.Ratio.Upper, 1.10) + require.Equal(t, 50*time.Microsecond, probe.AbsoluteGapUpper) + require.True(t, probe.Passed) +} + +// TestOrientationSelectorReportKeepsHoldoutEvaluationOnlyAndExcludesDiagnostic verifies orientation selector report keeps holdout evaluation only and excludes diagnostic behavior. +func TestOrientationSelectorReportKeepsHoldoutEvaluationOnlyAndExcludesDiagnostic(t *testing.T) { + for _, testCase := range []struct { + // split retains the split while anonymous record is assembled or evaluated. + split string + // role retains the role while anonymous record is assembled or evaluated. + role string + // qualificationEligible indicates whether qualification eligible applies. + qualificationEligible bool + // qualificationPassed indicates whether qualification passed applies. + qualificationPassed bool + }{ + { + split: "holdout", + role: "frozen_evaluation", + qualificationEligible: true, + qualificationPassed: false, + }, + { + split: "diagnostic", + role: "diagnostic_only", + qualificationEligible: false, + qualificationPassed: false, + }, + } { + t.Run(testCase.split, func(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + testCase.split, + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 17, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.Equal(t, testCase.role, report.Cases[0].QualificationRole) + require.False(t, report.Cases[0].ThresholdTuningEligible) + require.Equal(t, testCase.qualificationEligible, report.Cases[0].QualificationEligible) + require.Equal(t, testCase.qualificationPassed, report.QualificationPassed) + }) + } +} + +// TestOrientationSelectorReportRequiresPassingTrainingAndFrozenHoldout verifies orientation selector report requires passing training and frozen holdout behavior. +func TestOrientationSelectorReportRequiresPassingTrainingAndFrozenHoldout(t *testing.T) { + trainingShadow, trainingIncumbent, trainingReverse := orientationSelectorRecords( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, + ) + holdoutShadow, holdoutIncumbent, holdoutReverse := orientationSelectorRecords( + "holdout", string(optimize.ExpansionSearchStepwiseForward), 10*time.Millisecond, 10*time.Millisecond, time.Millisecond, + ) + renameOrientationRecords("training-pass", trainingShadow, trainingIncumbent, trainingReverse) + renameOrientationRecords("holdout-fail", holdoutShadow, holdoutIncumbent, holdoutReverse) + shadow := append(trainingShadow, holdoutShadow...) + incumbent := append(trainingIncumbent, holdoutIncumbent...) + reverse := append(trainingReverse, holdoutReverse...) + + report, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Seed: 19, + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) +} + +// TestOrientationSelectorReportRejectsSplitDriftAndNonIncumbentShadowRuntime verifies orientation selector report rejects split drift and non incumbent shadow runtime behavior. +func TestOrientationSelectorReportRejectsSplitDriftAndNonIncumbentShadowRuntime(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + reverse[0].Shape.QualificationSplit = "holdout" + _, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolConfirmation, + }) + require.ErrorContains(t, err, "changes qualification split") + + reverse[0].Shape.QualificationSplit = "training" + shadow[0].TraversalTelemetry.Summary.RuntimeIdentity = string(optimize.ExpansionSearchSuffixSeededReverse) + _, err = buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolConfirmation, + }) + require.ErrorContains(t, err, "incumbent-only") +} + +// TestOrientationSelectorReportRejectsUnbalancedThreeArmOrder verifies orientation selector report rejects unbalanced three arm order behavior. +func TestOrientationSelectorReportRejectsUnbalancedThreeArmOrder(t *testing.T) { + shadow, incumbent, reverse := orientationSelectorRecords( + "training", + string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, + 10*time.Millisecond, + 5*time.Millisecond, + ) + for recordIndex := range reverse { + for sampleIndex := range reverse[recordIndex].Stats.Samples { + reverse[recordIndex].Stats.Samples[sampleIndex].ArmOrder = 3 + } + } + _, err := buildOrientationSelectorReport(shadow, incumbent, reverse, testAAReportForRecords(t, incumbent), OrientationSelectorReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolConfirmation, + }) + require.ErrorContains(t, err, "duplicate three-arm order") +} + +// orientationSelectorRecords prepares or inspects test evidence for orientation selector records. +func orientationSelectorRecords( + split, wouldSelect string, + shadowDuration, incumbentDuration, reverseDuration time.Duration, +) (shadow, incumbent, reverse []CaseResult) { + const rounds = 12 + orders := [][3]int{ + {1, 2, 3}, + {2, 3, 1}, + {3, 1, 2}, + {1, 3, 2}, + {2, 1, 3}, + {3, 2, 1}, + } + for round := 1; round <= rounds; round++ { + order := orders[(round-1)%len(orders)] + shadow = append(shadow, orientationSelectorRecord(round, order[0], "shadow", split, wouldSelect, shadowDuration)) + incumbent = append(incumbent, orientationSelectorRecord(round, order[1], "incumbent", split, "", incumbentDuration)) + reverse = append(reverse, orientationSelectorRecord(round, order[2], "reverse", split, "", reverseDuration)) + } + return shadow, incumbent, reverse +} + +// orientationSelectorRecord prepares or inspects test evidence for orientation selector record. +func orientationSelectorRecord(round, armOrder int, arm, split, wouldSelect string, duration time.Duration) CaseResult { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + runtimeIdentity := forward + emittedIdentity := forward + selectorVersion := "static-lowering-v1" + runtimeBranch := "selected" + if arm == "shadow" { + emittedIdentity = string(optimize.ExpansionSearchPolicyOrientationProbeV1) + selectorVersion = emittedIdentity + runtimeBranch = "shadow_incumbent" + } + if arm == "reverse" { + runtimeIdentity = reverse + emittedIdentity = reverse + selectorVersion = "suffix-seeded-reverse-tool-v1" + } + fallback := false + overflow := false + record := CaseResult{ + Dataset: "orientation-fixture", + Name: "fixed-suffix", + Category: "generated_fixed_suffix_expansion_v2", + WorkloadSHA256: "orientation-workload-v1", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + QualificationSplit: split, + }, + RowCount: 1, + ObservedRows: []string{"[42]"}, + StableObservation: true, + SQLFingerprint: "orientation-" + arm + "-sql-v1", + PostgresEnvironment: &PostgresEnvironment{PlanCacheMode: "auto"}, + Environment: &RunEnvironment{ + Arm: arm, + ArmOrder: armOrder, + Block: round, + Round: round, + RunUUID: "orientation-run-" + fmt.Sprint(round), + BinarySHA256: "orientation-binary-v1", + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + WarmupIterations: 20, + }, + TraversalTelemetry: &TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: TraversalTelemetryLevelSummary, + Summary: TraversalExecutionSummary{ + RequestedIdentity: reverse, + PlannedIdentities: []string{forward, reverse}, + EmittedIdentity: emittedIdentity, + RuntimeIdentity: runtimeIdentity, + AppliedIdentity: runtimeIdentity, + SelectorVersion: selectorVersion, + SchedulerVersion: "not_applicable", + Caps: map[string]int64{}, + RuntimeBranch: runtimeBranch, + Overflow: &overflow, + FallbackExecuted: &fallback, + WouldSelectIdentity: wouldSelect, + Provenance: map[string]string{}, + }, + }, + } + record.Stats.WarmupIterations = 20 + for iteration := 1; iteration <= 50; iteration++ { + record.Stats.Samples = append(record.Stats.Samples, LatencySample{ + Round: round, + Block: round, + Arm: arm, + ArmOrder: armOrder, + RunUUID: record.Environment.RunUUID, + Iteration: iteration, + Classification: "warm", + Duration: duration, + }) + } + return record +} + +// renameOrientationRecords prepares or inspects test evidence for rename orientation records. +func renameOrientationRecords(name string, artifacts ...[]CaseResult) { + for _, records := range artifacts { + for index := range records { + records[index].Name = name + records[index].WorkloadSHA256 = "orientation-workload-" + name + } + } +} diff --git a/cmd/graphbench/orientation_selector_report_v2.go b/cmd/graphbench/orientation_selector_report_v2.go new file mode 100644 index 00000000..e0b31239 --- /dev/null +++ b/cmd/graphbench/orientation_selector_report_v2.go @@ -0,0 +1,1370 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// orientationSelectorReportV2Version reserves the stable protocol value used to recognize orientation selector report v2 version across artifacts and executions. +const orientationSelectorReportV2Version = 2 + +// OrientationSelectorV2FreezeManifest binds the immutable inputs authorized for orientation selector v2 freeze. +type OrientationSelectorV2FreezeManifest struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Policy identifies the policy. + Policy string `json:"policy"` + // Formula supplies the formula input to the OrientationSelectorV2FreezeManifest contract. + Formula string `json:"formula"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // SourceCommit supplies the source commit input to the OrientationSelectorV2FreezeManifest contract. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CohortDeclarationSHA256 binds the referenced cohort declaration content by SHA-256 digest. + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + // DiscoveryReportSHA256 binds the referenced discovery report content by SHA-256 digest. + DiscoveryReportSHA256 string `json:"discovery_report_sha256"` +} + +// OrientationSelectorV2ReportOptions configures the immutable four-arm v2 +// qualification workflow independently of the retained v1 shadow report. +type OrientationSelectorV2ReportOptions struct { + // Seed makes randomized statistical procedures reproducible. + Seed int64 + // Confidence sets the requested statistical confidence level. + Confidence float64 + // BootstrapCount records the number of bootstrap count. + BootstrapCount int + // Protocol identifies the protocol. + Protocol string + // Freeze supplies the freeze input to the OrientationSelectorV2ReportOptions contract. + Freeze *OrientationSelectorV2FreezeManifest + // Discovery supplies the discovery input to the OrientationSelectorV2ReportOptions contract. + Discovery *OrientationSelectorV2Report +} + +// OrientationLatencyGateV2 makes conditional applicability explicit. A +// reverse-selected shadow comparison remains visible but cannot qualify or +// disqualify the guarded selector. +type OrientationLatencyGateV2 struct { + // Applicable indicates whether applicable applies. + Applicable bool `json:"applicable"` + // OrientationLatencyGate supplies the orientation latency gate input to the OrientationLatencyGateV2 contract. + OrientationLatencyGate +} + +// OrientationSelectorV2Case records exact runtime attribution and the three +// frozen latency gates for one training, holdout, or diagnostic case. +type OrientationSelectorV2Case struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset"` + // Name identifies the name. + Name string `json:"name"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // QualificationRole supplies the qualification role input to the OrientationSelectorV2Case contract. + QualificationRole string `json:"qualification_role"` + // ThresholdTuningEligible indicates whether threshold tuning eligible applies. + ThresholdTuningEligible bool `json:"threshold_tuning_eligible"` + // QualificationEligible indicates whether qualification eligible applies. + QualificationEligible bool `json:"qualification_eligible"` + // Rounds records the number of rounds. + Rounds int `json:"matched_rounds"` + // WouldSelectIdentity identifies the would select identity. + WouldSelectIdentity string `json:"would_select_identity"` + // FastestExactIdentity identifies the fastest exact identity. + FastestExactIdentity string `json:"fastest_exact_identity"` + // GuardedRuntimeIdentity identifies the guarded runtime identity. + GuardedRuntimeIdentity string `json:"guarded_runtime_identity"` + // GuardedRuntimeBranch supplies the guarded runtime branch input to the OrientationSelectorV2Case contract. + GuardedRuntimeBranch string `json:"guarded_runtime_branch"` + // Overflow indicates whether overflow applies. + Overflow bool `json:"overflow"` + // FallbackExecuted indicates whether fallback executed applies. + FallbackExecuted bool `json:"fallback_executed"` + // ExactObservationsMatched indicates whether exact observations matched applies. + ExactObservationsMatched bool `json:"exact_observations_matched"` + // ShadowForwardOverhead supplies the shadow forward overhead input to the OrientationSelectorV2Case contract. + ShadowForwardOverhead OrientationLatencyGateV2 `json:"shadow_forward_overhead"` + // GuardedSelectedOverhead supplies the guarded selected overhead input to the OrientationSelectorV2Case contract. + GuardedSelectedOverhead OrientationLatencyGate `json:"guarded_selected_overhead"` + // GuardedFastestRegret supplies the guarded fastest regret input to the OrientationSelectorV2Case contract. + GuardedFastestRegret OrientationLatencyGate `json:"guarded_fastest_regret"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` +} + +// OrientationSelectorV2Report binds the immutable selector, source, binary, +// corpus, four timing artifacts, host A/A floor, and qualification outcome. +type OrientationSelectorV2Report struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Policy identifies the policy. + Policy string `json:"policy"` + // Protocol identifies the protocol. + Protocol string `json:"protocol"` + // Seed makes randomized statistical procedures reproducible. + Seed int64 `json:"seed"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // SourceCommit supplies the source commit input to the OrientationSelectorV2Report contract. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds the referenced corpus content by SHA-256 digest. + CorpusSHA256 string `json:"corpus_sha256"` + // CohortDeclarationSHA256 binds the referenced cohort declaration content by SHA-256 digest. + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + // FreezeManifestSHA256 binds the referenced freeze manifest content by SHA-256 digest. + FreezeManifestSHA256 string `json:"freeze_manifest_sha256,omitempty"` + // Formula supplies the formula input to the OrientationSelectorV2Report contract. + Formula string `json:"formula"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // ShadowArtifactSHA256 binds the referenced shadow artifact content by SHA-256 digest. + ShadowArtifactSHA256 string `json:"shadow_artifact_sha256,omitempty"` + // IncumbentArtifactSHA256 binds the referenced incumbent artifact content by SHA-256 digest. + IncumbentArtifactSHA256 string `json:"incumbent_artifact_sha256,omitempty"` + // ReverseArtifactSHA256 binds the referenced reverse artifact content by SHA-256 digest. + ReverseArtifactSHA256 string `json:"reverse_artifact_sha256,omitempty"` + // GuardedArtifactSHA256 binds the referenced guarded artifact content by SHA-256 digest. + GuardedArtifactSHA256 string `json:"guarded_artifact_sha256,omitempty"` + // AAReportSHA256 binds the referenced aa report content by SHA-256 digest. + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + // ShadowForwardRatioLimit supplies the shadow forward ratio limit input to the OrientationSelectorV2Report contract. + ShadowForwardRatioLimit float64 `json:"shadow_forward_ratio_upper_limit"` + // GuardedSelectedRatioLimit supplies the guarded selected ratio limit input to the OrientationSelectorV2Report contract. + GuardedSelectedRatioLimit float64 `json:"guarded_selected_ratio_upper_limit"` + // GuardedFastestRatioLimit supplies the guarded fastest ratio limit input to the OrientationSelectorV2Report contract. + GuardedFastestRatioLimit float64 `json:"guarded_fastest_ratio_upper_limit"` + // OverheadAbsoluteLimit supplies the overhead absolute limit input to the OrientationSelectorV2Report contract. + OverheadAbsoluteLimit time.Duration `json:"overhead_absolute_limit"` + // EvidencePassed indicates whether evidence passed applies. + EvidencePassed bool `json:"evidence_passed"` + // TrainingCases supplies the training cases input to the OrientationSelectorV2Report contract. + TrainingCases int `json:"training_cases"` + // HoldoutCases supplies the holdout cases input to the OrientationSelectorV2Report contract. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed indicates whether qualification passed applies. + QualificationPassed bool `json:"qualification_passed"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []OrientationSelectorV2Case `json:"cases"` +} + +// orientationSelectorV2Series accumulates matched observations used to evaluate orientation selector v2. +type orientationSelectorV2Series struct { + // shadow retains the shadow while orientationSelectorV2Series is assembled or evaluated. + shadow roundSamples + // incumbent retains the incumbent while orientationSelectorV2Series is assembled or evaluated. + incumbent roundSamples + // reverse retains the reverse while orientationSelectorV2Series is assembled or evaluated. + reverse roundSamples + // guarded retains the guarded while orientationSelectorV2Series is assembled or evaluated. + guarded roundSamples + // wouldSelect retains the would select while orientationSelectorV2Series is assembled or evaluated. + wouldSelect string + // shadowOverflow indicates whether shadow overflow applies. + shadowOverflow bool + // shadowObserved indicates whether shadow observed applies. + shadowObserved bool + // guardedRuntime retains the guarded runtime while orientationSelectorV2Series is assembled or evaluated. + guardedRuntime string + // guardedBranch retains the guarded branch while orientationSelectorV2Series is assembled or evaluated. + guardedBranch string + // overflow indicates whether overflow applies. + overflow bool + // fallback indicates whether fallback applies. + fallback bool + // guardedObserved indicates whether guarded observed applies. + guardedObserved bool +} + +// orientationSelectorV2Identity groups state that must remain consistent while processing orientation selector v2 identity. +type orientationSelectorV2Identity struct { + // sourceCommit retains the source commit while orientationSelectorV2Identity is assembled or evaluated. + sourceCommit string + // dirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + dirtyDiffSHA256 string + // binarySHA256 binds the referenced binary content by SHA-256 digest. + binarySHA256 string + // corpusSHA256 binds the referenced corpus content by SHA-256 digest. + corpusSHA256 string +} + +// buildOrientationSelectorV2Report evaluates matched shadow, exact forward, +// exact reverse, and actual guarded statements. V1 remains a separate schema +// and code path so new evidence cannot reinterpret its historical result. +func buildOrientationSelectorV2Report( + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, + aa *AAResolutionReport, + options OrientationSelectorV2ReportOptions, +) (OrientationSelectorV2Report, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return OrientationSelectorV2Report{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return OrientationSelectorV2Report{}, fmt.Errorf("bootstrap count must be positive") + } + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolConfirmation + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 20, 10, 20, 50 + if protocol == referencePairProtocolDiscovery { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 5, 5, 20, 10 + } else if protocol != referencePairProtocolConfirmation { + return OrientationSelectorV2Report{}, fmt.Errorf("unsupported orientation selector v2 protocol %q", protocol) + } + + if err := validateAAResolutionEvidence(aa, incumbentRecords, options.Confidence); err != nil { + return OrientationSelectorV2Report{}, fmt.Errorf("incumbent A/A evidence: %w", err) + } + if err := validateOrientationV2AAEvidence(aa, incumbentRecords); err != nil { + return OrientationSelectorV2Report{}, fmt.Errorf("incumbent A/A environment: %w", err) + } + incumbentHost, err := artifactHostFingerprint(incumbentRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + for name, records := range map[string][]CaseResult{ + "shadow": shadowRecords, "reverse": reverseRecords, "guarded": guardedRecords, + } { + host, err := artifactHostFingerprint(records) + if err != nil { + return OrientationSelectorV2Report{}, fmt.Errorf("%s artifact host: %w", name, err) + } + if host != incumbentHost { + return OrientationSelectorV2Report{}, fmt.Errorf("%s artifact host does not match incumbent host", name) + } + } + identity, err := validateOrientationV2EvidenceIdentity(shadowRecords, incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + + series, keys, err := collectOrientationSelectorV2Series(shadowRecords, incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + cohortDeclarationSHA256, err := validateOrientationV2Cohort(keys, shadowRecords, incumbentRecords, reverseRecords, guardedRecords, protocol) + if err != nil { + return OrientationSelectorV2Report{}, err + } + report := OrientationSelectorV2Report{ + Version: orientationSelectorReportV2Version, + Policy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + Protocol: protocol, + Seed: options.Seed, + Confidence: options.Confidence, + SourceCommit: identity.sourceCommit, + DirtyDiffSHA256: identity.dirtyDiffSHA256, + BinarySHA256: identity.binarySHA256, + CorpusSHA256: identity.corpusSHA256, + CohortDeclarationSHA256: cohortDeclarationSHA256, + Formula: "F2=root_rows+maximum_depth*forward_degree_rows;R2=suffix_rows+boundary_rows+reverse_degree_rows;reverse=complete&&4*R2<3*F2", + Caps: map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + }, + ShadowForwardRatioLimit: 1.10, + GuardedSelectedRatioLimit: 1.10, + GuardedFastestRatioLimit: 1.10, + OverheadAbsoluteLimit: 100 * time.Microsecond, + EvidencePassed: true, + } + if protocol == referencePairProtocolConfirmation { + if err := validateOrientationV2Freeze(options.Freeze, options.Discovery, report); err != nil { + return OrientationSelectorV2Report{}, err + } + } + trainingPassed, holdoutPassed := true, true + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for index, key := range keys { + current := series[key] + if err := requireOrientationV2RoundSets(key, current); err != nil { + return OrientationSelectorV2Report{}, err + } + rounds := sortedRounds(current.shadow) + if len(rounds) < minimumRounds || len(rounds) > maximumRounds { + return OrientationSelectorV2Report{}, fmt.Errorf("%s/%s requires %d-%d matched orientation-v2 rounds, got %d", key.dataset, key.name, minimumRounds, maximumRounds, len(rounds)) + } + for _, round := range rounds { + if len(current.shadow[round]) < minimumSamples || len(current.incumbent[round]) < minimumSamples || + len(current.reverse[round]) < minimumSamples || len(current.guarded[round]) < minimumSamples { + return OrientationSelectorV2Report{}, fmt.Errorf("%s/%s round %d requires %d samples per orientation-v2 arm", key.dataset, key.name, round, minimumSamples) + } + } + if err := validateOrientationV2ArmOrder(shadowRecords, incumbentRecords, reverseRecords, guardedRecords, key, rounds, minimumWarmups); err != nil { + return OrientationSelectorV2Report{}, err + } + + split, err := qualificationSplit(key, shadowRecords, incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return OrientationSelectorV2Report{}, err + } + role, tuningEligible, qualificationEligible := orientationQualificationRole(split, protocol) + if qualificationEligible && !strings.HasPrefix(key.dataset, "generated_fixed_suffix_expansion_v3_") { + return OrientationSelectorV2Report{}, fmt.Errorf("%s/%s qualification evidence is not from the frozen fixed-suffix v3 corpus", key.dataset, key.name) + } + fastestIdentity, fastest := fastestOrientationExactArm(current.incumbent, current.reverse) + selectedIdentity, selected := string(optimize.ExpansionSearchStepwiseForward), current.incumbent + if current.wouldSelect == string(optimize.ExpansionSearchSuffixSeededReverse) { + selectedIdentity, selected = string(optimize.ExpansionSearchSuffixSeededReverse), current.reverse + } + seed := options.Seed + int64(index)*7919 + _, selectorFloorAbsolute, err := aaTimingFloor(aa, key, false, 0) + if err != nil { + return OrientationSelectorV2Report{}, err + } + shadowGate := orientationLatencyGate( + string(optimize.ExpansionSearchStepwiseForward), + string(optimize.ExpansionSearchPolicyOrientationProbeV2)+":shadow", + current.incumbent, + current.shadow, + report.ShadowForwardRatioLimit, + report.OverheadAbsoluteLimit, + seed, + gateOptions, + ) + shadowApplicable := current.wouldSelect == string(optimize.ExpansionSearchStepwiseForward) + guardedSelected := orientationLatencyGate( + selectedIdentity, + string(optimize.ExpansionSearchPolicyOrientationProbeV2)+":"+current.guardedRuntime, + selected, + current.guarded, + report.GuardedSelectedRatioLimit, + report.OverheadAbsoluteLimit, + seed+3, + gateOptions, + ) + guardedFastest := orientationLatencyGate( + fastestIdentity, + string(optimize.ExpansionSearchPolicyOrientationProbeV2)+":"+current.guardedRuntime, + fastest, + current.guarded, + report.GuardedFastestRatioLimit, + selectorFloorAbsolute, + seed+6, + gateOptions, + ) + entry := OrientationSelectorV2Case{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + QualificationRole: role, + ThresholdTuningEligible: tuningEligible, + QualificationEligible: qualificationEligible, + Rounds: len(rounds), + WouldSelectIdentity: current.wouldSelect, + FastestExactIdentity: fastestIdentity, + GuardedRuntimeIdentity: current.guardedRuntime, + GuardedRuntimeBranch: current.guardedBranch, + Overflow: current.overflow, + FallbackExecuted: current.fallback, + ExactObservationsMatched: true, + ShadowForwardOverhead: OrientationLatencyGateV2{ + Applicable: shadowApplicable, + OrientationLatencyGate: shadowGate, + }, + GuardedSelectedOverhead: guardedSelected, + GuardedFastestRegret: guardedFastest, + Passed: (!shadowApplicable || shadowGate.Passed) && guardedSelected.Passed && guardedFastest.Passed, + } + if shadowApplicable && !shadowGate.Passed { + entry.Reasons = append(entry.Reasons, "forward-selected shadow overhead exceeds 10% and 100us") + } + if !guardedSelected.Passed { + entry.Reasons = append(entry.Reasons, "guarded selected-arm overhead exceeds 10% and 100us") + } + if !guardedFastest.Passed { + entry.Reasons = append(entry.Reasons, "guarded fastest-arm regret exceeds the 1.10/A/A floor") + } + if !entry.Passed { + report.EvidencePassed = false + } + if qualificationEligible { + switch split { + case "training": + report.TrainingCases++ + trainingPassed = trainingPassed && entry.Passed + case "holdout": + report.HoldoutCases++ + holdoutPassed = holdoutPassed && entry.Passed + } + } + report.Cases = append(report.Cases, entry) + } + report.TrainingPassed = protocol == referencePairProtocolConfirmation && report.TrainingCases > 0 && trainingPassed + report.HoldoutPassed = protocol == referencePairProtocolConfirmation && report.HoldoutCases > 0 && holdoutPassed + if protocol == referencePairProtocolConfirmation && (report.TrainingCases != 8 || report.HoldoutCases != 4) { + return OrientationSelectorV2Report{}, fmt.Errorf("orientation-v2 confirmation requires exactly 8 training and 4 holdout cases, got %d/%d", report.TrainingCases, report.HoldoutCases) + } + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +// collectOrientationSelectorV2Series collects orientation selector v2 series. +func collectOrientationSelectorV2Series( + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, +) (map[performanceKey]*orientationSelectorV2Series, []performanceKey, error) { + artifacts := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // records retains the records while anonymous record is assembled or evaluated. + records []CaseResult + }{ + { + name: "shadow", + records: shadowRecords, + }, + { + name: "incumbent", + records: incumbentRecords, + }, + { + name: "reverse", + records: reverseRecords, + }, + { + name: "guarded", + records: guardedRecords, + }, + } + keySets := make([]map[performanceKey]struct{}, len(artifacts)) + for index, artifact := range artifacts { + keys, err := orientationV2ArtifactKeys(artifact.name, artifact.records) + if err != nil { + return nil, nil, err + } + keySets[index] = keys + } + for index := 1; index < len(keySets); index++ { + if !orientationV2KeySetsEqual(keySets[0], keySets[index]) { + return nil, nil, fmt.Errorf("orientation-v2 %s artifact case set does not match shadow artifact", artifacts[index].name) + } + } + + series := make(map[performanceKey]*orientationSelectorV2Series, len(keySets[0])) + for key := range keySets[0] { + series[key] = &orientationSelectorV2Series{ + shadow: roundSamples{}, + incumbent: roundSamples{}, + reverse: roundSamples{}, + guarded: roundSamples{}, + } + } + for _, artifact := range artifacts { + seenRounds := map[performanceKey]map[int]struct{}{} + for _, record := range artifact.records { + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + current := series[key] + if current == nil { + return nil, nil, fmt.Errorf("orientation-v2 %s artifact contains unexpected case %s/%s", artifact.name, key.dataset, key.name) + } + if err := validateOrientationV2Record(record, artifact.name); err != nil { + return nil, nil, err + } + round, err := orientationV2RecordRound(record) + if err != nil { + return nil, nil, err + } + if seenRounds[key] == nil { + seenRounds[key] = map[int]struct{}{} + } + if _, duplicate := seenRounds[key][round]; duplicate { + return nil, nil, fmt.Errorf("%s/%s %s artifact duplicates round %d", key.dataset, key.name, artifact.name, round) + } + seenRounds[key][round] = struct{}{} + switch artifact.name { + case "shadow": + choice := record.TraversalTelemetry.Summary.WouldSelectIdentity + shadowOverflow := *record.TraversalTelemetry.Summary.Overflow + if current.shadowObserved && (current.wouldSelect != choice || current.shadowOverflow != shadowOverflow) { + return nil, nil, fmt.Errorf("%s/%s changes shadow would_select identity across rounds", key.dataset, key.name) + } + current.wouldSelect, current.shadowOverflow, current.shadowObserved = choice, shadowOverflow, true + appendOrientationWarmSamples(current.shadow, record) + case "incumbent": + appendOrientationWarmSamples(current.incumbent, record) + case "reverse": + appendOrientationWarmSamples(current.reverse, record) + case "guarded": + summary := record.TraversalTelemetry.Summary + if current.guardedObserved && + (current.guardedRuntime != summary.RuntimeIdentity || current.guardedBranch != summary.RuntimeBranch || + current.overflow != *summary.Overflow || current.fallback != *summary.FallbackExecuted) { + return nil, nil, fmt.Errorf("%s/%s changes guarded runtime outcome across rounds", key.dataset, key.name) + } + current.guardedRuntime, current.guardedBranch = summary.RuntimeIdentity, summary.RuntimeBranch + current.overflow, current.fallback, current.guardedObserved = *summary.Overflow, *summary.FallbackExecuted, true + appendOrientationWarmSamples(current.guarded, record) + } + } + } + + keys := sortedPerformanceKeys(keySets[0]) + for _, key := range keys { + current := series[key] + if !current.shadowObserved || current.wouldSelect == "" || !current.guardedObserved { + return nil, nil, fmt.Errorf("%s/%s lacks attributable shadow or guarded records", key.dataset, key.name) + } + if err := validateOrientationV2RuntimeConsistency(key, current); err != nil { + return nil, nil, err + } + if err := validateOrientationExactObservations(key, shadowRecords, incumbentRecords, reverseRecords, guardedRecords); err != nil { + return nil, nil, err + } + } + return series, keys, nil +} + +// orientationV2ArtifactKeys returns the lookup keys used for orientation v2 artifact. +func orientationV2ArtifactKeys(name string, records []CaseResult) (map[performanceKey]struct{}, error) { + keys := map[performanceKey]struct{}{} + if len(records) == 0 { + return nil, fmt.Errorf("orientation-v2 %s artifact is empty", name) + } + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + return nil, fmt.Errorf("orientation-v2 %s artifact contains non-PostgreSQL record %s/%s", name, record.Dataset, record.Name) + } + if record.Dataset == "" || record.Name == "" || !hasWarmLatencySample(record) { + return nil, fmt.Errorf("orientation-v2 %s artifact contains an incomplete timing record", name) + } + keys[performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + }] = struct{}{} + } + return keys, nil +} + +// orientationV2KeySetsEqual supports benchmark evidence processing for orientation v2 key sets equal. +func orientationV2KeySetsEqual(left, right map[performanceKey]struct{}) bool { + if len(left) != len(right) { + return false + } + for key := range left { + if _, found := right[key]; !found { + return false + } + } + return true +} + +// validateOrientationV2Cohort validates orientation v2 cohort. +func validateOrientationV2Cohort( + keys []performanceKey, + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, + protocol string, +) (string, error) { + cohortDeclarationSHA256 := "" + for name, records := range map[string][]CaseResult{ + "shadow": shadowRecords, "incumbent": incumbentRecords, "reverse": reverseRecords, "guarded": guardedRecords, + } { + selection, err := selectionIdentity(records) + if err != nil { + return "", fmt.Errorf("orientation-v2 %s selection: %w", name, err) + } + if cohortDeclarationSHA256 == "" { + cohortDeclarationSHA256 = selection.DeclarationSHA256 + } + if selection.Version != selectionManifestVersion || !lowercaseSHA256(selection.DeclarationSHA256) || + selection.DeclarationSHA256 != cohortDeclarationSHA256 || !selection.DiagnosticOnly || + selection.SelectedDeclarationCount != 2*len(keys) || len(selection.Resolved) != len(keys) || + selection.FullDeclarationCount != selection.SelectedDeclarationCount+selection.OmittedDeclarationCount { + return "", fmt.Errorf("orientation-v2 %s selection does not bind the exact measured cohort", name) + } + resolved := make(map[performanceKey]struct{}, len(selection.Resolved)) + for _, item := range selection.Resolved { + if item.Category != "generated_fixed_suffix_expansion" { + return "", fmt.Errorf("orientation-v2 %s selection contains a non-v3 category", name) + } + resolved[performanceKey{ + dataset: item.Dataset, + name: item.Name, + backend: ModePostgresSQL, + }] = struct{}{} + } + for _, key := range keys { + if _, found := resolved[key]; !found { + return "", fmt.Errorf("orientation-v2 %s selection omits %s/%s", name, key.dataset, key.name) + } + } + } + + if protocol == referencePairProtocolConfirmation { + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return "", err + } + if cohortDeclarationSHA256 != canonical.declarationSHA256 || !orientationV2KeySetsEqual(canonical.keys, performanceKeySet(keys)) { + return "", fmt.Errorf("orientation-v2 confirmation does not contain the exact frozen 8-training/4-holdout cohort") + } + } + return cohortDeclarationSHA256, nil +} + +// orientationV2CanonicalCohort groups state that must remain consistent while processing orientation v2 canonical cohort. +type orientationV2CanonicalCohort struct { + // keys retains the keys while orientationV2CanonicalCohort is assembled or evaluated. + keys map[performanceKey]struct{} + // trainingKeys retains the training keys while orientationV2CanonicalCohort is assembled or evaluated. + trainingKeys map[performanceKey]struct{} + // declarationSHA256 binds the referenced declaration content by SHA-256 digest. + declarationSHA256 string + // trainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + trainingDeclarationSHA256 string +} + +// orientationV2CanonicalCases contains the frozen orientation v2 canonical cases declaration consulted by package validation. +var orientationV2CanonicalCases = []struct { + // dataset retains the dataset while anonymous record is assembled or evaluated. + dataset string + // name retains the name while anonymous record is assembled or evaluated. + name string + // split retains the split while anonymous record is assembled or evaluated. + split string +}{ + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q1_z1_c0_s0_p0", "GFSE-V3-TRAIN-Q1-C0-S0-root_baseline", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s0_p0", "GFSE-V3-TRAIN-Q4-C0-S0-root_multiplicity", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s0_p0", "GFSE-V3-TRAIN-Q4-C1-S0-productive_cycle", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c0_s1_p0", "GFSE-V3-TRAIN-Q4-C0-S1-productive_self_loop", "training"}, + {"generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s1_p0", "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", "training"}, + {"generated_fixed_suffix_expansion_v3_d3_f6_r1_x0_i4_m2_q2_z0_c0_s0_p32", "GFSE-V3-TRAIN-D03-F006-R1-X0-I4-M2-Q2-endpoint", "training"}, + {"generated_fixed_suffix_expansion_v3_d5_f8_r4_x3_i0_m1_q3_z0_c0_s0_p0", "GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path", "training"}, + {"generated_fixed_suffix_expansion_v3_d6_f10_r10_x1_i7_m3_q1_z0_c0_s0_p64", "GFSE-V3-TRAIN-D06-F010-R10-X1-I7-M3-Q1-endpoint", "training"}, + {"generated_fixed_suffix_expansion_v3_d7_f5_r1_x3_i6_m2_q6_z0_c1_s1_p24", "GFSE-V3-HOLDOUT-D07-F005-R1-X3-I6-M2-Q6-C1-S1-path", "holdout"}, + {"generated_fixed_suffix_expansion_v3_d11_f7_r0_x4_i0_m3_q2_z1_c1_s0_p96", "GFSE-V3-HOLDOUT-D11-F007-R0-X4-I0-M3-Q2-C1-S0-endpoint", "holdout"}, + {"generated_fixed_suffix_expansion_v3_d13_f9_r4_x1_i2_m1_q7_z0_c0_s1_p8", "GFSE-V3-HOLDOUT-D13-F009-R4-X1-I2-M1-Q7-C0-S1-path", "holdout"}, + {"generated_fixed_suffix_expansion_v3_d15_f12_r6_x6_i9_m2_q3_z1_c0_s0_p128", "GFSE-V3-HOLDOUT-D15-F012-R6-X6-I9-M2-Q3-Z1-endpoint", "holdout"}, +} + +// canonicalOrientationV2Cohort resolves the frozen orientation-v2 training and holdout membership. +func canonicalOrientationV2Cohort() (orientationV2CanonicalCohort, error) { + keys := map[performanceKey]struct{}{} + trainingKeys := map[performanceKey]struct{}{} + declared := make([]DeclaredCaseBackend, 0, 24) + trainingDeclared := make([]DeclaredCaseBackend, 0, 16) + training, holdout := 0, 0 + for _, testCase := range orientationV2CanonicalCases { + key := performanceKey{ + dataset: testCase.dataset, + name: testCase.name, + backend: ModePostgresSQL, + } + if _, duplicate := keys[key]; duplicate || !strings.HasPrefix(testCase.dataset, "generated_fixed_suffix_expansion_v3_") { + return orientationV2CanonicalCohort{}, fmt.Errorf("frozen orientation-v2 cohort contains an invalid declaration") + } + keys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + declared = append(declared, DeclaredCaseBackend{ + Dataset: key.dataset, + Name: key.name, + Backend: backend, + }) + } + if testCase.split == "training" { + training++ + trainingKeys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + trainingDeclared = append(trainingDeclared, DeclaredCaseBackend{ + Dataset: key.dataset, + Name: key.name, + Backend: backend, + }) + } + } else if testCase.split == "holdout" { + holdout++ + } else { + return orientationV2CanonicalCohort{}, fmt.Errorf("frozen orientation-v2 cohort contains an invalid split") + } + } + if training != 8 || holdout != 4 || len(keys) != 12 { + return orientationV2CanonicalCohort{}, fmt.Errorf("frozen orientation-v2 cohort must contain exactly 8 training and 4 holdout cases") + } + return orientationV2CanonicalCohort{ + keys: keys, + trainingKeys: trainingKeys, + declarationSHA256: declarationSHA256(declared), + trainingDeclarationSHA256: declarationSHA256(trainingDeclared), + }, nil +} + +// performanceKeySet returns the lookup keys used for performance. +func performanceKeySet(keys []performanceKey) map[performanceKey]struct{} { + result := make(map[performanceKey]struct{}, len(keys)) + for _, key := range keys { + result[key] = struct{}{} + } + return result +} + +// validateOrientationV2Freeze validates orientation v2 freeze. +func validateOrientationV2Freeze(freeze *OrientationSelectorV2FreezeManifest, discovery *OrientationSelectorV2Report, report OrientationSelectorV2Report) error { + if freeze == nil || discovery == nil { + return fmt.Errorf("orientation-v2 confirmation requires a discovery report and freeze manifest") + } + if freeze.Version != 1 || freeze.Policy != report.Policy || freeze.Formula != report.Formula || + freeze.SourceCommit != report.SourceCommit || freeze.DirtyDiffSHA256 != report.DirtyDiffSHA256 || + freeze.BinarySHA256 != report.BinarySHA256 || freeze.CohortDeclarationSHA256 != report.CohortDeclarationSHA256 || + !lowercaseSHA256(freeze.DiscoveryReportSHA256) || len(freeze.Caps) != len(report.Caps) { + return fmt.Errorf("orientation-v2 confirmation identity differs from the frozen discovery") + } + if report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || discovery.Version != orientationSelectorReportV2Version || + discovery.Protocol != referencePairProtocolDiscovery || discovery.Policy != freeze.Policy || discovery.Formula != freeze.Formula || + discovery.SourceCommit != freeze.SourceCommit || discovery.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || + discovery.BinarySHA256 != freeze.BinarySHA256 || len(discovery.Cases) != 8 || + !lowercaseSHA256(discovery.ShadowArtifactSHA256) || !lowercaseSHA256(discovery.IncumbentArtifactSHA256) || + !lowercaseSHA256(discovery.ReverseArtifactSHA256) || !lowercaseSHA256(discovery.GuardedArtifactSHA256) || + !lowercaseSHA256(discovery.AAReportSHA256) { + return fmt.Errorf("orientation-v2 discovery report does not prove the frozen clean training-only identity") + } + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return err + } + discoveryKeys := map[performanceKey]struct{}{} + for _, entry := range discovery.Cases { + if entry.QualificationSplit != "training" { + return fmt.Errorf("orientation-v2 discovery report contains non-training timing") + } + discoveryKeys[performanceKey{ + dataset: entry.Dataset, + name: entry.Name, + backend: ModePostgresSQL, + }] = struct{}{} + } + if !orientationV2KeySetsEqual(discoveryKeys, canonical.trainingKeys) { + return fmt.Errorf("orientation-v2 discovery report does not contain the exact frozen training cohort") + } + if discovery.CohortDeclarationSHA256 != canonical.trainingDeclarationSHA256 { + return fmt.Errorf("orientation-v2 discovery report does not bind the exact frozen training declaration") + } + for name, value := range report.Caps { + if freeze.Caps[name] != value || discovery.Caps[name] != value { + return fmt.Errorf("orientation-v2 confirmation cap %s differs from the frozen discovery", name) + } + } + return nil +} + +// orientationV2RecordRound supports benchmark evidence processing for orientation v2 record round. +func orientationV2RecordRound(record CaseResult) (int, error) { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + current := sample.Round + if current == 0 { + current = round + } + if current < 1 || (round != 0 && current != round) { + return 0, fmt.Errorf("%s/%s has inconsistent orientation-v2 round metadata", record.Dataset, record.Name) + } + round = current + } + if round < 1 { + return 0, fmt.Errorf("%s/%s has no orientation-v2 round identity", record.Dataset, record.Name) + } + return round, nil +} + +// validateOrientationV2Record validates orientation v2 record. +func validateOrientationV2Record(record CaseResult, arm string) error { + if record.Status != StatusOK || record.Environment == nil || record.PostgresEnvironment == nil || record.TraversalTelemetry == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing PostgreSQL record", record.Dataset, record.Name, arm) + } + if record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || len(record.Environment.Concurrency) != 0 { + return fmt.Errorf("%s/%s %s arm lacks the schema-v2 single-session timing contract", record.Dataset, record.Name, arm) + } + if record.Environment.ExistingGraph || record.Fixture == nil || record.Fixture.Dataset != record.Dataset || + !lowercaseSHA256(record.Fixture.Checksum) || !record.Fixture.PhysicalValidated { + return fmt.Errorf("%s/%s %s arm lacks one exact physically validated corpus fixture", record.Dataset, record.Name, arm) + } + if !lowercaseSHA256(record.WorkloadSHA256) || !lowercaseSHA256(record.SQLFingerprint) { + return fmt.Errorf("%s/%s %s arm lacks canonical workload or SQL identity", record.Dataset, record.Name, arm) + } + if len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil { + return fmt.Errorf("%s/%s %s arm mixes selector timing with supplemental PostgreSQL measurements", record.Dataset, record.Name, arm) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") { + return fmt.Errorf("%s/%s %s arm was not measured under Repeatable Read", record.Dataset, record.Name, arm) + } + if err := record.TraversalTelemetry.Validate(); err != nil { + return fmt.Errorf("%s/%s %s arm telemetry: %w", record.Dataset, record.Name, arm, err) + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.Overflow == nil || summary.FallbackExecuted == nil { + return fmt.Errorf("%s/%s %s arm lacks a complete runtime outcome", record.Dataset, record.Name, arm) + } + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + v2 := string(optimize.ExpansionSearchPolicyOrientationProbeV2) + switch arm { + case "shadow": + if summary.EmittedIdentity != v2 || summary.SelectorVersion != v2 || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryInlineStatement || + summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || summary.RuntimeBranch != "shadow_incumbent" || + (summary.WouldSelectIdentity != forward && summary.WouldSelectIdentity != reverse) || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s shadow telemetry does not prove orientation-probe-v2 incumbent-only execution", record.Dataset, record.Name) + } + if *summary.Overflow && summary.WouldSelectIdentity != forward { + return fmt.Errorf("%s/%s overflowing shadow evidence did not fail closed to forward", record.Dataset, record.Name) + } + case "incumbent": + if summary.EmittedIdentity != forward || summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryInlineStatement || summary.WouldSelectIdentity != "" || *summary.Overflow { + return fmt.Errorf("%s/%s incumbent artifact did not execute one exact forward statement", record.Dataset, record.Name) + } + validSelected := summary.RuntimeBranch == "selected" && !*summary.FallbackExecuted + validCompileFallback := summary.RuntimeBranch == "compile_time_fallback" && *summary.FallbackExecuted && summary.FallbackIdentity == forward + if !validSelected && !validCompileFallback { + return fmt.Errorf("%s/%s incumbent artifact has an unsupported exact-arm runtime tuple", record.Dataset, record.Name) + } + if summary.SelectorVersion != "fixed-suffix-static-v1" { + return fmt.Errorf("%s/%s incumbent artifact has an unexpected selector identity", record.Dataset, record.Name) + } + case "reverse": + if summary.EmittedIdentity != reverse || summary.RuntimeIdentity != reverse || summary.AppliedIdentity != reverse || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryInlineStatement || summary.WouldSelectIdentity != "" || + summary.RuntimeBranch != "selected" || *summary.Overflow || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s reverse artifact did not execute one exact forced-reverse statement", record.Dataset, record.Name) + } + if summary.SelectorVersion != "suffix-seeded-reverse-tool-v1" { + return fmt.Errorf("%s/%s reverse artifact has an unexpected selector identity", record.Dataset, record.Name) + } + case "guarded": + if summary.EmittedIdentity != v2 || summary.SelectorVersion != v2 || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s guarded artifact does not prove the orientation-probe-v2 dual-arm boundary", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown orientation-v2 arm %q", arm) + } + if err := validateOrientationV2SampleRuntime(record, arm); err != nil { + return err + } + return nil +} + +// validateOrientationV2SampleRuntime validates orientation v2 sample runtime. +func validateOrientationV2SampleRuntime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || + sample.RuntimeBranch != summary.RuntimeBranch || sample.FallbackExecuted == nil || + *sample.FallbackExecuted != *summary.FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample contradicts its runtime summary", record.Dataset, record.Name, arm) + } + switch arm { + case "shadow", "guarded": + if sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s arm warm sample lacks timed-invocation attribution", record.Dataset, record.Name, arm) + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + case "incumbent", "reverse": + if sample.RuntimeAttestation != "same_case_invocation_local_replay" && sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s exact arm warm sample lacks runtime attribution", record.Dataset, record.Name, arm) + } + if sample.RuntimeAttestation == "timed_invocation" { + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s exact arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + } else if len(sample.RuntimeReceiptEvents) != 0 { + return fmt.Errorf("%s/%s %s exact arm replay must not claim a timed receipt", record.Dataset, record.Name, arm) + } + } + } + return nil +} + +// validateOrientationV2RuntimeConsistency validates orientation v2 runtime consistency. +func validateOrientationV2RuntimeConsistency(key performanceKey, current *orientationSelectorV2Series) error { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + if current.shadowOverflow && !current.overflow { + return fmt.Errorf("%s/%s guarded evidence lost shadow probe overflow", key.dataset, key.name) + } + if current.overflow { + choiceConsistent := current.shadowOverflow && current.wouldSelect == forward || !current.shadowOverflow && current.wouldSelect == reverse + if !choiceConsistent || current.guardedRuntime != forward || current.guardedBranch != "exact_forward_incumbent" || !current.fallback { + return fmt.Errorf("%s/%s guarded overflow did not execute the exact forward fallback", key.dataset, key.name) + } + return nil + } + if current.fallback { + return fmt.Errorf("%s/%s guarded artifact reports fallback without overflow", key.dataset, key.name) + } + if current.wouldSelect == reverse { + if current.guardedRuntime != reverse || current.guardedBranch != "suffix_seeded_reverse" { + return fmt.Errorf("%s/%s guarded runtime does not match the shadow reverse choice", key.dataset, key.name) + } + return nil + } + if current.wouldSelect == forward && current.guardedRuntime == forward && current.guardedBranch == "exact_forward_incumbent" { + return nil + } + return fmt.Errorf("%s/%s guarded runtime does not match the shadow forward choice", key.dataset, key.name) +} + +// requireOrientationV2RoundSets supports benchmark evidence processing for require orientation v2 round sets. +func requireOrientationV2RoundSets(key performanceKey, current *orientationSelectorV2Series) error { + expected := sortedRounds(current.shadow) + for name, rounds := range map[string][]int{ + "incumbent": sortedRounds(current.incumbent), + "reverse": sortedRounds(current.reverse), + "guarded": sortedRounds(current.guarded), + } { + if !slices.Equal(expected, rounds) { + return fmt.Errorf("%s/%s %s arm round set does not match shadow", key.dataset, key.name, name) + } + } + return nil +} + +// validateOrientationV2ArmOrder validates orientation v2 arm order. +func validateOrientationV2ArmOrder( + shadowRecords, incumbentRecords, reverseRecords, guardedRecords []CaseResult, + key performanceKey, + rounds []int, + minimumWarmups int, +) error { + armRecords := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // records retains the records while anonymous record is assembled or evaluated. + records []CaseResult + }{ + { + name: "shadow", + records: shadowRecords, + }, + { + name: "incumbent", + records: incumbentRecords, + }, + { + name: "reverse", + records: reverseRecords, + }, + { + name: "guarded", + records: guardedRecords, + }, + } + evidence := make([]map[int]pairedRoundEvidence, len(armRecords)) + positionCounts := make([][5]int, len(armRecords)) + for index, arm := range armRecords { + current, err := collectPairedRoundEvidence(arm.records, key) + if err != nil { + return err + } + evidence[index] = current + } + for _, round := range rounds { + seenPositions := map[int]struct{}{} + seenNames := map[string]struct{}{} + block, runUUID := 0, "" + for index, arm := range armRecords { + current, found := evidence[index][round] + if !found || current.Warmups < minimumWarmups || current.Arm != arm.name { + return fmt.Errorf("%s/%s round %d lacks %s arm identity or %d warmups", key.dataset, key.name, round, arm.name, minimumWarmups) + } + if current.ArmOrder < 1 || current.ArmOrder > 4 { + return fmt.Errorf("%s/%s round %d has invalid four-arm order", key.dataset, key.name, round) + } + if _, duplicate := seenPositions[current.ArmOrder]; duplicate { + return fmt.Errorf("%s/%s round %d has duplicate four-arm order", key.dataset, key.name, round) + } + if _, duplicate := seenNames[current.Arm]; duplicate { + return fmt.Errorf("%s/%s round %d has indistinct four-arm labels", key.dataset, key.name, round) + } + seenPositions[current.ArmOrder] = struct{}{} + seenNames[current.Arm] = struct{}{} + positionCounts[index][current.ArmOrder]++ + if block == 0 { + block, runUUID = current.Block, current.RunUUID + } else if current.Block != block || current.RunUUID != runUUID { + return fmt.Errorf("%s/%s round %d has mismatched four-arm block or run UUID", key.dataset, key.name, round) + } + } + if block < 1 || runUUID == "" || len(seenPositions) != 4 || len(seenNames) != 4 { + return fmt.Errorf("%s/%s round %d lacks a complete four-arm block", key.dataset, key.name, round) + } + } + for index, counts := range positionCounts { + minimum, maximum := counts[1], counts[1] + for position := 2; position <= 4; position++ { + minimum = min(minimum, counts[position]) + maximum = max(maximum, counts[position]) + } + if maximum-minimum > 1 { + return fmt.Errorf("%s/%s %s arm order is not position-balanced", key.dataset, key.name, armRecords[index].name) + } + } + return nil +} + +// validateOrientationV2EvidenceIdentity validates orientation v2 evidence identity. +func validateOrientationV2EvidenceIdentity(artifacts ...[]CaseResult) (orientationSelectorV2Identity, error) { + identity := orientationSelectorV2Identity{} + var postgresEnvironment *PostgresEnvironment + allRecords := make([]CaseResult, 0) + for _, records := range artifacts { + allRecords = append(allRecords, records...) + for _, record := range records { + if record.Environment == nil || record.PostgresEnvironment == nil { + return orientationSelectorV2Identity{}, fmt.Errorf("%s/%s lacks orientation-v2 environment identity", record.Dataset, record.Name) + } + current := orientationSelectorV2Identity{ + sourceCommit: strings.TrimSpace(record.Environment.SourceCommit), + dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, + binarySHA256: record.Environment.BinarySHA256, + corpusSHA256: record.Environment.CorpusSHA256, + } + if current.sourceCommit == "" || current.sourceCommit == "unknown" || + !lowercaseSHA256(current.dirtyDiffSHA256) || !lowercaseSHA256(current.binarySHA256) || !lowercaseSHA256(current.corpusSHA256) { + return orientationSelectorV2Identity{}, fmt.Errorf("%s/%s lacks frozen source, diff, binary, or corpus identity", record.Dataset, record.Name) + } + if identity.sourceCommit == "" { + identity = current + } else if identity != current { + return orientationSelectorV2Identity{}, fmt.Errorf("orientation-v2 artifacts mix source, diff, binary, or corpus identities") + } + if postgresEnvironment == nil { + copy := *record.PostgresEnvironment + postgresEnvironment = © + } else if !sameOrientationV2PostgresEnvironment(postgresEnvironment, record.PostgresEnvironment) { + return orientationSelectorV2Identity{}, fmt.Errorf("orientation-v2 artifacts mix PostgreSQL environments") + } + } + } + keys, err := orientationV2ArtifactKeys("combined", allRecords) + if err != nil { + return orientationSelectorV2Identity{}, err + } + for key := range keys { + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(allRecords, key) + if err != nil { + return orientationSelectorV2Identity{}, err + } + fixtureSHA256, err := fixtureSHA256ForKey(allRecords, key) + if err != nil { + return orientationSelectorV2Identity{}, err + } + if !lowercaseSHA256(postgresEnvironmentSHA256) || !lowercaseSHA256(fixtureSHA256) { + return orientationSelectorV2Identity{}, fmt.Errorf("%s/%s lacks frozen PostgreSQL or fixture identity", key.dataset, key.name) + } + } + return identity, nil +} + +// validateOrientationV2AAEvidence validates orientation v2aa evidence. +func validateOrientationV2AAEvidence(report *AAResolutionReport, records []CaseResult) error { + keys, err := orientationV2ArtifactKeys("incumbent", records) + if err != nil { + return err + } + entries := make(map[performanceKey]AAResolutionCase, len(report.Cases)) + for _, entry := range report.Cases { + entries[performanceKey{ + dataset: entry.Dataset, + name: entry.Name, + backend: entry.Backend, + }] = entry + } + for key := range keys { + entry, found := entries[key] + if !found { + return fmt.Errorf("A/A report has no environment evidence for %s/%s", key.dataset, key.name) + } + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(records, key) + if err != nil { + return err + } + fixtureSHA256, err := fixtureSHA256ForKey(records, key) + if err != nil { + return err + } + if !lowercaseSHA256(entry.PostgresEnvironmentSHA256) || entry.PostgresEnvironmentSHA256 != postgresEnvironmentSHA256 { + return fmt.Errorf("A/A PostgreSQL environment does not match %s/%s", key.dataset, key.name) + } + if !lowercaseSHA256(entry.FixtureSHA256) || entry.FixtureSHA256 != fixtureSHA256 { + return fmt.Errorf("A/A fixture does not match %s/%s", key.dataset, key.name) + } + } + return nil +} + +// lowercaseSHA256 reports whether a value is a canonical lowercase SHA-256 digest. +func lowercaseSHA256(value string) bool { + return value == strings.ToLower(value) && validSHA256(value) +} + +// sameOrientationV2PostgresEnvironment reports whether two runs share the PostgreSQL settings bound by orientation evidence. +func sameOrientationV2PostgresEnvironment(left, right *PostgresEnvironment) bool { + return left.Version == right.Version && left.Database == right.Database && + left.PlanCacheMode == right.PlanCacheMode && left.TransactionIsolation == right.TransactionIsolation && + left.WorkMem == right.WorkMem && left.TempFileLimit == right.TempFileLimit && + left.GraphPartitionCount == right.GraphPartitionCount && + left.DatabaseOID == right.DatabaseOID && left.PostmasterStartedAt.Equal(right.PostmasterStartedAt) && + left.Autovacuum == right.Autovacuum && + left.SchemaFingerprint == right.SchemaFingerprint && left.IndexFingerprint == right.IndexFingerprint +} + +// createOrientationSelectorV2Report loads four matched timing artifacts and +// one checksummed A/A report, then writes schema-v2 qualification evidence. +func createOrientationSelectorV2Report( + shadowPath, incumbentPath, reversePath, guardedPath, aaPath, freezePath, discoveryReportPath, freezeOutputPath, outputPath string, + options OrientationSelectorV2ReportOptions, +) (bool, error) { + paths := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // path retains the path while anonymous record is assembled or evaluated. + path string + }{ + { + name: "shadow", + path: shadowPath, + }, + { + name: "incumbent", + path: incumbentPath, + }, + { + name: "reverse", + path: reversePath, + }, + { + name: "guarded", + path: guardedPath, + }, + } + artifacts := make([][]CaseResult, len(paths)) + for index, input := range paths { + records, err := readJSONLFile(input.path) + if err != nil { + return false, fmt.Errorf("read orientation-v2 %s artifact: %w", input.name, err) + } + artifacts[index] = records + } + aa, aaSHA, err := loadAAResolutionReport(aaPath) + if err != nil { + return false, fmt.Errorf("read orientation-v2 A/A report: %w", err) + } + freezeSHA := "" + if freezePath != "" { + freeze, digest, err := loadOrientationSelectorV2FreezeManifest(freezePath) + if err != nil { + return false, fmt.Errorf("read orientation-v2 freeze manifest: %w", err) + } + options.Freeze = freeze + freezeSHA = digest + discovery, err := loadOrientationSelectorV2Report(discoveryReportPath) + if err != nil { + return false, fmt.Errorf("read orientation-v2 discovery report: %w", err) + } + if digest, err := fileSHA256(discoveryReportPath); err != nil { + return false, err + } else if digest != freeze.DiscoveryReportSHA256 { + return false, fmt.Errorf("orientation-v2 discovery report digest does not match freeze manifest") + } + options.Discovery = discovery + } + report, err := buildOrientationSelectorV2Report(artifacts[0], artifacts[1], artifacts[2], artifacts[3], aa, options) + if err != nil { + return false, err + } + for index, input := range paths { + digest, err := fileSHA256(input.path) + if err != nil { + return false, err + } + switch index { + case 0: + report.ShadowArtifactSHA256 = digest + case 1: + report.IncumbentArtifactSHA256 = digest + case 2: + report.ReverseArtifactSHA256 = digest + case 3: + report.GuardedArtifactSHA256 = digest + } + } + report.AAReportSHA256 = aaSHA + report.FreezeManifestSHA256 = freezeSHA + if err := writeOrientationSelectorV2Report(outputPath, report); err != nil { + return false, err + } + if options.Protocol == referencePairProtocolDiscovery { + if err := writeOrientationSelectorV2FreezeManifest(freezeOutputPath, outputPath, report, artifacts...); err != nil { + return false, err + } + } + return report.QualificationPassed, nil +} + +// loadOrientationSelectorV2Report loads orientation selector v2 report. +func loadOrientationSelectorV2Report(path string) (*OrientationSelectorV2Report, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + report := &OrientationSelectorV2Report{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, fmt.Errorf("decode orientation-v2 discovery report: %w", err) + } + return report, nil +} + +// loadOrientationSelectorV2FreezeManifest loads orientation selector v2 freeze manifest. +func loadOrientationSelectorV2FreezeManifest(path string) (*OrientationSelectorV2FreezeManifest, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + manifest := &OrientationSelectorV2FreezeManifest{} + if err := json.Unmarshal(raw, manifest); err != nil { + return nil, "", fmt.Errorf("decode orientation-v2 freeze manifest: %w", err) + } + digest := sha256.Sum256(raw) + return manifest, hex.EncodeToString(digest[:]), nil +} + +// writeOrientationSelectorV2FreezeManifest writes orientation selector v2 freeze manifest. +func writeOrientationSelectorV2FreezeManifest(path, discoveryReportPath string, report OrientationSelectorV2Report, artifacts ...[]CaseResult) error { + if path == "" || discoveryReportPath == "" { + return fmt.Errorf("orientation-v2 discovery freeze requires report and manifest output paths") + } + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return err + } + training := map[performanceKey]struct{}{} + for _, record := range artifacts[0] { + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + if record.Shape.QualificationSplit == "training" { + training[key] = struct{}{} + } + } + if !orientationV2KeySetsEqual(training, canonical.trainingKeys) || report.CohortDeclarationSHA256 != canonical.trainingDeclarationSHA256 { + return fmt.Errorf("orientation-v2 discovery freeze requires the exact eight canonical training cases and no holdouts") + } + for _, records := range artifacts { + for _, record := range records { + if record.Shape.QualificationSplit != "training" { + return fmt.Errorf("orientation-v2 discovery freeze cannot contain holdout or diagnostic timing") + } + } + } + if report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + return fmt.Errorf("orientation-v2 discovery freeze requires a clean source tree") + } + discoveryReportSHA256, err := fileSHA256(discoveryReportPath) + if err != nil { + return err + } + manifest := OrientationSelectorV2FreezeManifest{ + Version: 1, + Policy: report.Policy, + Formula: report.Formula, + Caps: report.Caps, + SourceCommit: report.SourceCommit, + DirtyDiffSHA256: report.DirtyDiffSHA256, + BinarySHA256: report.BinarySHA256, + CohortDeclarationSHA256: canonical.declarationSHA256, + DiscoveryReportSHA256: discoveryReportSHA256, + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(manifest) + closeErr := output.Close() + if encodeErr != nil { + return encodeErr + } + return closeErr +} + +// writeOrientationSelectorV2Report writes orientation selector v2 report. +func writeOrientationSelectorV2Report(path string, report OrientationSelectorV2Report) (err error) { + output := os.Stdout + if path != "" { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/orientation_selector_report_v2_test.go b/cmd/graphbench/orientation_selector_report_v2_test.go new file mode 100644 index 00000000..6efca169 --- /dev/null +++ b/cmd/graphbench/orientation_selector_report_v2_test.go @@ -0,0 +1,862 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestOrientationSelectorV2ReportPassesForwardAndReverseWithApplicableShadowGate verifies orientation selector v2 report passes forward and reverse with applicable shadow gate behavior. +func TestOrientationSelectorV2ReportPassesForwardAndReverseWithApplicableShadowGate(t *testing.T) { + artifacts := orientationSelectorV2Artifacts{} + for index := range 8 { + training := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond+50*time.Microsecond, 10*time.Millisecond, 14*time.Millisecond, 10*time.Millisecond+60*time.Microsecond, + false, + ) + renameOrientationV2Records(fmt.Sprintf("training-forward-%02d", index), training) + artifacts = appendOrientationV2Artifacts(artifacts, training) + } + for index := range 4 { + holdout := orientationSelectorV2Records( + "holdout", string(optimize.ExpansionSearchSuffixSeededReverse), + 30*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond+50*time.Microsecond, + false, + ) + renameOrientationV2Records(fmt.Sprintf("holdout-reverse-%02d", index), holdout) + artifacts = appendOrientationV2Artifacts(artifacts, holdout) + } + + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 7, + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + Protocol: referencePairProtocolDiscovery, + }, + ) + + require.NoError(t, err) + require.Equal(t, orientationSelectorReportV2Version, report.Version) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), report.Policy) + require.False(t, report.QualificationPassed) + require.Zero(t, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.Len(t, report.Cases, 12) + for _, entry := range report.Cases { + require.True(t, entry.Passed) + require.True(t, entry.GuardedSelectedOverhead.Passed) + require.True(t, entry.GuardedFastestRegret.Passed) + if entry.WouldSelectIdentity == string(optimize.ExpansionSearchStepwiseForward) { + require.True(t, entry.ShadowForwardOverhead.Applicable) + } else { + require.False(t, entry.ShadowForwardOverhead.Applicable) + require.Greater(t, entry.ShadowForwardOverhead.Ratio.Upper, 1.10) + require.False(t, entry.ShadowForwardOverhead.Passed) + } + } +} + +// TestOrientationSelectorV2ConfirmationBindsCanonicalCohortAndFrozenDiscovery verifies orientation selector v2 confirmation binds canonical cohort and frozen discovery behavior. +func TestOrientationSelectorV2ConfirmationBindsCanonicalCohortAndFrozenDiscovery(t *testing.T) { + training, full := canonicalOrientationV2TestArtifacts(t) + discovery, err := buildOrientationSelectorV2Report( + training.shadow, training.incumbent, training.reverse, training.guarded, + testAAReportForRecords(t, training.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 5, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.NoError(t, err) + discovery.ShadowArtifactSHA256, discovery.IncumbentArtifactSHA256 = testSHA("1"), testSHA("2") + discovery.ReverseArtifactSHA256, discovery.GuardedArtifactSHA256 = testSHA("3"), testSHA("4") + discovery.AAReportSHA256 = testSHA("5") + canonical, err := canonicalOrientationV2Cohort() + require.NoError(t, err) + freeze := testOrientationV2Freeze() + freeze.DirtyDiffSHA256 = cleanWorkingTreeSHA256() + freeze.CohortDeclarationSHA256 = canonical.declarationSHA256 + + report, err := buildOrientationSelectorV2Report( + full.shadow, full.incumbent, full.reverse, full.guarded, + testAAReportForRecords(t, full.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 7, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + Protocol: referencePairProtocolConfirmation, + Freeze: freeze, + Discovery: &discovery, + }, + ) + + require.NoError(t, err) + require.True(t, report.QualificationPassed) + require.Equal(t, 8, report.TrainingCases) + require.Equal(t, 4, report.HoldoutCases) + require.Equal(t, canonical.declarationSHA256, report.CohortDeclarationSHA256) +} + +// TestCreateOrientationSelectorV2DiscoveryWritesBoundFreeze verifies create orientation selector v2 discovery writes bound freeze behavior. +func TestCreateOrientationSelectorV2DiscoveryWritesBoundFreeze(t *testing.T) { + training, _ := canonicalOrientationV2TestArtifacts(t) + training = compactOrientationV2Artifacts(training, 5, 10) + directory := t.TempDir() + paths := map[string]string{ + "shadow": filepath.Join(directory, "shadow.jsonl"), "incumbent": filepath.Join(directory, "incumbent.jsonl"), + "reverse": filepath.Join(directory, "reverse.jsonl"), "guarded": filepath.Join(directory, "guarded.jsonl"), + "aa": filepath.Join(directory, "aa.json"), "report": filepath.Join(directory, "discovery.json"), + "freeze": filepath.Join(directory, "freeze.json"), + } + writeOrientationV2TestArtifact(t, paths["shadow"], training.shadow) + writeOrientationV2TestArtifact(t, paths["incumbent"], training.incumbent) + writeOrientationV2TestArtifact(t, paths["reverse"], training.reverse) + writeOrientationV2TestArtifact(t, paths["guarded"], training.guarded) + require.NoError(t, writeAAResolutionReport(paths["aa"], *testAAReportForRecords(t, training.incumbent))) + + passed, err := createOrientationSelectorV2Report( + paths["shadow"], paths["incumbent"], paths["reverse"], paths["guarded"], paths["aa"], "", "", paths["freeze"], paths["report"], + OrientationSelectorV2ReportOptions{ + Seed: 11, + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolDiscovery, + }, + ) + + require.NoError(t, err) + require.False(t, passed) + freeze, _, err := loadOrientationSelectorV2FreezeManifest(paths["freeze"]) + require.NoError(t, err) + report, err := loadOrientationSelectorV2Report(paths["report"]) + require.NoError(t, err) + reportSHA256, err := fileSHA256(paths["report"]) + require.NoError(t, err) + canonical, err := canonicalOrientationV2Cohort() + require.NoError(t, err) + require.Equal(t, reportSHA256, freeze.DiscoveryReportSHA256) + require.Equal(t, canonical.declarationSHA256, freeze.CohortDeclarationSHA256) + require.Equal(t, report.Policy, freeze.Policy) + require.Equal(t, cleanWorkingTreeSHA256(), freeze.DirtyDiffSHA256) +} + +// TestOrientationSelectorV2ReportEnforcesEachLatencyGate verifies orientation selector v2 report enforces each latency gate behavior. +func TestOrientationSelectorV2ReportEnforcesEachLatencyGate(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // choice retains the choice while anonymous record is assembled or evaluated. + choice string + // shadow retains the shadow while anonymous record is assembled or evaluated. + shadow time.Duration + // forward retains the forward while anonymous record is assembled or evaluated. + forward time.Duration + // reverse retains the reverse while anonymous record is assembled or evaluated. + reverse time.Duration + // guarded retains the guarded while anonymous record is assembled or evaluated. + guarded time.Duration + // reason retains the reason while anonymous record is assembled or evaluated. + reason string + // shadowFails indicates whether shadow fails applies. + shadowFails bool + // selectedFails indicates whether selected fails applies. + selectedFails bool + // fastestFails indicates whether fastest fails applies. + fastestFails bool + }{ + { + name: "forward shadow", + choice: string(optimize.ExpansionSearchStepwiseForward), + shadow: 12 * time.Millisecond, + forward: 10 * time.Millisecond, + reverse: 14 * time.Millisecond, + guarded: 10 * time.Millisecond, + reason: "forward-selected shadow overhead", + shadowFails: true, + }, + { + name: "guarded selected", + choice: string(optimize.ExpansionSearchSuffixSeededReverse), + shadow: 20 * time.Millisecond, + forward: 10 * time.Millisecond, + reverse: 5 * time.Millisecond, + guarded: 7 * time.Millisecond, + reason: "guarded selected-arm overhead", + selectedFails: true, + fastestFails: true, + }, + { + name: "guarded fastest", + choice: string(optimize.ExpansionSearchStepwiseForward), + shadow: 10 * time.Millisecond, + forward: 10 * time.Millisecond, + reverse: 5 * time.Millisecond, + guarded: 10 * time.Millisecond, + reason: "guarded fastest-arm regret", + fastestFails: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + artifacts := orientationSelectorV2Records("training", testCase.choice, testCase.shadow, testCase.forward, testCase.reverse, testCase.guarded, false) + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 11, + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.NoError(t, err) + require.False(t, report.Cases[0].Passed) + require.Contains(t, report.Cases[0].Reasons[0]+fmt.Sprint(report.Cases[0].Reasons[1:]), testCase.reason) + require.Equal(t, testCase.shadowFails, report.Cases[0].ShadowForwardOverhead.Applicable && !report.Cases[0].ShadowForwardOverhead.Passed) + require.Equal(t, testCase.selectedFails, !report.Cases[0].GuardedSelectedOverhead.Passed) + require.Equal(t, testCase.fastestFails, !report.Cases[0].GuardedFastestRegret.Passed) + }) + } +} + +// TestOrientationSelectorV2ReportAcceptsExactOverflowFallback verifies orientation selector v2 report accepts exact overflow fallback behavior. +func TestOrientationSelectorV2ReportAcceptsExactOverflowFallback(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, + true, + ) + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 13, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.NoError(t, err) + require.True(t, report.Cases[0].Overflow) + require.True(t, report.Cases[0].FallbackExecuted) + require.Equal(t, "exact_forward_incumbent", report.Cases[0].GuardedRuntimeBranch) +} + +// TestOrientationSelectorV2ReportAcceptsStateOverflowAfterReverseChoice verifies orientation selector v2 report accepts state overflow after reverse choice behavior. +func TestOrientationSelectorV2ReportAcceptsStateOverflowAfterReverseChoice(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 10*time.Millisecond, + true, + ) + for index := range artifacts.shadow { + artifacts.shadow[index].TraversalTelemetry.Summary.Overflow = boolPointer(false) + } + report, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Seed: 17, + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.NoError(t, err) + require.True(t, report.Cases[0].Overflow) + require.True(t, report.Cases[0].FallbackExecuted) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), report.Cases[0].GuardedRuntimeIdentity) +} + +// TestOrientationSelectorV2ReportRejectsIncompleteConfirmationCohort verifies orientation selector v2 report rejects incomplete confirmation cohort behavior. +func TestOrientationSelectorV2ReportRejectsIncompleteConfirmationCohort(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + renameOrientationV2Records("training-only", artifacts) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolConfirmation, + Freeze: testOrientationV2Freeze(), + }, + ) + require.ErrorContains(t, err, "exact frozen 8-training/4-holdout cohort") +} + +// TestOrientationSelectorV2ReportRequiresFrozenDiscoveryForConfirmation verifies orientation selector v2 report requires frozen discovery for confirmation behavior. +func TestOrientationSelectorV2ReportRequiresFrozenDiscoveryForConfirmation(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolConfirmation, + }, + ) + require.Error(t, err) +} + +// TestOrientationSelectorV2ReportRejectsRuntimeIdentityAndReceiptDrift verifies orientation selector v2 report rejects runtime identity and receipt drift behavior. +func TestOrientationSelectorV2ReportRejectsRuntimeIdentityAndReceiptDrift(t *testing.T) { + for _, mutate := range []func(*orientationSelectorV2Artifacts){ + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].TraversalTelemetry.Summary.RuntimeIdentity = string(optimize.ExpansionSearchStepwiseForward) + artifacts.guarded[0].TraversalTelemetry.Summary.AppliedIdentity = string(optimize.ExpansionSearchStepwiseForward) + artifacts.guarded[0].TraversalTelemetry.Summary.RuntimeBranch = "exact_forward_incumbent" + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].TraversalTelemetry.Summary.FallbackExecuted = boolPointer(true) + artifacts.guarded[0].TraversalTelemetry.Summary.FallbackIdentity = string(optimize.ExpansionSearchStepwiseForward) + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].Stats.Samples[1].RuntimeReceiptEvents = nil + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.shadow[0].TraversalTelemetry.Summary.EmittedIdentity = string(optimize.ExpansionSearchPolicyOrientationProbeV1) + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.incumbent[0].TraversalTelemetry.Summary.SelectorVersion = "static-lowering-v1" + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].TraversalTelemetry.Summary.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + }, + } { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond, false, + ) + mutate(&artifacts) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.Error(t, err) + } +} + +// TestOrientationSelectorV2ReportRejectsIdentityCaseObservationAndOrderDrift verifies orientation selector v2 report rejects identity case observation and order drift behavior. +func TestOrientationSelectorV2ReportRejectsIdentityCaseObservationAndOrderDrift(t *testing.T) { + mutations := []func(*orientationSelectorV2Artifacts){ + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].Environment.CorpusSHA256 = testSHA("9") + }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.reverse[0].WorkloadSHA256 = "changed" }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.guarded[0].ObservedRows = []string{"changed"} + }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.guarded[0].SQLFingerprint = "changed" }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.guarded = artifacts.guarded[1:] }, + func(artifacts *orientationSelectorV2Artifacts) { + for idx := range artifacts.guarded { + artifacts.guarded[idx].Name = "unexpected" + } + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.reverse[0].Shape.QualificationSplit = "holdout" + }, + func(artifacts *orientationSelectorV2Artifacts) { artifacts.guarded[0].Fixture.Checksum = "changed" }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.reverse[0].PostgresEnvironment.EdgeRelationBytes++ + }, + func(artifacts *orientationSelectorV2Artifacts) { + artifacts.shadow[0].PostgresEnvironment.AnalyzeState = "edge:never" + }, + func(artifacts *orientationSelectorV2Artifacts) { + for sampleIdx := range artifacts.guarded[0].Stats.Samples { + artifacts.guarded[0].Stats.Samples[sampleIdx].ArmOrder = 3 + } + }, + } + for _, mutate := range mutations { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchSuffixSeededReverse), + 10*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond, false, + ) + mutate(&artifacts) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.Error(t, err) + } +} + +// TestOrientationSelectorV2ReportRejectsUnboundAAEnvironment verifies orientation selector v2 report rejects unbound aa environment behavior. +func TestOrientationSelectorV2ReportRejectsUnboundAAEnvironment(t *testing.T) { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + for _, mutate := range []func(*AAResolutionReport){ + func(report *AAResolutionReport) { report.Cases[0].PostgresEnvironmentSHA256 = "" }, + func(report *AAResolutionReport) { report.Cases[0].FixtureSHA256 = testSHA("9") }, + } { + aa := testAAReportForRecords(t, artifacts.incumbent) + mutate(aa) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, aa, + OrientationSelectorV2ReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.ErrorContains(t, err, "incumbent A/A environment") + } +} + +// TestOrientationSelectorV2ReportRejectsSupplementalMeasurements verifies orientation selector v2 report rejects supplemental measurements behavior. +func TestOrientationSelectorV2ReportRejectsSupplementalMeasurements(t *testing.T) { + mutations := []func(*CaseResult){ + func(record *CaseResult) { record.Concurrency = []ConcurrencyBlock{{Concurrency: 2}} }, + func(record *CaseResult) { record.PostgresReferences = []PostgresReferenceResult{{Name: "unexpected"}} }, + func(record *CaseResult) { record.ClientWaterfall = &ClientWaterfall{} }, + func(record *CaseResult) { record.RawPGXWaterfall = &PostgresBoundaryWaterfall{} }, + func(record *CaseResult) { record.RawPGXRoundTrip = &PostgresBoundaryWaterfall{} }, + } + for _, mutate := range mutations { + artifacts := orientationSelectorV2Records( + "training", string(optimize.ExpansionSearchStepwiseForward), + 10*time.Millisecond, 10*time.Millisecond, 12*time.Millisecond, 10*time.Millisecond, false, + ) + mutate(&artifacts.shadow[0]) + _, err := buildOrientationSelectorV2Report( + artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded, + testAAReportForRecords(t, artifacts.incumbent), + OrientationSelectorV2ReportOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + Protocol: referencePairProtocolDiscovery, + }, + ) + require.ErrorContains(t, err, "mixes selector timing with supplemental PostgreSQL measurements") + } +} + +// orientationSelectorV2Artifacts groups state that must remain consistent while processing orientation selector v2 artifacts. +type orientationSelectorV2Artifacts struct { + // shadow retains the shadow while orientationSelectorV2Artifacts is assembled or evaluated. + shadow []CaseResult + // incumbent retains the incumbent while orientationSelectorV2Artifacts is assembled or evaluated. + incumbent []CaseResult + // reverse retains the reverse while orientationSelectorV2Artifacts is assembled or evaluated. + reverse []CaseResult + // guarded retains the guarded while orientationSelectorV2Artifacts is assembled or evaluated. + guarded []CaseResult +} + +// canonicalOrientationV2TestArtifacts builds a complete frozen artifact set for selector-v2 tests. +func canonicalOrientationV2TestArtifacts(t *testing.T) (orientationSelectorV2Artifacts, orientationSelectorV2Artifacts) { + t.Helper() + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + full := orientationSelectorV2Artifacts{} + for _, testCase := range corpus.Cases { + isTraining := false + if !slices.Contains(testCase.Tags, "orientation-v2-training") && !slices.Contains(testCase.Tags, "orientation-v2-holdout") { + continue + } + isTraining = testCase.Shape.QualificationSplit == "training" + choice := string(optimize.ExpansionSearchSuffixSeededReverse) + shadow, forward, reverse, guarded := 20*time.Millisecond, 10*time.Millisecond, 5*time.Millisecond, 5*time.Millisecond+50*time.Microsecond + if isTraining { + choice = string(optimize.ExpansionSearchStepwiseForward) + shadow, forward, reverse, guarded = 10*time.Millisecond+50*time.Microsecond, 10*time.Millisecond, 14*time.Millisecond, 10*time.Millisecond+60*time.Microsecond + } + current := orientationSelectorV2Records(testCase.Shape.QualificationSplit, choice, shadow, forward, reverse, guarded, false) + fixture, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err) + fixture.PhysicalValidated = true + fixture.PhysicalNodeCount, fixture.PhysicalEdgeCount = int64(fixture.NodeCount), int64(fixture.EdgeCount) + fixture.NodeRelationBytes, fixture.EdgeRelationBytes = int64(fixture.NodeCount*1024), int64(fixture.EdgeCount*1024) + for _, records := range [][]CaseResult{current.shadow, current.incumbent, current.reverse, current.guarded} { + for index := range records { + record := &records[index] + record.Source, record.Dataset, record.Name, record.Category, record.Shape = testCase.Source, testCase.Dataset, testCase.Name, testCase.Category, testCase.Shape + record.WorkloadSHA256 = scaleCaseWorkloadIdentity(testCase, ModePostgresSQL) + attachFixtureMetadata(record, fixture) + record.Environment.DirtyDiffSHA256 = cleanWorkingTreeSHA256() + record.PostgresEnvironment.NodeRelationBytes = fixture.NodeRelationBytes + record.PostgresEnvironment.EdgeRelationBytes = fixture.EdgeRelationBytes + record.PostgresEnvironment.AnalyzeState = "edge:analyzed,node:analyzed" + } + } + full = appendOrientationV2Artifacts(full, current) + } + training := orientationSelectorV2Artifacts{ + shadow: cloneOrientationV2Split(full.shadow, "training"), + incumbent: cloneOrientationV2Split(full.incumbent, "training"), + reverse: cloneOrientationV2Split(full.reverse, "training"), + guarded: cloneOrientationV2Split(full.guarded, "training"), + } + stampOrientationV2Selections(&training) + stampOrientationV2Selections(&full) + return training, full +} + +// cloneOrientationV2Split returns an independent copy of orientation v2 split. +func cloneOrientationV2Split(records []CaseResult, split string) []CaseResult { + result := make([]CaseResult, 0, len(records)) + for _, record := range records { + if record.Shape.QualificationSplit != split { + continue + } + copy := record + if record.Environment != nil { + environment := *record.Environment + copy.Environment = &environment + } + result = append(result, copy) + } + return result +} + +// compactOrientationV2Artifacts prepares or inspects test evidence for compact orientation v2 artifacts. +func compactOrientationV2Artifacts(artifacts orientationSelectorV2Artifacts, rounds, samples int) orientationSelectorV2Artifacts { + compact := func(records []CaseResult) []CaseResult { + result := make([]CaseResult, 0, len(records)) + for _, record := range records { + if record.Environment.Round > rounds { + continue + } + copy := record + copy.Stats.Samples = append([]LatencySample(nil), record.Stats.Samples[:samples]...) + result = append(result, copy) + } + return result + } + return orientationSelectorV2Artifacts{ + shadow: compact(artifacts.shadow), + incumbent: compact(artifacts.incumbent), + reverse: compact(artifacts.reverse), + guarded: compact(artifacts.guarded), + } +} + +// writeOrientationV2TestArtifact writes orientation v2 test artifact. +func writeOrientationV2TestArtifact(t *testing.T, path string, records []CaseResult) { + t.Helper() + output, err := os.Create(path) + require.NoError(t, err) + require.NoError(t, writeJSONL(output, records)) + require.NoError(t, output.Close()) +} + +// orientationSelectorV2Records prepares or inspects test evidence for orientation selector v2 records. +func orientationSelectorV2Records( + split, wouldSelect string, + shadowDuration, incumbentDuration, reverseDuration, guardedDuration time.Duration, + overflow bool, +) orientationSelectorV2Artifacts { + const rounds = 12 + orders := [][4]int{ + {1, 2, 3, 4}, {2, 3, 4, 1}, {3, 4, 1, 2}, {4, 1, 2, 3}, + {1, 3, 4, 2}, {2, 4, 1, 3}, {3, 1, 2, 4}, {4, 2, 3, 1}, + {1, 4, 2, 3}, {2, 1, 3, 4}, {3, 2, 4, 1}, {4, 3, 1, 2}, + } + artifacts := orientationSelectorV2Artifacts{} + for round := 1; round <= rounds; round++ { + order := orders[round-1] + artifacts.shadow = append(artifacts.shadow, orientationSelectorV2Record(round, order[0], "shadow", split, wouldSelect, shadowDuration, overflow)) + artifacts.incumbent = append(artifacts.incumbent, orientationSelectorV2Record(round, order[1], "incumbent", split, "", incumbentDuration, false)) + artifacts.reverse = append(artifacts.reverse, orientationSelectorV2Record(round, order[2], "reverse", split, "", reverseDuration, false)) + artifacts.guarded = append(artifacts.guarded, orientationSelectorV2Record(round, order[3], "guarded", split, wouldSelect, guardedDuration, overflow)) + } + stampOrientationV2Selections(&artifacts) + return artifacts +} + +// orientationSelectorV2Record prepares or inspects test evidence for orientation selector v2 record. +func orientationSelectorV2Record(round, armOrder int, arm, split, choice string, duration time.Duration, overflow bool) CaseResult { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + v2 := string(optimize.ExpansionSearchPolicyOrientationProbeV2) + runtimeIdentity, emittedIdentity, selectorVersion := forward, forward, "fixed-suffix-static-v1" + runtimeBranch, boundary, wouldSelect := "selected", optimize.ExpansionSearchExecutionBoundaryInlineStatement, "" + fallback := false + requested := forward + if arm == "shadow" { + emittedIdentity, selectorVersion, wouldSelect = v2, v2, choice + runtimeBranch, requested = "shadow_incumbent", reverse + } + if arm == "reverse" { + runtimeIdentity, emittedIdentity, selectorVersion, requested = reverse, reverse, "suffix-seeded-reverse-tool-v1", reverse + } + if arm == "guarded" { + emittedIdentity, selectorVersion, boundary, requested = v2, v2, optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, reverse + if choice == reverse && !overflow { + runtimeIdentity, runtimeBranch = reverse, "suffix_seeded_reverse" + } else { + runtimeIdentity, runtimeBranch = forward, "exact_forward_incumbent" + fallback = overflow + } + } + provenance := map[string]string{ + "requested_identity": "test", "planned_identities": "test", "emitted_identity": "test", + "runtime_identity": "test", "applied_identity": "test", "selector_version": "test", + "scheduler_version": "test", "runtime_branch": "test", "runtime_outcome_available": "test", + "overflow": "test", "fallback_executed": "test", "execution_boundary": "test", + } + if wouldSelect != "" { + provenance["would_select_identity"] = "test" + } + if fallback { + provenance["fallback_identity"] = "test" + } + available := true + record := CaseResult{ + Source: "cases/orientation-v2.json", + Dataset: "orientation-v2-fixture", + Name: "fixed-suffix", + Category: "generated_fixed_suffix_expansion", + WorkloadSHA256: sqlFingerprint("orientation-v2-workload"), + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + QualificationSplit: split, + }, + RowCount: 1, + ObservedRows: []string{"[42]"}, + StableObservation: true, + SQLFingerprint: sqlFingerprint("orientation-v2-" + arm + "-sql"), + Fixture: &FixtureMetadata{ + Dataset: "orientation-v2-fixture", + Checksum: sqlFingerprint("orientation-v2-fixture-checksum"), + NodeCount: 10, + EdgeCount: 12, + PhysicalValidated: true, + PhysicalNodeCount: 10, + PhysicalEdgeCount: 12, + Configuration: "orientation-v2-test", + }, + PostgresEnvironment: &PostgresEnvironment{ + Version: "PostgreSQL test", + Database: "dawgs", + PlanCacheMode: "auto", + TransactionIsolation: "repeatable read", + WorkMem: "4MB", + TempFileLimit: "-1", + GraphPartitionCount: 1, + DatabaseOID: 1, + Autovacuum: "on", + AnalyzeState: "stable", + SchemaFingerprint: "schema", + IndexFingerprint: "index", + }, + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: testSHA("c"), + SourceCommit: "deadbeef", + DirtyDiffSHA256: testSHA("d"), + BinarySHA256: testSHA("b"), + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + RunUUID: fmt.Sprintf("orientation-v2-run-%d", round), + Arm: arm, + ArmOrder: armOrder, + Block: round, + Round: round, + WarmupIterations: 20, + PoolSize: 1, + }, + TraversalTelemetry: &TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: TraversalTelemetryLevelSummary, + Summary: TraversalExecutionSummary{ + RequestedIdentity: requested, + PlannedIdentities: []string{forward, reverse}, + EmittedIdentity: emittedIdentity, + RuntimeIdentity: runtimeIdentity, + AppliedIdentity: runtimeIdentity, + SelectorVersion: selectorVersion, + SchedulerVersion: "not_applicable", + ExecutionBoundary: boundary, + Caps: map[string]int64{}, + RuntimeOutcomeAvailable: &available, + RuntimeBranch: runtimeBranch, + Overflow: boolPointer(overflow), + FallbackExecuted: boolPointer(fallback), + WouldSelectIdentity: wouldSelect, + Provenance: provenance, + }, + }, + } + if fallback { + record.TraversalTelemetry.Summary.FallbackIdentity = forward + } + record.Stats.WarmupIterations = 20 + for iteration := 1; iteration <= 50; iteration++ { + sample := LatencySample{ + Round: round, + Block: round, + Arm: arm, + ArmOrder: armOrder, + RunUUID: record.Environment.RunUUID, + Iteration: iteration, + Classification: "warm", + Duration: duration, + RequestedIdentity: requested, + RuntimeIdentity: runtimeIdentity, + RuntimeBranch: runtimeBranch, + FallbackExecuted: boolPointer(fallback), + } + if arm == "shadow" || arm == "guarded" { + sample.RuntimeAttestation = "timed_invocation" + sample.RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + Ordinal: 1, + RuntimeIdentity: runtimeIdentity, + RuntimeBranch: runtimeBranch, + FallbackExecuted: fallback, + }} + } else { + sample.RuntimeAttestation = "same_case_invocation_local_replay" + } + record.Stats.Samples = append(record.Stats.Samples, sample) + } + return record +} + +// renameOrientationV2Records prepares or inspects test evidence for rename orientation v2 records. +func renameOrientationV2Records(name string, artifacts orientationSelectorV2Artifacts) { + for _, records := range [][]CaseResult{artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded} { + for index := range records { + records[index].Name = name + records[index].Dataset = "generated_fixed_suffix_expansion_v3_" + name + records[index].WorkloadSHA256 = sqlFingerprint("orientation-v2-workload-" + name) + records[index].Fixture.Dataset = records[index].Dataset + records[index].Fixture.Checksum = sqlFingerprint("orientation-v2-fixture-" + name) + records[index].PostgresEnvironment.NodeRelationBytes = int64(len(name) * 1024) + records[index].PostgresEnvironment.EdgeRelationBytes = int64(len(name) * 2048) + } + } + stampOrientationV2Selections(&artifacts) +} + +// appendOrientationV2Artifacts appends orientation v2 artifacts. +func appendOrientationV2Artifacts(values ...orientationSelectorV2Artifacts) orientationSelectorV2Artifacts { + result := orientationSelectorV2Artifacts{} + for _, value := range values { + result.shadow = append(result.shadow, value.shadow...) + result.incumbent = append(result.incumbent, value.incumbent...) + result.reverse = append(result.reverse, value.reverse...) + result.guarded = append(result.guarded, value.guarded...) + } + stampOrientationV2Selections(&result) + return result +} + +// stampOrientationV2Selections prepares or inspects test evidence for stamp orientation v2 selections. +func stampOrientationV2Selections(artifacts *orientationSelectorV2Artifacts) { + if artifacts == nil { + return + } + keys := map[performanceKey]struct{}{} + for _, record := range artifacts.shadow { + keys[performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + }] = struct{}{} + } + declared := make([]DeclaredCaseBackend, 0, 2*len(keys)) + resolved := make([]ResolvedCaseSelector, 0, len(keys)) + for _, key := range sortedPerformanceKeys(keys) { + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + declared = append(declared, DeclaredCaseBackend{ + Dataset: key.dataset, + Name: key.name, + Backend: backend, + }) + } + resolved = append(resolved, ResolvedCaseSelector{ + Dataset: key.dataset, + Name: key.name, + Category: "generated_fixed_suffix_expansion", + }) + } + selection := &SelectionManifest{ + Version: selectionManifestVersion, + Resolved: resolved, + DiagnosticOnly: true, + FullDeclarationCount: 2 * len(keys), + SelectedDeclarationCount: 2 * len(keys), + DeclarationSHA256: declarationSHA256(declared), + } + for _, records := range [][]CaseResult{artifacts.shadow, artifacts.incumbent, artifacts.reverse, artifacts.guarded} { + for index := range records { + copy := *selection + copy.Resolved = append([]ResolvedCaseSelector(nil), selection.Resolved...) + records[index].Environment.Selection = © + } + } +} + +// boolPointer returns an addressable representation of bool. +func boolPointer(value bool) *bool { return &value } + +// testSHA prepares or inspects test evidence for test sha. +func testSHA(digit string) string { + value := "" + for len(value) < 64 { + value += digit + } + return value[:64] +} + +// testOrientationV2Freeze prepares or inspects test evidence for test orientation v2 freeze. +func testOrientationV2Freeze() *OrientationSelectorV2FreezeManifest { + return &OrientationSelectorV2FreezeManifest{ + Version: 1, + Policy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + Formula: "F2=root_rows+maximum_depth*forward_degree_rows;R2=suffix_rows+boundary_rows+reverse_degree_rows;reverse=complete&&4*R2<3*F2", + Caps: map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, "state_limit": optimize.ExpansionSearchOrientationStateLimit, + }, + SourceCommit: "deadbeef", + DirtyDiffSHA256: testSHA("d"), + BinarySHA256: testSHA("b"), + DiscoveryReportSHA256: testSHA("e"), + } +} diff --git a/cmd/graphbench/p5_adjacency_feasibility.go b/cmd/graphbench/p5_adjacency_feasibility.go new file mode 100644 index 00000000..914a90d2 --- /dev/null +++ b/cmd/graphbench/p5_adjacency_feasibility.go @@ -0,0 +1,1307 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "math" + "os" + "sort" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs" + "github.com/specterops/dawgs/drivers/pg" + pgquery "github.com/specterops/dawgs/drivers/pg/query" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" + "github.com/specterops/dawgs/util/size" +) + +const ( + p5AdjacencyFeasibilitySchema = "p5-adjacency-materialization-feasibility-v2" + p5AdjacencyFeasibilityProtocol = "benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2.json" + p5WarmupIterations = 1 + p5TimedIterations = 5 + p5Blocks = 4 +) + +var p5AdjacencyGraphSequence int64 + +// P5AdjacencyFeasibilityReport is the intentionally non-promotional physical +// evidence artifact for the P5 shadow relation. Its layout separates writes, +// WAL, storage, and raw lookup observations so it cannot be mistaken for a +// Cypher performance qualification. +type P5AdjacencyFeasibilityReport struct { + Schema string `json:"schema"` + Status string `json:"status"` + ProtocolSHA256 string `json:"protocol_sha256"` + SourceArchiveSHA256 string `json:"source_archive_sha256"` + Environment RunEnvironment `json:"environment"` + Postgres PostgresEnvironment `json:"postgres"` + WALAttribution P5AdjacencyWALAttribution `json:"wal_attribution"` + Conditions []P5AdjacencyConditionResult `json:"conditions"` + Calibrations []P5AdjacencyCalibration `json:"committed_calibrations"` + Cancellation P5AdjacencyCancellation `json:"cancellation_and_pool_reuse"` + Passed bool `json:"passed"` + NoBudgetDecision bool `json:"no_budget_decision"` + NoCypherReadPath bool `json:"no_cypher_read_path"` +} + +// P5AdjacencyConditionResult records one counterbalanced fixture condition. +type P5AdjacencyConditionResult struct { + Block int `json:"block"` + Condition string `json:"condition"` + Targets int `json:"targets"` + GraphID int32 `json:"graph_id"` + SetupWAL int64 `json:"setup_wal_lsn_delta_bytes"` + BaseStorage P5AdjacencyRelationSize `json:"base_edge_storage"` + ShadowStorage *P5AdjacencyRelationSize `json:"shadow_storage,omitempty"` + Operations []P5AdjacencyOperationMeasurement `json:"operations"` + ReadProbes []P5AdjacencyReadProbe `json:"read_probes"` +} + +// P5AdjacencyWALAttribution describes the capture-only PostgreSQL facility +// used to account for a top-level mutation and all trigger maintenance it +// invokes. It is not a runtime dependency of the driver or the shadow schema. +type P5AdjacencyWALAttribution struct { + Source string `json:"source"` + ExtensionVersion string `json:"extension_version"` + Track string `json:"track"` + ExtensionWasPresent bool `json:"extension_was_present"` + ExtensionCreated bool `json:"extension_created_for_capture"` +} + +// P5AdjacencyRelationSize reports heap and index bytes for one physical relation. +type P5AdjacencyRelationSize struct { + Relation string `json:"relation"` + HeapBytes int64 `json:"heap_bytes"` + IndexBytes int64 `json:"index_bytes"` + TotalBytes int64 `json:"total_bytes"` +} + +// P5AdjacencyOperationMeasurement records one rollback-only timing series. +type P5AdjacencyOperationMeasurement struct { + Operation string `json:"operation"` + Warmup P5AdjacencyLatencySample `json:"warmup"` + Samples []P5AdjacencyLatencySample `json:"samples"` + Median time.Duration `json:"median"` + P95 time.Duration `json:"p95"` + Observed P5AdjacencyMutationObservation `json:"observed"` +} + +// P5AdjacencyLatencySample supplies one rollback-only physical write timing. +type P5AdjacencyLatencySample struct { + Iteration int `json:"iteration"` + Duration time.Duration `json:"duration"` +} + +// P5AdjacencyMutationObservation records state transition evidence outside +// the timed statement itself. +type P5AdjacencyMutationObservation struct { + AffectedRows int64 `json:"affected_rows"` + BaseEdgesBefore int64 `json:"base_edges_before"` + BaseEdgesAfter int64 `json:"base_edges_after"` + ShadowRowsBefore int64 `json:"shadow_rows_before,omitempty"` + ShadowRowsAfter int64 `json:"shadow_rows_after,omitempty"` + MaintenanceRowsChanged int64 `json:"maintenance_rows_changed,omitempty"` + PropertyRowsUnchanged bool `json:"property_rows_unchanged"` + RollbackRestoredFixture bool `json:"rollback_restored_fixture"` +} + +// P5AdjacencyReadProbe preserves a raw SQL lookup, its result cardinality, +// elapsed time, and a PostgreSQL plan containing buffer observations. +type P5AdjacencyReadProbe struct { + Relation string `json:"relation"` + Duration time.Duration `json:"duration"` + ResultCardinality int64 `json:"result_cardinality"` + Plan json.RawMessage `json:"plan"` +} + +// P5AdjacencyCalibration records one committed setup or mutation calibration. +type P5AdjacencyCalibration struct { + Condition string `json:"condition"` + Targets int `json:"targets"` + Operation string `json:"operation"` + GraphID int32 `json:"graph_id"` + SetupWAL int64 `json:"setup_wal_lsn_delta_bytes"` + MutationWALLSN int64 `json:"mutation_wal_lsn_delta_bytes"` + StatementWAL P5AdjacencyStatementWAL `json:"statement_wal"` + WALQuiescent bool `json:"wal_quiescent"` + Duration time.Duration `json:"duration"` + Observed P5AdjacencyMutationObservation `json:"observed"` +} + +// P5AdjacencyStatementWAL is the pg_stat_statements delta for one committed +// top-level mutation. PostgreSQL attributes trigger writes to that statement, +// unlike EXPLAIN's plan-node WAL counters. +type P5AdjacencyStatementWAL struct { + Tag string `json:"tag"` + Calls int64 `json:"calls"` + Records int64 `json:"records"` + FPI int64 `json:"full_page_images"` + Bytes int64 `json:"bytes"` +} + +// P5AdjacencyCancellation records recovery after a cancelled write. pgx may +// close the cancelled connection, so the replay PID is recorded explicitly. +type P5AdjacencyCancellation struct { + Ran bool `json:"ran"` + CancelledBackendPID uint32 `json:"cancelled_backend_pid"` + ReplayBackendPID uint32 `json:"replay_backend_pid"` + PoolReuseSucceeded bool `json:"pool_reuse_succeeded"` + RollbackObserved bool `json:"rollback_observed"` +} + +type p5AdjacencyGraph struct { + db graph.Database + pool *pgxpool.Pool + graphID int32 +} + +type p5AdjacencyFixture struct { + graphID int32 + rootID int64 + targetIDs []int64 + deleteEdgeIDs []int64 + updateEdgeIDs []int64 + deleteKindID int16 + updateKindID int16 + createKindID int16 + nodes int64 + edges int64 +} + +type p5AdjacencyRowQueryer interface { + QueryRow(context.Context, string, ...any) pgx.Row +} + +// runP5AdjacencyFeasibilityCapture performs the frozen physical study. It is +// deliberately separate from the normal corpus runner and does not load, +// translate, or execute any Cypher query. +func runP5AdjacencyFeasibilityCapture(ctx context.Context, cfg config, connection string, args []string) (_ P5AdjacencyFeasibilityReport, err error) { + if cfg.PoolSize != 1 { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("P5 adjacency feasibility capture requires pool-size 1") + } + if current := workingTreeSHA256(); current != cleanWorkingTreeSHA256() { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("P5 adjacency feasibility capture requires a clean source tree") + } + protocolSHA, err := fileSHA256(p5AdjacencyFeasibilityProtocol) + if err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("checksum P5 protocol: %w", err) + } + sourceArchive, err := sourceArchiveSHA256() + if err != nil { + return P5AdjacencyFeasibilityReport{}, err + } + + startedAt := time.Now().UTC() + environment := resolveRunEnvironment(cfg, args, SelectionManifest{}, startedAt, startedAt) + environment.Selection = nil + environment.Protocol = p5AdjacencyFeasibilitySchema + environment.WarmupIterations = p5WarmupIterations + environment.PoolSize = 1 + + control, err := openP5AdjacencyGraph(ctx, connection) + if err != nil { + return P5AdjacencyFeasibilityReport{}, err + } + defer func() { + cleanupErr := dropP5AdjacencyShadow(ctx, control.db) + closeErr := control.db.Close(ctx) + if err == nil && cleanupErr != nil { + err = fmt.Errorf("remove P5 adjacency shadow after capture: %w", cleanupErr) + } + if err == nil && closeErr != nil { + err = fmt.Errorf("close P5 adjacency control graph: %w", closeErr) + } + }() + if err := dropP5AdjacencyShadow(ctx, control.db); err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("reset P5 adjacency shadow: %w", err) + } + if err := cleanupP5AdjacencyOwnedGraphs(ctx, control); err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("reset abandoned P5 adjacency graphs: %w", err) + } + if err := disableP5AdjacencyAutovacuum(ctx, control.pool, false); err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("disable autovacuum for P5 WAL attribution: %w", err) + } + defer func() { + if restoreErr := restoreP5AdjacencyAutovacuum(ctx, control.pool); err == nil && restoreErr != nil { + err = fmt.Errorf("restore autovacuum after P5 capture: %w", restoreErr) + } + }() + walAttribution, releaseWALAttribution, err := prepareP5AdjacencyWALAttribution(ctx, control.pool) + if err != nil { + return P5AdjacencyFeasibilityReport{}, err + } + defer func() { + if releaseErr := releaseWALAttribution(); err == nil && releaseErr != nil { + err = fmt.Errorf("release P5 WAL attribution extension: %w", releaseErr) + } + }() + + postgres, err := captureP5PostgresEnvironment(ctx, control.pool) + if err != nil { + return P5AdjacencyFeasibilityReport{}, err + } + report := P5AdjacencyFeasibilityReport{ + Schema: p5AdjacencyFeasibilitySchema, + Status: "physical_feasibility_capture_no_budget_decision", + ProtocolSHA256: protocolSHA, + SourceArchiveSHA256: sourceArchive, + Environment: environment, + Postgres: postgres, + WALAttribution: walAttribution, + NoBudgetDecision: true, + NoCypherReadPath: true, + } + + conditions := []string{"base", "shadow"} + for block := 1; block <= p5Blocks; block++ { + ordered := append([]string(nil), conditions...) + if block%2 == 0 { + ordered[0], ordered[1] = ordered[1], ordered[0] + } + for _, condition := range ordered { + shadow := condition == "shadow" + if !shadow { + if err := dropP5AdjacencyShadow(ctx, control.db); err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("prepare base block %d: %w", block, err) + } + } + for _, targets := range []int{1, 1_000, 2_000} { + result, cancellation, runErr := runP5AdjacencyTimedCondition(ctx, connection, block, condition, targets, report.Cancellation.Ran) + if runErr != nil { + return P5AdjacencyFeasibilityReport{}, runErr + } + report.Conditions = append(report.Conditions, result) + if cancellation.Ran { + report.Cancellation = cancellation + } + } + } + } + + for _, condition := range conditions { + shadow := condition == "shadow" + if !shadow { + if err := dropP5AdjacencyShadow(ctx, control.db); err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("prepare base calibrations: %w", err) + } + } + for _, targets := range []int{1, 1_000, 2_000} { + for _, operation := range p5AdjacencyOperations() { + calibration, runErr := runP5AdjacencyCalibration(ctx, connection, condition, targets, operation) + if runErr != nil { + return P5AdjacencyFeasibilityReport{}, runErr + } + report.Calibrations = append(report.Calibrations, calibration) + } + } + } + if !report.Cancellation.Ran || !report.Cancellation.PoolReuseSucceeded { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("P5 cancellation and pool reuse proof did not complete") + } + if err := dropP5AdjacencyShadow(ctx, control.db); err != nil { + return P5AdjacencyFeasibilityReport{}, fmt.Errorf("remove P5 adjacency shadow before artifact write: %w", err) + } + if err := deleteP5AdjacencyGraph(ctx, control, false); err != nil { + return P5AdjacencyFeasibilityReport{}, err + } + + report.Environment.EndedAt = time.Now().UTC() + report.Passed = true + if err := writeP5AdjacencyFeasibilityReport(cfg.P5AdjacencyFeasibilityOutput, report); err != nil { + return P5AdjacencyFeasibilityReport{}, err + } + return report, nil +} + +func prepareP5AdjacencyWALAttribution(ctx context.Context, pool *pgxpool.Pool) (P5AdjacencyWALAttribution, func() error, error) { + attribution := P5AdjacencyWALAttribution{Source: "pg_stat_statements"} + if err := pool.QueryRow(ctx, `select exists (select 1 from pg_extension where extname = 'pg_stat_statements')`).Scan(&attribution.ExtensionWasPresent); err != nil { + return P5AdjacencyWALAttribution{}, nil, fmt.Errorf("check pg_stat_statements extension: %w", err) + } + if !attribution.ExtensionWasPresent { + if _, err := pool.Exec(ctx, `create extension pg_stat_statements`); err != nil { + return P5AdjacencyWALAttribution{}, nil, fmt.Errorf("install capture-only pg_stat_statements extension: %w", err) + } + attribution.ExtensionCreated = true + } + release := func() error { + if !attribution.ExtensionCreated { + return nil + } + _, err := pool.Exec(ctx, `drop extension pg_stat_statements`) + return err + } + if err := pool.QueryRow(ctx, `select extversion from pg_extension where extname = 'pg_stat_statements'`).Scan(&attribution.ExtensionVersion); err != nil { + _ = release() + return P5AdjacencyWALAttribution{}, nil, fmt.Errorf("read pg_stat_statements extension version: %w", err) + } + if err := pool.QueryRow(ctx, `show pg_stat_statements.track`).Scan(&attribution.Track); err != nil { + _ = release() + return P5AdjacencyWALAttribution{}, nil, fmt.Errorf("read pg_stat_statements tracking mode: %w", err) + } + if attribution.Track != "top" && attribution.Track != "all" { + _ = release() + return P5AdjacencyWALAttribution{}, nil, fmt.Errorf("pg_stat_statements.track must be top or all, got %q", attribution.Track) + } + var entries int64 + if err := pool.QueryRow(ctx, `select count(*) from pg_stat_statements`).Scan(&entries); err != nil { + _ = release() + return P5AdjacencyWALAttribution{}, nil, fmt.Errorf("query pg_stat_statements; add it to shared_preload_libraries before starting PostgreSQL: %w", err) + } + return attribution, release, nil +} + +func p5AdjacencyOperations() []string { + return []string{ + "batch_relationship_create", + "relationship_upsert_conflict_merge", + "relationship_property_only_update", + "batched_relationship_delete", + "batched_node_delete_cascade", + "graph_clear_reload", + "graph_drop", + } +} + +func runP5AdjacencyTimedCondition(ctx context.Context, connection string, block int, condition string, targets int, cancellationComplete bool) (P5AdjacencyConditionResult, P5AdjacencyCancellation, error) { + shadow := condition == "shadow" + graphState, err := openP5AdjacencyGraph(ctx, connection) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + defer graphState.db.Close(ctx) + fixture, setupWAL, err := setupP5AdjacencyFixture(ctx, graphState, targets, shadow) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + baseStorage, err := p5AdjacencyRelationSize(ctx, graphState.pool, fmt.Sprintf("edge_%d", fixture.graphID)) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + result := P5AdjacencyConditionResult{ + Block: block, + Condition: condition, + Targets: targets, + GraphID: fixture.graphID, + SetupWAL: setupWAL, + BaseStorage: baseStorage, + } + if shadow { + shadowStorage, err := p5AdjacencyRelationSize(ctx, graphState.pool, fmt.Sprintf("p5_adjacency_v1_%d", fixture.graphID)) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + result.ShadowStorage = &shadowStorage + } + + for _, operation := range p5AdjacencyOperations() { + measurement, err := measureP5AdjacencyOperation(ctx, graphState, fixture, shadow, operation) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + result.Operations = append(result.Operations, measurement) + } + baseProbe, err := p5AdjacencyReadProbe(ctx, graphState, fixture, false) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + result.ReadProbes = append(result.ReadProbes, baseProbe) + if shadow { + shadowProbe, err := p5AdjacencyReadProbe(ctx, graphState, fixture, true) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + result.ReadProbes = append(result.ReadProbes, shadowProbe) + } + + cancellation := P5AdjacencyCancellation{} + if shadow && !cancellationComplete { + cancellation, err = p5AdjacencyCancellationAndReuse(ctx, graphState, fixture) + if err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + } + if err := deleteP5AdjacencyGraph(ctx, graphState, shadow); err != nil { + return P5AdjacencyConditionResult{}, P5AdjacencyCancellation{}, err + } + return result, cancellation, nil +} + +func runP5AdjacencyCalibration(ctx context.Context, connection, condition string, targets int, operation string) (P5AdjacencyCalibration, error) { + shadow := condition == "shadow" + graphState, err := openP5AdjacencyGraph(ctx, connection) + if err != nil { + return P5AdjacencyCalibration{}, err + } + defer graphState.db.Close(ctx) + fixture, setupWAL, err := setupP5AdjacencyFixture(ctx, graphState, targets, shadow) + if err != nil { + return P5AdjacencyCalibration{}, err + } + if err := waitForP5WALQuiescence(ctx, graphState.pool); err != nil { + return P5AdjacencyCalibration{}, err + } + beforeWAL, err := p5AdjacencyWALPosition(ctx, graphState.pool) + if err != nil { + return P5AdjacencyCalibration{}, err + } + tx, err := graphState.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return P5AdjacencyCalibration{}, err + } + statementWALTag := p5AdjacencyStatementWALTag(operation) + observation, duration, statementWAL, runErr := runP5AdjacencyMutationWithWAL(ctx, tx, fixture, shadow, operation, statementWALTag) + if runErr == nil { + runErr = tx.Commit(ctx) + } else { + _ = tx.Rollback(ctx) + } + if runErr != nil { + return P5AdjacencyCalibration{}, fmt.Errorf("committed %s calibration: %w", operation, runErr) + } + if err := waitForP5WALQuiescence(ctx, graphState.pool); err != nil { + return P5AdjacencyCalibration{}, err + } + afterWAL, err := p5AdjacencyWALPosition(ctx, graphState.pool) + if err != nil { + return P5AdjacencyCalibration{}, err + } + if afterWAL-beforeWAL < statementWAL.Bytes { + return P5AdjacencyCalibration{}, fmt.Errorf("%s statement WAL bytes exceed its LSN delta", operation) + } + if shadow { + if err := assertP5AdjacencyExact(ctx, graphState.pool, fixture.graphID); err != nil { + return P5AdjacencyCalibration{}, err + } + } else if err := assertP5AdjacencyAbsent(ctx, graphState.pool); err != nil { + return P5AdjacencyCalibration{}, err + } + if err := deleteP5AdjacencyGraph(ctx, graphState, shadow); err != nil { + return P5AdjacencyCalibration{}, err + } + return P5AdjacencyCalibration{ + Condition: condition, + Targets: targets, + Operation: operation, + GraphID: fixture.graphID, + SetupWAL: setupWAL, + MutationWALLSN: afterWAL - beforeWAL, + StatementWAL: statementWAL, + WALQuiescent: true, + Duration: duration, + Observed: observation, + }, nil +} + +func measureP5AdjacencyOperation(ctx context.Context, graphState *p5AdjacencyGraph, fixture p5AdjacencyFixture, shadow bool, operation string) (P5AdjacencyOperationMeasurement, error) { + measurement := P5AdjacencyOperationMeasurement{Operation: operation} + var expected P5AdjacencyMutationObservation + for iteration := 0; iteration <= p5TimedIterations; iteration++ { + tx, err := graphState.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return P5AdjacencyOperationMeasurement{}, err + } + observation, duration, runErr := runP5AdjacencyMutationTimed(ctx, tx, fixture, shadow, operation) + rollbackErr := tx.Rollback(ctx) + if runErr != nil { + return P5AdjacencyOperationMeasurement{}, runErr + } + if rollbackErr != nil { + return P5AdjacencyOperationMeasurement{}, rollbackErr + } + if shadow { + if err := assertP5AdjacencyExact(ctx, graphState.pool, fixture.graphID); err != nil { + return P5AdjacencyOperationMeasurement{}, fmt.Errorf("verify rollback %s: %w", operation, err) + } + } else if err := assertP5AdjacencyAbsent(ctx, graphState.pool); err != nil { + return P5AdjacencyOperationMeasurement{}, err + } + observation.RollbackRestoredFixture = true + if iteration == 0 { + expected = observation + measurement.Warmup = P5AdjacencyLatencySample{Iteration: iteration, Duration: duration} + continue + } + if !p5AdjacencyObservationEqual(expected, observation) { + return P5AdjacencyOperationMeasurement{}, fmt.Errorf("%s iteration %d changed state observations", operation, iteration) + } + measurement.Samples = append(measurement.Samples, P5AdjacencyLatencySample{Iteration: iteration, Duration: duration}) + } + measurement.Observed = expected + durations := make([]time.Duration, 0, len(measurement.Samples)) + for _, sample := range measurement.Samples { + durations = append(durations, sample.Duration) + } + measurement.Median, measurement.P95 = p5AdjacencyQuantiles(durations) + return measurement, nil +} + +func p5AdjacencyObservationEqual(left, right P5AdjacencyMutationObservation) bool { + return left.AffectedRows == right.AffectedRows && + left.BaseEdgesBefore == right.BaseEdgesBefore && + left.BaseEdgesAfter == right.BaseEdgesAfter && + left.ShadowRowsBefore == right.ShadowRowsBefore && + left.ShadowRowsAfter == right.ShadowRowsAfter && + left.MaintenanceRowsChanged == right.MaintenanceRowsChanged && + left.PropertyRowsUnchanged == right.PropertyRowsUnchanged +} + +func p5AdjacencyQuantiles(durations []time.Duration) (time.Duration, time.Duration) { + values := append([]time.Duration(nil), durations...) + sort.Slice(values, func(left, right int) bool { return values[left] < values[right] }) + median := values[len(values)/2] + p95 := values[int(math.Ceil(float64(len(values))*0.95))-1] + return median, p95 +} + +type p5AdjacencyMutationExecution struct { + observation P5AdjacencyMutationObservation + duration time.Duration + statementWAL P5AdjacencyStatementWAL +} + +func runP5AdjacencyMutationTimed(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture, shadow bool, operation string) (P5AdjacencyMutationObservation, time.Duration, error) { + execution, err := runP5AdjacencyMutationInternal(ctx, tx, fixture, shadow, operation, "") + return execution.observation, execution.duration, err +} + +func runP5AdjacencyMutationWithWAL(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture, shadow bool, operation, walTag string) (P5AdjacencyMutationObservation, time.Duration, P5AdjacencyStatementWAL, error) { + execution, err := runP5AdjacencyMutationInternal(ctx, tx, fixture, shadow, operation, walTag) + return execution.observation, execution.duration, execution.statementWAL, err +} + +func runP5AdjacencyMutation(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture, shadow bool, operation string) (P5AdjacencyMutationObservation, error) { + execution, err := runP5AdjacencyMutationInternal(ctx, tx, fixture, shadow, operation, "") + return execution.observation, err +} + +func runP5AdjacencyMutationInternal(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture, shadow bool, operation, walTag string) (p5AdjacencyMutationExecution, error) { + if shadow { + if err := assertP5AdjacencyExact(ctx, tx, fixture.graphID); err != nil { + return p5AdjacencyMutationExecution{}, err + } + } else if err := assertP5AdjacencyAbsent(ctx, tx); err != nil { + return p5AdjacencyMutationExecution{}, err + } + observation := P5AdjacencyMutationObservation{} + if err := tx.QueryRow(ctx, `select count(*) from edge where graph_id = $1`, fixture.graphID).Scan(&observation.BaseEdgesBefore); err != nil { + return p5AdjacencyMutationExecution{}, err + } + if shadow { + if err := tx.QueryRow(ctx, `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, fixture.graphID).Scan(&observation.ShadowRowsBefore); err != nil { + return p5AdjacencyMutationExecution{}, err + } + } + identityBefore := "" + if shadow && (operation == "relationship_upsert_conflict_merge" || operation == "relationship_property_only_update") { + var err error + identityBefore, err = p5AdjacencyShadowIdentity(ctx, tx, fixture) + if err != nil { + return p5AdjacencyMutationExecution{}, err + } + } + + statement, arguments, err := p5AdjacencyMutationStatement(fixture, operation) + if err != nil { + return p5AdjacencyMutationExecution{}, err + } + var ( + affected int64 + duration time.Duration + statementWAL P5AdjacencyStatementWAL + ) + if walTag != "" { + if statementWAL, err = p5AdjacencyStatementWALStats(ctx, tx, walTag); err != nil { + return p5AdjacencyMutationExecution{}, err + } + var result pgconn.CommandTag + start := time.Now() + result, err = tx.Exec(ctx, "with "+walTag+" as (select 1) "+statement, arguments...) + duration = time.Since(start) + affected = result.RowsAffected() + if err == nil { + afterWAL, statsErr := p5AdjacencyStatementWALStats(ctx, tx, walTag) + if statsErr != nil { + err = statsErr + } else { + statementWAL, err = p5AdjacencyStatementWALDelta(statementWAL, afterWAL) + } + } + if err == nil && (statementWAL.Calls != 1 || statementWAL.Bytes <= 0) { + err = fmt.Errorf("P5 statement WAL tag %q recorded calls=%d bytes=%d", walTag, statementWAL.Calls, statementWAL.Bytes) + } + } else { + var result pgconn.CommandTag + start := time.Now() + result, err = tx.Exec(ctx, statement, arguments...) + duration = time.Since(start) + affected = result.RowsAffected() + } + if err != nil { + return p5AdjacencyMutationExecution{}, err + } + observation.AffectedRows = affected + if err := tx.QueryRow(ctx, `select count(*) from edge where graph_id = $1`, fixture.graphID).Scan(&observation.BaseEdgesAfter); err != nil { + return p5AdjacencyMutationExecution{}, err + } + if err := assertP5AdjacencyMutationPostState(ctx, tx, fixture, operation, observation); err != nil { + return p5AdjacencyMutationExecution{}, err + } + if shadow { + if err := tx.QueryRow(ctx, `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, fixture.graphID).Scan(&observation.ShadowRowsAfter); err != nil { + return p5AdjacencyMutationExecution{}, err + } + observation.MaintenanceRowsChanged = absInt64(observation.ShadowRowsAfter - observation.ShadowRowsBefore) + if err := assertP5AdjacencyExact(ctx, tx, fixture.graphID); err != nil { + return p5AdjacencyMutationExecution{}, err + } + if identityBefore != "" { + identityAfter, err := p5AdjacencyShadowIdentity(ctx, tx, fixture) + if err != nil { + return p5AdjacencyMutationExecution{}, err + } + observation.PropertyRowsUnchanged = identityBefore == identityAfter + if !observation.PropertyRowsUnchanged { + return p5AdjacencyMutationExecution{}, fmt.Errorf("%s rewrote P5 shadow rows", operation) + } + } + } + return p5AdjacencyMutationExecution{observation: observation, duration: duration, statementWAL: statementWAL}, nil +} + +func p5AdjacencyMutationStatement(fixture p5AdjacencyFixture, operation string) (string, []any, error) { + switch operation { + case "batch_relationship_create": + return ` + insert into edge(graph_id, start_id, end_id, kind_id, properties) + select $1, $2, target_id, $3, '{"p5_create":true}'::jsonb + from unnest($4::bigint[]) as targets(target_id)`, + []any{fixture.graphID, fixture.rootID, fixture.createKindID, fixture.targetIDs}, nil + case "relationship_upsert_conflict_merge": + return ` + insert into edge(graph_id, start_id, end_id, kind_id, properties) + select $1, $2, target_id, $3, '{"p5_upsert":true}'::jsonb + from unnest($4::bigint[]) as targets(target_id) + on conflict (start_id, end_id, kind_id, graph_id) do update + set properties = edge.properties || excluded.properties`, + []any{fixture.graphID, fixture.rootID, fixture.updateKindID, fixture.targetIDs}, nil + case "relationship_property_only_update": + return `update edge set properties = properties || '{"p5_property_only":true}'::jsonb where graph_id = $1 and id = any($2::bigint[])`, + []any{fixture.graphID, fixture.updateEdgeIDs}, nil + case "batched_relationship_delete": + return `delete from edge where graph_id = $1 and id = any($2::bigint[])`, []any{fixture.graphID, fixture.deleteEdgeIDs}, nil + case "batched_node_delete_cascade": + return `delete from node where graph_id = $1 and id = any($2::bigint[])`, []any{fixture.graphID, fixture.targetIDs}, nil + case "graph_clear_reload": + return `delete from node where graph_id = $1`, []any{fixture.graphID}, nil + case "graph_drop": + return `delete from graph where id = $1`, []any{fixture.graphID}, nil + default: + return "", nil, fmt.Errorf("unknown P5 mutation %q", operation) + } +} + +func p5AdjacencyStatementWALTag(operation string) string { + return "p5_adjacency_v2_wal_" + operation +} + +func p5AdjacencyStatementWALStats(ctx context.Context, queryer p5AdjacencyRowQueryer, tag string) (P5AdjacencyStatementWAL, error) { + measurement := P5AdjacencyStatementWAL{Tag: tag} + err := queryer.QueryRow(ctx, ` + select + coalesce(sum(calls), 0)::bigint, + coalesce(sum(wal_records), 0)::bigint, + coalesce(sum(wal_fpi), 0)::bigint, + coalesce(sum(wal_bytes), 0)::bigint + from pg_stat_statements + where query like 'with ' || $1 || '%'`, tag).Scan( + &measurement.Calls, + &measurement.Records, + &measurement.FPI, + &measurement.Bytes, + ) + if err != nil { + return P5AdjacencyStatementWAL{}, fmt.Errorf("read pg_stat_statements WAL for %q: %w", tag, err) + } + return measurement, nil +} + +func p5AdjacencyStatementWALDelta(before, after P5AdjacencyStatementWAL) (P5AdjacencyStatementWAL, error) { + if before.Tag != after.Tag { + return P5AdjacencyStatementWAL{}, fmt.Errorf("P5 statement WAL tags differ: %q and %q", before.Tag, after.Tag) + } + if after.Calls < before.Calls || after.Records < before.Records || after.FPI < before.FPI || after.Bytes < before.Bytes { + return P5AdjacencyStatementWAL{}, fmt.Errorf("P5 statement WAL counters moved backwards for %q", before.Tag) + } + return P5AdjacencyStatementWAL{ + Tag: before.Tag, + Calls: after.Calls - before.Calls, + Records: after.Records - before.Records, + FPI: after.FPI - before.FPI, + Bytes: after.Bytes - before.Bytes, + }, nil +} + +func assertP5AdjacencyMutationPostState(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture, operation string, observation P5AdjacencyMutationObservation) error { + expectedAffected := int64(len(fixture.targetIDs)) + expectedEdges := fixture.edges + switch operation { + case "batch_relationship_create": + expectedEdges += expectedAffected + case "relationship_upsert_conflict_merge", "relationship_property_only_update": + // These mutate properties only and retain the base-edge cardinality. + case "batched_relationship_delete": + expectedEdges -= expectedAffected + case "batched_node_delete_cascade": + expectedEdges = 2 + case "graph_clear_reload": + expectedAffected = fixture.nodes + expectedEdges = 0 + case "graph_drop": + expectedAffected = 1 + expectedEdges = 0 + default: + return fmt.Errorf("unknown P5 mutation %q", operation) + } + if observation.AffectedRows != expectedAffected { + return fmt.Errorf("%s affected %d rows, expected %d", operation, observation.AffectedRows, expectedAffected) + } + if observation.BaseEdgesAfter != expectedEdges { + return fmt.Errorf("%s left %d base edges, expected %d", operation, observation.BaseEdgesAfter, expectedEdges) + } + if operation != "batched_node_delete_cascade" && operation != "graph_clear_reload" && operation != "graph_drop" { + return nil + } + var nodes int64 + if err := tx.QueryRow(ctx, `select count(*) from node where graph_id = $1`, fixture.graphID).Scan(&nodes); err != nil { + return err + } + expectedNodes := int64(2) + if operation == "graph_clear_reload" || operation == "graph_drop" { + expectedNodes = 0 + } + if nodes != expectedNodes { + return fmt.Errorf("%s left %d nodes, expected %d", operation, nodes, expectedNodes) + } + return nil +} + +func p5AdjacencyCreateRelationships(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, ` + insert into edge(graph_id, start_id, end_id, kind_id, properties) + select $1, $2, target_id, $3, '{"p5_create":true}'::jsonb + from unnest($4::bigint[]) as targets(target_id)`, + fixture.graphID, fixture.rootID, fixture.createKindID, fixture.targetIDs, + ) + return result.RowsAffected(), err +} + +func p5AdjacencyUpsertRelationships(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, ` + insert into edge(graph_id, start_id, end_id, kind_id, properties) + select $1, $2, target_id, $3, '{"p5_upsert":true}'::jsonb + from unnest($4::bigint[]) as targets(target_id) + on conflict (start_id, end_id, kind_id, graph_id) do update + set properties = edge.properties || excluded.properties`, + fixture.graphID, fixture.rootID, fixture.updateKindID, fixture.targetIDs, + ) + return result.RowsAffected(), err +} + +func p5AdjacencyUpdateRelationshipProperties(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, + `update edge set properties = properties || '{"p5_property_only":true}'::jsonb where graph_id = $1 and id = any($2::bigint[])`, + fixture.graphID, fixture.updateEdgeIDs, + ) + return result.RowsAffected(), err +} + +func p5AdjacencyDeleteRelationships(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, `delete from edge where graph_id = $1 and id = any($2::bigint[])`, fixture.graphID, fixture.deleteEdgeIDs) + return result.RowsAffected(), err +} + +func p5AdjacencyDeleteNodes(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, `delete from node where graph_id = $1 and id = any($2::bigint[])`, fixture.graphID, fixture.targetIDs) + return result.RowsAffected(), err +} + +func p5AdjacencyClearGraph(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, `delete from node where graph_id = $1`, fixture.graphID) + return result.RowsAffected(), err +} + +func p5AdjacencyDropGraph(ctx context.Context, tx pgx.Tx, fixture p5AdjacencyFixture) (int64, error) { + result, err := tx.Exec(ctx, `delete from graph where id = $1`, fixture.graphID) + return result.RowsAffected(), err +} + +func setupP5AdjacencyFixture(ctx context.Context, graphState *p5AdjacencyGraph, targets int, shadow bool) (p5AdjacencyFixture, int64, error) { + if err := disableP5AdjacencyFixtureAutovacuum(ctx, graphState.pool, graphState.graphID, false); err != nil { + return p5AdjacencyFixture{}, 0, err + } + if shadow { + if err := installP5AdjacencyShadow(ctx, graphState.db); err != nil { + return p5AdjacencyFixture{}, 0, err + } + if err := disableP5AdjacencyFixtureAutovacuum(ctx, graphState.pool, graphState.graphID, true); err != nil { + return p5AdjacencyFixture{}, 0, err + } + } + if err := waitForP5WALQuiescence(ctx, graphState.pool); err != nil { + return p5AdjacencyFixture{}, 0, err + } + beforeWAL, err := p5AdjacencyWALPosition(ctx, graphState.pool) + if err != nil { + return p5AdjacencyFixture{}, 0, err + } + fixtureGraph := testutil.NewDirectWriteScaleFixture(targets) + idMap, err := opengraph.WriteGraph(ctx, graphState.db, fixtureGraph) + if err != nil { + return p5AdjacencyFixture{}, 0, err + } + if err := waitForP5WALQuiescence(ctx, graphState.pool); err != nil { + return p5AdjacencyFixture{}, 0, err + } + afterWAL, err := p5AdjacencyWALPosition(ctx, graphState.pool) + if err != nil { + return p5AdjacencyFixture{}, 0, err + } + fixture := p5AdjacencyFixture{ + graphID: graphState.graphID, + rootID: idMap["write-root"].Int64(), + targetIDs: make([]int64, 0, targets), + } + for _, name := range testutil.FixtureNames("write-target", targets) { + fixture.targetIDs = append(fixture.targetIDs, idMap[name].Int64()) + } + for name, destination := range map[string]*int16{ + "WriteDeleteRelationship": &fixture.deleteKindID, + "WriteUpdateRelationship": &fixture.updateKindID, + "WriteSurvivor": &fixture.createKindID, + } { + if err := graphState.pool.QueryRow(ctx, `select id from kind where name = $1`, name).Scan(destination); err != nil { + return p5AdjacencyFixture{}, 0, err + } + } + if err := graphState.pool.QueryRow(ctx, `select coalesce(array_agg(id order by id), '{}'::bigint[]) from edge where graph_id = $1 and kind_id = $2 and properties ->> 'deletebatch' = 'true'`, fixture.graphID, fixture.deleteKindID).Scan(&fixture.deleteEdgeIDs); err != nil { + return p5AdjacencyFixture{}, 0, err + } + if err := graphState.pool.QueryRow(ctx, `select coalesce(array_agg(id order by id), '{}'::bigint[]) from edge where graph_id = $1 and kind_id = $2 and start_id = $3 and end_id = any($4::bigint[])`, fixture.graphID, fixture.updateKindID, fixture.rootID, fixture.targetIDs).Scan(&fixture.updateEdgeIDs); err != nil { + return p5AdjacencyFixture{}, 0, err + } + if int64(len(fixture.deleteEdgeIDs)) != int64(targets) || int64(len(fixture.updateEdgeIDs)) != int64(targets) { + return p5AdjacencyFixture{}, 0, fmt.Errorf("direct-write fixture %d did not produce the required mutation targets", targets) + } + if err := graphState.pool.QueryRow(ctx, `select count(*) from node where graph_id = $1`, fixture.graphID).Scan(&fixture.nodes); err != nil { + return p5AdjacencyFixture{}, 0, err + } + if err := graphState.pool.QueryRow(ctx, `select count(*) from edge where graph_id = $1`, fixture.graphID).Scan(&fixture.edges); err != nil { + return p5AdjacencyFixture{}, 0, err + } + if shadow { + if err := assertP5AdjacencyExact(ctx, graphState.pool, fixture.graphID); err != nil { + return p5AdjacencyFixture{}, 0, err + } + } else if err := assertP5AdjacencyAbsent(ctx, graphState.pool); err != nil { + return p5AdjacencyFixture{}, 0, err + } + statements := fmt.Sprintf("vacuum (analyze) node_%d, edge_%d", fixture.graphID, fixture.graphID) + if shadow { + statements += fmt.Sprintf(", p5_adjacency_v1_%d", fixture.graphID) + } + if _, err := graphState.pool.Exec(ctx, statements); err != nil { + return p5AdjacencyFixture{}, 0, err + } + return fixture, afterWAL - beforeWAL, nil +} + +func openP5AdjacencyGraph(ctx context.Context, connection string) (*p5AdjacencyGraph, error) { + poolConfig, err := pgxpool.ParseConfig(connection) + if err != nil { + return nil, err + } + poolConfig.MinConns = 1 + poolConfig.MaxConns = 1 + poolConfig.AfterConnect = pg.AfterPooledConnectionEstablished + poolConfig.AfterRelease = pg.AfterPooledConnectionRelease + pool, err := pgxpool.NewWithConfig(ctx, poolConfig) + if err != nil { + return nil, err + } + db, err := dawgs.Open(ctx, pg.DriverName, dawgs.Config{ + GraphQueryMemoryLimit: size.Gibibyte, + ConnectionString: connection, + Pool: pool, + }) + if err != nil { + pool.Close() + return nil, err + } + fixture := testutil.NewDirectWriteScaleFixture(1) + nodeKinds, edgeKinds := fixture.Kinds() + graphSchema := graph.Graph{ + Name: fmt.Sprintf("p5_adjacency_%d", atomic.AddInt64(&p5AdjacencyGraphSequence, 1)), + Nodes: nodeKinds, + Edges: edgeKinds, + } + if err := db.AssertSchema(ctx, graph.Schema{Graphs: []graph.Graph{graphSchema}, DefaultGraph: graphSchema}); err != nil { + _ = db.Close(ctx) + return nil, err + } + driver, ok := db.(*pg.Driver) + if !ok { + _ = db.Close(ctx) + return nil, fmt.Errorf("expected PostgreSQL graph driver, got %T", db) + } + defaultGraph, ok := driver.DefaultGraph() + if !ok { + _ = db.Close(ctx) + return nil, fmt.Errorf("P5 adjacency graph was not selected") + } + return &p5AdjacencyGraph{db: db, pool: pool, graphID: defaultGraph.ID}, nil +} + +func installP5AdjacencyShadow(ctx context.Context, db graph.Database) error { + return db.WriteTransaction(ctx, func(tx graph.Transaction) error { + return pgquery.On(tx).InstallP5AdjacencyShadow() + }, pg.OptionSetQueryExecMode(pgx.QueryExecModeSimpleProtocol)) +} + +func dropP5AdjacencyShadow(ctx context.Context, db graph.Database) error { + return db.WriteTransaction(ctx, func(tx graph.Transaction) error { + return pgquery.On(tx).DropP5AdjacencyShadow() + }, pg.OptionSetQueryExecMode(pgx.QueryExecModeSimpleProtocol)) +} + +func deleteP5AdjacencyGraph(ctx context.Context, graphState *p5AdjacencyGraph, shadow bool) error { + if _, err := graphState.pool.Exec(ctx, `delete from graph where id = $1`, graphState.graphID); err != nil { + return err + } + if shadow { + var rows int64 + if err := graphState.pool.QueryRow(ctx, `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, graphState.graphID).Scan(&rows); err != nil { + return err + } + if rows != 0 { + return fmt.Errorf("graph %d left %d committed P5 adjacency rows", graphState.graphID, rows) + } + } + return nil +} + +// cleanupP5AdjacencyOwnedGraphs removes only prior runner fixtures that may +// remain if a process is interrupted before its normal graph-drop cleanup. +// The current control graph remains available for P5 schema setup and removal. +func cleanupP5AdjacencyOwnedGraphs(ctx context.Context, control *p5AdjacencyGraph) error { + _, err := control.pool.Exec(ctx, `delete from graph where name like 'p5_adjacency_%' and id <> $1`, control.graphID) + return err +} + +func assertP5AdjacencyAbsent(ctx context.Context, queryer p5AdjacencyRowQueryer) error { + var absent bool + if err := queryer.QueryRow(ctx, `select to_regclass('public.p5_adjacency_v1') is null`).Scan(&absent); err != nil { + return err + } + if !absent { + return fmt.Errorf("P5 adjacency relation is present in a base condition") + } + return nil +} + +func assertP5AdjacencyExact(ctx context.Context, queryer p5AdjacencyRowQueryer, graphID int32) error { + var baseEdges, shadowRows, mismatches int64 + if err := queryer.QueryRow(ctx, `select count(*) from edge where graph_id = $1`, graphID).Scan(&baseEdges); err != nil { + return err + } + if err := queryer.QueryRow(ctx, `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, graphID).Scan(&shadowRows); err != nil { + return err + } + if shadowRows != baseEdges*2 { + return fmt.Errorf("graph %d has %d base edges but %d P5 rows", graphID, baseEdges, shadowRows) + } + if err := queryer.QueryRow(ctx, ` + select + (select count(*) from edge e where e.graph_id = $1 and ( + not exists (select 1 from public.p5_adjacency_v1 a where a.graph_id = e.graph_id and a.edge_id = e.id and a.direction = 1 and a.anchor_id = e.start_id and a.neighbor_id = e.end_id and a.kind_id = e.kind_id) + or not exists (select 1 from public.p5_adjacency_v1 a where a.graph_id = e.graph_id and a.edge_id = e.id and a.direction = -1 and a.anchor_id = e.end_id and a.neighbor_id = e.start_id and a.kind_id = e.kind_id) + )) + + + (select count(*) from public.p5_adjacency_v1 a where a.graph_id = $1 and not exists ( + select 1 from edge e where e.graph_id = a.graph_id and e.id = a.edge_id and e.kind_id = a.kind_id and ( + (a.direction = 1 and a.anchor_id = e.start_id and a.neighbor_id = e.end_id) + or (a.direction = -1 and a.anchor_id = e.end_id and a.neighbor_id = e.start_id) + ) + ))`, graphID).Scan(&mismatches); err != nil { + return err + } + if mismatches != 0 { + return fmt.Errorf("graph %d has %d P5 base/shadow mismatches", graphID, mismatches) + } + return nil +} + +func p5AdjacencyShadowIdentity(ctx context.Context, queryer p5AdjacencyRowQueryer, fixture p5AdjacencyFixture) (string, error) { + var identity string + err := queryer.QueryRow(ctx, ` + select coalesce(string_agg(edge_id::text || ':' || direction::text || ':' || ctid::text, ',' order by edge_id, direction), '') + from public.p5_adjacency_v1 + where graph_id = $1 and edge_id = any($2::bigint[])`, fixture.graphID, fixture.updateEdgeIDs).Scan(&identity) + return identity, err +} + +func p5AdjacencyReadProbe(ctx context.Context, graphState *p5AdjacencyGraph, fixture p5AdjacencyFixture, shadow bool) (P5AdjacencyReadProbe, error) { + if shadow { + if err := assertP5AdjacencyExact(ctx, graphState.pool, fixture.graphID); err != nil { + return P5AdjacencyReadProbe{}, err + } + } + statement := `select id, end_id, kind_id from edge where graph_id = $1 and start_id = $2 and kind_id = $3 order by id` + relation := "edge" + if shadow { + statement = `select edge_id, neighbor_id, kind_id from public.p5_adjacency_v1 where graph_id = $1 and direction = 1 and anchor_id = $2 and kind_id = $3 order by edge_id` + relation = "p5_adjacency_v1" + } + args := []any{fixture.graphID, fixture.rootID, fixture.updateKindID} + start := time.Now() + rows, err := graphState.pool.Query(ctx, statement, args...) + if err != nil { + return P5AdjacencyReadProbe{}, err + } + var cardinality int64 + for rows.Next() { + var first, second int64 + var kind int16 + if err := rows.Scan(&first, &second, &kind); err != nil { + rows.Close() + return P5AdjacencyReadProbe{}, err + } + cardinality++ + } + if err := rows.Err(); err != nil { + rows.Close() + return P5AdjacencyReadProbe{}, err + } + rows.Close() + if cardinality != int64(len(fixture.targetIDs)) { + return P5AdjacencyReadProbe{}, fmt.Errorf("%s read probe returned %d rows, expected %d", relation, cardinality, len(fixture.targetIDs)) + } + var plan []byte + if err := graphState.pool.QueryRow(ctx, "explain (analyze, buffers, format json) "+statement, args...).Scan(&plan); err != nil { + return P5AdjacencyReadProbe{}, err + } + if !json.Valid(plan) { + return P5AdjacencyReadProbe{}, fmt.Errorf("%s read probe returned invalid plan JSON", relation) + } + return P5AdjacencyReadProbe{Relation: relation, Duration: time.Since(start), ResultCardinality: cardinality, Plan: plan}, nil +} + +func p5AdjacencyRelationSize(ctx context.Context, pool *pgxpool.Pool, relation string) (P5AdjacencyRelationSize, error) { + size := P5AdjacencyRelationSize{Relation: relation} + err := pool.QueryRow(ctx, ` + select coalesce(pg_relation_size(to_regclass($1)), 0), + coalesce(pg_indexes_size(to_regclass($1)), 0), + coalesce(pg_total_relation_size(to_regclass($1)), 0)`, relation).Scan(&size.HeapBytes, &size.IndexBytes, &size.TotalBytes) + return size, err +} + +func p5AdjacencyWALPosition(ctx context.Context, pool *pgxpool.Pool) (int64, error) { + var position int64 + err := pool.QueryRow(ctx, `select pg_wal_lsn_diff(pg_current_wal_lsn(), '0/0')::bigint`).Scan(&position) + return position, err +} + +// disableP5AdjacencyAutovacuum prevents disposable fixture cleanup from +// changing the global LSN during the setup calibration. Statement-level WAL +// remains the authoritative mutation value, while the LSN deltas are retained +// as a quiescent cross-check. +func disableP5AdjacencyAutovacuum(ctx context.Context, pool *pgxpool.Pool, shadow bool) error { + relations := []string{"node", "edge"} + if shadow { + relations = append(relations, "public.p5_adjacency_v1") + } + for _, relation := range relations { + children, err := p5AdjacencyPartitions(ctx, pool, relation) + if err != nil { + return err + } + for _, child := range children { + if _, err := pool.Exec(ctx, "alter table "+child+" set (autovacuum_enabled = false)"); err != nil { + return err + } + } + } + return nil +} + +func disableP5AdjacencyFixtureAutovacuum(ctx context.Context, pool *pgxpool.Pool, graphID int32, shadow bool) error { + relations := []string{fmt.Sprintf("node_%d", graphID), fmt.Sprintf("edge_%d", graphID)} + if shadow { + relations = append(relations, fmt.Sprintf("p5_adjacency_v1_%d", graphID)) + } + for _, relation := range relations { + if _, err := pool.Exec(ctx, "alter table "+relation+" set (autovacuum_enabled = false)"); err != nil { + return err + } + } + return nil +} + +// restoreP5AdjacencyAutovacuum restores the database-wide parent settings +// after the disposable capture has finished. The shadow relation may already +// have been removed, so only core relations are reset here. +func restoreP5AdjacencyAutovacuum(ctx context.Context, pool *pgxpool.Pool) error { + for _, relation := range []string{"node", "edge"} { + children, err := p5AdjacencyPartitions(ctx, pool, relation) + if err != nil { + return err + } + for _, child := range children { + if _, err := pool.Exec(ctx, "alter table "+child+" reset (autovacuum_enabled)"); err != nil { + return err + } + } + } + return nil +} + +func p5AdjacencyPartitions(ctx context.Context, pool *pgxpool.Pool, parent string) ([]string, error) { + rows, err := pool.Query(ctx, ` + select child.oid::regclass::text + from pg_inherits + join pg_class child on child.oid = inhrelid + where inhparent = to_regclass($1) + order by child.oid`, parent) + if err != nil { + return nil, err + } + defer rows.Close() + var children []string + for rows.Next() { + var child string + if err := rows.Scan(&child); err != nil { + return nil, err + } + children = append(children, child) + } + return children, rows.Err() +} + +func waitForP5WALQuiescence(ctx context.Context, pool *pgxpool.Pool) error { + deadline := time.Now().Add(30 * time.Second) + for { + var activeAutovacuum int + if err := pool.QueryRow(ctx, ` + select count(*) + from pg_stat_activity + where datname = current_database() + and backend_type = 'autovacuum worker' + and state <> 'idle'`).Scan(&activeAutovacuum); err != nil { + return err + } + if activeAutovacuum == 0 { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("autovacuum remained active during P5 WAL calibration") + } + timer := time.NewTimer(100 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func captureP5PostgresEnvironment(ctx context.Context, pool *pgxpool.Pool) (PostgresEnvironment, error) { + var environment PostgresEnvironment + err := pool.QueryRow(ctx, ` + select version(), current_database(), current_setting('plan_cache_mode'), current_setting('transaction_isolation'), + current_setting('work_mem'), current_setting('temp_file_limit'), (select count(*) from graph), + pg_postmaster_start_time(), (select oid::int8 from pg_database where datname = current_database()), current_setting('autovacuum')`).Scan( + &environment.Version, + &environment.Database, + &environment.PlanCacheMode, + &environment.TransactionIsolation, + &environment.WorkMem, + &environment.TempFileLimit, + &environment.GraphPartitionCount, + &environment.PostmasterStartedAt, + &environment.DatabaseOID, + &environment.Autovacuum, + ) + return environment, err +} + +func p5AdjacencyCancellationAndReuse(ctx context.Context, graphState *p5AdjacencyGraph, fixture p5AdjacencyFixture) (P5AdjacencyCancellation, error) { + proof := P5AdjacencyCancellation{Ran: true} + if err := graphState.pool.QueryRow(ctx, `select pg_backend_pid()`).Scan(&proof.CancelledBackendPID); err != nil { + return P5AdjacencyCancellation{}, err + } + tx, err := graphState.pool.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return P5AdjacencyCancellation{}, err + } + cancelledContext, cancel := context.WithCancel(ctx) + timer := time.AfterFunc(20*time.Millisecond, cancel) + _, executeErr := tx.Exec(cancelledContext, ` + with delayed as materialized (select pg_sleep(1)) + insert into edge(graph_id, start_id, end_id, kind_id, properties) + select $1, $2, $3, $4, '{"p5_cancel":true}'::jsonb from delayed`, + fixture.graphID, fixture.rootID, fixture.targetIDs[0], fixture.createKindID, + ) + timer.Stop() + cancel() + rollbackErr := tx.Rollback(ctx) + proof.RollbackObserved = rollbackErr == nil || executeErr != nil + if executeErr == nil { + return P5AdjacencyCancellation{}, fmt.Errorf("P5 cancellation statement committed unexpectedly") + } + if err := graphState.pool.QueryRow(ctx, `select pg_backend_pid()`).Scan(&proof.ReplayBackendPID); err != nil { + return P5AdjacencyCancellation{}, fmt.Errorf("reacquire pool connection after cancellation: %w", err) + } + proof.PoolReuseSucceeded = true + if err := assertP5AdjacencyExact(ctx, graphState.pool, fixture.graphID); err != nil { + return P5AdjacencyCancellation{}, err + } + return proof, nil +} + +func writeP5AdjacencyFeasibilityReport(path string, report P5AdjacencyFeasibilityReport) error { + if path == "" { + return fmt.Errorf("P5 adjacency feasibility output path must not be empty") + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return fmt.Errorf("create immutable P5 adjacency feasibility artifact: %w", err) + } + defer output.Close() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +func absInt64(value int64) int64 { + if value < 0 { + return -value + } + return value +} diff --git a/cmd/graphbench/p5_adjacency_feasibility_integration_test.go b/cmd/graphbench/p5_adjacency_feasibility_integration_test.go new file mode 100644 index 00000000..bc7cea39 --- /dev/null +++ b/cmd/graphbench/p5_adjacency_feasibility_integration_test.go @@ -0,0 +1,119 @@ +//go:build manual_integration && integration + +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" +) + +func isPostgreSQLConnection(connection string) bool { + normalized := strings.ToLower(connection) + return strings.HasPrefix(normalized, "postgres://") || strings.HasPrefix(normalized, "postgresql://") +} + +func TestP5AdjacencyTaggedPGXStatementVisibleToPGStatStatements(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + if !isPostgreSQLConnection(connection) { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + ctx := context.Background() + graphState, err := openP5AdjacencyGraph(ctx, connection) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, deleteP5AdjacencyGraph(ctx, graphState, false)) + require.NoError(t, graphState.db.Close(ctx)) + }) + _, release, err := prepareP5AdjacencyWALAttribution(ctx, graphState.pool) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, release()) + }) + + tag := p5AdjacencyStatementWALTag("pgx_visibility_probe") + tx, err := graphState.pool.Begin(ctx) + require.NoError(t, err) + defer func() { + _ = tx.Rollback(ctx) + }() + before, err := p5AdjacencyStatementWALStats(ctx, tx, tag) + require.NoError(t, err) + _, err = tx.Exec(ctx, "with "+tag+" as (select 1) insert into graph(name) values ($1)", pgx.QueryExecModeSimpleProtocol, tag) + require.NoError(t, err) + after, err := p5AdjacencyStatementWALStats(ctx, tx, tag) + require.NoError(t, err) + stats, err := p5AdjacencyStatementWALDelta(before, after) + require.NoError(t, err) + require.NoError(t, tx.Rollback(ctx)) + require.Equal(t, int64(1), stats.Calls) + require.Positive(t, stats.Bytes) +} + +func TestP5AdjacencyCalibrationAttributesTaggedStatementWAL(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + if !isPostgreSQLConnection(connection) { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + ctx := context.Background() + resetP5AdjacencyTestState(t, ctx, connection) + t.Cleanup(func() { + resetP5AdjacencyTestState(t, ctx, connection) + }) + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + attribution, release, err := prepareP5AdjacencyWALAttribution(ctx, pool) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, release()) + }) + require.Equal(t, "pg_stat_statements", attribution.Source) + + calibration, err := runP5AdjacencyCalibration(ctx, connection, "base", 1, "batch_relationship_create") + require.NoError(t, err) + require.Equal(t, int64(1), calibration.StatementWAL.Calls) + require.Positive(t, calibration.StatementWAL.Records) + require.Positive(t, calibration.StatementWAL.Bytes) + require.GreaterOrEqual(t, calibration.MutationWALLSN, calibration.StatementWAL.Bytes) + + control, err := openP5AdjacencyGraph(ctx, connection) + require.NoError(t, err) + require.NoError(t, installP5AdjacencyShadow(ctx, control.db)) + shadowCalibration, err := runP5AdjacencyCalibration(ctx, connection, "shadow", 1, "batch_relationship_create") + require.NoError(t, err) + require.Equal(t, int64(1), shadowCalibration.StatementWAL.Calls) + require.Positive(t, shadowCalibration.StatementWAL.Records) + require.Greater(t, shadowCalibration.StatementWAL.Bytes, calibration.StatementWAL.Bytes) + require.NoError(t, dropP5AdjacencyShadow(ctx, control.db)) + require.NoError(t, deleteP5AdjacencyGraph(ctx, control, false)) + require.NoError(t, control.db.Close(ctx)) +} + +func resetP5AdjacencyTestState(t *testing.T, ctx context.Context, connection string) { + t.Helper() + control, err := openP5AdjacencyGraph(ctx, connection) + require.NoError(t, err) + require.NoError(t, dropP5AdjacencyShadow(ctx, control.db)) + require.NoError(t, cleanupP5AdjacencyOwnedGraphs(ctx, control)) + require.NoError(t, deleteP5AdjacencyGraph(ctx, control, false)) + require.NoError(t, control.db.Close(ctx)) +} diff --git a/cmd/graphbench/p5_adjacency_feasibility_test.go b/cmd/graphbench/p5_adjacency_feasibility_test.go new file mode 100644 index 00000000..7de91abc --- /dev/null +++ b/cmd/graphbench/p5_adjacency_feasibility_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestParseConfigP5AdjacencyFeasibility(t *testing.T) { + cfg, err := parseConfig([]string{"-p5-adjacency-feasibility-output", "p5.json"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "p5.json", cfg.P5AdjacencyFeasibilityOutput) + + _, err = parseConfig([]string{"-p5-adjacency-feasibility-output", "p5.json", "-pool-size", "2"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires pool-size 1") + + _, err = parseConfig([]string{"-p5-adjacency-feasibility-output", "p5.json", "-modes", "neo4j"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires only postgres_sql mode") + + _, err = parseConfig([]string{"-p5-adjacency-feasibility-output", "p5.json", "-cases", "SP-S0-DIRECT"}, func(string) string { return "" }) + require.ErrorContains(t, err, "does not accept corpus selectors") +} + +func TestP5AdjacencyQuantiles(t *testing.T) { + median, p95 := p5AdjacencyQuantiles([]time.Duration{5, 1, 3, 2, 4}) + require.Equal(t, 3*time.Nanosecond, median) + require.Equal(t, 5*time.Nanosecond, p95) +} + +func TestP5AdjacencyStatementWALTag(t *testing.T) { + tag := p5AdjacencyStatementWALTag("batched_relationship_delete") + require.Equal(t, "p5_adjacency_v2_wal_batched_relationship_delete", tag) + + delta, err := p5AdjacencyStatementWALDelta( + P5AdjacencyStatementWAL{Tag: tag, Calls: 3, Records: 10, FPI: 1, Bytes: 100}, + P5AdjacencyStatementWAL{Tag: tag, Calls: 4, Records: 13, FPI: 2, Bytes: 140}, + ) + require.NoError(t, err) + require.Equal(t, P5AdjacencyStatementWAL{Tag: tag, Calls: 1, Records: 3, FPI: 1, Bytes: 40}, delta) +} + +func TestWriteP5AdjacencyFeasibilityReportIsImmutable(t *testing.T) { + path := filepath.Join(t.TempDir(), "p5.json") + report := P5AdjacencyFeasibilityReport{Schema: p5AdjacencyFeasibilitySchema, Passed: true} + require.NoError(t, writeP5AdjacencyFeasibilityReport(path, report)) + require.ErrorContains(t, writeP5AdjacencyFeasibilityReport(path, report), "create immutable") +} diff --git a/cmd/graphbench/perf_gate.go b/cmd/graphbench/perf_gate.go new file mode 100644 index 00000000..e353ae66 --- /dev/null +++ b/cmd/graphbench/perf_gate.go @@ -0,0 +1,964 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "math/rand" + "os" + "sort" + "time" +) + +const ( + // perfGateVersion identifies the serialized schema revision for perf gate. + perfGateVersion = 5 + + // defaultBootstrapCount sets the fallback number of resamples used to estimate confidence bounds. + defaultBootstrapCount = 10_000 + + // minimumGateRounds requires this many independent matched rounds before a workload may pass. + minimumGateRounds = 5 + + // minimumP95Samples requires this many warm samples per arm before the P95 ratio is gated. + minimumP95Samples = 150 + + // minimumDiscoveryWarmups requires the discovery protocol's untimed warmup floor. + minimumDiscoveryWarmups = 5 +) + +// PerfGateOptions defines statistical confidence, materiality, targets, and declared backend coverage for gating. +type PerfGateOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // RegressionThreshold sets the largest median ratio that is not considered a regression. + RegressionThreshold float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // DeclaredBackends lists case/backend declarations that the performance gate must cover. + DeclaredBackends []DeclaredCaseBackend + // TargetNames restricts materiality requirements to the named workloads. + TargetNames []string + // MaterialityRatio sets the relative change required before a difference is material. + MaterialityRatio float64 + // MaterialityAbsolute sets the absolute duration change required before a difference is material. + MaterialityAbsolute time.Duration + // DiagnosticMode allows incomplete diagnostic selections that cannot produce a release-gate pass. + DiagnosticMode bool + // AAReportPath selects the host A/A evidence loaded by artifact comparison mode. + AAReportPath string + // AAReport contains host-specific per-case timing resolution required for promotion. + AAReport *AAResolutionReport + // AAReportSHA256 identifies the exact A/A report supplied to the gate. + AAReportSHA256 string +} + +// RatioInterval describes a point estimate and confidence bounds for a latency ratio. +type RatioInterval struct { + // Estimate supplies the estimate input to the RatioInterval contract. + Estimate float64 `json:"estimate"` + // Lower supplies the lower input to the RatioInterval contract. + Lower float64 `json:"lower"` + // Upper supplies the upper input to the RatioInterval contract. + Upper float64 `json:"upper"` +} + +// DurationInterval describes a duration estimate and its confidence bounds. +type DurationInterval struct { + // Estimate supplies the estimate input to the DurationInterval contract. + Estimate time.Duration `json:"estimate"` + // Lower supplies the lower input to the DurationInterval contract. + Lower time.Duration `json:"lower"` + // Upper supplies the upper input to the DurationInterval contract. + Upper time.Duration `json:"upper"` +} + +// PerfGateCase reports matched sample evidence, bootstrap intervals, and classification for one gated workload. +type PerfGateCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // Tier identifies whether timing is gated or stress-diagnostic. + Tier string `json:"tier"` + // QualificationSplit identifies training, frozen holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // TimingGated reports whether latency evidence contributes to promotion. + TimingGated bool `json:"timing_gated"` + // Rounds records the number of rounds. + Rounds int `json:"rounds"` + // BaselineSamples records warm timing samples available from the baseline arm. + BaselineSamples int `json:"baseline_samples"` + // CandidateSamples records warm timing samples available from the candidate arm. + CandidateSamples int `json:"candidate_samples"` + // BaselineStatus supplies the baseline status input to the PerfGateCase contract. + BaselineStatus string `json:"baseline_status,omitempty"` + // CandidateStatus supplies the candidate status input to the PerfGateCase contract. + CandidateStatus string `json:"candidate_status,omitempty"` + // OracleOnly marks a backend as a correctness oracle excluded from latency regression decisions. + OracleOnly bool `json:"oracle_only,omitempty"` + // MedianRatio reports the candidate-to-baseline median latency ratio and confidence bounds. + MedianRatio RatioInterval `json:"median_ratio"` + // P95Ratio reports the candidate-to-baseline P95 latency ratio and confidence bounds. + P95Ratio *RatioInterval `json:"p95_ratio,omitempty"` + // MedianSaving reports absolute median latency saved by the candidate. + MedianSaving *DurationInterval `json:"median_saving,omitempty"` + // MedianChange reports candidate-minus-baseline median latency. + MedianChange *DurationInterval `json:"median_change,omitempty"` + // P95Change reports candidate-minus-baseline P95 latency. + P95Change *DurationInterval `json:"p95_change,omitempty"` + // P50NoiseRatio supplies the p50 noise ratio input to the PerfGateCase contract. + P50NoiseRatio float64 `json:"p50_noise_ratio,omitempty"` + // P50NoiseAbsolute supplies the p50 noise absolute input to the PerfGateCase contract. + P50NoiseAbsolute time.Duration `json:"p50_noise_absolute,omitempty"` + // P95NoiseRatio supplies the p95 noise ratio input to the PerfGateCase contract. + P95NoiseRatio float64 `json:"p95_noise_ratio,omitempty"` + // P95NoiseAbsolute supplies the p95 noise absolute input to the PerfGateCase contract. + P95NoiseAbsolute time.Duration `json:"p95_noise_absolute,omitempty"` + // MaterialityRatio sets the relative change required before a difference is material. + MaterialityRatio *float64 `json:"materiality_ratio_upper_limit,omitempty"` + // MaterialityAbsolute sets the absolute duration change required before a difference is material. + MaterialityAbsolute *time.Duration `json:"materiality_absolute_lower_limit,omitempty"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Reasons lists explanations for the reported disposition. + Reasons []string `json:"reasons,omitempty"` + // CandidateRuntimeReceiptChains preserves complete measured candidate + // branch chains used by the performance decision. + CandidateRuntimeReceiptChains [][]RuntimeReceiptEvent `json:"candidate_runtime_receipt_chains,omitempty"` +} + +// PerfGateReport contains baseline and candidate identities, gate policy, and every workload disposition. +type PerfGateReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // RegressionThreshold sets the largest median ratio that is not considered a regression. + RegressionThreshold float64 `json:"regression_threshold"` + // BaselineSHA256 identifies the exact baseline artifact evaluated by the gate. + BaselineSHA256 string `json:"baseline_sha256"` + // CandidateSHA256 identifies the exact candidate artifact evaluated by the gate. + CandidateSHA256 string `json:"candidate_sha256"` + // AAReportSHA256 identifies the exact host A/A resolution report evaluated by the gate. + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + // DeclarationSHA256 identifies the canonical set of declared workloads. + DeclarationSHA256 string `json:"declaration_sha256,omitempty"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // PromotionEligible reports whether this complete, non-diagnostic evidence may support production promotion. + PromotionEligible bool `json:"promotion_eligible"` + // MaterialityRequired reports that promotion requires at least one explicitly named improvement target. + MaterialityRequired bool `json:"materiality_required"` + // MaterialityTargets supplies the materiality targets input to the PerfGateReport contract. + MaterialityTargets int `json:"materiality_targets"` + // MaterialityPassed reports whether every resolved target cleared the configured A/A-aware improvement floor. + MaterialityPassed bool `json:"materiality_passed"` + // QualificationRequired reports whether the artifact contains a prioritized traversal candidate that requires independent training and frozen-holdout gates. + QualificationRequired bool `json:"qualification_required"` + // TrainingCases records prioritized traversal cases gated on the selector-training partition. + TrainingCases int `json:"training_cases"` + // HoldoutCases records prioritized traversal cases gated on the frozen topology holdout. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed reports whether every observed prioritized training case passed. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed reports whether every observed prioritized holdout case passed. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed reports whether nonempty training and holdout partitions independently passed. + QualificationPassed bool `json:"qualification_passed"` + // QualificationFamilies contains the independent split disposition for each concrete traversal candidate family. + QualificationFamilies []TraversalQualificationStatus `json:"qualification_families,omitempty"` + // Cases contains the gate disposition and statistical evidence for each declared workload. + Cases []PerfGateCase `json:"cases"` +} + +// performanceKey identifies one dataset, case, and backend across performance artifacts. +type performanceKey struct { + // dataset names the fixture shared by matched baseline and candidate records. + dataset string + // name identifies the workload case within its dataset. + name string + // backend separates independently gated execution modes for the same workload. + backend ExecutionMode +} + +// roundSamples groups positive warm durations by independent measurement round. +type roundSamples map[int][]time.Duration + +// comparePerformanceArtifacts validates two artifacts, writes their performance-gate report, and returns its pass status. +func comparePerformanceArtifacts(baselinePath, candidatePath, outputPath string, options PerfGateOptions) (bool, error) { + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return false, fmt.Errorf("read baseline: %w", err) + } + + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return false, fmt.Errorf("read candidate: %w", err) + } + if err := validatePerformanceArtifactSelections(baseline, candidate, options.DiagnosticMode); err != nil { + return false, err + } + if options.AAReportPath != "" { + options.AAReport, options.AAReportSHA256, err = loadAAResolutionReport(options.AAReportPath) + if err != nil { + return false, fmt.Errorf("load performance-gate A/A evidence: %w", err) + } + } + + baselineChecksum, err := fileSHA256(baselinePath) + if err != nil { + return false, err + } + candidateChecksum, err := fileSHA256(candidatePath) + if err != nil { + return false, err + } + + report, err := buildPerfGateReport(baseline, candidate, options) + if err != nil { + return false, err + } + report.BaselineSHA256 = baselineChecksum + report.CandidateSHA256 = candidateChecksum + + if err := writePerfGateReport(outputPath, report); err != nil { + return false, err + } + return report.Passed && report.PromotionEligible, nil +} + +// validatePerformanceArtifactSelections rejects adaptive or diagnostic artifacts when complete-gate input is required. +func validatePerformanceArtifactSelections(baseline, candidate []CaseResult, diagnosticMode bool) error { + if !diagnosticMode && (hasAdaptiveDiscoveryRecord(baseline) || hasAdaptiveDiscoveryRecord(candidate)) { + return fmt.Errorf("adaptive-discovery artifacts are refused by the complete performance gate") + } + baselineSelection, baselineErr := selectionIdentity(baseline) + candidateSelection, candidateErr := selectionIdentity(candidate) + if baselineErr != nil || candidateErr != nil { + if diagnosticMode { + return fmt.Errorf("diagnostic comparison requires selection manifests in both artifacts") + } + return fmt.Errorf("complete performance gate requires selection manifests in both artifacts") + } + if err := validateSelectionManifestAccounting(baselineSelection); err != nil { + return fmt.Errorf("baseline artifact %w", err) + } + if err := validateSelectionManifestAccounting(candidateSelection); err != nil { + return fmt.Errorf("candidate artifact %w", err) + } + if baselineSelection.ProtectedDeclarationCount != candidateSelection.ProtectedDeclarationCount || + baselineSelection.ProtectedDeclarationSHA256 != candidateSelection.ProtectedDeclarationSHA256 { + return fmt.Errorf("artifact protected declaration omissions differ") + } + if baselineSelection.DiagnosticOnly || candidateSelection.DiagnosticOnly { + if !diagnosticMode { + return fmt.Errorf("diagnostic-only artifacts are refused by the complete performance gate") + } + if !baselineSelection.DiagnosticOnly || !candidateSelection.DiagnosticOnly { + return fmt.Errorf("diagnostic comparison requires two diagnostic-only artifacts") + } + if baselineSelection.DeclarationSHA256 != candidateSelection.DeclarationSHA256 { + return fmt.Errorf("diagnostic artifact declarations differ: %s != %s", baselineSelection.DeclarationSHA256, candidateSelection.DeclarationSHA256) + } + return nil + } + if diagnosticMode { + return fmt.Errorf("diagnostic comparison mode requires filtered diagnostic-only artifacts") + } + return nil +} + +// hasAdaptiveDiscoveryRecord reports whether any record was produced by adaptive existing-graph discovery. +func hasAdaptiveDiscoveryRecord(records []CaseResult) bool { + for _, record := range records { + if record.ExistingGraph != nil && record.ExistingGraph.Adaptive { + return true + } + if record.Environment != nil && record.Environment.Protocol == "adaptive_discovery" { + return true + } + } + return false +} + +// buildPerfGateReport compares matched baseline and candidate samples and classifies each declared workload. +func buildPerfGateReport(baseline, candidate []CaseResult, options PerfGateOptions) (PerfGateReport, error) { + if err := validatePerformanceWorkloadIdentity(baseline, candidate); err != nil { + return PerfGateReport{}, err + } + if options.Confidence <= 0 || options.Confidence >= 1 { + return PerfGateReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.RegressionThreshold < 0 { + return PerfGateReport{}, fmt.Errorf("regression threshold must not be negative") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return PerfGateReport{}, fmt.Errorf("bootstrap count must be positive") + } + if options.MaterialityRatio == 0 { + options.MaterialityRatio = 0.95 + } + if options.MaterialityRatio <= 0 || options.MaterialityRatio >= 1 { + return PerfGateReport{}, fmt.Errorf("materiality ratio must be between 0 and 1") + } + if options.MaterialityAbsolute == 0 { + options.MaterialityAbsolute = 100 * time.Microsecond + } + if options.MaterialityAbsolute < 0 { + return PerfGateReport{}, fmt.Errorf("materiality absolute duration must not be negative") + } + + baselineSeries := collectWarmSeries(baseline) + candidateSeries := collectWarmSeries(candidate) + keys := declaredPerformanceKeys(options.DeclaredBackends, baseline, candidate) + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + if keys[i].name != keys[j].name { + return keys[i].name < keys[j].name + } + return keys[i].backend < keys[j].backend + }) + if len(keys) == 0 { + return PerfGateReport{}, fmt.Errorf("artifacts and declaration contain no PostgreSQL or Neo4j cases") + } + tiers := make(map[performanceKey]string, len(keys)) + splits := make(map[performanceKey]string, len(keys)) + hasPromotionTiming := false + for _, key := range keys { + tier, err := timingTier(key, baseline, candidate) + if err != nil { + return PerfGateReport{}, err + } + tiers[key] = tier + split, err := qualificationSplit(key, baseline, candidate) + if err != nil { + return PerfGateReport{}, err + } + splits[key] = split + if key.backend == ModePostgresSQL && (tier == "normal" || tier == "envelope") && promotionTimingSplit(split) { + hasPromotionTiming = true + } + } + if hasPromotionTiming && !options.DiagnosticMode { + if !validSHA256(options.AAReportSHA256) { + return PerfGateReport{}, fmt.Errorf("complete performance gate requires a checksummed host A/A report") + } + if err := validateAAResolutionEvidence(options.AAReport, baseline, options.Confidence); err != nil { + return PerfGateReport{}, fmt.Errorf("baseline A/A evidence: %w", err) + } + if err := validateAAResolutionEvidence(options.AAReport, candidate, options.Confidence); err != nil { + return PerfGateReport{}, fmt.Errorf("candidate A/A evidence: %w", err) + } + } else if options.AAReport != nil { + if !validSHA256(options.AAReportSHA256) { + return PerfGateReport{}, fmt.Errorf("supplied A/A report checksum is malformed") + } + if err := validateAAResolutionEvidence(options.AAReport, baseline, options.Confidence); err != nil { + return PerfGateReport{}, err + } + } + targetNames := make(map[string]struct{}, len(options.TargetNames)) + for _, name := range options.TargetNames { + targetNames[name] = struct{}{} + } + + report := PerfGateReport{ + Version: perfGateVersion, + Seed: options.Seed, + Confidence: options.Confidence, + RegressionThreshold: options.RegressionThreshold, + AAReportSHA256: options.AAReportSHA256, + Passed: true, + PromotionEligible: !options.DiagnosticMode && hasPromotionTiming && len(targetNames) > 0, + MaterialityRequired: hasPromotionTiming && !options.DiagnosticMode, + MaterialityPassed: len(targetNames) > 0, + TrainingPassed: true, + HoldoutPassed: true, + } + resolvedMaterialityTargets := map[string]struct{}{} + qualification := map[string]*TraversalQualificationStatus{} + if len(options.DeclaredBackends) > 0 { + report.DeclarationSHA256 = declarationSHA256(options.DeclaredBackends) + } + for idx, key := range keys { + baselineStatus := artifactCaseStatus(baseline, key) + candidateStatus := artifactCaseStatus(candidate, key) + baselineRounds, candidateRounds := matchedRounds(baselineSeries[key], candidateSeries[key]) + gateCase := PerfGateCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + Tier: tiers[key], + QualificationSplit: splits[key], + TimingGated: key.backend == ModePostgresSQL && (tiers[key] == "normal" || tiers[key] == "envelope") && promotionTimingSplit(splits[key]) && !options.DiagnosticMode, + Rounds: len(baselineRounds), + BaselineSamples: sampleCount(baselineRounds), + CandidateSamples: sampleCount(candidateRounds), + BaselineStatus: baselineStatus, + CandidateStatus: candidateStatus, + OracleOnly: key.backend == ModeNeo4j, + Passed: true, + CandidateRuntimeReceiptChains: caseRuntimeReceiptChains(candidate, key), + } + if candidateStatus != StatusOK { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("required candidate record status is %s", candidateStatus)) + } + if gateCase.TimingGated { + if err := validateCandidateRuntimeEvidence(candidate, key); err != nil { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, err.Error()) + } + } + // Neo4j is a correctness oracle. A successful record means its untimed + // exact observation checks passed; its latency never affects this gate. + if key.backend == ModeNeo4j { + if !gateCase.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, gateCase) + continue + } + if baselineStatus != StatusOK { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("required baseline record status is %s", baselineStatus)) + } + if gateCase.TimingGated && len(baselineRounds) < minimumGateRounds { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("need at least %d matched rounds, got %d", minimumGateRounds, len(baselineRounds))) + } + if gateCase.TimingGated && len(baselineRounds) > 0 { + if err := validatePairedOrderEvidence(baseline, candidate, key, sortedRounds(baselineRounds), minimumDiscoveryWarmups); err != nil { + return PerfGateReport{}, fmt.Errorf("invalid promotion evidence: %w", err) + } + } + if tiers[key] == "stress" { + gateCase.Reasons = append(gateCase.Reasons, "stress tier timing is diagnostic") + } + if splits[key] == "diagnostic" { + gateCase.Reasons = append(gateCase.Reasons, "diagnostic qualification split is excluded from promotion timing") + } + + gateCase.P50NoiseRatio, gateCase.P50NoiseAbsolute = minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + gateCase.P95NoiseRatio, gateCase.P95NoiseAbsolute = minimumTimingNoiseRatio, minimumTimingNoiseAbsolute + if options.AAReport != nil { + if ratio, absolute, err := aaTimingFloor(options.AAReport, key, false, options.RegressionThreshold); err == nil { + gateCase.P50NoiseRatio, gateCase.P50NoiseAbsolute = ratio, absolute + } else if gateCase.TimingGated { + return PerfGateReport{}, err + } + if ratio, absolute, err := aaTimingFloor(options.AAReport, key, true, options.RegressionThreshold); err == nil { + gateCase.P95NoiseRatio, gateCase.P95NoiseAbsolute = ratio, absolute + } else if gateCase.TimingGated { + return PerfGateReport{}, err + } + } else { + gateCase.P50NoiseRatio = max(gateCase.P50NoiseRatio, options.RegressionThreshold) + gateCase.P95NoiseRatio = max(gateCase.P95NoiseRatio, options.RegressionThreshold) + } + + seed := options.Seed + int64(idx)*7919 + if len(baselineRounds) > 0 { + gateCase.MedianRatio = bootstrapRoundMedianRatio(baselineRounds, candidateRounds, seed, options) + saving := bootstrapRoundMedianSaving(baselineRounds, candidateRounds, seed+3, options) + gateCase.MedianSaving = &saving + change := negateDurationInterval(saving) + gateCase.MedianChange = &change + if gateCase.TimingGated && gateCase.MedianRatio.Lower > 1+gateCase.P50NoiseRatio && change.Lower > gateCase.P50NoiseAbsolute { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("median regression exceeds host A/A floors: ratio lower %.4f > %.4f and change lower %s > %s", gateCase.MedianRatio.Lower, 1+gateCase.P50NoiseRatio, change.Lower, gateCase.P50NoiseAbsolute)) + } + } + + if gateCase.BaselineSamples >= minimumP95Samples && gateCase.CandidateSamples >= minimumP95Samples { + interval := bootstrapStratifiedP95Ratio(baselineRounds, candidateRounds, seed+1, options) + gateCase.P95Ratio = &interval + change := bootstrapStratifiedQuantileChange(baselineRounds, candidateRounds, 0.95, seed+2, options) + gateCase.P95Change = &change + if gateCase.TimingGated && interval.Lower > 1+gateCase.P95NoiseRatio && change.Lower > gateCase.P95NoiseAbsolute { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("p95 regression exceeds host A/A floors: ratio lower %.4f > %.4f and change lower %s > %s", interval.Lower, 1+gateCase.P95NoiseRatio, change.Lower, gateCase.P95NoiseAbsolute)) + } + } else if gateCase.TimingGated { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("need at least %d warm samples per side for p95, got %d/%d", minimumP95Samples, gateCase.BaselineSamples, gateCase.CandidateSamples)) + } + + if _, isTarget := targetNames[key.name]; isTarget && gateCase.TimingGated && len(baselineRounds) > 0 { + resolvedMaterialityTargets[key.name] = struct{}{} + effectiveRatio := min(options.MaterialityRatio, 1-gateCase.P50NoiseRatio) + effectiveAbsolute := max(options.MaterialityAbsolute, gateCase.P50NoiseAbsolute) + gateCase.MaterialityRatio = &effectiveRatio + gateCase.MaterialityAbsolute = &effectiveAbsolute + materialRatio := gateCase.MedianRatio.Upper <= effectiveRatio + materialAbsolute := gateCase.MedianSaving != nil && gateCase.MedianSaving.Lower >= effectiveAbsolute + if !materialRatio && !materialAbsolute { + gateCase.Passed = false + report.MaterialityPassed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("target improvement is not material: median ratio upper %.4f > %.4f and saving lower %s < %s", gateCase.MedianRatio.Upper, effectiveRatio, gateCase.MedianSaving.Lower, effectiveAbsolute)) + } + } + + if !gateCase.Passed { + report.Passed = false + } + if prioritizedTraversalKey(key, baseline, candidate) && gateCase.TimingGated { + report.QualificationRequired = true + family := traversalQualificationFamily(key, baseline, candidate) + status := qualification[family] + if status == nil { + status = &TraversalQualificationStatus{ + Family: family, + TrainingPassed: true, + HoldoutPassed: true, + } + qualification[family] = status + } + switch gateCase.QualificationSplit { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && gateCase.Passed + status.TrainingCases++ + status.TrainingPassed = status.TrainingPassed && gateCase.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && gateCase.Passed + status.HoldoutCases++ + status.HoldoutPassed = status.HoldoutPassed && gateCase.Passed + } + } + report.Cases = append(report.Cases, gateCase) + } + report.MaterialityTargets = len(resolvedMaterialityTargets) + if report.MaterialityRequired { + if len(targetNames) == 0 { + report.MaterialityPassed = false + } + if report.MaterialityTargets != len(targetNames) { + return PerfGateReport{}, fmt.Errorf("materiality targets resolved to %d timing-gated cases, expected %d", report.MaterialityTargets, len(targetNames)) + } + } + if report.QualificationRequired { + families := make([]string, 0, len(qualification)) + for family := range qualification { + families = append(families, family) + } + sort.Strings(families) + for _, family := range families { + status := qualification[family] + status.TrainingPassed = status.TrainingPassed && status.TrainingCases > 0 + status.HoldoutPassed = status.HoldoutPassed && status.HoldoutCases > 0 + status.Passed = status.TrainingPassed && status.HoldoutPassed + report.TrainingPassed = report.TrainingPassed && status.TrainingPassed + report.HoldoutPassed = report.HoldoutPassed && status.HoldoutPassed + report.QualificationFamilies = append(report.QualificationFamilies, *status) + } + report.QualificationPassed = report.TrainingPassed && report.HoldoutPassed + report.Passed = report.Passed && report.QualificationPassed + } else { + report.TrainingPassed = false + report.HoldoutPassed = false + } + if options.DiagnosticMode { + report.Passed = false + } + report.PromotionEligible = report.PromotionEligible && report.Passed && report.MaterialityPassed + + return report, nil +} + +// validatePerformanceWorkloadIdentity ensures matched artifacts describe identical logical workloads per case and backend. +func validatePerformanceWorkloadIdentity(baseline, candidate []CaseResult) error { + collect := func(label string, records []CaseResult) (map[performanceKey]string, error) { + identities := map[performanceKey]string{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL && record.ExecutionMode != ModeNeo4j { + continue + } + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + if record.WorkloadSHA256 == "" { + return nil, fmt.Errorf("%s artifact case %s/%s/%s has no workload identity", label, key.dataset, key.name, key.backend) + } + identityPayload := struct { + // WorkloadSHA256 binds the compared samples to one logical workload declaration. + WorkloadSHA256 string `json:"workload_sha256"` + // ManifestSHA256 identifies the anchor manifest that authorized the run. + ManifestSHA256 string `json:"manifest_sha256,omitempty"` + // ContentIdentity binds resumable work to the logical contents of the live graph. + ContentIdentity string `json:"content_identity,omitempty"` + // FixtureChecksum identifies the loaded fixture contents. + FixtureChecksum string `json:"fixture_checksum,omitempty"` + // FixtureConfiguration captures generator settings used to construct the loaded fixture. + FixtureConfiguration string `json:"fixture_configuration,omitempty"` + }{WorkloadSHA256: record.WorkloadSHA256} + if record.ExistingGraph != nil { + identityPayload.ManifestSHA256 = record.ExistingGraph.ManifestSHA256 + identityPayload.ContentIdentity = record.ExistingGraph.ContentIdentity + } + if record.Fixture != nil { + identityPayload.FixtureChecksum = record.Fixture.Checksum + identityPayload.FixtureConfiguration = record.Fixture.Configuration + } + raw, _ := json.Marshal(identityPayload) + digest := sha256.Sum256(raw) + identity := hex.EncodeToString(digest[:]) + if present, found := identities[key]; found && present != identity { + return nil, fmt.Errorf("%s artifact case %s/%s/%s mixes workload identities", label, key.dataset, key.name, key.backend) + } + identities[key] = identity + } + return identities, nil + } + + baselineIdentities, err := collect("baseline", baseline) + if err != nil { + return err + } + candidateIdentities, err := collect("candidate", candidate) + if err != nil { + return err + } + for key, baselineIdentity := range baselineIdentities { + if candidateIdentity, found := candidateIdentities[key]; found && candidateIdentity != baselineIdentity { + return fmt.Errorf("logical workload differs for %s/%s/%s", key.dataset, key.name, key.backend) + } + } + return nil +} + +// declaredPerformanceKeys returns the unique case/backend keys that the performance gate must evaluate. +func declaredPerformanceKeys(declared []DeclaredCaseBackend, baseline, candidate []CaseResult) []performanceKey { + unique := map[performanceKey]struct{}{} + for _, item := range declared { + if item.UnsupportedReason != "" { + continue + } + if item.Backend == ModePostgresSQL || item.Backend == ModeNeo4j { + unique[performanceKey{ + dataset: item.Dataset, + name: item.Name, + backend: item.Backend, + }] = struct{}{} + } + } + + if len(declared) == 0 { + for _, records := range [][]CaseResult{baseline, candidate} { + for _, record := range records { + if record.ExecutionMode == ModePostgresSQL || record.ExecutionMode == ModeNeo4j { + unique[performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + }] = struct{}{} + } + } + } + } + + keys := make([]performanceKey, 0, len(unique)) + for key := range unique { + keys = append(keys, key) + } + return keys +} + +// artifactCaseStatus returns the first non-OK status for a declared case/backend pair, "missing" when no record exists, or OK when every matching record succeeded. +func artifactCaseStatus(records []CaseResult, key performanceKey) string { + found := false + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + found = true + if record.Status != StatusOK { + return record.Status + } + } + if !found { + return "missing" + } + return StatusOK +} + +// declarationSHA256 sorts declared case/backend contracts and hashes their canonical JSON so compared artifacts must describe the same workload set. +func declarationSHA256(declared []DeclaredCaseBackend) string { + items := append([]DeclaredCaseBackend(nil), declared...) + sort.Slice(items, func(i, j int) bool { + if items[i].Dataset != items[j].Dataset { + return items[i].Dataset < items[j].Dataset + } + if items[i].Name != items[j].Name { + return items[i].Name < items[j].Name + } + if items[i].Backend != items[j].Backend { + return items[i].Backend < items[j].Backend + } + return items[i].UnsupportedReason < items[j].UnsupportedReason + }) + digest := sha256.New() + for _, item := range items { + fmt.Fprintf(digest, "%s\x00%s\x00%s\x00%s\n", item.Dataset, item.Name, item.Backend, item.UnsupportedReason) + } + + return hex.EncodeToString(digest.Sum(nil)) +} + +// collectWarmSeries groups positive warm durations by case, backend, and round. +func collectWarmSeries(records []CaseResult) map[performanceKey]roundSamples { + series := map[performanceKey]roundSamples{} + for _, record := range records { + if record.Status != StatusOK { + continue + } + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + + if series[key] == nil { + series[key] = roundSamples{} + } + series[key][sample.Round] = append(series[key][sample.Round], sample.Duration) + } + } + + return series +} + +// matchedRounds returns round numbers present in both measurement series. +func matchedRounds(baseline, candidate roundSamples) (roundSamples, roundSamples) { + matchedBaseline := roundSamples{} + matchedCandidate := roundSamples{} + for round, baselineSamples := range baseline { + candidateSamples, found := candidate[round] + if !found || len(baselineSamples) == 0 || len(candidateSamples) == 0 { + continue + } + + matchedBaseline[round] = baselineSamples + matchedCandidate[round] = candidateSamples + } + + return matchedBaseline, matchedCandidate +} + +// bootstrapRoundMedianRatio bootstraps the ratio between paired round medians. +func bootstrapRoundMedianRatio(baseline, candidate roundSamples, seed int64, options PerfGateOptions) RatioInterval { + rounds := sortedRounds(baseline) + baselineMedians := make([]float64, len(rounds)) + candidateMedians := make([]float64, len(rounds)) + for idx, round := range rounds { + baselineMedians[idx] = durationQuantile(baseline[round], 0.5) + candidateMedians[idx] = durationQuantile(candidate[round], 0.5) + } + estimate := quantile(candidateMedians, 0.5) / quantile(baselineMedians, 0.5) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + ratios := make([]float64, options.BootstrapCount) + resampledBaseline := make([]float64, len(rounds)) + resampledCandidate := make([]float64, len(rounds)) + for iteration := range ratios { + for idx := range rounds { + selected := rng.Intn(len(rounds)) + resampledBaseline[idx] = baselineMedians[selected] + resampledCandidate[idx] = candidateMedians[selected] + } + ratios[iteration] = quantile(resampledCandidate, 0.5) / quantile(resampledBaseline, 0.5) + } + return confidenceInterval(estimate, ratios, options.Confidence) +} + +// bootstrapRoundMedianSaving bootstraps the absolute duration saved between paired round medians. +func bootstrapRoundMedianSaving(baseline, candidate roundSamples, seed int64, options PerfGateOptions) DurationInterval { + rounds := sortedRounds(baseline) + baselineMedians := make([]float64, len(rounds)) + candidateMedians := make([]float64, len(rounds)) + for idx, round := range rounds { + baselineMedians[idx] = durationQuantile(baseline[round], 0.5) + candidateMedians[idx] = durationQuantile(candidate[round], 0.5) + } + estimate := quantile(baselineMedians, 0.5) - quantile(candidateMedians, 0.5) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + savings := make([]float64, options.BootstrapCount) + resampledBaseline := make([]float64, len(rounds)) + resampledCandidate := make([]float64, len(rounds)) + for iteration := range savings { + for idx := range rounds { + selected := rng.Intn(len(rounds)) + resampledBaseline[idx] = baselineMedians[selected] + resampledCandidate[idx] = candidateMedians[selected] + } + savings[iteration] = quantile(resampledBaseline, 0.5) - quantile(resampledCandidate, 0.5) + } + interval := confidenceInterval(estimate, savings, options.Confidence) + return DurationInterval{ + Estimate: time.Duration(interval.Estimate), + Lower: time.Duration(interval.Lower), + Upper: time.Duration(interval.Upper), + } +} + +// bootstrapStratifiedP95Ratio bootstraps a P95 ratio while preserving round strata. +func bootstrapStratifiedP95Ratio(baseline, candidate roundSamples, seed int64, options PerfGateOptions) RatioInterval { + rounds := sortedRounds(baseline) + estimate := durationQuantile(flattenSamples(candidate, rounds), 0.95) / durationQuantile(flattenSamples(baseline, rounds), 0.95) + rng := rand.New(rand.NewSource(seed)) // #nosec G404 -- deterministic statistical resampling + ratios := make([]float64, options.BootstrapCount) + for iteration := range ratios { + var resampledBaseline, resampledCandidate []time.Duration + for _, round := range rounds { + resampledBaseline = append(resampledBaseline, resampleDurations(rng, baseline[round])...) + resampledCandidate = append(resampledCandidate, resampleDurations(rng, candidate[round])...) + } + ratios[iteration] = durationQuantile(resampledCandidate, 0.95) / durationQuantile(resampledBaseline, 0.95) + } + return confidenceInterval(estimate, ratios, options.Confidence) +} + +// confidenceInterval returns the requested central interval from sorted bootstrap estimates. +func confidenceInterval(estimate float64, samples []float64, confidence float64) RatioInterval { + alpha := (1 - confidence) / 2 + return RatioInterval{ + Estimate: estimate, + Lower: quantile(samples, alpha), + Upper: quantile(samples, 1-alpha), + } +} + +// durationQuantile returns a nearest-rank duration quantile from a copy of the samples. +func durationQuantile(values []time.Duration, probability float64) float64 { + numeric := make([]float64, len(values)) + for idx, value := range values { + numeric[idx] = float64(value) + } + return quantile(numeric, probability) +} + +// quantile returns a nearest-rank quantile from sorted floating-point samples. +func quantile(values []float64, probability float64) float64 { + ordered := append([]float64(nil), values...) + sort.Float64s(ordered) + if len(ordered) == 0 { + return math.NaN() + } + index := int(math.Ceil(probability*float64(len(ordered)))) - 1 + if index < 0 { + index = 0 + } + if index >= len(ordered) { + index = len(ordered) - 1 + } + return ordered[index] +} + +// sortedRounds returns measurement round keys in ascending order. +func sortedRounds(samples roundSamples) []int { + rounds := make([]int, 0, len(samples)) + for round := range samples { + rounds = append(rounds, round) + } + sort.Ints(rounds) + return rounds +} + +// flattenSamples concatenates samples from the requested rounds in the supplied round order. +func flattenSamples(samples roundSamples, rounds []int) []time.Duration { + var flattened []time.Duration + for _, round := range rounds { + flattened = append(flattened, samples[round]...) + } + return flattened +} + +// resampleDurations draws a same-size bootstrap sample of durations with replacement. +func resampleDurations(rng *rand.Rand, values []time.Duration) []time.Duration { + resampled := make([]time.Duration, len(values)) + for idx := range resampled { + resampled[idx] = values[rng.Intn(len(values))] + } + return resampled +} + +// sampleCount returns the total number of durations across all measurement rounds. +func sampleCount(samples roundSamples) int { + count := 0 + for _, values := range samples { + count += len(values) + } + return count +} + +// fileSHA256 returns the SHA-256 digest of a file's contents. +func fileSHA256(path string) (string, error) { + content, err := os.ReadFile(path) + if err != nil { + return "", err + } + digest := sha256.Sum256(content) + return hex.EncodeToString(digest[:]), nil +} + +// writePerfGateReport writes a performance-gate report to stdout or the requested file. +func writePerfGateReport(path string, report PerfGateReport) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/perf_gate_test.go b/cmd/graphbench/perf_gate_test.go new file mode 100644 index 00000000..9f930963 --- /dev/null +++ b/cmd/graphbench/perf_gate_test.go @@ -0,0 +1,774 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestBuildPerfGateReportTreatsNeo4jAsCorrectnessOracle verifies that PostgreSQL receives latency ratios while Neo4j contributes correctness observations without performance gating. +func TestBuildPerfGateReportTreatsNeo4jAsCorrectnessOracle(t *testing.T) { + baseline := []CaseResult{ + perfGateRecord("one_shortest_path_bound_pair", ModePostgresSQL, 10*time.Millisecond, 5, 30), + perfGateRecord("one_shortest_path_bound_pair", ModeNeo4j, 3*time.Millisecond, 5, 30), + } + candidate := []CaseResult{ + perfGateRecord("one_shortest_path_bound_pair", ModePostgresSQL, 3*time.Millisecond, 5, 30), + perfGateRecord("one_shortest_path_bound_pair", ModeNeo4j, 2*time.Millisecond, 5, 30), + } + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 42, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 250, + })) + + require.NoError(t, err) + require.True(t, report.Passed) + require.Len(t, report.Cases, 2) + postgres := findPerfGateCase(t, report.Cases, ModePostgresSQL) + require.InDelta(t, 0.3, postgres.MedianRatio.Estimate, 0.0001) + require.NotNil(t, postgres.P95Ratio) + neo4j := findPerfGateCase(t, report.Cases, ModeNeo4j) + require.True(t, neo4j.OracleOnly) + require.Nil(t, neo4j.P95Ratio) +} + +// TestBuildPerfGateReportFailsMissingDeclaredPostgresCase verifies that every declared PostgreSQL workload must have a candidate record and that the declaration set is fingerprinted. +func TestBuildPerfGateReportFailsMissingDeclaredPostgresCase(t *testing.T) { + baseline := []CaseResult{perfGateRecord("present", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("present", ModePostgresSQL, time.Millisecond, 5, 30)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + DeclaredBackends: []DeclaredCaseBackend{ + { + Dataset: "fixture", + Name: "present", + Backend: ModePostgresSQL, + }, + { + Dataset: "fixture", + Name: "missing", + Backend: ModePostgresSQL, + }, + }, + })) + + require.NoError(t, err) + require.False(t, report.Passed) + require.NotEmpty(t, report.DeclarationSHA256) + var missing PerfGateCase + for _, gateCase := range report.Cases { + if gateCase.Name == "missing" { + missing = gateCase + } + } + require.Equal(t, "missing", missing.CandidateStatus) + require.ErrorContains(t, reasonsError(missing.Reasons), "required candidate record status is missing") +} + +// TestBuildPerfGateReportAppliesMaterialityOnlyToDeclaredTargets verifies that a named target passes only when the confidence-bound saving clears both ratio and absolute thresholds. +func TestBuildPerfGateReportAppliesMaterialityOnlyToDeclaredTargets(t *testing.T) { + baseline := []CaseResult{perfGateRecord("target", ModePostgresSQL, 10*time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("target", ModePostgresSQL, 9_700*time.Microsecond, 5, 30)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + TargetNames: []string{"target"}, + MaterialityRatio: 0.95, + MaterialityAbsolute: 100 * time.Microsecond, + })) + + require.NoError(t, err) + require.True(t, report.Passed, "%v", report.Cases[0].Reasons) + require.NotNil(t, report.Cases[0].MedianSaving) + require.Equal(t, 300*time.Microsecond, report.Cases[0].MedianSaving.Lower) +} + +// TestBuildPerfGateReportFailsRegressionAndInsufficientP95 verifies that an excessive median slowdown and fewer than 150 warm samples independently fail a PostgreSQL gate case. +func TestBuildPerfGateReportFailsRegressionAndInsufficientP95(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 10*time.Millisecond, 5, 10)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 13*time.Millisecond, 5, 10)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 7, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + })) + + require.NoError(t, err) + require.False(t, report.Passed) + require.Len(t, report.Cases, 1) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "median regression") + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "at least 150 warm samples") +} + +// TestBuildPerfGateReportRequiresMatchedRounds verifies that four baseline/candidate rounds are insufficient for an inferential gate even with ample samples. +func TestBuildPerfGateReportRequiresMatchedRounds(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 10*time.Millisecond, 4, 40)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 9*time.Millisecond, 4, 40)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + })) + + require.NoError(t, err) + require.False(t, report.Passed) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "at least 5 matched rounds") +} + +// TestBuildPerfGateReportRequiresHostAAEvidence verifies that a non-diagnostic promotion cannot substitute fixed defaults for a checksummed host calibration. +func TestBuildPerfGateReportRequiresHostAAEvidence(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + stampPairedEvidence(baseline, candidate, minimumDiscoveryWarmups) + + _, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + }) + + require.ErrorContains(t, err, "checksummed host A/A report") +} + +// TestBuildPerfGateReportRequiresMaterialityTargetForPromotion verifies a +// containment-only comparison can pass without authorizing a no-win rollout. +func TestBuildPerfGateReportRequiresMaterialityTargetForPromotion(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + })) + + require.NoError(t, err) + require.True(t, report.Passed) + require.True(t, report.MaterialityRequired) + require.False(t, report.MaterialityPassed) + require.False(t, report.PromotionEligible) +} + +// TestBuildPerfGateReportRejectsMismatchedAAHost verifies a syntactically valid calibration from another host cannot qualify production timing. +func TestBuildPerfGateReportRejectsMismatchedAAHost(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + options := qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + }) + options.AAReport.HostFingerprint = strings.Repeat("c", 64) + + _, err := buildPerfGateReport(baseline, candidate, options) + + require.ErrorContains(t, err, "host fingerprint does not match") +} + +// TestBuildPerfGateReportUsesP95AbsoluteFloor verifies a relative regression below 100us remains inside the mandatory fast-case floor while preserving the absolute interval in the report. +func TestBuildPerfGateReportUsesP95AbsoluteFloor(t *testing.T) { + baseline := []CaseResult{perfGateRecord("fast", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("fast", ModePostgresSQL, 1060*time.Microsecond, 5, 30)} + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 100, + })) + + require.NoError(t, err) + require.True(t, report.Passed, "%v", report.Cases[0].Reasons) + require.Equal(t, minimumTimingNoiseAbsolute, report.Cases[0].P95NoiseAbsolute) + require.Equal(t, 60*time.Microsecond, report.Cases[0].P95Change.Lower) +} + +// TestBuildPerfGateReportRejectsUnbalancedPromotionEvidence verifies matched rounds with one fixed arm order cannot support promotion. +func TestBuildPerfGateReportRejectsUnbalancedPromotionEvidence(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 900*time.Microsecond, 5, 30)} + options := qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + }) + for idx := range baseline[0].Stats.Samples { + baseline[0].Stats.Samples[idx].ArmOrder = 1 + candidate[0].Stats.Samples[idx].ArmOrder = 2 + } + + _, err := buildPerfGateReport(baseline, candidate, options) + + require.ErrorContains(t, err, "arm order is not balanced") +} + +// TestBuildPerfGateReportKeepsStressTimingDiagnostic verifies stress latency cannot fail production timing gates even without A/A or paired-order evidence. +func TestBuildPerfGateReportKeepsStressTimingDiagnostic(t *testing.T) { + baseline := []CaseResult{perfGateRecord("stress", ModePostgresSQL, time.Millisecond, 1, 1)} + candidate := []CaseResult{perfGateRecord("stress", ModePostgresSQL, 10*time.Millisecond, 1, 1)} + baseline[0].Shape.FixtureTier = "stress" + candidate[0].Shape.FixtureTier = "stress" + + report, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.False(t, report.Cases[0].TimingGated) + require.Contains(t, report.Cases[0].Reasons, "stress tier timing is diagnostic") +} + +// TestBuildPerfGateReportKeepsDiagnosticSplitOutOfPromotion verifies a normal +// fixture explicitly reserved for boundary diagnostics needs no A/A evidence +// and cannot make the report promotion eligible. +func TestBuildPerfGateReportKeepsDiagnosticSplitOutOfPromotion(t *testing.T) { + baseline := []CaseResult{perfGateRecord("boundary", ModePostgresSQL, time.Millisecond, 1, 1)} + candidate := []CaseResult{perfGateRecord("boundary", ModePostgresSQL, 10*time.Millisecond, 1, 1)} + baseline[0].Shape.QualificationSplit = "diagnostic" + candidate[0].Shape.QualificationSplit = "diagnostic" + + report, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 10, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.False(t, report.PromotionEligible) + require.False(t, report.Cases[0].TimingGated) + require.Contains(t, report.Cases[0].Reasons, "diagnostic qualification split is excluded from promotion timing") +} + +// TestBuildPerfGateReportRejectsChangedLogicalWorkload verifies that baseline and candidate records with different workload digests cannot be compared. +func TestBuildPerfGateReportRejectsChangedLogicalWorkload(t *testing.T) { + baseline := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 10*time.Millisecond, 5, 30)} + candidate := []CaseResult{perfGateRecord("ordinary_case", ModePostgresSQL, 9*time.Millisecond, 5, 30)} + candidate[0].WorkloadSHA256 = "changed-workload" + + _, err := buildPerfGateReport(baseline, candidate, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 100, + }) + require.ErrorContains(t, err, "logical workload differs") +} + +// TestUnsupportedDeclarationAffectsChecksumWithoutRequiringARecord verifies that an explicitly unsupported backend needs no measurement but its reason remains part of declaration identity. +func TestUnsupportedDeclarationAffectsChecksumWithoutRequiringARecord(t *testing.T) { + declared := []DeclaredCaseBackend{ + { + Dataset: "fixture", + Name: "directionless", + Backend: ModeNeo4j, + }, + { + Dataset: "fixture", + Name: "directionless", + Backend: ModePostgresSQL, + UnsupportedReason: "unsupported form", + }, + } + records := []CaseResult{perfGateRecord("directionless", ModeNeo4j, time.Millisecond, 1, 1)} + + report, err := buildPerfGateReport(records, records, PerfGateOptions{ + Seed: 1, + Confidence: 0.95, + RegressionThreshold: 0.20, + BootstrapCount: 10, + DeclaredBackends: declared, + }) + require.NoError(t, err) + require.True(t, report.Passed) + require.Len(t, report.Cases, 1) + + changed := append([]DeclaredCaseBackend(nil), declared...) + changed[1].UnsupportedReason = "different reason" + require.NotEqual(t, declarationSHA256(declared), declarationSHA256(changed)) +} + +// TestValidatePerformanceArtifactSelectionsRefusesDiagnosticsFromCompleteGate verifies that subset artifacts require an explicit diagnostic override and still must share the same declaration digest. +func TestValidatePerformanceArtifactSelectionsRefusesDiagnosticsFromCompleteGate(t *testing.T) { + manifest := &SelectionManifest{ + Version: selectionManifestVersion, + DiagnosticOnly: true, + FullDeclarationCount: 1, + SelectedDeclarationCount: 1, + DeclarationSHA256: strings.Repeat("a", 64), + } + left := []CaseResult{{ + Dataset: "fixture", + Name: "case", + Environment: &RunEnvironment{ + Selection: manifest, + }, + }} + right := []CaseResult{{ + Dataset: "fixture", + Name: "case", + Environment: &RunEnvironment{ + Selection: manifest, + }, + }} + + require.ErrorContains(t, validatePerformanceArtifactSelections(left, right, false), "refused") + require.NoError(t, validatePerformanceArtifactSelections(left, right, true)) + right[0].Environment.Selection = &SelectionManifest{ + Version: selectionManifestVersion, + DiagnosticOnly: true, + FullDeclarationCount: 1, + SelectedDeclarationCount: 1, + DeclarationSHA256: strings.Repeat("b", 64), + } + require.ErrorContains(t, validatePerformanceArtifactSelections(left, right, true), "declarations differ") +} + +// perfGateRecord returns one successful workload observation with identical warm samples arranged into the requested rounds. +func perfGateRecord(name string, mode ExecutionMode, duration time.Duration, rounds, samplesPerRound int) CaseResult { + record := CaseResult{ + Dataset: "fixture", + Name: name, + WorkloadSHA256: fmt.Sprintf("workload:%s:%s", name, mode), + ExecutionMode: mode, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + Environment: &RunEnvironment{ + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + WarmupIterations: minimumDiscoveryWarmups, + }, + } + record.Stats.WarmupIterations = minimumDiscoveryWarmups + for round := 1; round <= rounds; round++ { + for iteration := 1; iteration <= samplesPerRound; iteration++ { + record.Stats.Samples = append(record.Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: duration, + }) + } + } + return record +} + +// qualifiedPerfGateOptions stamps balanced pairing metadata and supplies host-matched A/A evidence. +func qualifiedPerfGateOptions(t *testing.T, baseline, candidate []CaseResult, options PerfGateOptions) PerfGateOptions { + t.Helper() + stampPairedEvidence(baseline, candidate, minimumDiscoveryWarmups) + options.AAReport = testAAReportForRecords(t, baseline) + options.AAReportSHA256 = strings.Repeat("b", 64) + return options +} + +// testAAReportForRecords prepares or inspects test evidence for test aa report for records. +func testAAReportForRecords(t *testing.T, records []CaseResult) *AAResolutionReport { + t.Helper() + hostFingerprint, err := artifactHostFingerprint(records) + require.NoError(t, err) + + keys := map[performanceKey]struct{}{} + for _, record := range records { + if record.ExecutionMode == ModePostgresSQL && hasWarmLatencySample(record) { + keys[performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + }] = struct{}{} + } + } + aa := &AAResolutionReport{ + Version: aaReportVersion, + Confidence: defaultConfidenceLevel, + ArtifactSHA256: strings.Repeat("a", 64), + HostFingerprint: hostFingerprint, + MinimumRounds: minimumGateRounds, + MinimumSamplesPerArmPerRound: 10, + OrderBalanced: true, + PhysicalChronology: &AAPhysicalChronology{ + Version: aaPhysicalChronologyVersion, + Validated: true, + ArtifactSHA256: strings.Repeat("a", 64), + Rounds: minimumGateRounds, + Arms: []string{"aa-a", "aa-b"}, + }, + } + for _, key := range sortedPerformanceKeys(keys) { + workloadSHA256, err := workloadSHA256ForKey(records, key) + require.NoError(t, err) + postgresEnvironmentSHA256, err := postgresTimingEnvironmentSHA256ForKey(records, key) + require.NoError(t, err) + fixtureSHA256, err := fixtureSHA256ForKey(records, key) + require.NoError(t, err) + aa.Cases = append(aa.Cases, AAResolutionCase{ + Dataset: key.dataset, + Name: key.name, + Backend: key.backend, + WorkloadSHA256: workloadSHA256, + PostgresEnvironmentSHA256: postgresEnvironmentSHA256, + FixtureSHA256: fixtureSHA256, + Rounds: minimumGateRounds, + SamplesPerArm: minimumGateRounds * 10, + P50: testAAMetricResolution(), + P95: testAAMetricResolution(), + }) + } + return aa +} + +// testAAMetricResolution prepares or inspects test evidence for test aa metric resolution. +func testAAMetricResolution() AAMetricResolution { + return AAMetricResolution{ + Ratio: RatioInterval{ + Estimate: 1, + Lower: 0.99, + Upper: 1.01, + }, + RatioResolution: 0.01, + AbsoluteChange: DurationInterval{ + Estimate: 0, + Lower: -10 * time.Microsecond, + Upper: 10 * time.Microsecond, + }, + AbsoluteResolution: 10 * time.Microsecond, + } +} + +// stampPairedEvidence prepares or inspects test evidence for stamp paired evidence. +func stampPairedEvidence(left, right []CaseResult, warmups int) { + stamp := func(records []CaseResult, arm string, leftArm bool) { + for recordIdx := range records { + record := &records[recordIdx] + record.Stats.WarmupIterations = warmups + if record.Environment == nil { + record.Environment = &RunEnvironment{} + } + record.Environment.WarmupIterations = warmups + record.Environment.Arm = arm + for sampleIdx := range record.Stats.Samples { + sample := &record.Stats.Samples[sampleIdx] + leftFirst := sample.Round%2 == 1 + order := 2 + if leftArm == leftFirst { + order = 1 + } + sample.Block = sample.Round + sample.Arm = arm + sample.ArmOrder = order + sample.RunUUID = fmt.Sprintf("pair-%s-%d", record.Name, sample.Round) + } + } + } + stamp(left, "baseline", true) + stamp(right, "candidate", false) +} + +// findPerfGateCase returns the report entry for a backend or fails the calling test when the gate omitted it. +func findPerfGateCase(t *testing.T, cases []PerfGateCase, mode ExecutionMode) PerfGateCase { + t.Helper() + for _, gateCase := range cases { + if gateCase.Backend == mode { + return gateCase + } + } + t.Fatalf("missing %s gate case", mode) + return PerfGateCase{} +} + +// reasonsError joins gate-failure reasons into one diagnostic error. +func reasonsError(reasons []string) error { + return fmt.Errorf("%s", strings.Join(reasons, "; ")) +} + +// TestQualificationSplitFailsClosedOnMissingOrDriftingTraversalPartitions +// verifies benchmark artifacts cannot silently reclassify selector training as +// frozen holdout evidence. +func TestQualificationSplitFailsClosedOnMissingOrDriftingTraversalPartitions(t *testing.T) { + key := performanceKey{ + dataset: "fixture", + name: "sp", + backend: ModePostgresSQL, + } + left := []CaseResult{{ + Dataset: "fixture", + Name: "sp", + Category: "generated_shortest_path_v2", + ExecutionMode: ModePostgresSQL, + }} + _, err := qualificationSplit(key, left) + require.ErrorContains(t, err, "no frozen qualification split") + + left[0].Shape.QualificationSplit = "training" + right := append([]CaseResult(nil), left...) + right[0].Shape.QualificationSplit = "holdout" + _, err = qualificationSplit(key, left, right) + require.ErrorContains(t, err, "changes qualification split") + + right[0].Shape.QualificationSplit = "training" + split, err := qualificationSplit(key, left, right) + require.NoError(t, err) + require.Equal(t, "training", split) +} + +// TestQualificationSplitRecognizesCompatibleFixedSuffixV2Categories verifies +// the v2 dataset cannot bypass partition enforcement through its intentionally +// backwards-compatible category name. +func TestQualificationSplitRecognizesCompatibleFixedSuffixV2Categories(t *testing.T) { + key := performanceKey{ + dataset: "generated_fixed_suffix_expansion_v2_d8_f16", + name: "GFSE-V2-D08-F016", + backend: ModePostgresSQL, + } + records := []CaseResult{{ + Dataset: key.dataset, + Name: key.name, + Category: "generated_fixed_suffix_expansion", + ExecutionMode: key.backend, + }} + + _, err := qualificationSplit(key, records) + require.ErrorContains(t, err, "no frozen qualification split") +} + +// TestTraversalQualificationFamilyRecognizesFixedSuffixV3WithoutTelemetry verifies traversal qualification family recognizes fixed suffix v3 without telemetry behavior. +func TestTraversalQualificationFamilyRecognizesFixedSuffixV3WithoutTelemetry(t *testing.T) { + key := performanceKey{ + dataset: "generated_fixed_suffix_expansion_v3_d8_f16", + name: "GFSE-V3-D08-F016", + backend: ModePostgresSQL, + } + records := []CaseResult{{ + Dataset: key.dataset, + Name: key.name, + Category: "generated_fixed_suffix_expansion", + ExecutionMode: key.backend, + }} + + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), traversalQualificationFamily(key, records)) +} + +// TestTraversalQualificationUsesOrientationPolicyBeforeRequestedArm verifies traversal qualification uses orientation policy before requested arm behavior. +func TestTraversalQualificationUsesOrientationPolicyBeforeRequestedArm(t *testing.T) { + key := performanceKey{ + dataset: "generated_fixed_suffix_expansion_v3_d8_f16", + name: "GFSE-V3-D08-F016", + backend: ModePostgresSQL, + } + record := CaseResult{ + Dataset: key.dataset, + Name: key.name, + Category: "generated_fixed_suffix_expansion", + ExecutionMode: key.backend, + TraversalTelemetry: &TraversalExecutionTelemetry{Summary: TraversalExecutionSummary{ + RequestedIdentity: string(optimize.ExpansionSearchSuffixSeededReverse), + EmittedIdentity: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + RuntimeBranch: "suffix_seeded_reverse", + }}, + } + + require.True(t, requiresCandidateRuntimeEvidence(record)) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), traversalQualificationFamily(key, []CaseResult{record})) +} + +// TestBuildPerfGateReportRequiresIndependentTraversalHoldout verifies a +// complete release gate cannot be assembled from selector-training topology +// alone even when every measured case passes. +func TestBuildPerfGateReportRequiresIndependentTraversalHoldout(t *testing.T) { + baseline := []CaseResult{ + perfGateRecord("sp-training", ModePostgresSQL, 10*time.Millisecond, minimumGateRounds, 30), + perfGateRecord("sp-holdout", ModePostgresSQL, 10*time.Millisecond, minimumGateRounds, 30), + } + candidate := []CaseResult{ + perfGateRecord("sp-training", ModePostgresSQL, 5*time.Millisecond, minimumGateRounds, 30), + perfGateRecord("sp-holdout", ModePostgresSQL, 5*time.Millisecond, minimumGateRounds, 30), + } + for _, records := range [][]CaseResult{baseline, candidate} { + records[0].Category = "generated_shortest_path_v2" + records[0].Shape.QualificationSplit = "training" + records[1].Category = "generated_shortest_path_v2" + records[1].Shape.QualificationSplit = "holdout" + } + + report, err := buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + TargetNames: []string{"sp-training", "sp-holdout"}, + })) + require.NoError(t, err) + require.True(t, report.QualificationRequired) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.True(t, report.Passed) + require.True(t, report.PromotionEligible) + require.Equal(t, []TraversalQualificationStatus{{ + Family: "SP", + TrainingCases: 1, + HoldoutCases: 1, + TrainingPassed: true, + HoldoutPassed: true, + Passed: true, + }}, report.QualificationFamilies) + + // A passing ASP holdout may not qualify an SP candidate's training data. + baseline[1].Cypher = "RETURN allShortestPaths((a)-[:E*1..3]->(b))" + candidate[1].Cypher = baseline[1].Cypher + report, err = buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + TargetNames: []string{"sp-training", "sp-holdout"}, + })) + require.NoError(t, err) + require.False(t, report.QualificationPassed) + require.False(t, report.Passed) + require.False(t, report.PromotionEligible) + baseline[1].Cypher = "" + candidate[1].Cypher = "" + + for idx := range baseline { + baseline[idx].Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{ + { + TargetKind: "traversal", + Family: "SP", + Applied: "SP-S4-C-D", + Selected: "SP-S4-C-D", + }, + { + TargetKind: "endpoint_resolution", + Family: "endpoint_resolution", + TraversalFamily: "SP", + Applied: "ENDPOINT-RESOLUTION-INCUMBENT", + }, + }} + candidate[idx].Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{ + { + TargetKind: "traversal", + Family: "SP", + Applied: "SP-B1-C-ALT-NODE-D", + Selected: "SP-B1-C-ALT-NODE-D", + }, + { + TargetKind: "endpoint_resolution", + Family: "endpoint_resolution", + TraversalFamily: "SP", + Applied: "ENDPOINT-RESOLUTION-INCUMBENT", + }, + }} + fallback := false + available := true + candidate[idx].TraversalTelemetry = &TraversalExecutionTelemetry{Summary: TraversalExecutionSummary{ + RequestedIdentity: "SP-B1-C-ALT-NODE-D", + RuntimeIdentity: "SP-B1-C-ALT-NODE-D", + RuntimeBranch: "bidirectional_search", + RuntimeOutcomeAvailable: &available, + FallbackExecuted: &fallback, + }} + setSampleTraversalRuntimeMetadata(&candidate[idx].Stats, candidate[idx].TraversalTelemetry) + for sampleIdx := range candidate[idx].Stats.Samples { + candidate[idx].Stats.Samples[sampleIdx].RuntimeAttestation = "timed_invocation" + candidate[idx].Stats.Samples[sampleIdx].RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + Ordinal: 1, + RuntimeIdentity: "SP-B1-C-ALT-NODE-D", + RuntimeBranch: "bidirectional_search", + FallbackExecuted: false, + }} + } + } + report, err = buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + TargetNames: []string{"sp-training", "sp-holdout"}, + })) + require.NoError(t, err) + require.True(t, report.QualificationPassed) + require.Equal(t, "SP-B1-C-ALT-NODE-D@bidirectional_search", report.QualificationFamilies[0].Family) + candidate[0].Stats.Samples[0].RuntimeAttestation = "same_case_invocation_local_replay" + require.ErrorContains(t, validateCandidateRuntimeEvidence(candidate, performanceKey{ + dataset: candidate[0].Dataset, + name: candidate[0].Name, + backend: candidate[0].ExecutionMode, + }), "runtime attribution") + candidate[0].Stats.Samples[0].RuntimeAttestation = "timed_invocation" + for idx := range baseline { + baseline[idx].Optimization = nil + candidate[idx].Optimization = nil + } + + baseline = baseline[:1] + candidate = candidate[:1] + report, err = buildPerfGateReport(baseline, candidate, qualifiedPerfGateOptions(t, baseline, candidate, PerfGateOptions{ + Confidence: defaultConfidenceLevel, + BootstrapCount: 50, + TargetNames: []string{"sp-training"}, + })) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.False(t, report.Passed) + require.False(t, report.PromotionEligible) +} + +// TestValidateRuntimeReceiptEventsPreservesNestedFallbackChain verifies validate runtime receipt events preserves nested fallback chain behavior. +func TestValidateRuntimeReceiptEventsPreservesNestedFallbackChain(t *testing.T) { + fallback := true + events := []RuntimeReceiptEvent{ + { + Ordinal: 1, + RuntimeIdentity: "SP-I1-C-WE+MAT-M0", + RuntimeBranch: "candidate_overflow", + FallbackExecuted: true, + }, + { + Ordinal: 2, + RuntimeIdentity: "SP-S4-C-WE+MAT-M0", + RuntimeBranch: "workspace_overflow", + FallbackExecuted: true, + }, + { + Ordinal: 3, + RuntimeIdentity: "SP-S3-U-E+MAT-M0", + RuntimeBranch: "exact_fallback", + FallbackExecuted: true, + }, + } + require.NoError(t, validateRuntimeReceiptEvents(events, "SP-S3-U-E+MAT-M0", "exact_fallback", &fallback)) + + events[1].Ordinal = 3 + require.ErrorContains(t, validateRuntimeReceiptEvents(events, "SP-S3-U-E+MAT-M0", "exact_fallback", &fallback), "not contiguous") +} diff --git a/cmd/graphbench/postgres.go b/cmd/graphbench/postgres.go index 355b6bc3..72e62830 100644 --- a/cmd/graphbench/postgres.go +++ b/cmd/graphbench/postgres.go @@ -18,14 +18,23 @@ package main import ( "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" "fmt" + "os" "regexp" + "slices" "strconv" "strings" + "time" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/specterops/dawgs" "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" @@ -33,19 +42,257 @@ import ( "github.com/specterops/dawgs/util/size" ) +// postgresSQLRunner owns PostgreSQL translation, connection, graph, and executor settings. type postgresSQLRunner struct { + // datasetDir locates fixture and corpus files on disk. datasetDir string - db graph.Database - pgDriver *pg.Driver - graphID int32 + // db provides graph transactions for fixture preparation and query execution. + db graph.Database + // pgDriver provides PostgreSQL graph access and kind mapping. + pgDriver *pg.Driver + // pool supplies PostgreSQL connections for translated and raw execution. + pool *pgxpool.Pool + // graphID selects the PostgreSQL graph partition used for translation, fixture validation, and execution. + graphID int32 + // backendPID identifies the backend pid. + backendPID string + // poolSize retains the pool size while postgresSQLRunner is assembled or evaluated. + poolSize int + // round identifies the measurement round used to balance execution order. + round int + // concurrency lists worker counts measured by the PostgreSQL runner. + concurrency []int + // environment accumulates PostgreSQL environment evidence for the current runner. + environment PostgresEnvironment + // references enables independent PostgreSQL reference execution for the runner. + references bool + // referenceArms lists independent PostgreSQL reference arms measured by the runner. + referenceArms []string + // toolOptions carries forced translation-executor selections for diagnostic runs. + toolOptions translate.ToolOptions + // productionManifest supplies the immutable guarded candidate identity used + // for pre-closure production-boundary measurement. + productionManifest *PromotionManifest + // repeatableRead measures an incumbent or tool arm under an explicit stable + // snapshot for comparison with an admission-equivalent production candidate. + repeatableRead bool + // traversalTelemetry selects opt-in summary or untimed diagnostic traversal evidence. + traversalTelemetry string + // suffixRouteComponentClosure records the measurement-only boundary closure + // required by the fixed-suffix routing preflight. It does not influence SQL + // selection or execution. + suffixRouteComponentClosure bool + // sessionMemoryCeilingBytes bounds workspace observed by a closure on one + // physical PostgreSQL session. + sessionMemoryCeilingBytes int64 + // poolMemoryCeilingBytes bounds workspace observed by all sessions in a + // closure's PostgreSQL pool. + poolMemoryCeilingBytes int64 + // existingGraph supplies live-graph anchors, checkpoints, and callbacks to the runner. + existingGraph *existingGraphRunnerOptions } -func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus) (*postgresSQLRunner, error) { +// existingGraphRunnerOptions supplies live-graph anchors and completed-workload state to the PostgreSQL runner. +type existingGraphRunnerOptions struct { + // Manifest supplies validated live-graph anchors and identity metadata to the runner. + Manifest ExistingGraphAnchorManifest + // ProgressPath selects the append-only progress artifact written by the runner. + ProgressPath string + // Discovery enables adaptive live-graph discovery instead of the fixed confirmation protocol. + Discovery bool + // TimeoutClasses lists the increasing per-attempt deadlines applied during adaptive discovery. + TimeoutClasses []time.Duration + // SampleFloor sets the minimum timed samples required for each live-graph attempt. + SampleFloor int + // Completed maps completed live-graph case keys to fixture-bound identities. + Completed map[string]string + // OnRecord receives each completed live-graph CaseResult for immediate persistence. + OnRecord func(CaseResult) error + // OnComplete records final live-graph node and relationship counts after successful execution. + OnComplete func(int64, int64) error +} + +// setProductionManifest loads a provisional promotion manifest. Evidence may +// be empty because this mode exists to produce that evidence; all fields that +// determine SQL selection and runtime behavior are still validated here. +func (s *postgresSQLRunner) setProductionManifest(path string) error { + if path == "" { + return nil + } + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var manifest PromotionManifest + if err := decodePromotionEvidence(raw, &manifest); err != nil { + return fmt.Errorf("decode provisional promotion manifest: %w", err) + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) || manifest.SelectorVersion == optimize.ShortestPathSelectorStaticV8HiddenFanIn { + return fmt.Errorf("SP-I2 V1 selector-v8 production activation is terminally rejected") + } + if manifest.Version != promotionManifestVersion || manifest.ExecutionBoundary != "guarded_dual_arm" || strings.TrimSpace(manifest.SelectorVersion) == "" { + return fmt.Errorf("provisional manifest must be version 2 with a selector and guarded_dual_arm boundary") + } + expectedFallback := map[string]string{ + string(optimize.ShortestPathExecutorASPI1DAG): string(optimize.ShortestPathExecutorASPA1DAG), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI2GuardedDistance): string(optimize.ShortestPathExecutorS4CanonicalDistance), + string(optimize.ExpansionSearchPolicyOrientationProbeV1): string(optimize.ExpansionSearchStepwiseForward), + string(optimize.ExpansionSearchPolicyOrientationProbeV2): string(optimize.ExpansionSearchStepwiseForward), + }[manifest.Candidate] + if expectedFallback == "" || manifest.FallbackExecutor != expectedFallback { + return fmt.Errorf("unsupported candidate/fallback pair %s -> %s", manifest.Candidate, manifest.FallbackExecutor) + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) && manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + return fmt.Errorf("canonical SP-I1 provisional manifest requires selector %q", optimize.ShortestPathSelectorStaticV6) + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) && manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV8HiddenFanIn { + return fmt.Errorf("SP-I2 distance provisional manifest requires selector %q", optimize.ShortestPathSelectorStaticV8HiddenFanIn) + } + if isOrientationProbePolicy(manifest.Candidate) { + if manifest.SelectorVersion != manifest.Candidate { + return fmt.Errorf("orientation candidate %q requires the same selector version", manifest.Candidate) + } + expectedCaps := orientationPromotionCaps() + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("%s requires exactly four immutable caps", manifest.Candidate) + } + for name, expected := range expectedCaps { + if manifest.Caps[name] != expected { + return fmt.Errorf("%s cap %s must equal %d", manifest.Candidate, name, expected) + } + } + } else if manifest.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) { + expectedCaps := spI2PromotionCaps() + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("SP-I2 distance candidate requires exactly state and frontier caps") + } + for name, expected := range expectedCaps { + if actual, found := manifest.Caps[name]; !found || actual != expected { + return fmt.Errorf("SP-I2 distance candidate cap %s must equal %d", name, expected) + } + } + } else { + expectedCaps := []string{"state_limit", "predecessor_limit", "enumeration_limit", "output_bytes_limit"} + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("guarded shortest candidate requires exactly four immutable caps") + } + for _, name := range expectedCaps { + if manifest.Caps[name] <= 0 { + return fmt.Errorf("guarded shortest candidate cap %s must be positive", name) + } + } + } + seenQueries := map[string]struct{}{} + seenBuckets := map[string]struct{}{} + for _, bucket := range manifest.Buckets { + if strings.TrimSpace(bucket.Name) == "" || len(bucket.QuerySHA256) == 0 { + return fmt.Errorf("every provisional production bucket requires a nonempty unique name and exact query cohort") + } + if _, duplicate := seenBuckets[bucket.Name]; duplicate { + return fmt.Errorf("provisional production bucket %q is duplicated", bucket.Name) + } + seenBuckets[bucket.Name] = struct{}{} + if !slices.Equal(bucket.QualificationSplit, []string{"training", "holdout"}) { + return fmt.Errorf("production bucket %q must bind exactly one training and one holdout qualification split in canonical order", bucket.Name) + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) { + if err := validateStaticV6CanonicalInboundBucket(bucket); err != nil { + return err + } + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) && + (bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationDistance) || bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship) { + return fmt.Errorf("SP-I2 distance bucket %q must be inbound, typed single-kind, distance-only, and depth-bounded", bucket.Name) + } + for _, digest := range bucket.QuerySHA256 { + if !isLowerHexSHA256(digest) { + return fmt.Errorf("production bucket %q contains an invalid query digest", bucket.Name) + } + if _, found := seenQueries[digest]; found { + return fmt.Errorf("production query digest %s is authorized more than once", digest) + } + seenQueries[digest] = struct{}{} + } + } + if len(seenQueries) != 1 { + return fmt.Errorf("provisional manifest requires exactly one authorized query digest") + } + if manifest.OperationalCandidateSQLSHA256 != "" && !isLowerHexSHA256(manifest.OperationalCandidateSQLSHA256) { + return fmt.Errorf("provisional manifest operational_candidate_sql_sha256 must be a lowercase SHA-256 digest when present") + } + s.productionManifest = &manifest + return nil +} + +// productionOptions derives execution options for production. +func (s *postgresSQLRunner) productionOptions(cypherQuery string) (translate.ProductionOptions, error) { + manifest := s.productionManifest + if manifest == nil { + return translate.ProductionOptions{}, fmt.Errorf("production manifest is not configured") + } + digest := pg.TraversalPolicyQuerySHA256(cypherQuery) + for _, bucket := range manifest.Buckets { + if !slices.Contains(bucket.QuerySHA256, digest) { + continue + } + options := translate.ProductionOptions{ + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: int64(bucket.MinimumDepth), + MaximumDepth: int64(bucket.MaximumDepth), + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + }, + SelectorVersion: manifest.SelectorVersion, + } + if isOrientationProbePolicy(manifest.Candidate) { + options.EnableExpansionOrientation = true + options.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicy(manifest.Candidate) + } else { + options.ShortestPathExecutor = optimize.ShortestPathExecutor(manifest.Candidate) + options.ShortestPathCaps = &translate.ProductionShortestPathCaps{ + StateLimit: manifest.Caps["state_limit"], + FrontierLimit: manifest.Caps["frontier_limit"], + PredecessorLimit: manifest.Caps["predecessor_limit"], + EnumerationLimit: manifest.Caps["enumeration_limit"], + OutputBytesLimit: manifest.Caps["output_bytes_limit"], + } + } + return options, nil + } + return translate.ProductionOptions{}, fmt.Errorf("query SHA-256 %s is absent from the provisional production manifest", digest) +} + +// newPostgresSQLRunner opens a PostgreSQL benchmark runner for managed-fixture execution. +func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus, poolSize, round int, concurrency []int, references bool, referenceArms []string, forceShortest, forceExpansion string) (*postgresSQLRunner, error) { + return newPostgresSQLRunnerWithExistingGraph(ctx, datasetDir, connection, corpus, poolSize, round, concurrency, references, referenceArms, forceShortest, forceExpansion, nil) +} + +// newPostgresSQLRunnerWithExistingGraph opens a PostgreSQL benchmark runner with optional live-graph state. +func newPostgresSQLRunnerWithExistingGraph(ctx context.Context, datasetDir, connection string, corpus ScaleCorpus, poolSize, round int, concurrency []int, references bool, referenceArms []string, forceShortest, forceExpansion string, existing *existingGraphRunnerOptions) (*postgresSQLRunner, error) { poolCfg, err := pgxpool.ParseConfig(connection) if err != nil { return nil, fmt.Errorf("parse PostgreSQL pool configuration: %w", err) } - pool, err := pg.NewPool(poolCfg) + // GraphBench needs first-call and steady-state samples from an identifiable + // physical session. A single-connection pool makes that relationship + // deterministic while retaining the production pool hooks. + poolCfg.MinConns = int32(poolSize) + poolCfg.MaxConns = int32(poolSize) + if compactBidirectionalSnapshotRequired(references, referenceArms, forceShortest) { + if poolCfg.ConnConfig.RuntimeParams == nil { + poolCfg.ConnConfig.RuntimeParams = map[string]string{} + } + poolCfg.ConnConfig.RuntimeParams["default_transaction_isolation"] = "repeatable read" + } + // pg.NewPool applies the production driver's fixed 5/50 pool sizing. The + // benchmark must preserve the requested size so a size-one run can prove + // that all samples in a case used the same physical session. + poolCfg.AfterConnect = pg.AfterPooledConnectionEstablished + poolCfg.AfterRelease = pg.AfterPooledConnectionRelease + pool, err := pgxpool.NewWithConfig(ctx, poolCfg) if err != nil { return nil, fmt.Errorf("create PostgreSQL pool: %w", err) } @@ -60,15 +307,17 @@ func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, co return nil, fmt.Errorf("open PostgreSQL database: %w", err) } - nodeKinds, edgeKinds, err := scanDatasetKinds(datasetDir, scaleCorpusDatasets(corpus)) - if err != nil { - _ = db.Close(ctx) - return nil, err - } + if existing == nil { + nodeKinds, edgeKinds, err := scanDatasetKinds(datasetDir, scaleCorpusDatasets(corpus)) + if err != nil { + _ = db.Close(ctx) + return nil, err + } - if err := db.AssertSchema(ctx, benchmarkSchema(nodeKinds, edgeKinds)); err != nil { - _ = db.Close(ctx) - return nil, fmt.Errorf("assert PostgreSQL schema: %w", err) + if err := db.AssertSchema(ctx, benchmarkSchema(nodeKinds, edgeKinds)); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("assert PostgreSQL schema: %w", err) + } } pgDriver, ok := db.(*pg.Driver) @@ -76,21 +325,104 @@ func newPostgresSQLRunner(ctx context.Context, datasetDir, connection string, co _ = db.Close(ctx) return nil, fmt.Errorf("expected *pg.Driver, got %T", db) } + if existing != nil { + if err := pgDriver.SetDefaultGraph(ctx, graph.Graph{ + Name: existing.Manifest.Graph, + }); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("select existing PostgreSQL graph: %w", err) + } + if err := pgDriver.Fetch(ctx); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("fetch existing PostgreSQL kinds: %w", err) + } + } defaultGraph, ok := pgDriver.DefaultGraph() if !ok { _ = db.Close(ctx) return nil, fmt.Errorf("PostgreSQL default graph is not set") } + if existing != nil && existing.Manifest.Graph != "" && existing.Manifest.Graph != defaultGraph.Name { + _ = db.Close(ctx) + return nil, fmt.Errorf("anchor manifest graph %q does not match PostgreSQL default graph %q", existing.Manifest.Graph, defaultGraph.Name) + } + var backendPID int32 + if err := pool.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("identify PostgreSQL benchmark connection: %w", err) + } + var postgresEnvironment PostgresEnvironment + if err := pool.QueryRow(ctx, `select version(), current_database(), current_setting('plan_cache_mode'), current_setting('transaction_isolation'), current_setting('work_mem'), current_setting('temp_file_limit'), (select count(*) from graph), pg_postmaster_start_time(), (select oid::int8 from pg_database where datname = current_database()), current_setting('autovacuum')`).Scan( + &postgresEnvironment.Version, + &postgresEnvironment.Database, + &postgresEnvironment.PlanCacheMode, + &postgresEnvironment.TransactionIsolation, + &postgresEnvironment.WorkMem, + &postgresEnvironment.TempFileLimit, + &postgresEnvironment.GraphPartitionCount, + &postgresEnvironment.PostmasterStartedAt, + &postgresEnvironment.DatabaseOID, + &postgresEnvironment.Autovacuum, + ); err != nil { + _ = db.Close(ctx) + return nil, fmt.Errorf("capture PostgreSQL environment: %w", err) + } return &postgresSQLRunner{ - datasetDir: datasetDir, - db: db, - pgDriver: pgDriver, - graphID: defaultGraph.ID, + datasetDir: datasetDir, + db: db, + pgDriver: pgDriver, + pool: pool, + graphID: defaultGraph.ID, + backendPID: strconv.FormatInt(int64(backendPID), 10), + poolSize: poolSize, + round: round, + concurrency: append([]int(nil), concurrency...), + environment: postgresEnvironment, + references: references, + referenceArms: append([]string(nil), referenceArms...), + toolOptions: translate.ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutor(forceShortest), + ForceExpansionSearchStrategy: optimize.ExpansionSearchStrategy(forceExpansion), + }, + existingGraph: existing, }, nil } +// compactBidirectionalSnapshotRequired reports whether any selected production +// or reference arm can execute the multi-statement B1/B2 workspace kernel. +func compactBidirectionalSnapshotRequired(references bool, referenceArms []string, forceShortest string) bool { + switch optimize.ShortestPathExecutor(forceShortest) { + case optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return true + } + if !references { + return false + } + if len(referenceArms) == 0 { + return true + } + for _, arm := range referenceArms { + switch arm { + case "sp_b1_strict_alternating_distance", + "sp_b1_strict_alternating_witness_m0", + "sp_b2_smaller_frontier_distance", + "sp_b2_smaller_frontier_witness_m0", + "asp_b1_bidirectional_dag_strict_m0", + "asp_b2_bidirectional_dag_smaller_frontier_m0": + return true + } + } + return false +} + +// Close releases the graph database and PostgreSQL pool owned by the runner. func (s *postgresSQLRunner) Close(ctx context.Context) error { if s.db == nil { return nil @@ -99,13 +431,21 @@ func (s *postgresSQLRunner) Close(ctx context.Context) error { return s.db.Close(ctx) } -func (s *postgresSQLRunner) Run(ctx context.Context, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { +// Run measures supported corpus cases against managed fixtures or the configured preexisting graph. +func (s *postgresSQLRunner) Run(ctx context.Context, warmupIterations, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { + if s.existingGraph != nil { + return s.runExistingGraph(ctx, warmupIterations, iterations, corpus) + } var ( records []CaseResult casesByDataset = scaleCasesByDataset(corpus) ) for _, datasetName := range scaleCorpusDatasets(corpus) { + fixture, err := fixtureMetadata(s.datasetDir, datasetName) + if err != nil { + return nil, err + } if err := clearGraph(ctx, s.db); err != nil { return nil, fmt.Errorf("clear graph for %s: %w", datasetName, err) } @@ -114,13 +454,32 @@ func (s *postgresSQLRunner) Run(ctx context.Context, iterations int, corpus Scal if err != nil { return nil, err } + if err := s.captureAndValidateFixture(ctx, &fixture); err != nil { + return nil, fmt.Errorf("validate %s fixture: %w", datasetName, err) + } + activePartitions := fmt.Sprintf("vacuum (analyze) node_%d, edge_%d", s.graphID, s.graphID) + if _, err := s.pool.Exec(ctx, activePartitions); err != nil { + return nil, fmt.Errorf("vacuum and analyze %s fixture: %w", datasetName, err) + } + if err := s.pool.QueryRow(ctx, `select pg_total_relation_size(format('node_%s', $1::int4)::regclass), pg_total_relation_size(format('edge_%s', $1::int4)::regclass), coalesce((select string_agg(relname || ':' || coalesce(last_analyze::text, 'never'), ',' order by relname) from pg_stat_all_tables where relname in (format('node_%s', $1::int4), format('edge_%s', $1::int4))), '')`, s.graphID).Scan( + &fixture.NodeRelationBytes, &fixture.EdgeRelationBytes, &s.environment.AnalyzeState, + ); err != nil { + return nil, fmt.Errorf("capture %s fixture relation sizes: %w", datasetName, err) + } + s.environment.NodeRelationBytes = fixture.NodeRelationBytes + s.environment.EdgeRelationBytes = fixture.EdgeRelationBytes for _, testCase := range casesByDataset[datasetName] { if !testCase.Supports(ModePostgresSQL) { continue } - record := s.runCase(ctx, iterations, testCase, idMap) + if err := s.resetCaseSession(ctx); err != nil { + return nil, fmt.Errorf("reset PostgreSQL session for %s: %w", testCase.Name, err) + } + + record := s.runCase(ctx, warmupIterations, iterations, testCase, idMap) + attachFixtureMetadata(&record, fixture) records = append(records, record) } } @@ -128,27 +487,427 @@ func (s *postgresSQLRunner) Run(ctx context.Context, iterations int, corpus Scal return records, nil } -func (s *postgresSQLRunner) runCase(ctx context.Context, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { - params, err := resolveCaseParams(testCase, idMap) - record := newCaseResult(testCase, ModePostgresSQL, params) +// runExistingGraph executes eligible live-graph cases, honoring checkpoints and progress callbacks. +func (s *postgresSQLRunner) runExistingGraph(ctx context.Context, warmupIterations, iterations int, corpus ScaleCorpus) ([]CaseResult, error) { + options := s.existingGraph + if err := validateExistingGraphCorpus(corpus, options.Manifest); err != nil { + return nil, err + } + anchors, err := s.resolveExistingGraphAnchors(ctx, options.Manifest) if err != nil { - record.Status = StatusError - record.Error = err.Error() - return record + return nil, err + } + idMap := idMapForManifest(anchors) + preNodes, preEdges, err := s.existingGraphCounts(ctx) + if err != nil { + return nil, err + } + if err := s.captureExistingGraphEnvironment(ctx); err != nil { + return nil, err + } + databaseDigest := sha256.Sum256([]byte(s.environment.Database)) + s.environment.Database = "sha256:" + hex.EncodeToString(databaseDigest[:]) + fixture := FixtureMetadata{ + Dataset: "existing_graph", + Checksum: strings.Join([]string{ + options.Manifest.Checksum, + options.Manifest.ContentIdentity, + s.environment.SchemaFingerprint, + s.environment.IndexFingerprint, + }, ":"), + PhysicalValidated: true, + PhysicalNodeCount: preNodes, + PhysicalEdgeCount: preEdges, + NodeRelationBytes: s.environment.NodeRelationBytes, + EdgeRelationBytes: s.environment.EdgeRelationBytes, + Configuration: "existing_graph_read_only", + } + if err := validateCompletedWorkloads(options.Completed, corpus, fixture); err != nil { + return nil, err + } + var records []CaseResult + for _, testCase := range corpus.Cases { + if !testCase.Supports(ModePostgresSQL) { + continue + } + caseKey := existingGraphCaseKey(ModePostgresSQL, testCase) + if _, completed := options.Completed[caseKey]; completed { + continue + } + if err := appendExistingGraphProgress(options.ProgressPath, ExistingGraphProgress{ + Stage: "case", + CaseKey: caseKey, + }); err != nil { + return nil, err + } + if err := s.resetCaseSession(ctx); err != nil { + return nil, fmt.Errorf("reset PostgreSQL session for %s: %w", testCase.Name, err) + } + record := s.runExistingGraphCase(ctx, warmupIterations, iterations, testCase, idMap) + attachFixtureMetadata(&record, fixture) + record.ExistingGraph.PreNodeCount, record.ExistingGraph.PreEdgeCount = preNodes, preEdges + redactExistingGraphRecord(&record, options.Manifest, anchors) + records = append(records, record) + if options.OnRecord != nil { + if err := options.OnRecord(record); err != nil { + return nil, err + } + } + } + postNodes, postEdges, err := s.existingGraphCounts(ctx) + if err != nil { + return nil, err + } + if preNodes != postNodes || preEdges != postEdges { + return nil, fmt.Errorf("existing graph cardinality changed: nodes %d -> %d, edges %d -> %d", preNodes, postNodes, preEdges, postEdges) + } + for idx := range records { + records[idx].ExistingGraph.PostNodeCount, records[idx].ExistingGraph.PostEdgeCount = postNodes, postEdges + } + if options.OnComplete != nil { + if err := options.OnComplete(postNodes, postEdges); err != nil { + return nil, err + } + } + if err := appendExistingGraphProgress(options.ProgressPath, ExistingGraphProgress{ + Stage: "complete", + Detail: fmt.Sprintf("nodes=%d edges=%d", postNodes, postEdges), + }); err != nil { + return nil, err + } + return records, nil +} + +// runExistingGraphCase executes the fixed-confirmation or adaptive timeout protocol for one read-only workload against a preexisting graph. +func (s *postgresSQLRunner) runExistingGraphCase(ctx context.Context, warmupIterations, iterations int, testCase ScaleCase, idMap opengraph.IDMap) CaseResult { + options := s.existingGraph + timeouts := options.TimeoutClasses + if len(timeouts) == 0 { + timeouts = []time.Duration{0} + } + live := &ExistingGraphRun{ + ManifestSHA256: options.Manifest.Checksum, + ContentIdentity: options.Manifest.ContentIdentity, + Protocol: "fixed_confirmation", + Adaptive: options.Discovery, + } + if options.Discovery { + live.Protocol = "adaptive_discovery" + } + var record CaseResult + for idx, timeout := range timeouts { + measured := iterations + warmups := warmupIterations + if options.Discovery && idx > 0 { + measured = max(options.SampleFloor, iterations>>idx) + warmups = warmupIterations >> idx + } + attemptCtx := ctx + cancel := func() {} + if timeout > 0 { + attemptCtx, cancel = context.WithTimeout(ctx, timeout) + } + record = s.runCase(attemptCtx, warmups, measured, testCase, idMap) + attemptErr := attemptCtx.Err() + cancel() + attempt := ExistingGraphAttempt{ + Timeout: timeout, + WarmupSamples: warmups, + MeasuredSamples: measured, + Status: record.Status, + Error: record.Error, + } + live.Attempts = append(live.Attempts, attempt) + if attemptErr == nil || !options.Discovery { + break + } + _ = appendExistingGraphProgress(options.ProgressPath, ExistingGraphProgress{ + Stage: "timeout", + CaseKey: existingGraphCaseKey(ModePostgresSQL, testCase), + Detail: timeout.String(), + }) + } + record.ExistingGraph = live + return record +} + +// resolveLogicalExistingGraphAnchor looks up one logical-key anchor and rejects missing or ambiguous matches. +func (s *postgresSQLRunner) resolveLogicalExistingGraphAnchor(ctx context.Context, name, logicalKey string) ([]int64, error) { + rows, err := s.pool.Query(ctx, `select id from node where graph_id = $1 and properties ->> 'logical_key' = $2 order by id limit 2`, s.graphID, logicalKey) + if err != nil { + return nil, fmt.Errorf("resolve anchor %s: %w", name, err) + } + defer rows.Close() + + var ids []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("resolve anchor %s rows: %w", name, err) + } + + return ids, nil +} + +// resolveExistingGraphAnchors resolves every manifest anchor to exactly one PostgreSQL node identifier. +func (s *postgresSQLRunner) resolveExistingGraphAnchors(ctx context.Context, manifest ExistingGraphAnchorManifest) (map[string]graph.ID, error) { + anchors := make(map[string]graph.ID, len(manifest.Anchors)) + for name, anchor := range manifest.Anchors { + var ids []int64 + if anchor.PhysicalID == nil { + if resolvedIDs, err := s.resolveLogicalExistingGraphAnchor(ctx, name, anchor.LogicalKey); err != nil { + return nil, err + } else { + ids = resolvedIDs + } + } else { + var ( + kindIDs string + properties string + id int64 + ) + + if err := s.pool.QueryRow(ctx, `select id, kind_ids::text, properties::text from node where graph_id = $1 and id = $2`, s.graphID, *anchor.PhysicalID).Scan(&id, &kindIDs, &properties); err != nil { + return nil, fmt.Errorf("resolve physical anchor %s: %w", name, err) + } + + digest := sha256.Sum256([]byte(kindIDs + "\n" + properties)) + actual := "sha256:" + hex.EncodeToString(digest[:]) + if actual != anchor.ContentSHA256 { + return nil, fmt.Errorf("physical anchor %s content identity mismatch", name) + } + + ids = append(ids, id) + } + + if len(ids) != 1 { + return nil, fmt.Errorf("anchor %s resolved to %d nodes; exactly one is required", name, len(ids)) + } + + if anchor.Kind != "" { + var matches bool + if err := s.pool.QueryRow(ctx, `select exists(select 1 from node n join kind k on k.id = any(n.kind_ids) where n.graph_id = $1 and n.id = $2 and k.name = $3)`, s.graphID, ids[0], anchor.Kind).Scan(&matches); err != nil { + return nil, err + } + + if !matches { + return nil, fmt.Errorf("anchor %s does not have declared kind %s", name, anchor.Kind) + } + } + + anchors[name] = graph.ID(ids[0]) + } + + return anchors, nil +} + +// existingGraphCounts returns node and relationship counts for the selected PostgreSQL graph. +func (s *postgresSQLRunner) existingGraphCounts(ctx context.Context) (int64, int64, error) { + var nodes, edges int64 + if err := s.pool.QueryRow(ctx, `select (select count(*) from node where graph_id = $1), (select count(*) from edge where graph_id = $1)`, s.graphID).Scan(&nodes, &edges); err != nil { + return 0, 0, err + } + + return nodes, edges, nil +} + +// captureExistingGraphEnvironment records live graph relation sizes and normalized schema and index fingerprints. +func (s *postgresSQLRunner) captureExistingGraphEnvironment(ctx context.Context) error { + if err := s.pool.QueryRow(ctx, `select pg_total_relation_size(format('node_%s', $1::int4)::regclass), pg_total_relation_size(format('edge_%s', $1::int4)::regclass)`, s.graphID).Scan(&s.environment.NodeRelationBytes, &s.environment.EdgeRelationBytes); err != nil { + return err + } + return s.pool.QueryRow(ctx, `select + md5(coalesce((select string_agg(table_name || ':' || column_name || ':' || data_type, ',' order by table_name, ordinal_position) from information_schema.columns where table_schema = current_schema() and table_name in ('graph','kind','node','edge')), '')), + md5(coalesce((select string_agg(indexname || ':' || indexdef, ',' order by indexname) from pg_indexes where schemaname = current_schema() and (tablename in ('node','edge') or tablename in (format('node_%s',$1::int4), format('edge_%s',$1::int4)))), ''))`, s.graphID).Scan(&s.environment.SchemaFingerprint, &s.environment.IndexFingerprint) +} + +// captureAndValidateFixture records physical fixture sizes and rejects cardinality or checksum drift. +func (s *postgresSQLRunner) captureAndValidateFixture(ctx context.Context, fixture *FixtureMetadata) error { + if err := s.pool.QueryRow(ctx, `select (select count(*) from node where graph_id = $1), (select count(*) from edge where graph_id = $1)`, s.graphID).Scan( + &fixture.PhysicalNodeCount, + &fixture.PhysicalEdgeCount, + ); err != nil { + return fmt.Errorf("count physical graph rows: %w", err) + } + if fixture.PhysicalNodeCount != int64(fixture.NodeCount) || fixture.PhysicalEdgeCount != int64(fixture.EdgeCount) { + return fmt.Errorf( + "physical cardinality mismatch: nodes=%d want=%d edges=%d want=%d", + fixture.PhysicalNodeCount, + fixture.NodeCount, + fixture.PhysicalEdgeCount, + fixture.EdgeCount, + ) } + fixture.PhysicalValidated = true - rowCount, stats, err := measureCypher(ctx, s.db, testCase.Cypher, params, iterations) + return nil +} + +// resetCaseSession supports benchmark evidence processing for reset case session. +func (s *postgresSQLRunner) resetCaseSession(ctx context.Context) error { + s.pool.Reset() + if s.poolSize != 1 { + s.backendPID = "" + return nil + } + + var backendPID int32 + if err := s.pool.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + return err + } + s.backendPID = strconv.FormatInt(int64(backendPID), 10) + return nil +} + +// runCase resolves fixture parameters, executes the PostgreSQL read or write measurement path, captures plans and cache statistics, and returns one CaseResult. +func (s *postgresSQLRunner) runCase(ctx context.Context, warmupIterations, iterations int, testCase ScaleCase, idMap opengraph.IDMap) (record CaseResult) { + params, err := resolveCaseParams(testCase, idMap) + record = newCaseResult(testCase, ModePostgresSQL, params) + defer func() { + stats := s.pgDriver.ParseCacheStats() + record.ParseCache = &stats + }() if err != nil { record.Status = StatusError record.Error = err.Error() return record } - record.RowCount = rowCount - record.Stats = stats - applyRowExpectation(&record) + if testCase.WriteScenario == nil { + var ( + rowCount int64 + observedRows []string + stats DurationStats + ) + readOptions := s.readTransactionOptions() + + if !hasForcedToolOptions(s.toolOptions) && s.productionManifest == nil { + if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureCypherWithWarmups(ctx, s.db, testCase.Cypher, params, testCase.Expected, idMap, warmupIterations, iterations) + } else { + rowCount, observedRows, stats, err = measureCypherWithWarmupsOptions(ctx, s.db, testCase.Cypher, params, testCase.Expected, idMap, warmupIterations, iterations, readOptions...) + } + } else { + translation, sqlQuery, translateErr := s.translateCypher(ctx, testCase.Cypher, params) + if translateErr != nil { + err = translateErr + } else { + measurementDB := s.db + if s.toolOptions.EnableExpansionSuffixReverseRetry { + fallbackTranslation, fallbackSQL, fallbackErr := s.translateIncumbentCypher(ctx, testCase.Cypher, params) + if fallbackErr != nil { + err = fallbackErr + } else { + decision, found := suffixReverseRetryDecision(translation) + if !found { + err = fmt.Errorf("suffix reverse retry translation did not expose one retry decision") + } else { + measurementDB = &suffixReverseRetryDatabase{ + Database: s.db, + candidateSQL: sqlQuery, + fallbackSQL: fallbackSQL, + candidateParameters: translation.Parameters, + fallbackParameters: fallbackTranslation.Parameters, + limits: pg.SuffixReverseRetryLimits{ + OutputRows: decision.Admission.OutputRowLimit, + OutputBytes: decision.Admission.OutputBytesLimit, + }, + } + } + } + } + if err == nil { + requestedIdentity := timedRuntimeAttestationIdentity(translation) + if requestedIdentity == "" { + if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureRawSQLWithWarmups(ctx, measurementDB, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations) + } else { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsOptions(ctx, measurementDB, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, readOptions...) + } + } else if s.poolSize != 1 { + // Exact per-sample receipts require one physical session. Larger + // pools remain useful for operational smoke testing, but their + // samples intentionally lack promotion-grade attestation. + if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureRawSQLWithWarmups(ctx, measurementDB, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations) + } else { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsOptions(ctx, measurementDB, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, readOptions...) + } + } else if attestor, attestorErr := newPostgresTimedReadAttestor(s.pool, s.poolSize, requestedIdentity); attestorErr != nil { + err = attestorErr + } else if len(readOptions) == 0 { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsAndAttestation(ctx, measurementDB, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, attestor) + } else { + rowCount, observedRows, stats, err = measureRawSQLWithWarmupsAndAttestationOptions(ctx, measurementDB, sqlQuery, translation.Parameters, testCase.Expected, idMap, warmupIterations, iterations, attestor, readOptions...) + } + } + } + } + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + record.RowCount = rowCount + record.ObservedRows = observedRows + record.Stats = stats + labelLatencySamples(&record.Stats, ModePostgresSQL, testCase) + for idx := range record.Stats.Samples { + record.Stats.Samples[idx].ConnectionID = s.backendPID + } + applyRowExpectation(&record) + } else { + scenario, err := resolveWriteScenario(testCase, idMap) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + measurement, stats, err := measureWriteCypherWithWarmups(ctx, s.db, testCase.Cypher, params, scenario, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + + record.MatchedCount = &measurement.Matched + record.AffectedCount = &measurement.Affected + record.PostState = measurement.PostState + record.Stats = stats + labelLatencySamples(&record.Stats, ModePostgresSQL, testCase) + for idx := range record.Stats.Samples { + record.Stats.Samples[idx].ConnectionID = s.backendPID + } + } + if s.poolSize == 1 { + var backendPID int32 + if err := s.pool.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("verify PostgreSQL benchmark connection: %v", err) + return record + } + if current := strconv.FormatInt(int64(backendPID), 10); current != s.backendPID { + record.Status = StatusError + record.Error = fmt.Sprintf("PostgreSQL physical connection changed during case: %s -> %s", s.backendPID, current) + return record + } + } - explain, err := s.explain(ctx, testCase.Cypher, params) + if s.existingGraph != nil { + _ = appendExistingGraphProgress(s.existingGraph.ProgressPath, ExistingGraphProgress{ + Stage: "plan", + CaseKey: existingGraphCaseKey(ModePostgresSQL, testCase), + }) + } + explain, err := s.explain(ctx, testCase.Cypher, params, testCase.WriteScenario != nil) if err != nil { if record.Status == StatusOK { record.Status = StatusError @@ -158,37 +917,253 @@ func (s *postgresSQLRunner) runCase(ctx context.Context, iterations int, testCas } record.SQL = explain.SQL + record.SQLFingerprint = sqlFingerprint(explain.SQL) + postgresEnvironment := s.environment + if len(s.readTransactionOptions()) > 0 { + postgresEnvironment.TransactionIsolation = "repeatable read" + } + record.PostgresEnvironment = &postgresEnvironment record.PostgresPlan = explain.Plan + record.PostgresPlanJSON = explain.PlanJSON record.PostgresMetrics = &explain.Metrics record.Optimization = &explain.Optimization + if explain.Optimization.LoweringPlan != nil { + var fallbackReasons []string + for _, decision := range explain.Optimization.LoweringPlan.ShortestPathExecutor { + if decision.FallbackReason != "" && !slices.Contains(fallbackReasons, decision.FallbackReason) { + fallbackReasons = append(fallbackReasons, decision.FallbackReason) + } + } + for _, decision := range explain.Optimization.LoweringPlan.ExpansionSearchStrategy { + if decision.FallbackReason != "" && !slices.Contains(fallbackReasons, decision.FallbackReason) { + fallbackReasons = append(fallbackReasons, decision.FallbackReason) + } + } + record.FallbackReason = strings.Join(fallbackReasons, ",") + } + if s.suffixRouteComponentClosure && testCase.WriteScenario == nil { + var rawIsolation []pgx.TxIsoLevel + if len(s.readTransactionOptions()) > 0 { + rawIsolation = []pgx.TxIsoLevel{pgx.RepeatableRead} + } + waterfall, err := measureCompileWaterfall(ctx, testCase.Cypher, params, s.pgDriver.KindMapper(), s.graphID, iterations, s.toolOptions) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("suffix-route closure client compile waterfall: %v", err) + return record + } + record.ClientWaterfall = &waterfall + normalizer := postgresBoundaryObservationNormalizer{ + mapper: pg.NewValueMapper(ctx, s.pgDriver.KindMapper()), + reversedIDs: reverseIDMap(idMap), + scalarNodeIDs: resultContainsNodeIDs(testCase.Expected), + pathValues: resultContainsPaths(testCase.Expected), + } + closure, err := measurePostgresBoundaryClosure( + ctx, + s.pool, + explain.SQL, + explain.Parameters, + normalizer, + iterations, + s.sessionMemoryCeilingBytes, + s.poolMemoryCeilingBytes, + rawIsolation..., + ) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("suffix-route closure raw pgx waterfall: %v", err) + return record + } + expectedObservationSHA256, err := stableObservationSHA256(record.ObservedRows) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("suffix-route closure encode public observation: %v", err) + return record + } + for _, sample := range postgresBoundaryClosureSamples(closure) { + if sample.Rows != record.RowCount { + record.Status = StatusError + record.Error = fmt.Sprintf("suffix-route closure raw pgx row count %d differs from CySQL row count %d", sample.Rows, record.RowCount) + return record + } + if sample.ObservationSHA256 != expectedObservationSHA256 { + record.Status = StatusError + record.Error = "suffix-route closure raw pgx observation differs from CySQL public observation" + return record + } + } + record.PostgresBoundaryClosure = &closure + } + if s.references && testCase.WriteScenario == nil { + var rawIsolation []pgx.TxIsoLevel + if len(s.readTransactionOptions()) > 0 { + rawIsolation = []pgx.TxIsoLevel{pgx.RepeatableRead} + } + waterfall, err := measureCompileWaterfall(ctx, testCase.Cypher, params, s.pgDriver.KindMapper(), s.graphID, iterations, s.toolOptions) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("client compile waterfall: %v", err) + return record + } + record.ClientWaterfall = &waterfall + productionOrder, referenceOrder := referenceClosureMeasurementOrder(len(s.referenceArms) == 1, s.round) + var references []PostgresReferenceResult + if referenceOrder == 1 { + references, err = s.measureReferences(ctx, testCase, params, idMap, record.ObservedRows, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("PostgreSQL references: %v", err) + return record + } + setReferenceMeasurementOrder(references, referenceOrder) + } + rawWaterfall, err := measureRawPGXWaterfall(ctx, s.pool, explain.SQL, explain.Parameters, warmupIterations, iterations, rawIsolation...) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("raw pgx waterfall: %v", err) + return record + } + if len(rawWaterfall.Samples) > 0 && rawWaterfall.Samples[0].Rows != record.RowCount { + record.Status = StatusError + record.Error = fmt.Sprintf("raw pgx row count %d differs from CySQL row count %d", rawWaterfall.Samples[0].Rows, record.RowCount) + return record + } + rawWaterfall.MeasurementOrder = productionOrder + record.RawPGXWaterfall = &rawWaterfall + if referenceOrder != 1 { + references, err = s.measureReferences(ctx, testCase, params, idMap, record.ObservedRows, warmupIterations, iterations) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("PostgreSQL references: %v", err) + return record + } + setReferenceMeasurementOrder(references, referenceOrder) + } + record.PostgresReferences = references + roundTrip, err := measureRawPGXWaterfall(ctx, s.pool, "select 1", nil, warmupIterations, iterations, rawIsolation...) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("raw pgx round trip: %v", err) + return record + } + record.RawPGXRoundTrip = &roundTrip + } + if testCase.WriteScenario == nil && len(s.concurrency) > 0 { + if s.existingGraph != nil { + _ = appendExistingGraphProgress(s.existingGraph.ProgressPath, ExistingGraphProgress{ + Stage: "concurrency", + CaseKey: existingGraphCaseKey(ModePostgresSQL, testCase), + }) + } + var concurrencyIsolation []pgx.TxIsoLevel + if len(s.readTransactionOptions()) > 0 { + concurrencyIsolation = []pgx.TxIsoLevel{pgx.RepeatableRead} + } + blocks, err := measurePostgresConcurrency(ctx, s.pool, explain.SQL, explain.Parameters, s.poolSize, s.concurrency, iterations, concurrencyIsolation...) + if err != nil { + record.Status = StatusError + record.Error = fmt.Sprintf("concurrency smoke: %v", err) + return record + } + record.Concurrency = blocks + } + if testCase.WriteScenario == nil { + if err := s.attachPostgresTraversalTelemetry(ctx, &record, explain.Parameters); err != nil { + record.Status = StatusError + record.Error = err.Error() + return record + } + setSampleTraversalRuntimeMetadata(&record.Stats, record.TraversalTelemetry) + } return record } -type postgresExplain struct { - SQL string - Plan []string - Metrics PostgresPlanMetrics - Optimization translate.OptimizationSummary +// readTransactionOptions returns the one stable-snapshot contract shared by +// every PostgreSQL timing and plan-replay path. Provisional production +// manifests always require Repeatable Read; tool tournaments opt into the same +// isolation with -postgres-repeatable-read. +func (s *postgresSQLRunner) readTransactionOptions() []graph.TransactionOption { + if s.productionManifest == nil && !s.repeatableRead { + return nil + } + options := []graph.TransactionOption{pg.OptionSetTransactionIsolation(pgx.RepeatableRead)} + if s.toolOptions.EnableExpansionSuffixReverseRetry { + options = append(options, pg.OptionSkipStableSnapshotTraversalWorkspacesForTool()) + } + return options } -func (s *postgresSQLRunner) explain(ctx context.Context, cypherQuery string, params map[string]any) (postgresExplain, error) { - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) - if err != nil { - return postgresExplain{}, err +// timedRuntimeAttestationIdentity derives the stable identity used to compare timed runtime attestation. +func timedRuntimeAttestationIdentity(translation translate.Result) string { + outcome, ok := singleTraversalOutcome(translation.Optimization.TargetOutcomes) + if !ok { + return "" + } + requested := outcome.Candidate + if requested == "" { + requested = outcome.Selected + } + if strings.HasPrefix(requested, "SP-B1-") || strings.HasPrefix(requested, "SP-B2-") || + strings.HasPrefix(requested, "ASP-B1-") || strings.HasPrefix(requested, "ASP-B2-") || + requested == string(optimize.ShortestPathExecutorS4CanonicalDistance) || + requested == string(optimize.ShortestPathExecutorS4CanonicalWitness) || + requested == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) || + requested == string(optimize.ShortestPathExecutorI2GuardedDistance) || + isV2GraphBenchExecutor(requested) || + requested == string(optimize.ShortestPathExecutorASPI1DAG) || + outcome.SelectionMode == "component_tool" || + isOrientationProbePolicy(outcome.EmittedPolicy) || + isSuffixReverseGuardPolicy(outcome.EmittedPolicy) || + isSuffixReverseRetryPolicy(outcome.EmittedPolicy) { + return requested } + return "" +} - translation, err := translate.Translate(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID) - if err != nil { - return postgresExplain{}, err +// referenceClosureMeasurementOrder returns the balanced production/reference order for a measurement round. +func referenceClosureMeasurementOrder(singleSelectedReference bool, round int) (production, reference int) { + if singleSelectedReference && round > 0 && round%2 == 0 { + return 2, 1 } + return 1, 2 +} - sqlQuery, err := translate.Translated(translation) +// setReferenceMeasurementOrder assigns consecutive execution positions to reference results beginning at order. +func setReferenceMeasurementOrder(references []PostgresReferenceResult, order int) { + for idx := range references { + references[idx].MeasurementOrder = order + idx + } +} + +// postgresExplain contains translated SQL and normalized PostgreSQL EXPLAIN evidence. +type postgresExplain struct { + // SQL contains the rendered SQL statement. + SQL string + // Plan contains normalized PostgreSQL text-plan lines. + Plan []string + // PlanJSON contains structured backend plan evidence. + PlanJSON json.RawMessage + // Metrics contains normalized PostgreSQL plan counters and resources. + Metrics PostgresPlanMetrics + // Optimization captures translation optimization and lowering decisions. + Optimization translate.OptimizationSummary + // Parameters contains translated SQL parameters keyed by placeholder name. + Parameters map[string]any +} + +// explain translates a Cypher query and returns normalized SQL and PostgreSQL EXPLAIN evidence. +func (s *postgresSQLRunner) explain(ctx context.Context, cypherQuery string, params map[string]any, write bool) (postgresExplain, error) { + translation, sqlQuery, err := s.translateCypher(ctx, cypherQuery, params) if err != nil { return postgresExplain{}, err } - var plan []string - if err := s.db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var ( + plan []string + planJSON json.RawMessage + ) + runExplain := func(tx graph.Transaction) error { result := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, TIMING OFF) "+sqlQuery, translation.Parameters) defer result.Close() @@ -201,25 +1176,178 @@ func (s *postgresSQLRunner) explain(ctx context.Context, cypherQuery string, par plan = append(plan, fmt.Sprint(values[0])) } - return result.Error() - }); err != nil { - return postgresExplain{}, err + if err := result.Error(); err != nil { + return err + } + if !write { + jsonResult := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, TIMING ON, FORMAT JSON) "+sqlQuery, translation.Parameters) + defer jsonResult.Close() + if jsonResult.Next() && len(jsonResult.Values()) > 0 { + planJSON, err = encodePostgresPlanJSON(jsonResult.Values()[0]) + if err != nil { + return err + } + } + if err := jsonResult.Error(); err != nil { + return err + } + } + if write { + return errScaleWriteRollback + } + return nil + } + + var explainErr error + if write { + explainErr = s.db.WriteTransaction(ctx, runExplain) + if errors.Is(explainErr, errScaleWriteRollback) { + explainErr = nil + } + } else if readOptions := s.readTransactionOptions(); len(readOptions) > 0 { + explainErr = s.db.ReadTransaction(ctx, runExplain, readOptions...) + } else { + explainErr = s.db.ReadTransaction(ctx, runExplain) + } + if explainErr != nil { + return postgresExplain{}, explainErr } + metrics := parsePostgresPlanMetrics(plan) + if len(planJSON) > 0 { + if structured, err := parsePostgresPlanJSONMetrics(planJSON); err == nil { + metrics = structured + } + } return postgresExplain{ SQL: sqlQuery, Plan: plan, - Metrics: parsePostgresPlanMetrics(plan), + PlanJSON: planJSON, + Metrics: metrics, Optimization: translation.Optimization, + Parameters: translation.Parameters, }, nil } +// translateCypher parses and translates Cypher, applying forced tool options when configured. +func (s *postgresSQLRunner) translateCypher(ctx context.Context, cypherQuery string, params map[string]any) (translate.Result, string, error) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + return translate.Result{}, "", err + } + + var translation translate.Result + if s.productionManifest != nil { + options, optionsErr := s.productionOptions(cypherQuery) + if optionsErr != nil { + return translate.Result{}, "", optionsErr + } + translation, err = translate.TranslateWithProductionOptions(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID, options) + } else if !hasForcedToolOptions(s.toolOptions) { + translation, err = translate.Translate(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID) + } else { + translation, err = translate.TranslateForTool(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID, s.toolOptions) + } + if err != nil { + return translate.Result{}, "", err + } + + sqlQuery, err := translate.Translated(translation) + if err != nil { + return translate.Result{}, "", err + } + if err := verifyProductionManifestSQLAnchor(s.productionManifest, sqlQuery); err != nil { + return translate.Result{}, "", err + } + return translation, sqlQuery, nil +} + +func (s *postgresSQLRunner) translateIncumbentCypher(ctx context.Context, cypherQuery string, params map[string]any) (translate.Result, string, error) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + return translate.Result{}, "", err + } + translation, err := translate.Translate(ctx, regularQuery, s.pgDriver.KindMapper(), params, s.graphID) + if err != nil { + return translate.Result{}, "", err + } + formatted, err := translate.Translated(translation) + return translation, formatted, err +} + +func suffixReverseRetryDecision(translation translate.Result) (optimize.ExpansionSearchStrategyDecision, bool) { + if translation.Optimization.LoweringPlan == nil { + return optimize.ExpansionSearchStrategyDecision{}, false + } + var selected []optimize.ExpansionSearchStrategyDecision + for _, decision := range translation.Optimization.LoweringPlan.ExpansionSearchStrategy { + if decision.EmittedPolicy == optimize.ExpansionSearchPolicySuffixReverseRetryV1 { + selected = append(selected, decision) + } + } + if len(selected) != 1 { + return optimize.ExpansionSearchStrategyDecision{}, false + } + return selected[0], true +} + +// verifyProductionManifestSQLAnchor permits an unanchored preflight and makes +// every subsequent provisional-manifest capture fail before SQL execution when +// production translation drifts from the frozen operational statement. +func verifyProductionManifestSQLAnchor(manifest *PromotionManifest, sqlQuery string) error { + if manifest == nil || manifest.OperationalCandidateSQLSHA256 == "" { + return nil + } + actual := sqlFingerprint(sqlQuery) + if actual != manifest.OperationalCandidateSQLSHA256 { + return fmt.Errorf( + "production traversal SQL SHA-256 %s does not match provisional manifest anchor %s", + actual, + manifest.OperationalCandidateSQLSHA256, + ) + } + return nil +} + +// hasForcedToolOptions reports whether either executor-selection override is configured. +func hasForcedToolOptions(options translate.ToolOptions) bool { + return options.ForceShortestPathExecutor != "" || options.ForceExpansionSearchStrategy != "" || + options.GuardedDistanceStateLimit != 0 || options.GuardedDistanceFrontierLimit != 0 || + options.EnableExpansionOrientationTournament || options.EnableExpansionOrientationShadow || + options.ExpansionOrientationPolicy != "" || options.EnableExpansionSuffixReverseGuard || + options.EnableExpansionSuffixReverseRetry || options.EnableExpansionSuffixRouteComponent || options.SuffixReverseGuardSuffixRowLimit != 0 || options.SuffixReverseGuardStateLimit != 0 || + options.SuffixReverseRetryOutputRowLimit != 0 || options.SuffixReverseRetryOutputBytesLimit != 0 +} + +// encodePostgresPlanJSON normalizes byte, string, or structured EXPLAIN JSON into json.RawMessage. +func encodePostgresPlanJSON(value any) (json.RawMessage, error) { + switch typed := value.(type) { + case []byte: + return append(json.RawMessage(nil), typed...), nil + case string: + return append(json.RawMessage(nil), typed...), nil + default: + encoded, err := json.Marshal(value) + if err != nil { + return nil, err + } + + return json.RawMessage(encoded), nil + } +} + var ( - postgresPlanningPattern = regexp.MustCompile(`Planning Time: ([0-9.]+) ms`) + // postgresPlanningPattern extracts milliseconds from a PostgreSQL Planning Time summary line. + postgresPlanningPattern = regexp.MustCompile(`Planning Time: ([0-9.]+) ms`) + + // postgresExecutionPattern extracts milliseconds from a PostgreSQL Execution Time summary line. postgresExecutionPattern = regexp.MustCompile(`Execution Time: ([0-9.]+) ms`) - postgresBufferPattern = regexp.MustCompile(`(?:(shared|temp) )?(hit|read|dirtied|written)=([0-9]+)`) + + // postgresBufferPattern extracts storage class, operation, and page count from PostgreSQL buffer counters. + postgresBufferPattern = regexp.MustCompile(`(?:(shared|local|temp) )?(hit|read|dirtied|written)=([0-9]+)`) ) +// parsePostgresPlanMetrics extracts planning, execution, and buffer counters from PostgreSQL text-plan lines. func parsePostgresPlanMetrics(plan []string) PostgresPlanMetrics { var metrics PostgresPlanMetrics for _, line := range plan { @@ -247,6 +1375,7 @@ func parsePostgresPlanMetrics(plan []string) PostgresPlanMetrics { return metrics } +// parsePostgresBuffers extracts shared, local, and temporary buffer counters from one plan line. func parsePostgresBuffers(line string) Buffers { var ( buffers Buffers @@ -270,6 +1399,16 @@ func parsePostgresBuffers(line string) Buffers { buffers.SharedRead = value case "shared_dirtied": buffers.SharedDirtied = value + case "shared_written": + buffers.SharedWritten = value + case "local_hit": + buffers.LocalHit = value + case "local_read": + buffers.LocalRead = value + case "local_dirtied": + buffers.LocalDirtied = value + case "local_written": + buffers.LocalWritten = value case "temp_read": buffers.TempRead = value case "temp_written": diff --git a/cmd/graphbench/postgres_plan.go b/cmd/graphbench/postgres_plan.go new file mode 100644 index 00000000..63e38594 --- /dev/null +++ b/cmd/graphbench/postgres_plan.go @@ -0,0 +1,191 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "strings" +) + +// parsePostgresPlanJSONMetrics extracts only fields PostgreSQL exposes. Every +// derived counter remains explicitly plan-derived; fixture expectations and +// benchmark-only state diagnostics are recorded elsewhere. +func parsePostgresPlanJSONMetrics(raw json.RawMessage) (PostgresPlanMetrics, error) { + var documents []map[string]any + if err := json.Unmarshal(raw, &documents); err != nil { + return PostgresPlanMetrics{}, fmt.Errorf("decode PostgreSQL JSON plan: %w", err) + } + if len(documents) != 1 { + return PostgresPlanMetrics{}, fmt.Errorf("PostgreSQL JSON plan has %d documents, expected 1", len(documents)) + } + + metrics := PostgresPlanMetrics{Provenance: map[string]string{}} + metrics.PlanningMS = jsonFloatPointer(documents[0]["Planning Time"]) + metrics.ExecutionMS = jsonFloatPointer(documents[0]["Execution Time"]) + if metrics.PlanningMS != nil { + metrics.Provenance["planning_ms"] = "measured_plan_json" + } + if metrics.ExecutionMS != nil { + metrics.Provenance["execution_ms"] = "measured_plan_json" + } + plan, ok := documents[0]["Plan"].(map[string]any) + if !ok { + return PostgresPlanMetrics{}, fmt.Errorf("PostgreSQL JSON plan is missing its root Plan object") + } + walkPostgresPlanNode(plan, &metrics, 0) + if len(metrics.PlanNodes) > 0 { + metrics.Buffers = metrics.PlanNodes[0].Buffers + metrics.Provenance["buffers"] = "measured_plan_json_root_inclusive" + } + return metrics, nil +} + +// walkPostgresPlanNode flattens one EXPLAIN node into aggregate metrics, then recursively visits child plans and CTE subplans. + +// walkPostgresPlanNode supports benchmark evidence processing for walk postgres plan node. +func walkPostgresPlanNode(node map[string]any, metrics *PostgresPlanMetrics, parentPlanNodeID int64) { + planNodeID := int64(len(metrics.PlanNodes) + 1) + metric := PostgresPlanNodeMetric{ + PlanNodeID: planNodeID, + ParentPlanNodeID: parentPlanNodeID, + NodeType: jsonString(node["Node Type"]), + ParentRelationship: jsonString(node["Parent Relationship"]), + CTEName: jsonString(node["CTE Name"]), + RelationName: jsonString(node["Relation Name"]), + Alias: jsonString(node["Alias"]), + IndexName: jsonString(node["Index Name"]), + FunctionName: jsonString(node["Function Name"]), + SubplanName: jsonString(node["Subplan Name"]), + PlanRows: jsonInt64(node["Plan Rows"]), + PlanWidth: jsonInt64(node["Plan Width"]), + ActualRows: jsonInt64(node["Actual Rows"]), + ActualLoops: jsonInt64(node["Actual Loops"]), + RowsRemovedByFilter: jsonInt64(node["Rows Removed by Filter"]), + ActualTotalMS: jsonFloat64(node["Actual Total Time"]), + Buffers: postgresJSONBuffers(node), + Provenance: "measured_plan_json", + } + metrics.PlanNodes = append(metrics.PlanNodes, metric) + + rows := metric.ActualRows * metric.ActualLoops + lowerIdentity := strings.ToLower(strings.Join([]string{metric.NodeType, metric.CTEName, metric.RelationName, metric.Alias, metric.IndexName, metric.FunctionName, metric.SubplanName, jsonString(node["Index Cond"])}, " ")) + if strings.Contains(lowerIdentity, "endpoint_seeded_endpoints") && rows > metrics.EndpointProbeRows { + metrics.EndpointProbeRows = rows + metrics.EndpointGuardOverflow = rows >= 33 + metrics.Provenance["endpoint_probe_rows"] = "plan_derived_endpoint_seed_cte_rows" + } + if strings.Contains(lowerIdentity, "endpoint_seeded_states") && rows > metrics.ReverseStateProbeRows { + metrics.ReverseStateProbeRows = rows + metrics.StateGuardOverflow = rows >= 4097 + metrics.Provenance["reverse_state_probe_rows"] = "plan_derived_reverse_state_probe_cte_rows" + } + if strings.Contains(lowerIdentity, "endpoint_seeded_incumbent") && metric.ActualLoops > 0 { + metrics.ExpansionFallbackExecuted = true + metrics.Provenance["expansion_fallback_executed"] = "plan_derived_incumbent_cte_scan_loops" + } + if strings.Contains(lowerIdentity, "recursive union") { + metrics.RecursiveRows += rows + metrics.RecursiveLoops += metric.ActualLoops + metrics.Provenance["recursive_rows"] = "measured_plan_json" + metrics.Provenance["recursive_loops"] = "measured_plan_json" + } + for identity, target := range map[string]*int64{ + "frontier": &metrics.FrontierRows, + "witness": &metrics.WitnessRows, + "meeting": &metrics.MeetingRows, + } { + if strings.Contains(lowerIdentity, identity) { + *target += rows + metrics.Provenance[identity+"_rows"] = "plan_derived_labeled_state_rows" + } + } + if strings.Contains(lowerIdentity, "hydrated") || strings.Contains(lowerIdentity, "materializ") { + metrics.HydrationRows += rows + metrics.Provenance["hydration_rows"] = "plan_derived_labeled_state_rows" + } + if metric.CTEName == "roots" || (strings.Contains(lowerIdentity, " roots") && strings.Contains(lowerIdentity, "cte scan")) { + metrics.RootRows += rows + metrics.Provenance["root_rows"] = "measured_plan_json" + } + if strings.Contains(lowerIdentity, "edge") && strings.Contains(lowerIdentity, "start_id") { + metrics.ForwardEdgeProbes += metric.ActualLoops + metrics.Provenance["forward_edge_probes"] = "plan_derived_index_loops" + } + if strings.Contains(lowerIdentity, "edge") && strings.Contains(lowerIdentity, "end_id") { + metrics.ReverseEdgeProbes += metric.ActualLoops + metrics.Provenance["reverse_edge_probes"] = "plan_derived_index_loops" + } + if metric.RelationName == "node" || strings.HasPrefix(metric.RelationName, "node_") { + switch { + case strings.Contains(strings.ToLower(metric.Alias), "root"): + metrics.RootLookupLoops += metric.ActualLoops + metrics.Provenance["root_lookup_loops"] = "plan_derived_alias_loops" + case strings.Contains(strings.ToLower(metric.Alias), "boundary") || strings.Contains(strings.ToLower(metric.Alias), "next"): + metrics.BoundaryLookupLoops += metric.ActualLoops + metrics.Provenance["boundary_lookup_loops"] = "plan_derived_alias_loops" + default: + metrics.HydrationLoops += metric.ActualLoops + metrics.Provenance["hydration_loops"] = "plan_derived_node_relation_loops" + } + } + metrics.WALRecords += jsonInt64(node["WAL Records"]) + metrics.WALBytes += jsonInt64(node["WAL Bytes"]) + + children, _ := node["Plans"].([]any) + for _, child := range children { + if childNode, ok := child.(map[string]any); ok { + walkPostgresPlanNode(childNode, metrics, planNodeID) + } + } +} + +// postgresJSONBuffers converts optional JSON buffer counters to integer metrics. +func postgresJSONBuffers(node map[string]any) Buffers { + return Buffers{ + SharedHit: jsonInt64(node["Shared Hit Blocks"]), + SharedRead: jsonInt64(node["Shared Read Blocks"]), + SharedDirtied: jsonInt64(node["Shared Dirtied Blocks"]), + SharedWritten: jsonInt64(node["Shared Written Blocks"]), + LocalHit: jsonInt64(node["Local Hit Blocks"]), + LocalRead: jsonInt64(node["Local Read Blocks"]), + LocalDirtied: jsonInt64(node["Local Dirtied Blocks"]), + LocalWritten: jsonInt64(node["Local Written Blocks"]), + TempRead: jsonInt64(node["Temp Read Blocks"]), + TempWritten: jsonInt64(node["Temp Written Blocks"]), + } +} + +// jsonFloatPointer decodes a JSON number as an optional floating-point value. +func jsonFloatPointer(value any) *float64 { + if value == nil { + return nil + } + parsed := jsonFloat64(value) + return &parsed +} + +// jsonFloat64 decodes a JSON number as a floating-point value, returning zero when absent or invalid. +func jsonFloat64(value any) float64 { + switch typed := value.(type) { + case float64: + return typed + case json.Number: + parsed, _ := typed.Float64() + return parsed + default: + return 0 + } +} + +// jsonInt64 decodes a JSON number as an integer, returning zero when absent or invalid. +func jsonInt64(value any) int64 { return int64(jsonFloat64(value)) } + +// jsonString decodes a JSON string, returning an empty string for other values. +func jsonString(value any) string { + valueString, _ := value.(string) + return valueString +} diff --git a/cmd/graphbench/postgres_plan_test.go b/cmd/graphbench/postgres_plan_test.go new file mode 100644 index 00000000..93f8e727 --- /dev/null +++ b/cmd/graphbench/postgres_plan_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestParsePostgresPlanJSONMetricsWalksStructuredNodes verifies extraction of root timings, buffer use, recursive cardinality, labeled CTE rows, index probes, and provenance from nested plan JSON. +func TestParsePostgresPlanJSONMetricsWalksStructuredNodes(t *testing.T) { + raw := json.RawMessage(`[{ + "Plan": { + "Node Type": "Recursive Union", "Plan Rows": 12, "Plan Width": 64, + "Actual Rows": 19, "Actual Loops": 1, "Shared Hit Blocks": 40, + "Plans": [ + {"Node Type":"CTE Scan", "CTE Name":"roots", "Alias":"roots", "Actual Rows":1, "Actual Loops":1}, + {"Node Type":"Index Only Scan", "Relation Name":"edge_1", "Alias":"e", "Index Name":"edge_1_end_id_kind_id_idx", "Index Cond":"(end_id = reverse_trails.node_id)", "Actual Rows":1, "Actual Loops":18, "Shared Hit Blocks":36}, + {"Node Type":"Index Scan", "Relation Name":"node_1", "Alias":"boundary", "Actual Rows":2, "Actual Loops":1} + ] + }, + "Planning Time": 1.25, + "Execution Time": 2.5 +}]`) + + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Equal(t, 1.25, *metrics.PlanningMS) + require.Equal(t, 2.5, *metrics.ExecutionMS) + require.Equal(t, int64(40), metrics.Buffers.SharedHit) + require.Equal(t, int64(19), metrics.RecursiveRows) + require.Equal(t, int64(18), metrics.ReverseEdgeProbes) + require.Equal(t, int64(1), metrics.RootRows) + require.Equal(t, int64(1), metrics.BoundaryLookupLoops) + require.Len(t, metrics.PlanNodes, 4) + require.Equal(t, int64(1), metrics.PlanNodes[0].PlanNodeID) + require.Zero(t, metrics.PlanNodes[0].ParentPlanNodeID) + for idx := 1; idx < len(metrics.PlanNodes); idx++ { + require.Equal(t, int64(idx+1), metrics.PlanNodes[idx].PlanNodeID) + require.Equal(t, int64(1), metrics.PlanNodes[idx].ParentPlanNodeID) + } + require.Equal(t, "measured_plan_json", metrics.PlanNodes[0].Provenance) + require.Equal(t, "plan_derived_index_loops", metrics.Provenance["reverse_edge_probes"]) +} + +// TestParsePostgresPlanJSONMetricsRejectsMissingPlan verifies that timing metadata alone is not accepted as a PostgreSQL execution plan. +func TestParsePostgresPlanJSONMetricsRejectsMissingPlan(t *testing.T) { + _, err := parsePostgresPlanJSONMetrics(json.RawMessage(`[{"Planning Time":1}]`)) + require.ErrorContains(t, err, "missing its root Plan") +} + +// TestParsePostgresPlanJSONMetricsRetainsDirectPlanParentage verifies parse postgres plan json metrics retains direct plan parentage behavior. +func TestParsePostgresPlanJSONMetricsRetainsDirectPlanParentage(t *testing.T) { + raw := json.RawMessage(`[{ + "Plan": {"Node Type":"Append","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"Nested Loop","Parent Relationship":"InitPlan","Subplan Name":"CTE asp_i1_candidate_rows","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"CTE Scan","Parent Relationship":"Outer","CTE Name":"asp_i1_candidate_marker","Actual Rows":1,"Actual Loops":1}, + {"Node Type":"Result","Parent Relationship":"Inner","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"Function Scan","Parent Relationship":"Outer","Function Name":"shortest_path_compact","Actual Rows":1,"Actual Loops":1} + ]} + ]} + ]} +}]`) + + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Len(t, metrics.PlanNodes, 5) + require.Equal(t, int64(2), metrics.PlanNodes[1].PlanNodeID) + require.Equal(t, int64(1), metrics.PlanNodes[1].ParentPlanNodeID) + require.Equal(t, int64(2), metrics.PlanNodes[2].ParentPlanNodeID) + require.Equal(t, "Outer", metrics.PlanNodes[2].ParentRelationship) + require.Equal(t, int64(2), metrics.PlanNodes[3].ParentPlanNodeID) + require.Equal(t, "Inner", metrics.PlanNodes[3].ParentRelationship) + require.Equal(t, int64(4), metrics.PlanNodes[4].ParentPlanNodeID) +} + +// TestParsePostgresPlanJSONMetricsAttributesLabeledS4State verifies that repeated frontier loops and labeled witness, meeting, and hydration nodes populate their dedicated counters. +func TestParsePostgresPlanJSONMetricsAttributesLabeledS4State(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Node Type":"Result","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"CTE Scan","CTE Name":"forward_frontier","Actual Rows":3,"Actual Loops":2}, + {"Node Type":"CTE Scan","CTE Name":"selected_witness","Actual Rows":4,"Actual Loops":1}, + {"Node Type":"CTE Scan","CTE Name":"shortest_meeting","Actual Rows":1,"Actual Loops":1}, + {"Node Type":"Subquery Scan","Alias":"m0_hydrated","Actual Rows":5,"Actual Loops":1} + ]}}]`) + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Equal(t, int64(6), metrics.FrontierRows) + require.Equal(t, int64(4), metrics.WitnessRows) + require.Equal(t, int64(1), metrics.MeetingRows) + require.Equal(t, int64(5), metrics.HydrationRows) + require.Equal(t, "plan_derived_labeled_state_rows", metrics.Provenance["witness_rows"]) +} + +// TestParsePostgresPlanJSONMetricsAttributesEndpointGuardState verifies endpoint/state guard overflow detection and fallback attribution from labeled seeded-search CTEs. +func TestParsePostgresPlanJSONMetricsAttributesEndpointGuardState(t *testing.T) { + raw := json.RawMessage(`[{"Plan":{"Node Type":"Result","Actual Rows":1,"Actual Loops":1,"Plans":[ + {"Node Type":"CTE Scan","CTE Name":"s4_endpoint_seeded_endpoints","Actual Rows":33,"Actual Loops":1}, + {"Node Type":"CTE Scan","CTE Name":"s4_endpoint_seeded_states","Actual Rows":4097,"Actual Loops":1}, + {"Node Type":"CTE Scan","CTE Name":"s4_endpoint_seeded_incumbent","Actual Rows":10,"Actual Loops":1} + ]}}]`) + metrics, err := parsePostgresPlanJSONMetrics(raw) + require.NoError(t, err) + require.Equal(t, int64(33), metrics.EndpointProbeRows) + require.Equal(t, int64(4097), metrics.ReverseStateProbeRows) + require.True(t, metrics.EndpointGuardOverflow) + require.True(t, metrics.StateGuardOverflow) + require.True(t, metrics.ExpansionFallbackExecuted) +} diff --git a/cmd/graphbench/postgres_test.go b/cmd/graphbench/postgres_test.go index 54470e60..eb27815c 100644 --- a/cmd/graphbench/postgres_test.go +++ b/cmd/graphbench/postgres_test.go @@ -17,13 +17,432 @@ package main import ( + "encoding/json" + "os" + "path/filepath" + "strings" "testing" + "time" + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" "github.com/stretchr/testify/require" ) +// TestPostgresProductionManifestBuildsExactGuardedOptions verifies postgres production manifest builds exact guarded options behavior. +func TestPostgresProductionManifestBuildsExactGuardedOptions(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: "ASP-I1-U-DAG+MAT-M0", + SelectorVersion: "asp-i1-test-v1", + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: "ASP-A1-DAG", + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 10, "predecessor_limit": 20, "enumeration_limit": 30, "output_bytes_limit": 40}, + Buckets: []PromotionBucket{{ + Name: "outbound-depth8", + QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "outbound", + ObservationMode: "all_paths", + MinimumDepth: 1, + MaximumDepth: 8, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(path)) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.Equal(t, "ASP-I1-U-DAG+MAT-M0", string(options.ShortestPathExecutor)) + require.Equal(t, int64(10), options.ShortestPathCaps.StateLimit) + require.Equal(t, int64(8), options.AuthorizedBucket.MaximumDepth) + require.Equal(t, "asp-i1-test-v1", options.SelectorVersion) + _, err = runner.productionOptions(query + " RETURN 1") + require.ErrorContains(t, err, "absent from the provisional production manifest") +} + +func TestPostgresProductionManifestRejectsV1GuardedDistanceActivation(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:Traverse*1..32]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "hidden-fan-in-depth32", QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, + RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + runner := &postgresSQLRunner{} + require.ErrorContains(t, runner.setProductionManifest(path), "terminally rejected") +} + +// TestPostgresProductionManifestRequiresStaticV6CanonicalInboundBucket verifies postgres production manifest requires static v6 canonical inbound bucket behavior. +func TestPostgresProductionManifestRequiresStaticV6CanonicalInboundBucket(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:Traverse*1..64]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + digest := strings.Repeat("0", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalWitness), + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 10, "predecessor_limit": 20, "enumeration_limit": 30, "output_bytes_limit": 40}, + Buckets: []PromotionBucket{{ + Name: "canonical-inbound-depth64", + QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + write := func(t *testing.T, manifest PromotionManifest) string { + t.Helper() + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + return path + } + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(write(t, base))) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.Equal(t, optimize.ShortestPathSelectorStaticV6, options.SelectorVersion) + require.Equal(t, int64(64), options.AuthorizedBucket.MaximumDepth) + + tests := map[string]func(*PromotionManifest){ + "selector": func(manifest *PromotionManifest) { manifest.SelectorVersion = "sp-static-v5-contained" }, + "outbound": func(manifest *PromotionManifest) { manifest.Buckets[0].Direction = "outbound" }, + "maximum": func(manifest *PromotionManifest) { manifest.Buckets[0].MaximumDepth = 63 }, + "kinds": func(manifest *PromotionManifest) { manifest.Buckets[0].RelationshipKindCount = 2 }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + manifest.Buckets = clonePromotionBuckets(base.Buckets) + mutate(&manifest) + require.Error(t, (&postgresSQLRunner{}).setProductionManifest(write(t, manifest))) + }) + } +} + +// TestPostgresProductionManifestBuildsOrientationOptionsWithoutShortestPathFields verifies postgres production manifest builds orientation options without shortest path fields behavior. +func TestPostgresProductionManifestBuildsOrientationOptionsWithoutShortestPathFields(t *testing.T) { + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) WHERE id(r) = $root_id RETURN id(e)" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV1), + SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "outbound-fixed-suffix", + QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "outbound", + ObservationMode: "endpoint_ids", + MinimumDepth: 0, + MaximumDepth: 16, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(path)) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.True(t, options.EnableExpansionOrientation) + require.Empty(t, options.ShortestPathExecutor) + require.Nil(t, options.ShortestPathCaps) + require.Equal(t, int64(16), options.AuthorizedBucket.MaximumDepth) + require.Equal(t, "orientation-probe-v1", options.SelectorVersion) +} + +// TestPostgresProductionManifestRejectsNonExactOrientationContract verifies postgres production manifest rejects non exact orientation contract behavior. +func TestPostgresProductionManifestRejectsNonExactOrientationContract(t *testing.T) { + digest := strings.Repeat("0", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV1), + SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "fixed-suffix", + QuerySHA256: []string{digest}, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // mutate retains the mutate while anonymous record is assembled or evaluated. + mutate func(*PromotionManifest) + // err retains the err while anonymous record is assembled or evaluated. + err string + }{ + { + name: "fallback", + mutate: func(manifest *PromotionManifest) { manifest.FallbackExecutor = "EXPANSION-SUFFIX-SEEDED-REVERSE" }, + err: "unsupported candidate/fallback pair", + }, + { + name: "extra cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["extra_limit"] = 1 }, + err: "orientation-probe-v1 requires exactly four immutable caps", + }, + { + name: "missing cap", + mutate: func(manifest *PromotionManifest) { delete(manifest.Caps, "root_row_limit") }, + err: "orientation-probe-v1 requires exactly four immutable caps", + }, + { + name: "wrong cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["state_limit"]-- }, + err: "orientation-probe-v1 cap state_limit must equal 4096", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + test.mutate(&manifest) + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + err = (&postgresSQLRunner{}).setProductionManifest(path) + require.ErrorContains(t, err, test.err) + }) + } +} + +// TestPostgresProductionManifestCarriesOrientationProbeV2IntoTranslation verifies +// provisional production measurement cannot silently fall back to v1. +func TestPostgresProductionManifestCarriesOrientationProbeV2IntoTranslation(t *testing.T) { + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) WHERE id(r) = $root_id RETURN id(e)" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "outbound-fixed-suffix-v2", + QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "outbound", + ObservationMode: "endpoint_ids", + MinimumDepth: 0, + MaximumDepth: 16, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + + runner := &postgresSQLRunner{} + require.NoError(t, runner.setProductionManifest(path)) + options, err := runner.productionOptions(query) + require.NoError(t, err) + require.True(t, options.EnableExpansionOrientation) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV2, options.ExpansionOrientationPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), options.SelectorVersion) +} + +// TestPostgresProductionManifestSQLAnchorIsTwoPass verifies a provisional +// manifest may omit the SQL anchor only to derive it, while a populated anchor +// is checked against the exact SQL emitted by production translation. +func TestPostgresProductionManifestSQLAnchorIsTwoPass(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:Traverse*1..64]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + SelectorVersion: optimize.ShortestPathSelectorStaticV6, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalWitness), SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 10, "predecessor_limit": 20, "enumeration_limit": 30, "output_bytes_limit": 40}, + Buckets: []PromotionBucket{{ + Name: "canonical-inbound-depth64", QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, + Direction: "inbound", ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, + RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + write := func(value PromotionManifest) string { + raw, err := json.Marshal(value) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + return path + } + + preflight := &postgresSQLRunner{} + require.NoError(t, preflight.setProductionManifest(write(manifest))) + require.Empty(t, preflight.productionManifest.OperationalCandidateSQLSHA256) + + manifest.OperationalCandidateSQLSHA256 = strings.Repeat("f", 64) + formal := &postgresSQLRunner{} + require.NoError(t, formal.setProductionManifest(write(manifest))) + require.ErrorContains(t, verifyProductionManifestSQLAnchor(formal.productionManifest, "select 1"), "does not match provisional manifest anchor") + formal.productionManifest.OperationalCandidateSQLSHA256 = sqlFingerprint("select 1") + require.NoError(t, verifyProductionManifestSQLAnchor(formal.productionManifest, "select 1")) + + multipleQueries := manifest + multipleQueries.Buckets = clonePromotionBuckets(manifest.Buckets) + multipleQueries.Buckets[0].QuerySHA256 = append(multipleQueries.Buckets[0].QuerySHA256, strings.Repeat("e", 64)) + require.ErrorContains(t, (&postgresSQLRunner{}).setProductionManifest(write(multipleQueries)), "requires exactly one authorized query digest") + + manifest.OperationalCandidateSQLSHA256 = "NOT-A-DIGEST" + require.ErrorContains(t, (&postgresSQLRunner{}).setProductionManifest(write(manifest)), "must be a lowercase SHA-256 digest") +} + +func TestPostgresProductionManifestRejectsAmbiguousSetsAndJSON(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:Traverse*1..64]-(e)) RETURN p" + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + SelectorVersion: optimize.ShortestPathSelectorStaticV6, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalWitness), SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"state_limit": 10, "predecessor_limit": 20, "enumeration_limit": 30, "output_bytes_limit": 40}, + Buckets: []PromotionBucket{{Name: "qualified-query", QuerySHA256: []string{pg.TraversalPolicyQuerySHA256(query)}, Direction: "inbound", ObservationMode: "one_path", MinimumDepth: 1, MaximumDepth: 64, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + } + write := func(raw []byte) string { + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + return path + } + encode := func(manifest PromotionManifest) []byte { + raw, err := json.Marshal(manifest) + require.NoError(t, err) + return raw + } + + duplicateSplit := base + duplicateSplit.Buckets = clonePromotionBuckets(base.Buckets) + duplicateSplit.Buckets[0].QualificationSplit = []string{"training", "training", "holdout"} + require.ErrorContains(t, (&postgresSQLRunner{}).setProductionManifest(write(encode(duplicateSplit))), "exactly one training and one holdout") + + duplicateQuery := base + duplicateQuery.Buckets = clonePromotionBuckets(base.Buckets) + duplicateQuery.Buckets[0].QuerySHA256 = append(duplicateQuery.Buckets[0].QuerySHA256, duplicateQuery.Buckets[0].QuerySHA256[0]) + require.ErrorContains(t, (&postgresSQLRunner{}).setProductionManifest(write(encode(duplicateQuery))), "authorized more than once") + + duplicateKey := strings.Replace(string(encode(base)), `"version":2`, `"version":2,"version":2`, 1) + require.ErrorContains(t, (&postgresSQLRunner{}).setProductionManifest(write([]byte(duplicateKey))), "duplicate JSON object key") +} + +// TestPostgresReadTransactionOptionsMatchEveryStableSnapshotMode verifies postgres read transaction options match every stable snapshot mode behavior. +func TestPostgresReadTransactionOptionsMatchEveryStableSnapshotMode(t *testing.T) { + require.Empty(t, (&postgresSQLRunner{}).readTransactionOptions()) + + for name, runner := range map[string]*postgresSQLRunner{ + "explicit benchmark flag": {repeatableRead: true}, + "production manifest": {productionManifest: &PromotionManifest{}}, + } { + t.Run(name, func(t *testing.T) { + options := runner.readTransactionOptions() + require.Len(t, options, 1) + + pgConfig := &pg.Config{} + transactionConfig := &graph.TransactionConfig{DriverConfig: pgConfig} + options[0](transactionConfig) + require.Equal(t, pgx.RepeatableRead, pgConfig.Options.IsoLevel) + require.Equal(t, pgx.ReadWrite, pgConfig.Options.AccessMode) + }) + } +} + +func TestSuffixReverseGuardToolOptionsAreExecutableAndAttested(t *testing.T) { + options := translate.ToolOptions{EnableExpansionSuffixReverseGuard: true} + require.True(t, hasForcedToolOptions(options)) + + outcome := translate.TargetLoweringOutcome{ + TargetKind: "traversal", + Family: "fixed_suffix_expansion", + Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + EmittedPolicy: string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), + Selected: string(optimize.ExpansionSearchStepwiseForward), + } + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), timedRuntimeAttestationIdentity(translate.Result{ + Optimization: translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + })) +} + +func TestSuffixRouteComponentToolOptionsAreExecutableAndAttested(t *testing.T) { + options := translate.ToolOptions{EnableExpansionSuffixRouteComponent: true} + require.True(t, hasForcedToolOptions(options)) + + outcome := translate.TargetLoweringOutcome{ + TargetKind: "traversal", + Family: "fixed_suffix_expansion", + Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchSuffixSeededReverse), + Applied: string(optimize.ExpansionSearchSuffixSeededReverse), + SelectionMode: "component_tool", + } + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), timedRuntimeAttestationIdentity(translate.Result{ + Optimization: translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + })) +} + +// TestResolveCaseParams verifies that scalar, explicit-list, and generated-list fixture keys become ordered int64 IDs without disturbing ordinary parameters. func TestResolveCaseParams(t *testing.T) { params, err := resolveCaseParams(ScaleCase{ Params: map[string]any{ @@ -32,19 +451,52 @@ func TestResolveCaseParams(t *testing.T) { NodeParams: map[string]string{ "start_id": "n1", }, - }, opengraph.IDMap{"n1": graph.ID(42)}) + NodeListParams: map[string][]string{ + "end_ids": {"n2", "n1"}, + }, + GeneratedNodeListParams: map[string]testutil.GeneratedNodeListParam{ + "generated_ids": { + Prefix: "generated", + Count: 2, + Include: []string{"n2"}, + }, + }, + }, opengraph.IDMap{ + "n1": graph.ID(42), + "n2": graph.ID(84), + "generated-00": graph.ID(126), + "generated-01": graph.ID(168), + }) require.NoError(t, err) require.Equal(t, map[string]any{ - "name": "value", - "start_id": int64(42), + "name": "value", + "start_id": int64(42), + "end_ids": []int64{84, 42}, + "generated_ids": []int64{84, 126, 168}, }, params) } +// TestScaleCaseDecodesTypedDatetimeParameter verifies that the corpus JSON datetime envelope becomes a UTC time value rather than an untyped map. +func TestScaleCaseDecodesTypedDatetimeParameter(t *testing.T) { + var testCase ScaleCase + require.NoError(t, json.Unmarshal([]byte(`{ + "name":"typed-time", + "dataset":"base", + "category":"lookup", + "cypher":"MATCH (n) WHERE n.lastseen < $threshold RETURN n", + "params":{"threshold":{"$type":"datetime","value":"2026-01-02T03:04:05Z"}}, + "candidate_modes":["postgres_sql"] + }`), &testCase)) + + require.Equal(t, time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC), testCase.Params["threshold"]) +} + +// TestParsePostgresPlanMetrics verifies parsing of planning/execution milliseconds and every shared, local, and temporary buffer counter from text plans. func TestParsePostgresPlanMetrics(t *testing.T) { metrics := parsePostgresPlanMetrics([]string{ "Nested Loop (actual rows=1 loops=1)", - " Buffers: shared hit=12 read=3 dirtied=2, temp read=4 written=5", + " Buffers: shared hit=12 read=3 dirtied=2 written=1, local hit=7 read=6 dirtied=5 written=4, temp read=3 written=2", "Planning Time: 1.250 ms", "Execution Time: 9.750 ms", }) @@ -57,7 +509,48 @@ func TestParsePostgresPlanMetrics(t *testing.T) { SharedHit: 12, SharedRead: 3, SharedDirtied: 2, - TempRead: 4, - TempWritten: 5, + SharedWritten: 1, + LocalHit: 7, + LocalRead: 6, + LocalDirtied: 5, + LocalWritten: 4, + TempRead: 3, + TempWritten: 2, }, metrics.Buffers) } + +// TestGeneratedDatasetVariantsAreParameterizedAndRepeatable verifies deterministic generation for equal names and propagation of configured payload size into fixed-suffix nodes. +func TestGeneratedDatasetVariantsAreParameterizedAndRepeatable(t *testing.T) { + first := generatedDataset("generated_shortest_paths_d4_f16") + second := generatedDataset("generated_shortest_paths_d4_f16") + require.NotNil(t, first) + require.Equal(t, first, second) + + fixedSuffix := generatedDataset("generated_fixed_suffix_expansion_d2_f10_v2_p4096") + require.NotNil(t, fixedSuffix) + require.Contains(t, fixedSuffix.Nodes[0].Properties["payload"], "xxxx") +} + +// TestCompactBidirectionalRunsRequireRepeatableSnapshot verifies runner setup +// opts into stable snapshots exactly when a forced or reference B1/B2 arm can run. +func TestCompactBidirectionalRunsRequireRepeatableSnapshot(t *testing.T) { + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "SP-B1-C-ALT-NODE-D")) + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "SP-B2-C-MIN-LEVEL-WE+MAT-M0")) + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "ASP-B1-DAG-ALT-NODE")) + require.True(t, compactBidirectionalSnapshotRequired(false, nil, "ASP-B2-DAG-MIN-LEVEL")) + require.True(t, compactBidirectionalSnapshotRequired(true, nil, "")) + require.True(t, compactBidirectionalSnapshotRequired(true, []string{"sp_b1_strict_alternating_distance"}, "")) + require.True(t, compactBidirectionalSnapshotRequired(true, []string{"asp_b2_bidirectional_dag_smaller_frontier_m0"}, "")) + require.False(t, compactBidirectionalSnapshotRequired(false, nil, "SP-S4-C-D")) + require.False(t, compactBidirectionalSnapshotRequired(true, []string{"s4_canonical_source_distance"}, "")) +} + +// TestFixtureMetadataIncludesCardinalityAndChecksum verifies that generated fixtures expose their configuration, nonzero entity counts, and a full SHA-256 content digest. +func TestFixtureMetadataIncludesCardinalityAndChecksum(t *testing.T) { + metadata, err := fixtureMetadata("unused", "generated_shortest_paths_d4_f16") + require.NoError(t, err) + require.Equal(t, "generated_shortest_paths_d4_f16", metadata.Configuration) + require.Positive(t, metadata.NodeCount) + require.Positive(t, metadata.EdgeCount) + require.Len(t, metadata.Checksum, 64) +} diff --git a/cmd/graphbench/postgres_timed_attestation.go b/cmd/graphbench/postgres_timed_attestation.go new file mode 100644 index 00000000..773cdefb --- /dev/null +++ b/cmd/graphbench/postgres_timed_attestation.go @@ -0,0 +1,125 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// postgresTimedRuntimeDocument defines the serialized representation of postgres timed runtime. +type postgresTimedRuntimeDocument struct { + // SchemaVersion identifies the schema version for schema version. + SchemaVersion int `json:"schema_version"` + // InvocationID identifies the invocation id. + InvocationID string `json:"invocation_id"` + // RequestedIdentity identifies the requested identity. + RequestedIdentity string `json:"requested_identity"` + // RuntimeIdentity identifies the runtime identity. + RuntimeIdentity string `json:"runtime_identity"` + // RuntimeBranch supplies the runtime branch input to the postgresTimedRuntimeDocument contract. + RuntimeBranch string `json:"runtime_branch"` + // FallbackExecuted supplies the fallback executed input to the postgresTimedRuntimeDocument contract. + FallbackExecuted *bool `json:"fallback_executed"` + // RecordCount records the number of record count. + RecordCount int `json:"record_count"` + // Events supplies the events input to the postgresTimedRuntimeDocument contract. + Events []RuntimeReceiptEvent `json:"events"` +} + +// postgresTimedReadAttestor arms a lightweight session-local receipt before +// each timed query and reads it after the duration has been recorded. A +// size-one pool is required so arming, execution, and reading cannot migrate. +type postgresTimedReadAttestor struct { + // pool retains the pool while postgresTimedReadAttestor is assembled or evaluated. + pool *pgxpool.Pool + // requestedIdentity identifies the requested identity. + requestedIdentity string + // runID identifies the run id. + runID string + // activeInvocation retains the active invocation while postgresTimedReadAttestor is assembled or evaluated. + activeInvocation string +} + +// newPostgresTimedReadAttestor constructs postgres timed read attestor. +func newPostgresTimedReadAttestor(pool *pgxpool.Pool, poolSize int, requestedIdentity string) (*postgresTimedReadAttestor, error) { + if pool == nil { + return nil, fmt.Errorf("timed runtime attestation requires a PostgreSQL pool") + } + if poolSize != 1 { + return nil, fmt.Errorf("timed runtime attestation requires pool size 1, got %d", poolSize) + } + if strings.TrimSpace(requestedIdentity) == "" { + return nil, fmt.Errorf("timed runtime attestation requires a requested identity") + } + return &postgresTimedReadAttestor{ + pool: pool, + requestedIdentity: requestedIdentity, + runID: newRunUUID(), + }, nil +} + +// Begin supports benchmark evidence processing for begin. +func (s *postgresTimedReadAttestor) Begin(ctx context.Context, iteration int) error { + if s.activeInvocation != "" { + return fmt.Errorf("runtime attestation %q is still active", s.activeInvocation) + } + s.activeInvocation = fmt.Sprintf("%s-%d", s.runID, iteration) + if _, err := s.pool.Exec(ctx, "select public.begin_traversal_runtime_attestation_v1($1, $2)", s.activeInvocation, s.requestedIdentity); err != nil { + s.activeInvocation = "" + return err + } + return nil +} + +// Complete supports benchmark evidence processing for complete. +func (s *postgresTimedReadAttestor) Complete(ctx context.Context, _ int) (timedReadAttestation, error) { + invocationID := s.activeInvocation + if invocationID == "" { + return timedReadAttestation{}, fmt.Errorf("no runtime attestation is active") + } + s.activeInvocation = "" + var raw string + readErr := s.pool.QueryRow(ctx, "select coalesce(public.read_traversal_runtime_attestation_v1($1)::text, '')", invocationID).Scan(&raw) + _, clearErr := s.pool.Exec(ctx, "select public.clear_traversal_runtime_attestation_v1($1)", invocationID) + if readErr != nil { + return timedReadAttestation{}, readErr + } + if clearErr != nil { + return timedReadAttestation{}, clearErr + } + if strings.TrimSpace(raw) == "" { + return timedReadAttestation{}, fmt.Errorf("runtime invocation %q produced no receipt", invocationID) + } + var document postgresTimedRuntimeDocument + if err := json.Unmarshal([]byte(raw), &document); err != nil { + return timedReadAttestation{}, fmt.Errorf("decode runtime receipt: %w", err) + } + if document.SchemaVersion != 2 || document.InvocationID != invocationID || document.RequestedIdentity != s.requestedIdentity { + return timedReadAttestation{}, fmt.Errorf("runtime receipt identity does not match its armed invocation") + } + if document.RecordCount < 1 || len(document.Events) != document.RecordCount || document.RuntimeIdentity == "" || document.RuntimeBranch == "" || document.FallbackExecuted == nil { + return timedReadAttestation{}, fmt.Errorf("runtime receipt is incomplete or has a broken event chain: %s", raw) + } + for idx, event := range document.Events { + if event.Ordinal != idx+1 || event.RuntimeIdentity == "" || event.RuntimeBranch == "" { + return timedReadAttestation{}, fmt.Errorf("runtime receipt event chain is not contiguous") + } + document.Events[idx].InvocationID = invocationID + } + return timedReadAttestation{ + InvocationID: invocationID, + RequestedIdentity: document.RequestedIdentity, + RuntimeIdentity: document.RuntimeIdentity, + RuntimeBranch: document.RuntimeBranch, + FallbackExecuted: document.FallbackExecuted, + Events: append([]RuntimeReceiptEvent(nil), document.Events...), + }, nil +} diff --git a/cmd/graphbench/postgres_traversal_telemetry.go b/cmd/graphbench/postgres_traversal_telemetry.go new file mode 100644 index 00000000..ead756de --- /dev/null +++ b/cmd/graphbench/postgres_traversal_telemetry.go @@ -0,0 +1,3241 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +const ( + // postgresTraversalTelemetryOff reserves the stable protocol value used to recognize postgres traversal telemetry off across artifacts and executions. + postgresTraversalTelemetryOff = "off" + + // postgresTraversalTelemetrySummary reserves the stable protocol value used to recognize postgres traversal telemetry summary across artifacts and executions. + postgresTraversalTelemetrySummary = "summary" + + // postgresTraversalTelemetryDiagnostic reserves the stable protocol value used to recognize postgres traversal telemetry diagnostic across artifacts and executions. + postgresTraversalTelemetryDiagnostic = "diagnostic" + + // postgresTraversalPlanReplaySource reserves the stable protocol value used to recognize postgres traversal plan replay source across artifacts and executions. + postgresTraversalPlanReplaySource = "postgres_explain_analyze_json_timing_off" + + // postgresBidirectionalDiagnosticSource reserves the stable protocol value used to recognize postgres bidirectional diagnostic source across artifacts and executions. + postgresBidirectionalDiagnosticSource = "public.read_bidirectional_shortest_path_diagnostic_v1" + + // postgresBidirectionalAllShortestDiagnosticSource reserves the stable protocol value used to recognize postgres bidirectional all shortest diagnostic source across artifacts and executions. + postgresBidirectionalAllShortestDiagnosticSource = "public.read_bidirectional_all_shortest_path_diagnostic_v1" + + // postgresA1AllShortestDiagnosticSource identifies the invocation-local A1 + // workspace reader used only by untimed GraphBench diagnostic replays. + postgresA1AllShortestDiagnosticSource = "public.read_all_shortest_paths_a1_diagnostic_v1" +) + +// buildPostgresCaseTraversalTelemetry binds optimizer, emitted SQL, and +// separately replayed plan evidence into one validated traversal identity. +// A nil result means the statement has no unambiguous traversal target. +func buildPostgresCaseTraversalTelemetry( + optimization translate.OptimizationSummary, + metrics PostgresPlanMetrics, + connectionID string, + level TraversalTelemetryLevel, +) (*TraversalExecutionTelemetry, error) { + outcome, ok := singleTraversalOutcome(optimization.TargetOutcomes) + if !ok { + return nil, nil + } + + summary, family, err := traversalSummaryFromOutcome(outcome, metrics) + if err != nil { + return nil, err + } + telemetry := newPostgresTraversalTelemetry(summary, family, metrics, connectionID, level) + if functionBackedTraversal(metrics) && isBidirectionalTelemetryIdentity(summary) { + markTraversalSummaryUnavailable(&telemetry, "outer Function Scan does not expose the invocation-local runtime branch") + } + if telemetry.Diagnostic != nil && functionBackedTraversal(metrics) && (family == TraversalTelemetryFamilySP || family == TraversalTelemetryFamilyASP) { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"outer Function Scan does not expose invocation-local traversal work counters"} + } + if err := telemetry.Validate(); err != nil { + return nil, err + } + return &telemetry, nil +} + +// buildPostgresReferenceTraversalTelemetry binds an explicit reference +// architecture and implementation to its own untimed JSON EXPLAIN replay. +func buildPostgresReferenceTraversalTelemetry( + reference PostgresReferenceResult, + parameters map[string]any, + connectionID string, + level TraversalTelemetryLevel, +) (*TraversalExecutionTelemetry, error) { + if strings.TrimSpace(reference.Architecture) == "" || strings.TrimSpace(reference.ImplementationID) == "" || reference.PostgresMetrics == nil { + return nil, nil + } + if !isTraversalReferenceArchitecture(reference.Architecture) { + return nil, nil + } + + family := traversalFamilyForIdentity(reference.Architecture, "") + fallback := false + overflow := false + planned := []string{reference.Architecture} + fallbackIdentity := bidirectionalFallbackIdentity(reference.Architecture) + if fallbackIdentity != "" && fallbackIdentity != reference.Architecture { + planned = append(planned, fallbackIdentity) + } + summary := TraversalExecutionSummary{ + RequestedIdentity: reference.Architecture, + PlannedIdentities: planned, + EmittedIdentity: reference.ImplementationID, + RuntimeIdentity: reference.Architecture, + AppliedIdentity: reference.Architecture, + SelectorVersion: "explicit-reference-v1", + SchedulerVersion: schedulerForIdentity(reference.Architecture, ""), + ObservationMode: reference.ObservationShape, + Caps: referenceTraversalCaps(parameters), + RuntimeOutcomeAvailable: traversalTelemetryPointer(true), + RuntimeBranch: "explicit_reference", + Overflow: &overflow, + FallbackExecuted: &fallback, + Provenance: map[string]string{ + "requested_identity": "reference.architecture", + "planned_identities": "reference.architecture", + "emitted_identity": "reference.implementation_id", + "runtime_identity": postgresTraversalPlanReplaySource + ".reference_statement", + "applied_identity": "reference.architecture", + "selector_version": "reference.explicit_selection", + "scheduler_version": "reference.architecture", + "observation_mode": "reference.observation_shape", + "runtime_outcome_available": postgresTraversalPlanReplaySource + ".reference_statement", + "runtime_branch": postgresTraversalPlanReplaySource + ".reference_statement", + "overflow": postgresTraversalPlanReplaySource + ".visible_guards", + "fallback_executed": postgresTraversalPlanReplaySource + ".visible_branches", + }, + } + for name := range summary.Caps { + summary.Provenance["caps."+name] = "reference.parameters." + traversalCapParameterName(name) + } + + telemetry := newPostgresTraversalTelemetry(summary, family, *reference.PostgresMetrics, connectionID, level) + if functionBackedTraversal(*reference.PostgresMetrics) && isBidirectionalTelemetryIdentity(summary) { + markTraversalSummaryUnavailable(&telemetry, "outer Function Scan does not expose the invocation-local runtime branch") + } + if functionBackedTraversal(*reference.PostgresMetrics) && (family == TraversalTelemetryFamilySP || family == TraversalTelemetryFamilyASP) { + if telemetry.Diagnostic != nil { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"outer Function Scan does not expose invocation-local traversal work counters"} + } + } + if err := telemetry.Validate(); err != nil { + return nil, err + } + return &telemetry, nil +} + +// isTraversalReferenceArchitecture reports whether is traversal reference architecture. +func isTraversalReferenceArchitecture(identity string) bool { + return strings.HasPrefix(identity, "SP-") || + strings.HasPrefix(identity, "ASP-") || + strings.HasPrefix(identity, "EXPANSION-") || + strings.HasPrefix(identity, "EXPAND-INTO-") || + strings.HasPrefix(identity, "MAT-") || + identity == "hydration" +} + +// newPostgresTraversalTelemetry records either an optimizer/plan-derived +// summary or partial SQL-visible diagnostic evidence. It never converts +// absent executor counters into fabricated zero values. +func newPostgresTraversalTelemetry( + summary TraversalExecutionSummary, + family TraversalTelemetryFamily, + metrics PostgresPlanMetrics, + connectionID string, + level TraversalTelemetryLevel, +) TraversalExecutionTelemetry { + telemetry := TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: level, + Summary: summary, + } + if level == TraversalTelemetryLevelDiagnostic { + telemetry.Diagnostic = &TraversalExecutionDiagnostic{ + InvocationID: newRunUUID(), + ConnectionID: connectionID, + TimedSample: traversalTelemetryPointer(false), + RequiredFamilies: traversalRequiredFamilies(summary, family), + CounterStatus: TraversalTelemetryCounterStatusPlanPartial, + IncompleteReasons: []string{ + "JSON EXPLAIN exposes SQL plan work but not every qualification counter in the declared family", + }, + PlanReplay: postgresTraversalPlanReplay(metrics), + Provenance: map[string]string{}, + } + } + return telemetry +} + +// singleTraversalOutcome supports benchmark evidence processing for single traversal outcome. +func singleTraversalOutcome(outcomes []translate.TargetLoweringOutcome) (translate.TargetLoweringOutcome, bool) { + var shortest, expansion []translate.TargetLoweringOutcome + for _, outcome := range outcomes { + if outcome.TargetKind != "" && outcome.TargetKind != "traversal" { + continue + } + if outcome.Family == "SP" || outcome.Family == "ASP" { + shortest = append(shortest, outcome) + } else if strings.Contains(outcome.Family, "expansion") { + expansion = append(expansion, outcome) + } + } + // Shortest-path execution is the public traversal boundary even when its + // underlying variable step also produced ordinary-expansion analysis. + // Analysis-only endpoint/predicate outcomes must never make telemetry + // ambiguous or replace the executor identity. + if len(shortest) == 1 { + return shortest[0], true + } + if len(shortest) != 0 || len(expansion) != 1 { + return translate.TargetLoweringOutcome{}, false + } + return expansion[0], true +} + +// traversalSummaryFromOutcome supports benchmark evidence processing for traversal summary from outcome. +func traversalSummaryFromOutcome(outcome translate.TargetLoweringOutcome, metrics PostgresPlanMetrics) (TraversalExecutionSummary, TraversalTelemetryFamily, error) { + requested := outcome.Candidate + if requested == "" { + requested = outcome.Selected + } + applied := outcome.Applied + if applied == "" { + applied = outcome.Fallback + } + if requested == "" || applied == "" { + return TraversalExecutionSummary{}, "", fmt.Errorf("traversal target outcome has no requested or applied identity") + } + + planned := append([]string(nil), outcome.PlannedCandidates...) + for _, identity := range []string{requested, applied, outcome.Fallback} { + if identity != "" && !slices.Contains(planned, identity) { + planned = append(planned, identity) + } + } + emitted := outcome.EmittedPolicy + if emitted == "" { + if len(outcome.EmittedCandidates) > 1 { + emitted = strings.Join(outcome.EmittedCandidates, "+") + } else if len(outcome.EmittedCandidates) == 1 { + emitted = outcome.EmittedCandidates[0] + } else { + emitted = applied + } + } + + runtimeIdentity, runtimeBranch, fallbackExecuted, overflow := runtimeTraversalIdentity(outcome, metrics, requested, applied) + wouldSelectIdentity := "" + if outcome.SelectionMode == "shadow_tool" { + runtimeIdentity = applied + runtimeBranch = "shadow_incumbent" + fallbackExecuted = false + overflow = metrics.EndpointGuardOverflow || metrics.StateGuardOverflow || + orientationPlanOverflow(outcome, postgresTraversalPlanReplay(metrics)) + wouldSelectIdentity = shadowWouldSelectIdentity(outcome, metrics) + } + if outcome.EmittedPolicy != "" { + // Applied is a runtime fact for a same-statement policy; the translator + // can report emitted arms but cannot know which branch executed. + applied = runtimeIdentity + } + if runtimeIdentity != "" && !slices.Contains(planned, runtimeIdentity) { + planned = append(planned, runtimeIdentity) + } + if fallbackExecuted && outcome.Fallback != "" { + applied = outcome.Fallback + runtimeIdentity = outcome.Fallback + } + selectorVersion := outcome.SelectorVersion + if selectorVersion == "" { + selectorVersion = "static-lowering-v1" + } + summary := TraversalExecutionSummary{ + RequestedIdentity: requested, + PlannedIdentities: planned, + EmittedIdentity: emitted, + RuntimeIdentity: runtimeIdentity, + AppliedIdentity: applied, + SelectorVersion: selectorVersion, + SchedulerVersion: schedulerForIdentity(runtimeIdentity, outcome.Scheduler), + ExecutionBoundary: outcome.ExecutionBoundary, + ObservationMode: outcome.ObservationMode, + Caps: outcomeTraversalCaps(outcome), + RuntimeOutcomeAvailable: traversalTelemetryPointer(true), + RuntimeBranch: runtimeBranch, + Overflow: &overflow, + FallbackExecuted: &fallbackExecuted, + WouldSelectIdentity: wouldSelectIdentity, + Provenance: map[string]string{ + "requested_identity": "optimizer.target_outcome.candidate_or_selected", + "planned_identities": "optimizer.target_outcome.planned_candidates", + "emitted_identity": "translator.target_outcome.emitted_policy_or_candidates", + "runtime_identity": postgresTraversalPlanReplaySource + ".visible_branch_and_translator_applied", + "applied_identity": "translator.target_outcome.applied_or_fallback", + "execution_boundary": "optimizer.target_outcome.execution_boundary", + "selector_version": "optimizer.target_outcome.selector_version", + "scheduler_version": "optimizer.target_outcome.scheduler", + "observation_mode": "optimizer.target_outcome.observation_mode", + "runtime_outcome_available": postgresTraversalPlanReplaySource + ".visible_branch", + "runtime_branch": postgresTraversalPlanReplaySource + ".visible_branch", + "overflow": postgresTraversalPlanReplaySource + ".visible_guard", + "fallback_executed": postgresTraversalPlanReplaySource + ".visible_branch", + }, + } + if wouldSelectIdentity != "" { + summary.Provenance["would_select_identity"] = postgresTraversalPlanReplaySource + ".orientation_shadow_marker_rows" + } + for name := range summary.Caps { + summary.Provenance["caps."+name] = "optimizer.target_outcome." + traversalCapOutcomeField(name) + } + if fallbackExecuted { + summary.FallbackIdentity = applied + summary.Provenance["fallback_identity"] = "optimizer.target_outcome.fallback" + } + family := traversalFamilyForIdentity(runtimeIdentity, outcome.Family) + if isOrientationProbePolicy(outcome.EmittedPolicy) || + outcome.EmittedPolicy == string(optimize.ExpansionSearchPolicyEndpointGuardV1) { + family = TraversalTelemetryFamilyOrientation + } else if isSuffixReverseGuardPolicy(outcome.EmittedPolicy) || isSuffixReverseRetryPolicy(outcome.EmittedPolicy) { + family = TraversalTelemetryFamilySuffixGuard + } else if outcome.SelectorVersion == optimize.ExpansionSearchSelectorSuffixRouteComponentV1 { + family = TraversalTelemetryFamilySuffixComponent + } + if runtimeIdentity == "" { + telemetry := TraversalExecutionTelemetry{Summary: summary} + markTraversalSummaryUnavailable(&telemetry, "exact executed traversal marker is unavailable") + summary = telemetry.Summary + } + return summary, family, nil +} + +// traversalCapParameterName supports benchmark evidence processing for traversal cap parameter name. +func traversalCapParameterName(counterName string) string { + switch counterName { + case "state_rows": + return "state_limit" + case "frontier_rows", "queue_rows": + return "frontier_limit" + case "predecessor_rows": + return "predecessor_limit" + case "output_rows": + return "enumeration_limit" + case "output_bytes": + return "output_bytes_limit" + default: + return counterName + } +} + +// traversalCapOutcomeField supports benchmark evidence processing for traversal cap outcome field. +func traversalCapOutcomeField(counterName string) string { + switch counterName { + case "state_rows": + return "state_limit" + case "frontier_rows", "queue_rows": + return "frontier_limit" + case "predecessor_rows": + return "predecessor_limit" + case "endpoint_probe_rows": + return "endpoint_limit" + case "output_rows": + return "enumeration_limit" + case "output_bytes": + return "output_bytes_limit" + default: + return counterName + } +} + +// shadowWouldSelectIdentity derives the stable identity used to compare shadow would select. +func shadowWouldSelectIdentity(outcome translate.TargetLoweringOutcome, metrics PostgresPlanMetrics) string { + plan := postgresTraversalPlanReplay(metrics) + if plan.Counters["orientation_shadow_reverse_rows"] > 0 { + return outcome.Candidate + } + if plan.Counters["orientation_shadow_forward_rows"] > 0 { + if outcome.Fallback != "" { + return outcome.Fallback + } + return outcome.Applied + } + return "" +} + +// runtimeTraversalIdentity derives the stable identity used to compare runtime traversal. +func runtimeTraversalIdentity(outcome translate.TargetLoweringOutcome, metrics PostgresPlanMetrics, requested, applied string) (identity, branch string, fallback, overflow bool) { + identity, branch = applied, "selected" + if outcome.EmittedPolicy == "" && outcome.Fallback != "" && requested != applied && applied == outcome.Fallback { + return applied, "compile_time_fallback", true, false + } + overflow = metrics.EndpointGuardOverflow || metrics.StateGuardOverflow + if metrics.ExpansionFallbackExecuted { + identity = outcome.Fallback + if identity == "" { + identity = applied + } + return identity, "runtime_fallback", true, overflow + } + + plan := postgresTraversalPlanReplay(metrics) + if isSuffixReverseRetryPolicy(outcome.EmittedPolicy) { + candidateRows, candidatePresent := plan.Counters["suffix_guard_candidate_marker_rows"] + suffixOverflow, stateOverflow, overflowAvailable := suffixGuardPlanOverflows(outcome, plan) + if !candidatePresent || !overflowAvailable { + return "", "runtime_outcome_unavailable", false, false + } + if candidateRows == 1 && !suffixOverflow && !stateOverflow { + return string(optimize.ExpansionSearchSuffixSeededReverse), "reverse_complete", false, false + } + if candidateRows == 0 && suffixOverflow { + return string(optimize.ExpansionSearchSuffixSeededReverse), "forward_retry_suffix_overflow", false, true + } + if candidateRows == 0 && stateOverflow { + return string(optimize.ExpansionSearchSuffixSeededReverse), "forward_retry_state_overflow", false, true + } + return "", "runtime_outcome_unavailable", false, suffixOverflow || stateOverflow + } + if outcome.EmittedPolicy == optimize.ShortestPathPolicyASPI1GuardedV1 { + candidateRows, candidatePresent := plan.Counters["asp_i1_candidate_marker_rows"] + fallbackRows, fallbackPresent := plan.Counters["asp_i1_fallback_marker_rows"] + overflow = aspI1PlanOverflow(outcome, plan) + if !candidatePresent || !fallbackPresent { + return "", "runtime_outcome_unavailable", false, overflow + } + if candidateRows == 1 && fallbackRows == 0 { + return string(optimize.ShortestPathExecutorASPI1DAG), "inline_predecessor_dag", false, false + } + if fallbackRows == 1 && candidateRows == 0 { + return string(optimize.ShortestPathExecutorASPA1DAG), "exact_a1_fallback", true, true + } + return "", "runtime_outcome_unavailable", false, overflow + } + if outcome.EmittedPolicy == optimize.ShortestPathPolicyI1CanonicalGuardedV1 { + candidateRows, candidatePresent := plan.Counters["asp_i1_candidate_marker_rows"] + fallbackRows, fallbackPresent := plan.Counters["asp_i1_fallback_marker_rows"] + overflow = aspI1PlanOverflow(outcome, plan) + if !candidatePresent || !fallbackPresent { + return "", "runtime_outcome_unavailable", false, overflow + } + if candidateRows == 1 && fallbackRows == 0 { + outputRows, outputPresent := plan.Counters["asp_i1_output_rows"] + if !outputPresent { + return "", "runtime_outcome_unavailable", false, false + } + branch := "inline_canonical_witness" + if outputRows == 0 { + branch = "inline_canonical_no_path" + } + return string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), branch, false, false + } + if fallbackRows == 1 && candidateRows == 0 { + return string(optimize.ShortestPathExecutorS4CanonicalWitness), "exact_s4_fallback", true, true + } + return "", "runtime_outcome_unavailable", false, overflow + } + if outcome.EmittedPolicy == optimize.ShortestPathPolicyI2DistanceGuardedV1 || outcome.EmittedPolicy == optimize.ShortestPathPolicyI2DistanceGuardedV2 { + candidateRows, candidatePresent := plan.Counters["sp_i2_candidate_marker_rows"] + fallbackRows, fallbackPresent := plan.Counters["sp_i2_fallback_marker_rows"] + if !candidatePresent || !fallbackPresent { + return "", "runtime_outcome_unavailable", false, false + } + if candidateRows == 1 && fallbackRows == 0 { + outputRows, outputPresent := plan.Counters["sp_i2_output_rows"] + if !outputPresent { + return "", "runtime_outcome_unavailable", false, false + } + branch := "inline_canonical_distance" + if plan.Counters["sp_i2_direct_rows"] == 1 { + branch = "inline_direct_distance" + } else if outputRows == 0 { + branch = "inline_canonical_distance_no_path" + } + runtimeIdentity := outcome.Applied + if runtimeIdentity == "" { + runtimeIdentity = outcome.Selected + } + return runtimeIdentity, branch, false, false + } + if fallbackRows == 1 && candidateRows == 0 { + return string(optimize.ShortestPathExecutorS4CanonicalDistance), "exact_s4_distance_fallback", true, true + } + return "", "runtime_outcome_unavailable", false, overflow + } + if isSuffixReverseGuardPolicy(outcome.EmittedPolicy) { + candidateRows, candidatePresent := plan.Counters["suffix_guard_candidate_marker_rows"] + fallbackRows, fallbackPresent := plan.Counters["suffix_guard_fallback_marker_rows"] + suffixOverflow, stateOverflow, overflowAvailable := suffixGuardPlanOverflows(outcome, plan) + overflow = suffixOverflow || stateOverflow + if !candidatePresent || !fallbackPresent || !overflowAvailable { + return "", "runtime_outcome_unavailable", false, false + } + if candidateRows == 1 && fallbackRows == 0 { + if overflow { + return "", "runtime_outcome_unavailable", false, true + } + return string(optimize.ExpansionSearchSuffixSeededReverse), "suffix_seeded_reverse", false, false + } + if fallbackRows == 1 && candidateRows == 0 { + if suffixOverflow { + return string(optimize.ExpansionSearchStepwiseForward), "exact_forward_suffix_overflow", true, true + } + if stateOverflow { + return string(optimize.ExpansionSearchStepwiseForward), "exact_forward_state_overflow", true, true + } + return "", "runtime_outcome_unavailable", false, false + } + return "", "runtime_outcome_unavailable", false, overflow + } + if outcome.EmittedPolicy != "" { + candidateRows := plan.Counters["orientation_executed_candidate_rows"] + incumbentRows := plan.Counters["orientation_executed_incumbent_rows"] + overflow = overflow || orientationPlanOverflow(outcome, plan) + if candidateRows == 1 && incumbentRows == 0 && outcome.Candidate != "" { + if overflow { + return "", "runtime_outcome_unavailable", false, true + } + return outcome.Candidate, "suffix_seeded_reverse", false, false + } + if incumbentRows == 1 && candidateRows == 0 && outcome.Fallback != "" { + return outcome.Fallback, "exact_forward_incumbent", overflow, overflow + } + return "", "runtime_outcome_unavailable", false, overflow + } + return identity, branch, false, overflow +} + +// suffixGuardPlanOverflows derives both immutable cap+1 admission outcomes +// from named statement relations. Both counters and both configured caps must +// be present before marker rows can assert any runtime outcome. +func suffixGuardPlanOverflows(outcome translate.TargetLoweringOutcome, plan *TraversalPlanReplayEvidence) (suffix, state, available bool) { + if plan == nil || outcome.StateLimit <= 0 || outcome.ProbeCaps == nil || outcome.ProbeCaps.ReverseSeedRowLimit <= 0 { + return false, false, false + } + stateRows, statePresent := plan.Counters["suffix_guard_state_rows"] + suffixRows, suffixPresent := plan.Counters["suffix_guard_suffix_rows"] + if !statePresent || !suffixPresent { + return false, false, false + } + return suffixRows > outcome.ProbeCaps.ReverseSeedRowLimit, stateRows > outcome.StateLimit, true +} + +// aspI1PlanOverflow supports benchmark evidence processing for asp i1 plan overflow. +func aspI1PlanOverflow(outcome translate.TargetLoweringOutcome, plan *TraversalPlanReplayEvidence) bool { + for counter, limit := range map[string]int64{ + "asp_i1_distance_rows": outcome.StateLimit, + "asp_i1_predecessor_rows": outcome.PredecessorLimit, + "asp_i1_enumeration_rows": outcome.EnumerationLimit, + } { + if limit > 0 && plan.Counters[counter] > limit { + return true + } + } + return false +} + +// orientationPlanOverflow supports benchmark evidence processing for orientation plan overflow. +func orientationPlanOverflow(outcome translate.TargetLoweringOutcome, plan *TraversalPlanReplayEvidence) bool { + if outcome.StateLimit > 0 && plan.Counters["orientation_state_rows"] > outcome.StateLimit { + return true + } + if outcome.ProbeCaps == nil { + return false + } + for counter, limit := range map[string]int64{ + "orientation_root_probe_rows": outcome.ProbeCaps.RootRowLimit, + "orientation_suffix_probe_rows": outcome.ProbeCaps.ReverseSeedRowLimit, + "orientation_forward_degree_rows": outcome.ProbeCaps.DirectionalDegreeRowLimit, + "orientation_reverse_degree_rows": outcome.ProbeCaps.DirectionalDegreeRowLimit, + } { + if limit > 0 && plan.Counters[counter] > limit { + return true + } + } + return false +} + +// outcomeTraversalCaps returns the resource limits enforced for outcome traversal. +func outcomeTraversalCaps(outcome translate.TargetLoweringOutcome) map[string]int64 { + caps := map[string]int64{} + if outcome.StateLimit > 0 { + caps["state_rows"] = outcome.StateLimit + } + if outcome.FrontierLimit > 0 { + caps["frontier_rows"] = outcome.FrontierLimit + caps["queue_rows"] = outcome.FrontierLimit + } + if outcome.PredecessorLimit > 0 { + caps["predecessor_rows"] = outcome.PredecessorLimit + } + if outcome.EnumerationLimit > 0 { + caps["output_rows"] = outcome.EnumerationLimit + } + if outcome.OutputBytesLimit > 0 { + caps["output_bytes"] = outcome.OutputBytesLimit + } + if outcome.EndpointLimit > 0 { + caps["endpoint_probe_rows"] = outcome.EndpointLimit + } + if outcome.ProbeCaps != nil { + if outcome.ProbeCaps.RootRowLimit > 0 { + caps["forward_seed_rows"] = outcome.ProbeCaps.RootRowLimit + } + if outcome.ProbeCaps.ReverseSeedRowLimit > 0 { + if isSuffixReverseGuardPolicy(outcome.EmittedPolicy) || isSuffixReverseGuardPolicy(outcome.PlannedPolicy) || + isSuffixReverseRetryPolicy(outcome.EmittedPolicy) || isSuffixReverseRetryPolicy(outcome.PlannedPolicy) { + caps["suffix_rows"] = outcome.ProbeCaps.ReverseSeedRowLimit + } else { + caps["reverse_seed_rows"] = outcome.ProbeCaps.ReverseSeedRowLimit + } + } + if outcome.ProbeCaps.DirectionalDegreeRowLimit > 0 { + caps["directional_degree_rows"] = outcome.ProbeCaps.DirectionalDegreeRowLimit + } + if outcome.ProbeCaps.SurvivalRowLimit > 0 { + caps["survival_rows"] = outcome.ProbeCaps.SurvivalRowLimit + } + } + if outcome.Admission != nil { + if outcome.Admission.OutputRowLimit > 0 { + caps["output_rows"] = outcome.Admission.OutputRowLimit + } + if outcome.Admission.OutputBytesLimit > 0 { + caps["output_bytes"] = outcome.Admission.OutputBytesLimit + } + } + return caps +} + +// referenceTraversalCaps returns the resource limits enforced for reference traversal. +func referenceTraversalCaps(parameters map[string]any) map[string]int64 { + caps := map[string]int64{} + for _, name := range []string{"state_limit", "frontier_limit", "predecessor_limit", "enumeration_limit", "output_bytes_limit", "output_limit"} { + if value, ok := integerParameter(parameters[name]); ok && value > 0 { + counterName := strings.TrimSuffix(name, "_limit") + "_rows" + switch name { + case "enumeration_limit": + counterName = "output_rows" + case "output_bytes_limit": + counterName = "output_bytes" + } + caps[counterName] = value + if name == "frontier_limit" { + caps["queue_rows"] = value + } + } + } + return caps +} + +// integerParameter supports benchmark evidence processing for integer parameter. +func integerParameter(value any) (int64, bool) { + switch typed := value.(type) { + case int: + return int64(typed), true + case int32: + return int64(typed), true + case int64: + return typed, true + default: + return 0, false + } +} + +// traversalFamilyForIdentity derives the stable identity used to compare traversal family for. +func traversalFamilyForIdentity(identity, family string) TraversalTelemetryFamily { + if strings.HasPrefix(identity, "ASP-") || family == "ASP" { + return TraversalTelemetryFamilyASP + } + if strings.HasPrefix(identity, "SP-") || family == "SP" { + return TraversalTelemetryFamilySP + } + if isSuffixReverseGuardPolicy(identity) || isSuffixReverseRetryPolicy(identity) { + return TraversalTelemetryFamilySuffixGuard + } + if isOrientationProbePolicy(identity) || strings.Contains(identity, "ORIENTATION") { + return TraversalTelemetryFamilyOrientation + } + if strings.HasPrefix(identity, "MAT-") { + return TraversalTelemetryFamilyHydration + } + return TraversalTelemetryFamilyOrdinary +} + +// traversalRequiredFamilies derives the complete observation contract from +// the emitted policy and public result shape. Families are deliberately kept +// separate so search counters cannot stand in for hydration or workspace +// evidence. +func traversalRequiredFamilies(summary TraversalExecutionSummary, base TraversalTelemetryFamily) []TraversalTelemetryFamily { + var required []TraversalTelemetryFamily + add := func(family TraversalTelemetryFamily) { + if family != "" && !slices.Contains(required, family) { + required = append(required, family) + } + } + + identity := summary.RuntimeIdentity + if identity == "" { + identity = summary.RequestedIdentity + } + if summary.SelectorVersion == optimize.ExpansionSearchSelectorSuffixRouteComponentV1 { + add(TraversalTelemetryFamilySuffixComponent) + } else if isSuffixReverseGuardPolicy(summary.EmittedIdentity) || isSuffixReverseGuardPolicy(summary.SelectorVersion) || + isSuffixReverseRetryPolicy(summary.EmittedIdentity) || isSuffixReverseRetryPolicy(summary.SelectorVersion) { + add(TraversalTelemetryFamilySuffixGuard) + add(TraversalTelemetryFamilyOrdinary) + if observationRequiresHydration(summary.ObservationMode) { + add(TraversalTelemetryFamilyHydration) + } + } else if isOrientationProbePolicy(summary.EmittedIdentity) || isOrientationProbePolicy(summary.SelectorVersion) { + add(TraversalTelemetryFamilyOrientation) + add(TraversalTelemetryFamilyOrdinary) + if observationRequiresHydration(summary.ObservationMode) { + add(TraversalTelemetryFamilyHydration) + } + } else { + add(base) + } + if strings.HasPrefix(identity, "ASP-") || base == TraversalTelemetryFamilyASP { + add(TraversalTelemetryFamilyHydration) + } + if strings.Contains(identity, "WE+MAT") || strings.Contains(summary.RequestedIdentity, "WE+MAT") || + strings.HasPrefix(identity, "MAT-") || + (observationRequiresHydration(summary.ObservationMode) && + (strings.HasPrefix(identity, "SP-") || strings.HasPrefix(summary.RequestedIdentity, "SP-"))) { + add(TraversalTelemetryFamilyHydration) + } + if isBidirectionalSPIdentity(identity) || isBidirectionalASPIdentity(identity) || + isBidirectionalSPIdentity(summary.RequestedIdentity) || isBidirectionalASPIdentity(summary.RequestedIdentity) { + add(TraversalTelemetryFamilyWorkspace) + } + return required +} + +// observationRequiresHydration supports benchmark evidence processing for observation requires hydration. +func observationRequiresHydration(observation string) bool { + normalized := strings.ToLower(strings.TrimSpace(observation)) + return normalized == "one_path" || normalized == "all_paths" || normalized == "full_path" || + strings.Contains(normalized, "complete path") || strings.Contains(normalized, "all-shortest path") +} + +// isBidirectionalTelemetryIdentity reports whether is bidirectional telemetry identity. +func isBidirectionalTelemetryIdentity(summary TraversalExecutionSummary) bool { + return isBidirectionalSPIdentity(summary.RuntimeIdentity) || isBidirectionalASPIdentity(summary.RuntimeIdentity) || + isBidirectionalSPIdentity(summary.RequestedIdentity) || isBidirectionalASPIdentity(summary.RequestedIdentity) +} + +// isA1AllShortestTelemetryIdentity reports whether the exact A1 stored helper +// needs its separate single-ended diagnostic reader. +func isA1AllShortestTelemetryIdentity(summary TraversalExecutionSummary) bool { + identity := string(optimize.ShortestPathExecutorASPA1DAG) + return summary.RuntimeIdentity == identity || summary.RequestedIdentity == identity +} + +// bidirectionalTelemetryIdentity derives the stable identity used to compare bidirectional telemetry. +func bidirectionalTelemetryIdentity(summary TraversalExecutionSummary) string { + for _, identity := range []string{summary.RuntimeIdentity, summary.RequestedIdentity} { + if isBidirectionalSPIdentity(identity) || isBidirectionalASPIdentity(identity) { + return identity + } + } + return "" +} + +// schedulerForIdentity derives the stable identity used to compare scheduler for. +func schedulerForIdentity(identity, scheduler string) string { + if scheduler != "" { + return scheduler + } + switch { + case strings.Contains(identity, "ALT-NODE"): + return "strict_alternating_node" + case strings.Contains(identity, "MIN-LEVEL"): + return "smaller_current_level" + case strings.HasPrefix(identity, "SP-"), strings.HasPrefix(identity, "ASP-"): + return "single_ended_level" + default: + return "not_applicable" + } +} + +// functionBackedTraversal supports benchmark evidence processing for function backed traversal. +func functionBackedTraversal(metrics PostgresPlanMetrics) bool { + for _, node := range metrics.PlanNodes { + if node.NodeType == "Function Scan" && strings.TrimSpace(node.FunctionName) != "" { + return true + } + } + return false +} + +// postgresTraversalPlanReplay supports benchmark evidence processing for postgres traversal plan replay. +func postgresTraversalPlanReplay(metrics PostgresPlanMetrics) *TraversalPlanReplayEvidence { + replay := &TraversalPlanReplayEvidence{ + Source: postgresTraversalPlanReplaySource, + Counters: map[string]int64{"plan_nodes": int64(len(metrics.PlanNodes))}, + Flags: map[string]bool{}, + Provenance: map[string]string{"counters.plan_nodes": "postgres_metrics.plan_nodes"}, + } + addCounter := func(name string, value int64, metricName string) { + if provenance := metrics.Provenance[metricName]; provenance != "" { + replay.Counters[name] = value + replay.Provenance["counters."+name] = "postgres_metrics." + metricName + ":" + provenance + } + } + addCounter("root_rows", metrics.RootRows, "root_rows") + addCounter("recursive_rows", metrics.RecursiveRows, "recursive_rows") + addCounter("recursive_loops", metrics.RecursiveLoops, "recursive_loops") + addCounter("frontier_rows", metrics.FrontierRows, "frontier_rows") + addCounter("witness_rows", metrics.WitnessRows, "witness_rows") + addCounter("meeting_rows", metrics.MeetingRows, "meeting_rows") + addCounter("hydration_rows", metrics.HydrationRows, "hydration_rows") + addCounter("forward_edge_probe_loops", metrics.ForwardEdgeProbes, "forward_edge_probes") + addCounter("reverse_edge_probe_loops", metrics.ReverseEdgeProbes, "reverse_edge_probes") + addCounter("endpoint_probe_rows", metrics.EndpointProbeRows, "endpoint_probe_rows") + addCounter("reverse_state_probe_rows", metrics.ReverseStateProbeRows, "reverse_state_probe_rows") + addFlag := func(name string, value bool, metricName string) { + if provenance := metrics.Provenance[metricName]; provenance != "" { + replay.Flags[name] = value + replay.Provenance["flags."+name] = "postgres_metrics." + metricName + ":" + provenance + } + } + addFlag("endpoint_guard_overflow", metrics.EndpointGuardOverflow, "endpoint_probe_rows") + addFlag("state_guard_overflow", metrics.StateGuardOverflow, "reverse_state_probe_rows") + addFlag("fallback_executed", metrics.ExpansionFallbackExecuted, "expansion_fallback_executed") + + inlineCTECounters := map[string]string{ + "asp_i1_distance_bounded": "asp_i1_distance_rows", + "asp_i1_predecessor_bounded": "asp_i1_predecessor_rows", + "asp_i1_paths_bounded": "asp_i1_enumeration_rows", + "asp_i1_shortest": "asp_i1_output_rows", + "asp_i1_candidate_marker": "asp_i1_candidate_marker_rows", + "asp_i1_fallback_marker": "asp_i1_fallback_marker_rows", + "asp_i1_candidate_rows": "asp_i1_candidate_branch_rows", + "asp_i1_fallback_rows": "asp_i1_fallback_branch_rows", + "sp_i2_distance_bounded": "sp_i2_distance_rows", + "sp_i2_v2_direct": "sp_i2_direct_rows", + "sp_i2_admission": "sp_i2_admission_rows", + "sp_i2_target": "sp_i2_target_rows", + "sp_i2_candidate_marker": "sp_i2_candidate_marker_rows", + "sp_i2_fallback_marker": "sp_i2_fallback_marker_rows", + "sp_i2_candidate_rows": "sp_i2_candidate_branch_rows", + "sp_i2_fallback_rows": "sp_i2_fallback_branch_rows", + } + inlineCTEBodies := map[string][]PostgresPlanNodeMetric{} + suffixGuardCTEBodies := map[string][]PostgresPlanNodeMetric{} + suffixComponentCTEBodies := map[string][]PostgresPlanNodeMetric{} + for _, node := range metrics.PlanNodes { + rows := node.ActualRows * node.ActualLoops + for cteName := range inlineCTECounters { + if inlinePredecessorCTEBody(node, cteName) { + inlineCTEBodies[cteName] = append(inlineCTEBodies[cteName], node) + } + } + for suffix, name := range map[string]string{ + "orientation_root_probe": "orientation_root_probe_rows", + "orientation_suffix_probe": "orientation_suffix_probe_rows", + "orientation_boundaries": "orientation_boundary_rows", + "orientation_forward_degree_probe": "orientation_forward_degree_rows", + "orientation_reverse_degree_probe": "orientation_reverse_degree_rows", + "orientation_states": "orientation_state_rows", + } { + if orientationCTEBody(node, suffix) { + measuredRows := rows + if strings.HasSuffix(suffix, "_degree_probe") { + if boundedRows, ok := orientationBoundedDegreeRows(metrics.PlanNodes, node); ok { + measuredRows = boundedRows + } + } + replay.Counters[name] = measuredRows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix := range map[string]struct{}{ + "suffix_seeded_suffix": {}, + "suffix_seeded_boundaries": {}, + "suffix_seeded_reverse": {}, + "suffix_seeded_component_receipt": {}, + } { + if namedCTEBody(node, suffix) { + suffixComponentCTEBodies[suffix] = append(suffixComponentCTEBodies[suffix], node) + } + } + for suffix, name := range map[string]string{ + "suffix_guard_root_presence": "suffix_guard_root_presence_rows", + "suffix_guard_suffix_probe": "suffix_guard_suffix_rows", + "suffix_guard_boundaries": "suffix_guard_boundary_rows", + "suffix_guard_states": "suffix_guard_state_rows", + "suffix_guard_candidate_marker": "suffix_guard_candidate_marker_rows", + "suffix_guard_fallback_marker": "suffix_guard_fallback_marker_rows", + "suffix_guard_candidate_body": "suffix_guard_candidate_branch_rows", + "suffix_guard_fallback_body": "suffix_guard_fallback_branch_rows", + } { + if namedCTEBody(node, suffix) { + replay.Counters[name] = rows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.exact_suffix_guard_cte_materialization_body" + suffixGuardCTEBodies[suffix] = append(suffixGuardCTEBodies[suffix], node) + } + } + for suffix, name := range map[string]string{ + "orientation_shadow_forward": "orientation_shadow_forward_rows", + "orientation_shadow_reverse": "orientation_shadow_reverse_rows", + "orientation_shadow_selection": "orientation_shadow_selection_rows", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = rows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_executed_candidate": "orientation_executed_candidate_rows", + "orientation_executed_incumbent": "orientation_executed_incumbent_rows", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = rows + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_root_probe": "orientation_root_probe_loops", + "orientation_suffix_probe": "orientation_suffix_probe_loops", + "orientation_boundaries": "orientation_boundary_probe_loops", + "orientation_forward_degree_probe": "orientation_forward_degree_probe_loops", + "orientation_reverse_degree_probe": "orientation_reverse_degree_probe_loops", + "orientation_decision": "orientation_decision_loops", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = node.ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + for suffix, name := range map[string]string{ + "orientation_reverse": "orientation_candidate_branch_loops", + "orientation_incumbent": "orientation_incumbent_branch_loops", + } { + if orientationCTEBody(node, suffix) { + replay.Counters[name] = node.ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.measured_plan_json" + } + } + if node.NodeType == "Function Scan" && node.FunctionName != "" { + replay.Counters["function_scan_loops"] += node.ActualLoops + replay.Provenance["counters.function_scan_loops"] = "postgres_metrics.plan_nodes.function_scan_actual_loops" + } + } + for cteName, counterName := range inlineCTECounters { + bodies := inlineCTEBodies[cteName] + if len(bodies) != 1 { + continue + } + body := bodies[0] + replay.Counters[counterName] = body.ActualRows * body.ActualLoops + replay.Provenance["counters."+counterName] = "postgres_metrics.plan_nodes.exact_cte_materialization_body" + if cteName == "sp_i2_admission" { + replay.Counters["sp_i2_admission_loops"] = body.ActualLoops + replay.Provenance["counters.sp_i2_admission_loops"] = "postgres_metrics.plan_nodes.exact_cte_materialization_body" + } + if cteName == "sp_i2_v2_direct" { + replay.Counters["sp_i2_direct_loops"] = body.ActualLoops + replay.Provenance["counters.sp_i2_direct_loops"] = "postgres_metrics.plan_nodes.exact_cte_materialization_body" + } + + branch := "" + markerCTE := "" + switch cteName { + case "asp_i1_candidate_rows": + branch, markerCTE = "candidate", "asp_i1_candidate_marker" + case "asp_i1_fallback_rows": + branch, markerCTE = "fallback", "asp_i1_fallback_marker" + case "sp_i2_candidate_rows": + branch, markerCTE = "candidate", "sp_i2_candidate_marker" + case "sp_i2_fallback_rows": + branch, markerCTE = "fallback", "sp_i2_fallback_marker" + default: + continue + } + if body.PlanNodeID <= 0 { + continue + } + var directChildren, directOuterMarkers, directInnerExecutors []PostgresPlanNodeMetric + for _, node := range metrics.PlanNodes { + if node.ParentPlanNodeID != body.PlanNodeID { + continue + } + directChildren = append(directChildren, node) + switch { + case strings.EqualFold(strings.TrimSpace(node.ParentRelationship), "Outer") && + strings.EqualFold(strings.TrimSpace(node.NodeType), "CTE Scan") && + strings.EqualFold(strings.TrimSpace(node.CTEName), markerCTE): + directOuterMarkers = append(directOuterMarkers, node) + case strings.EqualFold(strings.TrimSpace(node.ParentRelationship), "Inner"): + directInnerExecutors = append(directInnerExecutors, node) + } + } + markerBodies := inlineCTEBodies[markerCTE] + if len(directChildren) != 2 || len(directOuterMarkers) != 1 || len(directInnerExecutors) != 1 || len(markerBodies) != 1 { + continue + } + markerRows := markerBodies[0].ActualRows * markerBodies[0].ActualLoops + outerMarkerRows := directOuterMarkers[0].ActualRows * directOuterMarkers[0].ActualLoops + if directOuterMarkers[0].ActualLoops != 1 || outerMarkerRows != markerRows { + continue + } + prefix := "asp_i1_" + if strings.HasPrefix(cteName, "sp_i2_") { + prefix = "sp_i2_" + } + name := prefix + branch + "_executor_loops" + replay.Counters[name] = directInnerExecutors[0].ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.marker_gated_direct_inner_child_actual_loops" + } + if candidateRows, candidateOK := replay.Counters["sp_i2_candidate_branch_rows"]; candidateOK { + if fallbackRows, fallbackOK := replay.Counters["sp_i2_fallback_branch_rows"]; fallbackOK { + replay.Counters["sp_i2_output_rows"] = candidateRows + fallbackRows + replay.Provenance["counters.sp_i2_output_rows"] = "postgres_metrics.plan_nodes.summed_sp_i2_output_branches" + } + } + addSuffixGuardExecutorEvidence(replay, metrics, suffixGuardCTEBodies) + if candidateRows, candidateOK := replay.Counters["suffix_guard_candidate_branch_rows"]; candidateOK { + if fallbackRows, fallbackOK := replay.Counters["suffix_guard_fallback_branch_rows"]; fallbackOK { + replay.Counters["suffix_guard_output_rows"] = candidateRows + fallbackRows + replay.Provenance["counters.suffix_guard_output_rows"] = "postgres_metrics.plan_nodes.summed_suffix_guard_output_branches" + } + } + for suffix, name := range map[string]string{ + "suffix_seeded_suffix": "suffix_component_suffix_rows", + "suffix_seeded_boundaries": "suffix_component_boundary_rows", + "suffix_seeded_reverse": "suffix_component_reverse_state_rows", + "suffix_seeded_component_receipt": "suffix_component_receipt_rows", + } { + bodies := suffixComponentCTEBodies[suffix] + if len(bodies) != 1 { + continue + } + body := bodies[0] + replay.Counters[name] = body.ActualRows * body.ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.exact_suffix_component_cte_materialization_body" + } + return replay +} + +// addSuffixGuardExecutorEvidence accepts only the same marker-outer, +// executor-inner plan shape used to prove inactive-arm suppression for guarded +// SP/ASP statements. Missing or ambiguous nodes intentionally leave counters +// absent so qualification fails closed. +func addSuffixGuardExecutorEvidence(replay *TraversalPlanReplayEvidence, metrics PostgresPlanMetrics, bodies map[string][]PostgresPlanNodeMetric) { + for _, branch := range []string{"candidate", "fallback"} { + bodySuffix := "suffix_guard_" + branch + "_body" + markerSuffix := "suffix_guard_" + branch + "_marker" + bodyNodes, markerNodes := bodies[bodySuffix], bodies[markerSuffix] + if len(bodyNodes) != 1 || len(markerNodes) != 1 || bodyNodes[0].PlanNodeID <= 0 { + continue + } + body := bodyNodes[0] + var directChildren, outerMarkers, innerExecutors []PostgresPlanNodeMetric + for _, node := range metrics.PlanNodes { + if node.ParentPlanNodeID != body.PlanNodeID { + continue + } + directChildren = append(directChildren, node) + switch { + case strings.EqualFold(strings.TrimSpace(node.ParentRelationship), "Outer") && + strings.EqualFold(strings.TrimSpace(node.NodeType), "CTE Scan") && + strings.HasSuffix(strings.ToLower(strings.TrimSpace(node.CTEName)), markerSuffix): + outerMarkers = append(outerMarkers, node) + case strings.EqualFold(strings.TrimSpace(node.ParentRelationship), "Inner"): + innerExecutors = append(innerExecutors, node) + } + } + markerRows := markerNodes[0].ActualRows * markerNodes[0].ActualLoops + if len(directChildren) != 2 || len(outerMarkers) != 1 || len(innerExecutors) != 1 || + outerMarkers[0].ActualLoops != 1 || outerMarkers[0].ActualRows*outerMarkers[0].ActualLoops != markerRows { + continue + } + name := "suffix_guard_" + branch + "_executor_loops" + replay.Counters[name] = innerExecutors[0].ActualLoops + replay.Provenance["counters."+name] = "postgres_metrics.plan_nodes.marker_gated_direct_inner_child_actual_loops" + } +} + +// orientationBoundedDegreeRows returns the rows observed at the degree probe's +// cap+1 Limit. Older plans put the Limit at the CTE body itself; scalar-summary +// plans put it directly beneath the one-row Aggregate materialization body. +func orientationBoundedDegreeRows(nodes []PostgresPlanNodeMetric, body PostgresPlanNodeMetric) (int64, bool) { + if strings.EqualFold(strings.TrimSpace(body.NodeType), "Limit") { + return body.ActualRows * body.ActualLoops, true + } + if body.PlanNodeID <= 0 { + return 0, false + } + for _, node := range nodes { + if node.ParentPlanNodeID == body.PlanNodeID && strings.EqualFold(strings.TrimSpace(node.NodeType), "Limit") { + return node.ActualRows * node.ActualLoops, true + } + } + return 0, false +} + +// orientationCTEBody matches the single materialization node PostgreSQL +// labels "CTE ". Consumer CTE scans may execute many times and aliases +// such as reverse_degree_probe contain shorter branch names, so substring +// attribution would over-count probes and invent work in inactive arms. +func orientationCTEBody(node PostgresPlanNodeMetric, suffix string) bool { + return namedCTEBody(node, suffix) +} + +// namedCTEBody matches a PostgreSQL CTE's single materialization body. CTEName +// and Alias identify consumer scans and are intentionally excluded. +func namedCTEBody(node PostgresPlanNodeMetric, suffix string) bool { + name := strings.ToLower(strings.TrimSpace(node.SubplanName)) + return strings.HasPrefix(name, "cte ") && strings.HasSuffix(name, suffix) +} + +// inlinePredecessorCTEBody uses an exact fixed name because its qualification +// contract is tied to one emitted statement shape, not stage-prefixed CTEs. +func inlinePredecessorCTEBody(node PostgresPlanNodeMetric, name string) bool { + return strings.EqualFold(strings.TrimSpace(node.SubplanName), "CTE "+name) +} + +// postgresBidirectionalDiagnosticDocument is the invocation-local document +// returned by read_bidirectional_shortest_path_diagnostic_v1. Pointer fields +// preserve the distinction between a measured zero and missing evidence. +type postgresBidirectionalDiagnosticDocument struct { + // SchemaVersion identifies the schema version for schema version. + SchemaVersion int `json:"schema_version"` + // InvocationID identifies the invocation id. + InvocationID string `json:"invocation_id"` + // Scheduler supplies the scheduler input to the postgresBidirectionalDiagnosticDocument contract. + Scheduler string `json:"scheduler"` + // StateLimit supplies the state limit input to the postgresBidirectionalDiagnosticDocument contract. + StateLimit *int64 `json:"state_limit"` + // FrontierLimit supplies the frontier limit input to the postgresBidirectionalDiagnosticDocument contract. + FrontierLimit *int64 `json:"frontier_limit"` + // PredecessorLimit supplies the predecessor limit input to the postgresBidirectionalDiagnosticDocument contract. + PredecessorLimit *int64 `json:"predecessor_limit"` + // SearchCalls supplies the search calls input to the postgresBidirectionalDiagnosticDocument contract. + SearchCalls *int64 `json:"search_calls"` + // RuntimeBranch supplies the runtime branch input to the postgresBidirectionalDiagnosticDocument contract. + RuntimeBranch string `json:"runtime_branch"` + // Overflowed supplies the overflowed input to the postgresBidirectionalDiagnosticDocument contract. + Overflowed *bool `json:"overflowed"` + // FallbackExecuted supplies the fallback executed input to the postgresBidirectionalDiagnosticDocument contract. + FallbackExecuted *bool `json:"fallback_executed"` + // Counters supplies the counters input to the postgresBidirectionalDiagnosticDocument contract. + Counters *postgresBidirectionalDiagnosticCounts `json:"counters"` + // Calls supplies the calls input to the postgresBidirectionalDiagnosticDocument contract. + Calls []postgresBidirectionalDiagnosticCall `json:"calls"` + // WorkspaceBytes supplies the workspace bytes input to the postgresBidirectionalDiagnosticDocument contract. + WorkspaceBytes int64 `json:"-"` +} + +// postgresBidirectionalDiagnosticCall groups state that must remain consistent while processing postgres bidirectional diagnostic call. +type postgresBidirectionalDiagnosticCall struct { + // SearchID identifies the search id. + SearchID *int64 `json:"search_id"` + // SourceID identifies the source id. + SourceID *int64 `json:"source_id"` + // TargetID identifies the target id. + TargetID *int64 `json:"target_id"` + // RuntimeBranch supplies the runtime branch input to the postgresBidirectionalDiagnosticCall contract. + RuntimeBranch string `json:"runtime_branch"` + // SchedulerActions supplies the scheduler actions input to the postgresBidirectionalDiagnosticCall contract. + SchedulerActions *int64 `json:"scheduler_actions"` + // CandidateEdges supplies the candidate edges input to the postgresBidirectionalDiagnosticCall contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the postgresBidirectionalDiagnosticCall contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenPeak supplies the seen peak input to the postgresBidirectionalDiagnosticCall contract. + SeenPeak *int64 `json:"seen_peak"` + // FrontierPeak supplies the frontier peak input to the postgresBidirectionalDiagnosticCall contract. + FrontierPeak *int64 `json:"frontier_peak"` + // QueuePeak supplies the queue peak input to the postgresBidirectionalDiagnosticCall contract. + QueuePeak *int64 `json:"queue_peak"` + // PredecessorPeak supplies the predecessor peak input to the postgresBidirectionalDiagnosticCall contract. + PredecessorPeak *int64 `json:"predecessor_peak"` + // MeetingCandidates supplies the meeting candidates input to the postgresBidirectionalDiagnosticCall contract. + MeetingCandidates *int64 `json:"meeting_candidates"` + // FrozenDistance supplies the frozen distance input to the postgresBidirectionalDiagnosticCall contract. + FrozenDistance *int64 `json:"frozen_distance"` + // WitnessRows records the number of witness rows. + WitnessRows *int64 `json:"witness_rows"` + // Overflowed supplies the overflowed input to the postgresBidirectionalDiagnosticCall contract. + Overflowed *bool `json:"overflowed"` + // FallbackExecuted supplies the fallback executed input to the postgresBidirectionalDiagnosticCall contract. + FallbackExecuted *bool `json:"fallback_executed"` +} + +// postgresBidirectionalDiagnosticCounts aggregates counters observed while evaluating postgres bidirectional diagnostic. +type postgresBidirectionalDiagnosticCounts struct { + // SchedulerActions supplies the scheduler actions input to the postgresBidirectionalDiagnosticCounts contract. + SchedulerActions *int64 `json:"scheduler_actions"` + // CandidateEdges supplies the candidate edges input to the postgresBidirectionalDiagnosticCounts contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the postgresBidirectionalDiagnosticCounts contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenPeak supplies the seen peak input to the postgresBidirectionalDiagnosticCounts contract. + SeenPeak *int64 `json:"seen_peak"` + // FrontierPeak supplies the frontier peak input to the postgresBidirectionalDiagnosticCounts contract. + FrontierPeak *int64 `json:"frontier_peak"` + // QueuePeak supplies the queue peak input to the postgresBidirectionalDiagnosticCounts contract. + QueuePeak *int64 `json:"queue_peak"` + // PredecessorPeak supplies the predecessor peak input to the postgresBidirectionalDiagnosticCounts contract. + PredecessorPeak *int64 `json:"predecessor_peak"` + // MeetingCandidates supplies the meeting candidates input to the postgresBidirectionalDiagnosticCounts contract. + MeetingCandidates *int64 `json:"meeting_candidates"` + // FrozenDistance supplies the frozen distance input to the postgresBidirectionalDiagnosticCounts contract. + FrozenDistance *int64 `json:"frozen_distance"` + // WitnessRows records the number of witness rows. + WitnessRows *int64 `json:"witness_rows"` + // Levels supplies the levels input to the postgresBidirectionalDiagnosticCounts contract. + Levels []postgresBidirectionalDiagnosticLevel `json:"levels"` +} + +// postgresBidirectionalDiagnosticLevel groups state that must remain consistent while processing postgres bidirectional diagnostic level. +type postgresBidirectionalDiagnosticLevel struct { + // SearchID identifies the search id. + SearchID *int64 `json:"search_id"` + // ActionIndex supplies the action index input to the postgresBidirectionalDiagnosticLevel contract. + ActionIndex *int64 `json:"action_index"` + // Side supplies the side input to the postgresBidirectionalDiagnosticLevel contract. + Side string `json:"side"` + // Action supplies the action input to the postgresBidirectionalDiagnosticLevel contract. + Action string `json:"action"` + // Depth supplies the depth input to the postgresBidirectionalDiagnosticLevel contract. + Depth *int64 `json:"depth"` + // FrontierRows records the number of frontier rows. + FrontierRows *int64 `json:"frontier_rows"` + // CandidateEdges supplies the candidate edges input to the postgresBidirectionalDiagnosticLevel contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the postgresBidirectionalDiagnosticLevel contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenRows records the number of seen rows. + SeenRows *int64 `json:"seen_rows"` + // QueueRows records the number of queue rows. + QueueRows *int64 `json:"queue_rows"` + // PredecessorRows records the number of predecessor rows. + PredecessorRows *int64 `json:"predecessor_rows"` + // MeetingCandidates supplies the meeting candidates input to the postgresBidirectionalDiagnosticLevel contract. + MeetingCandidates *int64 `json:"meeting_candidates"` +} + +// postgresBidirectionalAllShortestDiagnosticDocument defines the serialized representation of postgres bidirectional all shortest diagnostic. +type postgresBidirectionalAllShortestDiagnosticDocument struct { + // SchemaVersion identifies the schema version for schema version. + SchemaVersion int `json:"schema_version"` + // InvocationID identifies the invocation id. + InvocationID string `json:"invocation_id"` + // Scheduler supplies the scheduler input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + Scheduler string `json:"scheduler"` + // StateLimit supplies the state limit input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + StateLimit *int64 `json:"state_limit"` + // FrontierLimit supplies the frontier limit input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + FrontierLimit *int64 `json:"frontier_limit"` + // PredecessorLimit supplies the predecessor limit input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + PredecessorLimit *int64 `json:"predecessor_limit"` + // EnumerationLimit supplies the enumeration limit input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + EnumerationLimit *int64 `json:"enumeration_limit"` + // OutputBytesLimit supplies the output bytes limit input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + OutputBytesLimit *int64 `json:"output_bytes_limit"` + // SearchCalls supplies the search calls input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + SearchCalls *int64 `json:"search_calls"` + // RuntimeBranch supplies the runtime branch input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + RuntimeBranch string `json:"runtime_branch"` + // Overflowed supplies the overflowed input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + Overflowed *bool `json:"overflowed"` + // FallbackExecuted supplies the fallback executed input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + FallbackExecuted *bool `json:"fallback_executed"` + // Counters supplies the counters input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + Counters *postgresBidirectionalAllShortestDiagnosticCounts `json:"counters"` + // Calls supplies the calls input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + Calls []postgresBidirectionalAllShortestDiagnosticCall `json:"calls"` + // WorkspaceBytes supplies the workspace bytes input to the postgresBidirectionalAllShortestDiagnosticDocument contract. + WorkspaceBytes int64 `json:"-"` +} + +// postgresBidirectionalAllShortestDiagnosticCounts aggregates counters observed while evaluating postgres bidirectional all shortest diagnostic. +type postgresBidirectionalAllShortestDiagnosticCounts struct { + // SchedulerActions supplies the scheduler actions input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + SchedulerActions *int64 `json:"scheduler_actions"` + // CandidateEdges supplies the candidate edges input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenPeak supplies the seen peak input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + SeenPeak *int64 `json:"seen_peak"` + // FrontierPeak supplies the frontier peak input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + FrontierPeak *int64 `json:"frontier_peak"` + // QueuePeak supplies the queue peak input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + QueuePeak *int64 `json:"queue_peak"` + // PredecessorPeak supplies the predecessor peak input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + PredecessorPeak *int64 `json:"predecessor_peak"` + // MeetingCandidates supplies the meeting candidates input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + MeetingCandidates *int64 `json:"meeting_candidates"` + // FrozenDistance supplies the frozen distance input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + FrozenDistance *int64 `json:"frozen_distance"` + // WitnessRows records the number of witness rows. + WitnessRows *int64 `json:"witness_rows"` + // SameDepthPredecessorAdditions supplies the same depth predecessor additions input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + SameDepthPredecessorAdditions *int64 `json:"same_depth_predecessor_additions"` + // MeetingNodes supplies the meeting nodes input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + MeetingNodes *int64 `json:"meeting_nodes"` + // CutDepth supplies the cut depth input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + CutDepth *int64 `json:"cut_depth"` + // PathCountEstimate supplies the path count estimate input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + PathCountEstimate *int64 `json:"path_count_estimate"` + // PathCountSaturated supplies the path count saturated input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + PathCountSaturated *bool `json:"path_count_saturated"` + // EnumeratedCandidates supplies the enumerated candidates input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + EnumeratedCandidates *int64 `json:"enumerated_candidates"` + // DuplicateRejects supplies the duplicate rejects input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + DuplicateRejects *int64 `json:"duplicate_rejects"` + // OutputPaths identifies the filesystem output paths. + OutputPaths *int64 `json:"output_paths"` + // OutputEdgeCells supplies the output edge cells input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + OutputEdgeCells *int64 `json:"output_edge_cells"` + // OutputBytes supplies the output bytes input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + OutputBytes *int64 `json:"output_bytes"` + // Levels supplies the levels input to the postgresBidirectionalAllShortestDiagnosticCounts contract. + Levels []postgresBidirectionalDiagnosticLevel `json:"levels"` +} + +// postgresBidirectionalAllShortestDiagnosticCall groups state that must remain consistent while processing postgres bidirectional all shortest diagnostic call. +type postgresBidirectionalAllShortestDiagnosticCall struct { + // SearchID identifies the search id. + SearchID *int64 `json:"search_id"` + // SourceID identifies the source id. + SourceID *int64 `json:"source_id"` + // TargetID identifies the target id. + TargetID *int64 `json:"target_id"` + // RuntimeBranch supplies the runtime branch input to the postgresBidirectionalAllShortestDiagnosticCall contract. + RuntimeBranch string `json:"runtime_branch"` + // SchedulerActions supplies the scheduler actions input to the postgresBidirectionalAllShortestDiagnosticCall contract. + SchedulerActions *int64 `json:"scheduler_actions"` + // CandidateEdges supplies the candidate edges input to the postgresBidirectionalAllShortestDiagnosticCall contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the postgresBidirectionalAllShortestDiagnosticCall contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenPeak supplies the seen peak input to the postgresBidirectionalAllShortestDiagnosticCall contract. + SeenPeak *int64 `json:"seen_peak"` + // FrontierPeak supplies the frontier peak input to the postgresBidirectionalAllShortestDiagnosticCall contract. + FrontierPeak *int64 `json:"frontier_peak"` + // QueuePeak supplies the queue peak input to the postgresBidirectionalAllShortestDiagnosticCall contract. + QueuePeak *int64 `json:"queue_peak"` + // PredecessorPeak supplies the predecessor peak input to the postgresBidirectionalAllShortestDiagnosticCall contract. + PredecessorPeak *int64 `json:"predecessor_peak"` + // MeetingCandidates supplies the meeting candidates input to the postgresBidirectionalAllShortestDiagnosticCall contract. + MeetingCandidates *int64 `json:"meeting_candidates"` + // FrozenDistance supplies the frozen distance input to the postgresBidirectionalAllShortestDiagnosticCall contract. + FrozenDistance *int64 `json:"frozen_distance"` + // WitnessRows records the number of witness rows. + WitnessRows *int64 `json:"witness_rows"` + // SameDepthPredecessorAdditions supplies the same depth predecessor additions input to the postgresBidirectionalAllShortestDiagnosticCall contract. + SameDepthPredecessorAdditions *int64 `json:"same_depth_predecessor_additions"` + // MeetingNodes supplies the meeting nodes input to the postgresBidirectionalAllShortestDiagnosticCall contract. + MeetingNodes *int64 `json:"meeting_nodes"` + // CutDepth supplies the cut depth input to the postgresBidirectionalAllShortestDiagnosticCall contract. + CutDepth *int64 `json:"cut_depth"` + // PathCountEstimate supplies the path count estimate input to the postgresBidirectionalAllShortestDiagnosticCall contract. + PathCountEstimate *int64 `json:"path_count_estimate"` + // PathCountSaturated supplies the path count saturated input to the postgresBidirectionalAllShortestDiagnosticCall contract. + PathCountSaturated *bool `json:"path_count_saturated"` + // EnumeratedCandidates supplies the enumerated candidates input to the postgresBidirectionalAllShortestDiagnosticCall contract. + EnumeratedCandidates *int64 `json:"enumerated_candidates"` + // DuplicateRejects supplies the duplicate rejects input to the postgresBidirectionalAllShortestDiagnosticCall contract. + DuplicateRejects *int64 `json:"duplicate_rejects"` + // OutputPaths identifies the filesystem output paths. + OutputPaths *int64 `json:"output_paths"` + // OutputEdgeCells supplies the output edge cells input to the postgresBidirectionalAllShortestDiagnosticCall contract. + OutputEdgeCells *int64 `json:"output_edge_cells"` + // OutputBytes supplies the output bytes input to the postgresBidirectionalAllShortestDiagnosticCall contract. + OutputBytes *int64 `json:"output_bytes"` + // Overflowed supplies the overflowed input to the postgresBidirectionalAllShortestDiagnosticCall contract. + Overflowed *bool `json:"overflowed"` + // FallbackExecuted supplies the fallback executed input to the postgresBidirectionalAllShortestDiagnosticCall contract. + FallbackExecuted *bool `json:"fallback_executed"` +} + +// postgresA1AllShortestDiagnosticDocument is the single-ended A1 workspace +// receipt read after an untimed replay. It deliberately does not share the +// B1/B2 document: no bidirectional scheduler or meeting counters are implied. +type postgresA1AllShortestDiagnosticDocument struct { + SchemaVersion int `json:"schema_version"` + InvocationID string `json:"invocation_id"` + Scheduler string `json:"scheduler"` + SearchCalls *int64 `json:"search_calls"` + SourceID *int64 `json:"source_id"` + TargetID *int64 `json:"target_id"` + RuntimeBranch string `json:"runtime_branch"` + TargetDepth *int64 `json:"target_depth"` + OutputPaths *int64 `json:"output_paths"` + FallbackExecuted *bool `json:"fallback_executed"` + Levels []postgresA1AllShortestDiagnosticLevel `json:"levels"` + WorkspaceBytes int64 `json:"-"` +} + +// postgresA1AllShortestDiagnosticLevel records one exact single-ended A1 +// breadth-first layer from the session-local predecessor-DAG workspace. +type postgresA1AllShortestDiagnosticLevel struct { + ActionIndex *int64 `json:"action_index"` + Depth *int64 `json:"depth"` + CandidateEdges *int64 `json:"candidate_edges"` + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + SeenRows *int64 `json:"seen_rows"` + PredecessorRows *int64 `json:"predecessor_rows"` +} + +// attachPostgresTraversalTelemetry runs only after every timed case, +// reference, raw-PGX, and concurrency sample has completed. +func (s *postgresSQLRunner) attachPostgresTraversalTelemetry(ctx context.Context, record *CaseResult, parameters map[string]any) error { + if s.traversalTelemetry == "" || s.traversalTelemetry == postgresTraversalTelemetryOff { + for idx := range record.PostgresReferences { + record.PostgresReferences[idx].traversalTelemetryParameters = nil + } + return nil + } + + level := TraversalTelemetryLevel(s.traversalTelemetry) + if record.Optimization != nil && record.PostgresMetrics != nil { + telemetry, err := buildPostgresCaseTraversalTelemetry(*record.Optimization, *record.PostgresMetrics, s.backendPID, level) + if err != nil { + return fmt.Errorf("build PostgreSQL case traversal telemetry: %w", err) + } + if telemetry != nil { + if level == TraversalTelemetryLevelDiagnostic { + enrichSuffixGuardTraversalTelemetry(telemetry, *record.PostgresMetrics, record.RowCount, record.ObservedRows) + enrichSuffixRouteComponentTraversalTelemetry(telemetry, *record.PostgresMetrics, record.RowCount) + enrichSuffixRouteComponentClosureWorkspaceTelemetry(telemetry, record.PostgresBoundaryClosure) + enrichOrientationTraversalTelemetry( + telemetry, + *record.PostgresMetrics, + record.RowCount, + record.ObservedRows, + orientationPolicyMaximumDepth(*record.Optimization, telemetry.Summary.EmittedIdentity), + ) + enrichInlinePredecessorTraversalTelemetry(telemetry, *record.PostgresMetrics, record.RowCount, record.ObservedRows) + enrichInlineDistanceTraversalTelemetry(telemetry, record.RowCount) + if err := s.enrichBidirectionalTraversalTelemetry(ctx, telemetry, record.SQL, parameters, record.RowCount, record.ObservedRows, *record.PostgresMetrics); err != nil { + return fmt.Errorf("capture PostgreSQL case traversal telemetry: %w", err) + } + if err := s.enrichA1AllShortestTraversalTelemetry(ctx, telemetry, record.SQL, parameters, record.RowCount, record.ObservedRows, *record.PostgresMetrics); err != nil { + return fmt.Errorf("capture PostgreSQL A1 all-shortest traversal telemetry: %w", err) + } + } + record.TraversalTelemetry = telemetry + } + } + + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + parameters := reference.traversalTelemetryParameters + reference.traversalTelemetryParameters = nil + telemetry, err := buildPostgresReferenceTraversalTelemetry(*reference, parameters, s.backendPID, level) + if err != nil { + return fmt.Errorf("build PostgreSQL reference %s traversal telemetry: %w", reference.Name, err) + } + if telemetry == nil { + continue + } + if level == TraversalTelemetryLevelDiagnostic { + if err := s.enrichBidirectionalTraversalTelemetry(ctx, telemetry, reference.SQL, parameters, reference.RowCount, reference.ObservedRows, *reference.PostgresMetrics); err != nil { + return fmt.Errorf("capture PostgreSQL reference %s traversal telemetry: %w", reference.Name, err) + } + } + reference.TraversalTelemetry = telemetry + } + return nil +} + +// enrichSuffixRouteComponentTraversalTelemetry completes the direct component +// contract from the one emitted reverse statement. It fails closed unless all +// exact CTE materializations, the one receipt, and both replay timings are +// present. The output row count is the exact public observation from the +// production query; a consumer CTE scan may repeat it and is not authoritative. +func enrichSuffixRouteComponentTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64) { + if telemetry == nil || telemetry.Diagnostic == nil || + telemetry.Summary.SelectorVersion != optimize.ExpansionSearchSelectorSuffixRouteComponentV1 { + return + } + plan := telemetry.Diagnostic.PlanReplay + if plan == nil { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route component has no PostgreSQL plan replay") + return + } + required := []string{ + "suffix_component_suffix_rows", + "suffix_component_boundary_rows", + "suffix_component_reverse_state_rows", + "suffix_component_receipt_rows", + } + for _, name := range required { + if _, present := plan.Counters[name]; !present { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route component is missing exact plan counter "+name) + return + } + } + if plan.Counters["suffix_component_receipt_rows"] != 1 { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route component receipt must be exactly one row") + return + } + if metrics.PlanningMS == nil || metrics.ExecutionMS == nil || + metrics.Provenance["planning_ms"] == "" || metrics.Provenance["execution_ms"] == "" { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route component requires measured PostgreSQL planning and execution timings") + return + } + nodeHydrationLoops, nodeHydrationRows, edgeHydrationLoops, edgeHydrationRows, err := suffixComponentOrderedHydration( + metrics.PlanNodes, + observationRequiresHydration(telemetry.Summary.ObservationMode), + ) + if err != nil { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route component "+err.Error()) + return + } + planningNS := int64(*metrics.PlanningMS * float64(time.Millisecond)) + executionNS := int64(*metrics.ExecutionMS * float64(time.Millisecond)) + if planningNS < 0 || executionNS < 0 { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route component replay timings must not be negative") + return + } + telemetry.Diagnostic.Counters.SuffixComponent = &SuffixComponentTraversalCounters{ + SuffixRows: traversalTelemetryPointer(plan.Counters["suffix_component_suffix_rows"]), + BoundaryRows: traversalTelemetryPointer(plan.Counters["suffix_component_boundary_rows"]), + ReverseStateRows: traversalTelemetryPointer(plan.Counters["suffix_component_reverse_state_rows"]), + OrderedNodeHydrationLoops: traversalTelemetryPointer(nodeHydrationLoops), + OrderedNodeHydrationRows: traversalTelemetryPointer(nodeHydrationRows), + OrderedEdgeHydrationLoops: traversalTelemetryPointer(edgeHydrationLoops), + OrderedEdgeHydrationRows: traversalTelemetryPointer(edgeHydrationRows), + OutputRows: traversalTelemetryPointer(outputRows), + ReceiptRows: traversalTelemetryPointer(plan.Counters["suffix_component_receipt_rows"]), + PlanningTimeNS: traversalTelemetryPointer(planningNS), + ExecutionTimeNS: traversalTelemetryPointer(executionNS), + } + if telemetry.Diagnostic.Provenance == nil { + telemetry.Diagnostic.Provenance = map[string]string{} + } + for _, name := range []string{"suffix_rows", "boundary_rows", "reverse_state_rows", "receipt_rows"} { + telemetry.Diagnostic.Provenance["suffix_component."+name] = "untimed_timing_off_plan.exact_suffix_component_cte_materialization" + } + for _, name := range []string{ + "ordered_node_hydration_loops", "ordered_node_hydration_rows", + "ordered_edge_hydration_loops", "ordered_edge_hydration_rows", + } { + telemetry.Diagnostic.Provenance["suffix_component."+name] = "untimed_timing_off_plan.exact_ordered_path_hydration_alias" + } + telemetry.Diagnostic.Provenance["suffix_component.output_rows"] = "exact_public_observation.row_count" + telemetry.Diagnostic.Provenance["suffix_component.planning_time_ns"] = "postgres_metrics.planning_ms:" + metrics.Provenance["planning_ms"] + telemetry.Diagnostic.Provenance["suffix_component.execution_time_ns"] = "postgres_metrics.execution_ms:" + metrics.Provenance["execution_ms"] + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +// enrichSuffixRouteComponentClosureWorkspaceTelemetry binds the component's +// complete typed telemetry to separately measured raw-PGX workspace evidence. +// It is intentionally unavailable outside the explicit closure mode so an +// ordinary component capture cannot imply a measured high-water mark. +func enrichSuffixRouteComponentClosureWorkspaceTelemetry(telemetry *TraversalExecutionTelemetry, closure *PostgresBoundaryClosure) { + if telemetry == nil || telemetry.Diagnostic == nil || closure == nil || + telemetry.Summary.SelectorVersion != optimize.ExpansionSearchSelectorSuffixRouteComponentV1 { + return + } + if closure.Workspace.SessionPeakBytes < 0 || closure.Workspace.PoolPeakBytes < 0 { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-route closure workspace high-water must not be negative") + return + } + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) { + telemetry.Diagnostic.RequiredFamilies = append(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) + } + telemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: traversalTelemetryPointer(closure.Workspace.SessionPeakBytes), + PoolPeakBytes: traversalTelemetryPointer(closure.Workspace.PoolPeakBytes), + } + if telemetry.Diagnostic.Provenance == nil { + telemetry.Diagnostic.Provenance = map[string]string{} + } + telemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "postgres_boundary_closure.workspace.session_peak_bytes" + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "postgres_boundary_closure.workspace.pool_peak_bytes" +} + +// suffixComponentOrderedHydration reports only the two aliases emitted by the +// direct component's ordered-ID rehydration expressions. Root, suffix, and +// reverse-search node lookups are deliberately excluded. A path observation +// must expose exactly one node and one edge rehydration scan, even when both +// scans truthfully execute zero loops for a no-path result. +func suffixComponentOrderedHydration(nodes []PostgresPlanNodeMetric, required bool) (nodeLoops, nodeRows, edgeLoops, edgeRows int64, err error) { + var nodeMatches, edgeMatches int + for _, node := range nodes { + switch strings.ToLower(strings.TrimSpace(node.Alias)) { + case "_ordered_path_node": + nodeMatches++ + nodeLoops += node.ActualLoops + nodeRows += node.ActualRows * node.ActualLoops + case "_ordered_path_edge": + edgeMatches++ + edgeLoops += node.ActualLoops + edgeRows += node.ActualRows * node.ActualLoops + } + } + if nodeMatches > 1 || edgeMatches > 1 { + return 0, 0, 0, 0, fmt.Errorf("ordered hydration plan aliases are ambiguous") + } + if required && (nodeMatches != 1 || edgeMatches != 1) { + return 0, 0, 0, 0, fmt.Errorf("is missing exact ordered hydration plan aliases") + } + return nodeLoops, nodeRows, edgeLoops, edgeRows, nil +} + +// enrichSuffixGuardTraversalTelemetry completes the reverse-first guard's +// named-CTE contract. It deliberately does not synthesize the degree samples, +// scores, or shadow fields required by orientation policies. +func enrichSuffixGuardTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string) { + if telemetry == nil || telemetry.Diagnostic == nil || !isSuffixReverseGuardPolicy(telemetry.Summary.EmittedIdentity) { + return + } + plan := telemetry.Diagnostic.PlanReplay + if plan == nil { + return + } + get, present := func(name string) int64 { return plan.Counters[name] }, func(name string) bool { + _, ok := plan.Counters[name] + return ok + } + required := []string{ + "suffix_guard_root_presence_rows", "suffix_guard_suffix_rows", "suffix_guard_boundary_rows", "suffix_guard_state_rows", + "suffix_guard_candidate_marker_rows", "suffix_guard_fallback_marker_rows", "suffix_guard_candidate_branch_rows", + "suffix_guard_fallback_branch_rows", "suffix_guard_output_rows", "suffix_guard_candidate_executor_loops", "suffix_guard_fallback_executor_loops", + } + for _, name := range required { + if !present(name) { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-reverse guard is missing exact plan counter "+name) + return + } + } + if get("suffix_guard_output_rows") != int64(outputRows) { + markTraversalCountersUnavailable(telemetry.Diagnostic, "suffix-reverse guard plan output does not match the exact public observation") + return + } + suffixLimit := telemetry.Summary.Caps["suffix_rows"] + stateLimit := telemetry.Summary.Caps["state_rows"] + suffixOverflow := suffixLimit > 0 && get("suffix_guard_suffix_rows") > suffixLimit + stateOverflow := stateLimit > 0 && get("suffix_guard_state_rows") > stateLimit + counters := &SuffixGuardTraversalCounters{ + RootPresenceRows: traversalTelemetryPointer(get("suffix_guard_root_presence_rows")), + SuffixRows: traversalTelemetryPointer(get("suffix_guard_suffix_rows")), + DistinctBoundaryRows: traversalTelemetryPointer(get("suffix_guard_boundary_rows")), + StateRows: traversalTelemetryPointer(get("suffix_guard_state_rows")), + OutputRows: traversalTelemetryPointer(int64(outputRows)), + CandidateMarkerRows: traversalTelemetryPointer(get("suffix_guard_candidate_marker_rows")), + FallbackMarkerRows: traversalTelemetryPointer(get("suffix_guard_fallback_marker_rows")), + CandidateBranchRows: traversalTelemetryPointer(get("suffix_guard_candidate_branch_rows")), + FallbackBranchRows: traversalTelemetryPointer(get("suffix_guard_fallback_branch_rows")), + CandidateExecutorLoops: traversalTelemetryPointer(get("suffix_guard_candidate_executor_loops")), + FallbackExecutorLoops: traversalTelemetryPointer(get("suffix_guard_fallback_executor_loops")), + SuffixOverflow: traversalTelemetryPointer(suffixOverflow), + StateOverflow: traversalTelemetryPointer(stateOverflow), + } + telemetry.Diagnostic.Counters.SuffixGuard = counters + + var edgeCandidates, repeatRejects, hydrationLoops, hydrationRows, hydrationTimeNS int64 + for _, node := range metrics.PlanNodes { + identity := strings.ToLower(strings.Join([]string{node.CTEName, node.Alias, node.SubplanName}, " ")) + rows := node.ActualRows * node.ActualLoops + if node.RelationName == "edge" { + edgeCandidates += rows + node.RowsRemovedByFilter + repeatRejects += node.RowsRemovedByFilter + } + if strings.Contains(identity, "hydrat") || strings.Contains(identity, "materializ") { + hydrationLoops += node.ActualLoops + hydrationRows += rows + hydrationTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + } + } + telemetry.Diagnostic.Counters.Ordinary = &OrdinaryTraversalCounters{ + Roots: counters.RootPresenceRows, + EdgeCandidates: traversalTelemetryPointer(edgeCandidates), + AdmittedStates: counters.StateRows, + RelationshipRepeatRejects: traversalTelemetryPointer(repeatRejects), + RecursiveRows: traversalTelemetryPointer(metrics.RecursiveRows), + PeakState: counters.StateRows, + EmittedTrails: traversalTelemetryPointer(int64(outputRows)), + HydrationRows: traversalTelemetryPointer(metrics.HydrationRows), + } + if telemetry.Diagnostic.Provenance == nil { + telemetry.Diagnostic.Provenance = map[string]string{} + } + for _, name := range []string{ + "root_presence_rows", "suffix_rows", "distinct_boundary_rows", "state_rows", "output_rows", "candidate_marker_rows", + "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", "candidate_executor_loops", "fallback_executor_loops", + "suffix_overflow", "state_overflow", + } { + telemetry.Diagnostic.Provenance["suffix_guard."+name] = "untimed_timing_on_plan.suffix_guard_named_ctes" + } + for _, name := range []string{"roots", "edge_candidates", "admitted_states", "relationship_repeat_rejects", "recursive_rows", "peak_state", "emitted_trails", "hydration_rows"} { + telemetry.Diagnostic.Provenance["ordinary."+name] = "untimed_timing_on_plan.executed_suffix_guard_branch" + } + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + bytes := int64(0) + for _, row := range observedRows { + bytes += int64(len(row)) + } + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: traversalTelemetryPointer(int64(outputRows)), NodeLookups: traversalTelemetryPointer(metrics.HydrationLoops), + EdgeLookups: traversalTelemetryPointer(metrics.HydrationRows), Loops: traversalTelemetryPointer(hydrationLoops), + Rows: traversalTelemetryPointer(hydrationRows), TimeNS: traversalTelemetryPointer(hydrationTimeNS), Bytes: traversalTelemetryPointer(bytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "untimed_timing_on_plan_and_exact_public_observation" + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +func enrichInlineDistanceTraversalTelemetry(telemetry *TraversalExecutionTelemetry, outputRows int64) { + if telemetry == nil || telemetry.Diagnostic == nil || + (telemetry.Summary.EmittedIdentity != optimize.ShortestPathPolicyI2DistanceGuardedV1 && telemetry.Summary.EmittedIdentity != optimize.ShortestPathPolicyI2DistanceGuardedV2) || + telemetry.Diagnostic.PlanReplay == nil { + return + } + plan := telemetry.Diagnostic.PlanReplay + required := []string{"sp_i2_distance_rows", "sp_i2_target_rows", "sp_i2_output_rows", "sp_i2_candidate_marker_rows", "sp_i2_fallback_marker_rows", "sp_i2_candidate_branch_rows", "sp_i2_fallback_branch_rows", "sp_i2_candidate_executor_loops", "sp_i2_fallback_executor_loops"} + if telemetry.Summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV2 { + required = append(required, "sp_i2_admission_rows", "sp_i2_admission_loops") + if spI2DirectDevelopmentIdentity(telemetry.Summary.RequestedIdentity) { + required = append(required, "sp_i2_direct_rows", "sp_i2_direct_loops") + } + } + for _, name := range required { + if _, present := plan.Counters[name]; !present { + markTraversalCountersUnavailable(telemetry.Diagnostic, "inline distance plan replay is missing exact named counter: "+name) + return + } + } + if plan.Counters["sp_i2_output_rows"] != outputRows { + markTraversalCountersUnavailable(telemetry.Diagnostic, "inline distance plan output does not match the exact public observation") + return + } + get := func(name string) *int64 { return traversalTelemetryPointer(plan.Counters[name]) } + telemetry.Diagnostic.Counters.InlineShortestDistance = &InlineDistanceTraversalCounters{ + StateRows: get("sp_i2_distance_rows"), FrontierRows: get("sp_i2_distance_rows"), OutputRows: traversalTelemetryPointer(outputRows), + CandidateMarkerRows: get("sp_i2_candidate_marker_rows"), FallbackMarkerRows: get("sp_i2_fallback_marker_rows"), + CandidateBranchRows: get("sp_i2_candidate_branch_rows"), FallbackBranchRows: get("sp_i2_fallback_branch_rows"), + CandidateExecutorLoops: get("sp_i2_candidate_executor_loops"), FallbackExecutorLoops: get("sp_i2_fallback_executor_loops"), + } + for _, name := range []string{"state_rows", "frontier_rows", "output_rows", "candidate_marker_rows", "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", "candidate_executor_loops", "fallback_executor_loops"} { + telemetry.Diagnostic.Provenance["inline_shortest_distance."+name] = "untimed_timing_on_plan.inline_distance_named_ctes" + } + telemetry.Diagnostic.Provenance["inline_shortest_distance.frontier_rows"] = "untimed_timing_on_plan.inline_distance_state_relation_conservative_frontier_upper_bound" + if telemetry.Summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV2 { + stateLimit := telemetry.Summary.Caps["state_rows"] + frontierLimit := telemetry.Summary.Caps["frontier_rows"] + dominated := frontierLimit >= stateLimit + capRelationship := "frontier_limit stateLimit: + overflowReason = "state_observed_frontier_indeterminate" + default: + overflowReason = "frontier_observed" + } + } + inline := telemetry.Diagnostic.Counters.InlineShortestDistance + inline.AdmissionProbeRows = get("sp_i2_admission_rows") + inline.AdmissionProbeLoops = get("sp_i2_admission_loops") + if spI2DirectDevelopmentIdentity(telemetry.Summary.RequestedIdentity) { + inline.DirectProbeRows = get("sp_i2_direct_rows") + inline.DirectProbeLoops = get("sp_i2_direct_loops") + } + inline.TargetRows = get("sp_i2_target_rows") + inline.FrontierGuardDominated = &dominated + inline.CapRelationship = capRelationship + inline.ObservedOverflowReason = overflowReason + for _, name := range []string{"admission_probe_rows", "admission_probe_loops", "direct_probe_rows", "direct_probe_loops", "target_rows", "frontier_guard_dominated", "cap_relationship", "observed_overflow_reason"} { + telemetry.Diagnostic.Provenance["inline_shortest_distance."+name] = "untimed_timing_on_plan.inline_distance_v2_materialized_admission" + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +func spI2DirectDevelopmentIdentity(identity string) bool { + return identity == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) || + identity == string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP) +} + +// enrichInlineASPTraversalTelemetry maps the guarded statement's named CTEs +// to its dedicated bounded-work contract. Public observation bytes are a +// conservative ceiling for the staged edge-array bytes used by admission. +func enrichInlineASPTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string) { + enrichInlinePredecessorTraversalTelemetry(telemetry, metrics, outputRows, observedRows) +} + +// enrichInlinePredecessorTraversalTelemetry maps the shared guarded I1 +// statement's named CTEs to either the all-paths or canonical one-path counter +// family. The separate serialized fields prevent evidence from one public +// observation contract from satisfying the other. +func enrichInlinePredecessorTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string) { + if telemetry == nil || telemetry.Diagnostic == nil || + (telemetry.Summary.EmittedIdentity != optimize.ShortestPathPolicyASPI1GuardedV1 && + telemetry.Summary.EmittedIdentity != optimize.ShortestPathPolicyI1CanonicalGuardedV1) { + return + } + plan := telemetry.Diagnostic.PlanReplay + if plan == nil { + return + } + requiredPlanCounters := []string{ + "asp_i1_distance_rows", + "asp_i1_predecessor_rows", + "asp_i1_enumeration_rows", + "asp_i1_output_rows", + "asp_i1_candidate_marker_rows", + "asp_i1_fallback_marker_rows", + "asp_i1_candidate_branch_rows", + "asp_i1_fallback_branch_rows", + "asp_i1_candidate_executor_loops", + "asp_i1_fallback_executor_loops", + } + var missingPlanCounters []string + for _, name := range requiredPlanCounters { + if _, present := plan.Counters[name]; !present { + missingPlanCounters = append(missingPlanCounters, name) + } + } + if len(missingPlanCounters) > 0 { + markTraversalCountersUnavailable( + telemetry.Diagnostic, + "inline predecessor plan replay is missing exact named counters: "+strings.Join(missingPlanCounters, ", "), + ) + return + } + get := func(name string) int64 { return plan.Counters[name] } + outputBytes := int64(0) + for _, row := range observedRows { + outputBytes += int64(len(row)) + } + inline := &InlinePredecessorTraversalCounters{ + DistanceRows: traversalTelemetryPointer(get("asp_i1_distance_rows")), + PredecessorRows: traversalTelemetryPointer(get("asp_i1_predecessor_rows")), + EnumerationRows: traversalTelemetryPointer(get("asp_i1_enumeration_rows")), + OutputPaths: traversalTelemetryPointer(outputRows), + OutputBytes: traversalTelemetryPointer(outputBytes), + CandidateMarkerRows: traversalTelemetryPointer(get("asp_i1_candidate_marker_rows")), + FallbackMarkerRows: traversalTelemetryPointer(get("asp_i1_fallback_marker_rows")), + CandidateBranchRows: traversalTelemetryPointer(get("asp_i1_candidate_branch_rows")), + FallbackBranchRows: traversalTelemetryPointer(get("asp_i1_fallback_branch_rows")), + CandidateExecutorLoops: traversalTelemetryPointer(get("asp_i1_candidate_executor_loops")), + FallbackExecutorLoops: traversalTelemetryPointer(get("asp_i1_fallback_executor_loops")), + } + prefix := "inline_asp" + if telemetry.Summary.EmittedIdentity == optimize.ShortestPathPolicyI1CanonicalGuardedV1 { + prefix = "inline_shortest_path" + telemetry.Diagnostic.Counters.InlineShortestPath = inline + } else { + telemetry.Diagnostic.Counters.InlineASP = inline + } + if telemetry.Diagnostic.Provenance == nil { + telemetry.Diagnostic.Provenance = map[string]string{} + } + for _, name := range []string{ + "distance_rows", "predecessor_rows", "enumeration_rows", "candidate_marker_rows", + "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", + "candidate_executor_loops", "fallback_executor_loops", + } { + telemetry.Diagnostic.Provenance[prefix+"."+name] = "untimed_timing_on_plan.inline_predecessor_named_ctes" + } + telemetry.Diagnostic.Provenance[prefix+".output_paths"] = "exact_public_observation.row_count" + telemetry.Diagnostic.Provenance[prefix+".output_bytes"] = "exact_public_observation.conservative_serialized_bytes" + + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: traversalTelemetryPointer(outputRows), + NodeLookups: traversalTelemetryPointer(metrics.HydrationLoops), + EdgeLookups: traversalTelemetryPointer(metrics.HydrationRows), + Loops: traversalTelemetryPointer(metrics.HydrationLoops), + Rows: traversalTelemetryPointer(metrics.HydrationRows), + TimeNS: traversalTelemetryPointer(int64(0)), + Bytes: traversalTelemetryPointer(outputBytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "untimed_plan_and_exact_public_observation" + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +// enrichOrientationTraversalTelemetry turns explicitly named SQL probe and +// branch nodes into a complete, conservative diagnostic document. Probe times +// come from the untimed TIMING ON JSON EXPLAIN replay; hydration bytes use the +// captured public observation, never an estimated tuple width. +func enrichOrientationTraversalTelemetry(telemetry *TraversalExecutionTelemetry, metrics PostgresPlanMetrics, outputRows int64, observedRows []string, maximumDepth int64) { + if telemetry == nil || telemetry.Diagnostic == nil || !isOrientationProbePolicy(telemetry.Summary.EmittedIdentity) { + return + } + if telemetry.Summary.EmittedIdentity == string(optimize.ExpansionSearchPolicyOrientationProbeV2) && maximumDepth <= 0 { + markTraversalCountersUnavailable(telemetry.Diagnostic, "orientation-probe-v2 maximum depth is unavailable") + return + } + plan := telemetry.Diagnostic.PlanReplay + if plan == nil { + return + } + get := func(name string) int64 { return plan.Counters[name] } + forwardSeeds := get("orientation_root_probe_rows") + reverseSeeds := get("orientation_suffix_probe_rows") + boundaries := get("orientation_boundary_rows") + forwardDegree := get("orientation_forward_degree_rows") + reverseDegree := get("orientation_reverse_degree_rows") + stateRows := get("orientation_state_rows") + probeRows := forwardSeeds + reverseSeeds + boundaries + forwardDegree + reverseDegree + duplicateSeeds := max(reverseSeeds-boundaries, int64(0)) + shallowSurvivalRows := boundaries + shallowSurvival := float64(0) + if reverseSeeds > 0 { + shallowSurvival = float64(boundaries) / float64(reverseSeeds) + } + forwardScore := float64(forwardSeeds + forwardDegree) + if telemetry.Summary.EmittedIdentity == string(optimize.ExpansionSearchPolicyOrientationProbeV2) { + forwardScore = float64(forwardSeeds + maximumDepth*forwardDegree) + } + reverseScore := float64(reverseSeeds + boundaries + reverseDegree) + selectedSide := "forward" + if telemetry.Summary.RuntimeIdentity != telemetry.Summary.FallbackIdentity && strings.Contains(telemetry.Summary.RuntimeIdentity, "REVERSE") { + selectedSide = "reverse" + } + overflow := false + if telemetry.Summary.Overflow != nil { + overflow = *telemetry.Summary.Overflow + } + branchLoops := get("orientation_candidate_branch_loops") + get("orientation_incumbent_branch_loops") + + var probeTimeNS, probeHits, probeReads, edgeCandidates, repeatRejects, hydrationLoops, hydrationRows, hydrationTimeNS int64 + for _, node := range metrics.PlanNodes { + identity := strings.ToLower(strings.Join([]string{node.CTEName, node.Alias, node.SubplanName}, " ")) + rows := node.ActualRows * node.ActualLoops + if strings.Contains(identity, "orientation_") && (strings.Contains(identity, "_probe") || strings.Contains(identity, "_boundaries") || strings.Contains(identity, "_decision")) { + probeTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + probeHits += node.Buffers.SharedHit + node.Buffers.LocalHit + probeReads += node.Buffers.SharedRead + node.Buffers.LocalRead + } + if node.RelationName == "edge" { + edgeCandidates += rows + node.RowsRemovedByFilter + repeatRejects += node.RowsRemovedByFilter + } + if strings.Contains(identity, "hydrat") || strings.Contains(identity, "materializ") { + hydrationLoops += node.ActualLoops + hydrationRows += rows + hydrationTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + } + } + orientation := &OrientationTraversalCounters{ + ForwardSeeds: traversalTelemetryPointer(forwardSeeds), + ReverseSeeds: traversalTelemetryPointer(reverseSeeds), + DuplicateSeeds: traversalTelemetryPointer(duplicateSeeds), + SuffixRows: traversalTelemetryPointer(reverseSeeds), + DistinctBoundaries: traversalTelemetryPointer(boundaries), + TypedDirectionalDegreeSamples: traversalTelemetryPointer(forwardDegree + reverseDegree), + ForwardDegreeSamples: traversalTelemetryPointer(forwardDegree), + ReverseDegreeSamples: traversalTelemetryPointer(reverseDegree), + ShallowSurvivalRows: traversalTelemetryPointer(shallowSurvivalRows), + ShallowSurvival: traversalTelemetryPointer(shallowSurvival), + ProbeRows: traversalTelemetryPointer(probeRows), + ProbeTimeNS: traversalTelemetryPointer(probeTimeNS), + ProbeBufferHits: traversalTelemetryPointer(probeHits), + ProbeBufferReads: traversalTelemetryPointer(probeReads), + ForwardScore: traversalTelemetryPointer(forwardScore), + ReverseScore: traversalTelemetryPointer(reverseScore), + SelectedSide: selectedSide, + SentinelOverflow: traversalTelemetryPointer(overflow), + BranchLoops: traversalTelemetryPointer(branchLoops), + } + ordinary := &OrdinaryTraversalCounters{ + Roots: traversalTelemetryPointer(forwardSeeds), + EdgeCandidates: traversalTelemetryPointer(edgeCandidates), + AdmittedStates: traversalTelemetryPointer(stateRows), + RelationshipRepeatRejects: traversalTelemetryPointer(repeatRejects), + RecursiveRows: traversalTelemetryPointer(metrics.RecursiveRows), + PeakState: traversalTelemetryPointer(stateRows), + EmittedTrails: traversalTelemetryPointer(outputRows), + HydrationRows: traversalTelemetryPointer(metrics.HydrationRows), + } + telemetry.Diagnostic.Counters.Orientation = orientation + telemetry.Diagnostic.Counters.Ordinary = ordinary + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{"forward_seeds", "reverse_seeds", "duplicate_seeds", "suffix_rows", "distinct_boundaries", "typed_directional_degree_samples", "forward_degree_samples", "reverse_degree_samples", "shallow_survival_rows", "shallow_survival", "probe_rows", "probe_time_ns", "probe_buffer_hits", "probe_buffer_reads", "forward_score", "reverse_score", "selected_side", "sentinel_overflow", "branch_loops"} { + telemetry.Diagnostic.Provenance["orientation."+name] = "untimed_timing_on_plan.orientation_named_ctes" + } + for _, name := range []string{"roots", "edge_candidates", "admitted_states", "relationship_repeat_rejects", "recursive_rows", "peak_state", "emitted_trails", "hydration_rows"} { + telemetry.Diagnostic.Provenance["ordinary."+name] = "untimed_timing_on_plan.executed_orientation_branch" + } + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + bytes := int64(0) + for _, row := range observedRows { + bytes += int64(len(row)) + } + nodeLookups := metrics.HydrationLoops + edgeLookups := metrics.HydrationRows + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: traversalTelemetryPointer(outputRows), + NodeLookups: traversalTelemetryPointer(nodeLookups), + EdgeLookups: traversalTelemetryPointer(edgeLookups), + Loops: traversalTelemetryPointer(hydrationLoops), + Rows: traversalTelemetryPointer(hydrationRows), + TimeNS: traversalTelemetryPointer(hydrationTimeNS), + Bytes: traversalTelemetryPointer(bytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "untimed_timing_on_plan_and_exact_public_observation" + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +// orientationPolicyMaximumDepth supports benchmark evidence processing for orientation policy maximum depth. +func orientationPolicyMaximumDepth(summary translate.OptimizationSummary, policy string) int64 { + if !isOrientationProbePolicy(policy) { + return 0 + } + for _, outcome := range summary.TargetOutcomes { + if outcome.EmittedPolicy == policy && outcome.MaximumDepth != nil { + return *outcome.MaximumDepth + } + } + return 0 +} + +// enrichA1AllShortestTraversalTelemetry completes the A1 stored-helper +// diagnostic only from its own single-ended workspace receipt. The replay runs +// after timing, on the same physical connection, and cannot affect samples. +func (s *postgresSQLRunner) enrichA1AllShortestTraversalTelemetry( + ctx context.Context, + telemetry *TraversalExecutionTelemetry, + sqlQuery string, + parameters map[string]any, + expectedRows int64, + observedRows []string, + metrics PostgresPlanMetrics, +) error { + if telemetry == nil || telemetry.Level != TraversalTelemetryLevelDiagnostic || !isA1AllShortestTelemetryIdentity(telemetry.Summary) { + return nil + } + + invocationID := newRunUUID() + if telemetry.Diagnostic != nil { + invocationID = telemetry.Diagnostic.InvocationID + } + document, unavailableReason, err := s.replayA1AllShortestTraversalDiagnostic(ctx, invocationID, sqlQuery, parameters, expectedRows) + if err != nil { + if telemetry.Diagnostic != nil { + markTraversalCountersUnavailable(telemetry.Diagnostic, err.Error()) + return telemetry.Validate() + } + markTraversalSummaryUnavailable(telemetry, err.Error()) + return telemetry.Validate() + } + if unavailableReason != "" { + if telemetry.Diagnostic != nil { + markTraversalCountersUnavailable(telemetry.Diagnostic, unavailableReason) + return telemetry.Validate() + } + markTraversalSummaryUnavailable(telemetry, unavailableReason) + return telemetry.Validate() + } + if err := applyA1AllShortestTraversalDiagnostic(telemetry, document, invocationID, s.backendPID, observedRows, metrics); err != nil { + if telemetry.Diagnostic != nil { + markTraversalCountersUnavailable(telemetry.Diagnostic, err.Error()) + return telemetry.Validate() + } + markTraversalSummaryUnavailable(telemetry, err.Error()) + return telemetry.Validate() + } + return telemetry.Validate() +} + +// replayA1AllShortestTraversalDiagnostic runs the unmodified A1 statement +// after a diagnostic begin step has cleared the spd_* workspace. +func (s *postgresSQLRunner) replayA1AllShortestTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, +) (*postgresA1AllShortestDiagnosticDocument, string, error) { + rawDocument, workspaceBytes, unavailableReason, err := s.replayInvocationLocalTraversalDiagnostic( + ctx, invocationID, sqlQuery, parameters, expectedRows, + "select public.begin_all_shortest_paths_a1_diagnostic_v1($1)", + "select coalesce(public.read_all_shortest_paths_a1_diagnostic_v1($1)::text, '')", + "select public.clear_all_shortest_paths_a1_diagnostic_v1($1)", + []string{"spd_%"}, + ) + if err != nil || unavailableReason != "" { + return nil, unavailableReason, err + } + document := &postgresA1AllShortestDiagnosticDocument{} + if err := json.Unmarshal([]byte(rawDocument), document); err != nil { + return nil, "A1 all-shortest diagnostic reader returned malformed JSON: " + err.Error(), nil + } + document.WorkspaceBytes = workspaceBytes + return document, "", nil +} + +// applyA1AllShortestTraversalDiagnostic validates and maps the A1 receipt +// into the common all-shortest counter family without implying any two-sided +// search work. +func applyA1AllShortestTraversalDiagnostic( + telemetry *TraversalExecutionTelemetry, + document *postgresA1AllShortestDiagnosticDocument, + expectedInvocationID string, + expectedConnectionID string, + observedRows []string, + metrics PostgresPlanMetrics, +) error { + if telemetry == nil || document == nil { + return fmt.Errorf("A1 all-shortest diagnostic document is missing") + } + if document.SchemaVersion != 1 || document.InvocationID != expectedInvocationID { + return fmt.Errorf("A1 all-shortest diagnostic identity is invalid") + } + if telemetry.Diagnostic != nil && telemetry.Diagnostic.ConnectionID != expectedConnectionID { + return fmt.Errorf("A1 all-shortest diagnostic connection identity differs from replay connection") + } + if document.Scheduler != "single_ended_level" || document.Scheduler != telemetry.Summary.SchedulerVersion { + return fmt.Errorf("A1 all-shortest diagnostic scheduler %q differs from planned scheduler %q", document.Scheduler, telemetry.Summary.SchedulerVersion) + } + if document.SearchCalls == nil || *document.SearchCalls != 1 || document.SourceID == nil || document.TargetID == nil || + document.TargetDepth == nil || document.OutputPaths == nil || document.FallbackExecuted == nil || *document.FallbackExecuted { + return fmt.Errorf("A1 all-shortest diagnostic invocation state is incomplete") + } + if *document.OutputPaths < 0 || int64(len(observedRows)) != *document.OutputPaths { + return fmt.Errorf("A1 all-shortest diagnostic output count differs from the exact public observation") + } + if len(document.Levels) == 0 { + return fmt.Errorf("A1 all-shortest diagnostic levels are missing") + } + + var candidateEdges, distinctNewNodes, seenPeak, frontierPeak, predecessorPeak int64 + for idx, level := range document.Levels { + if level.ActionIndex == nil || level.Depth == nil || level.CandidateEdges == nil || level.DistinctNewNodes == nil || + level.SeenRows == nil || level.PredecessorRows == nil || *level.ActionIndex != int64(idx+1) || *level.Depth < 0 || + *level.CandidateEdges < 0 || *level.DistinctNewNodes < 0 || *level.SeenRows < 0 || *level.PredecessorRows < 0 { + return fmt.Errorf("A1 all-shortest diagnostic level %d is incomplete", idx) + } + candidateEdges += *level.CandidateEdges + distinctNewNodes += *level.DistinctNewNodes + seenPeak = max(seenPeak, *level.SeenRows) + frontierPeak = max(frontierPeak, *level.DistinctNewNodes) + predecessorPeak = max(predecessorPeak, *level.PredecessorRows) + } + + validBranch := false + switch document.RuntimeBranch { + case "one_hop_preflight": + validBranch = *document.TargetDepth == 1 && *document.OutputPaths > 0 + case "two_hop_preflight": + validBranch = *document.TargetDepth == 2 && *document.OutputPaths > 0 + case "preflight_no_path", "search_no_path": + validBranch = *document.TargetDepth == -1 && *document.OutputPaths == 0 + case "single_ended_search": + validBranch = *document.TargetDepth >= 3 && *document.OutputPaths > 0 + } + if !validBranch { + return fmt.Errorf("A1 all-shortest diagnostic runtime branch %q contradicts its result", document.RuntimeBranch) + } + + sameDepthPredecessors := max(predecessorPeak-distinctNewNodes, int64(0)) + edgeCells := int64(0) + if *document.TargetDepth > 0 { + edgeCells = *document.TargetDepth * *document.OutputPaths + } + outputBytes := int64(0) + for _, row := range observedRows { + outputBytes += int64(len(row)) + } + fallback := false + frozenDistance := *document.TargetDepth + pathCountSaturated := false + levels := make([]ShortestPathLevelCounters, len(document.Levels)) + for idx, level := range document.Levels { + levels[idx] = ShortestPathLevelCounters{ + SearchID: int64(1), + ActionIndex: *level.ActionIndex, + Side: "forward", + Action: "expand_level", + Depth: level.Depth, + FrontierRows: level.DistinctNewNodes, + CandidateEdges: level.CandidateEdges, + DistinctNewNodes: level.DistinctNewNodes, + SeenRows: level.SeenRows, + QueueRows: level.DistinctNewNodes, + PredecessorRows: level.PredecessorRows, + MeetingCandidates: traversalTelemetryPointer(int64(0)), + Provenance: fmt.Sprintf("%s.levels[%d]", postgresA1AllShortestDiagnosticSource, idx), + } + } + + telemetry.Summary.RuntimeIdentity = string(optimize.ShortestPathExecutorASPA1DAG) + telemetry.Summary.AppliedIdentity = string(optimize.ShortestPathExecutorASPA1DAG) + telemetry.Summary.FallbackIdentity = "" + telemetry.Summary.RuntimeBranch = document.RuntimeBranch + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(true) + telemetry.Summary.Overflow = traversalTelemetryPointer(false) + telemetry.Summary.FallbackExecuted = traversalTelemetryPointer(false) + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "runtime_outcome_available", "overflow", "fallback_executed", "scheduler_version"} { + telemetry.Summary.Provenance[name] = postgresA1AllShortestDiagnosticSource + } + if telemetry.Diagnostic == nil { + return nil + } + telemetry.Diagnostic.RequiredFamilies = traversalRequiredFamilies(telemetry.Summary, TraversalTelemetryFamilyASP) + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) { + telemetry.Diagnostic.RequiredFamilies = append(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) + } + telemetry.Diagnostic.Counters = TraversalDiagnosticCounters{AllShortestPaths: &AllShortestPathsTraversalCounters{ + Search: ShortestPathTraversalCounters{ + SchedulerActions: traversalTelemetryPointer(int64(len(levels))), + Levels: levels, + CandidateEdges: traversalTelemetryPointer(candidateEdges), + DistinctNewNodes: traversalTelemetryPointer(distinctNewNodes), + SeenPeak: traversalTelemetryPointer(seenPeak), + FrontierPeak: traversalTelemetryPointer(frontierPeak), + QueuePeak: traversalTelemetryPointer(frontierPeak), + PredecessorPeak: traversalTelemetryPointer(predecessorPeak), + MeetingCandidates: traversalTelemetryPointer(int64(0)), + FrozenDistance: traversalTelemetryPointer(frozenDistance), + WitnessRows: document.OutputPaths, + FallbackExecuted: traversalTelemetryPointer(fallback), + }, + SameDepthPredecessorAdditions: traversalTelemetryPointer(sameDepthPredecessors), + PredecessorPeak: traversalTelemetryPointer(predecessorPeak), + MeetingNodes: traversalTelemetryPointer(int64(0)), + CutDepth: traversalTelemetryPointer(max(frozenDistance, int64(0))), + PathCountEstimate: document.OutputPaths, + PathCountSaturated: traversalTelemetryPointer(pathCountSaturated), + EnumeratedCandidates: document.OutputPaths, + DuplicateRejects: traversalTelemetryPointer(int64(0)), + OutputPaths: document.OutputPaths, + OutputEdgeCells: traversalTelemetryPointer(edgeCells), + OutputBytes: traversalTelemetryPointer(outputBytes), + }} + telemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + PoolPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + } + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{"scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", "frontier_peak", "queue_peak", "predecessor_peak", "meeting_candidates", "frozen_distance", "witness_rows", "fallback_executed"} { + telemetry.Diagnostic.Provenance["all_shortest_paths.search."+name] = postgresA1AllShortestDiagnosticSource + } + for _, name := range []string{"same_depth_predecessor_additions", "predecessor_peak", "meeting_nodes", "cut_depth", "path_count_estimate", "path_count_saturated", "enumerated_candidates", "duplicate_rejects", "output_paths", "output_edge_cells", "output_bytes"} { + telemetry.Diagnostic.Provenance["all_shortest_paths."+name] = postgresA1AllShortestDiagnosticSource + } + telemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "pg_total_relation_size(pg_temp.spd_*)" + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "single_connection_diagnostic_pool.session_peak_bytes" + enrichA1AllShortestHydrationTelemetry(telemetry, document.OutputPaths, traversalTelemetryPointer(edgeCells), observedRows, metrics) + return nil +} + +// enrichA1AllShortestHydrationTelemetry binds outer path materialization to +// the exact A1 output counts without claiming function-internal plan nodes. +func enrichA1AllShortestHydrationTelemetry( + telemetry *TraversalExecutionTelemetry, + pathCount, edgeCells *int64, + observedRows []string, + metrics PostgresPlanMetrics, +) { + if telemetry == nil || telemetry.Diagnostic == nil || pathCount == nil || edgeCells == nil { + return + } + bytes := int64(0) + for _, row := range observedRows { + bytes += int64(len(row)) + } + nodeLookups := *pathCount + *edgeCells + rows := metrics.HydrationRows + if rows == 0 { + rows = nodeLookups + *edgeCells + } + var timeNS int64 + for _, node := range metrics.PlanNodes { + if node.RelationName == "node" { + timeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + } + } + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: pathCount, NodeLookups: traversalTelemetryPointer(nodeLookups), EdgeLookups: edgeCells, + Loops: traversalTelemetryPointer(metrics.HydrationLoops), Rows: traversalTelemetryPointer(rows), + TimeNS: traversalTelemetryPointer(timeNS), Bytes: traversalTelemetryPointer(bytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "A1_invocation_local_path_counts+untimed_timing_on_plan+exact_public_observation" + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil +} + +// enrichBidirectionalTraversalTelemetry replaces opaque Function Scan +// evidence only when the exact SP-B1/B2 statement reports a validated, +// invocation-local diagnostic document. Other hidden functions stay +// explicitly unavailable. +func (s *postgresSQLRunner) enrichBidirectionalTraversalTelemetry( + ctx context.Context, + telemetry *TraversalExecutionTelemetry, + sqlQuery string, + parameters map[string]any, + expectedRows int64, + observedRows []string, + metrics PostgresPlanMetrics, +) error { + if telemetry == nil || telemetry.Level != TraversalTelemetryLevelDiagnostic || !isBidirectionalTelemetryIdentity(telemetry.Summary) { + return nil + } + identity := bidirectionalTelemetryIdentity(telemetry.Summary) + + invocationID := newRunUUID() + if telemetry.Diagnostic != nil { + invocationID = telemetry.Diagnostic.InvocationID + } + var ( + unavailableReason string + err error + ) + if isBidirectionalASPIdentity(identity) { + var document *postgresBidirectionalAllShortestDiagnosticDocument + document, unavailableReason, err = s.replayBidirectionalAllShortestTraversalDiagnostic(ctx, invocationID, sqlQuery, parameters, expectedRows) + if err == nil && unavailableReason == "" { + err = applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, invocationID, s.backendPID) + if err == nil { + enrichBidirectionalHydrationTelemetry(telemetry, document.Counters.OutputPaths, document.Counters.OutputEdgeCells, observedRows, metrics) + } + } + } else { + var document *postgresBidirectionalDiagnosticDocument + document, unavailableReason, err = s.replayBidirectionalTraversalDiagnostic(ctx, invocationID, sqlQuery, parameters, expectedRows) + if err == nil && unavailableReason == "" { + err = applyBidirectionalTraversalDiagnostic(telemetry, document, invocationID, s.backendPID) + if err == nil { + pathCount := document.Counters.WitnessRows + edgeCells := int64(0) + if document.Counters.FrozenDistance != nil && *document.Counters.FrozenDistance > 0 && pathCount != nil { + edgeCells = *document.Counters.FrozenDistance * *pathCount + } + enrichBidirectionalHydrationTelemetry(telemetry, pathCount, traversalTelemetryPointer(edgeCells), observedRows, metrics) + } + } + } + if err != nil { + if telemetry.Diagnostic == nil { + markTraversalSummaryUnavailable(telemetry, err.Error()) + return telemetry.Validate() + } + markTraversalCountersUnavailable(telemetry.Diagnostic, err.Error()) + return telemetry.Validate() + } + if unavailableReason != "" { + if telemetry.Diagnostic == nil { + markTraversalSummaryUnavailable(telemetry, unavailableReason) + return telemetry.Validate() + } + markTraversalCountersUnavailable(telemetry.Diagnostic, unavailableReason) + return telemetry.Validate() + } + return telemetry.Validate() +} + +// enrichBidirectionalHydrationTelemetry supports benchmark evidence processing for enrich bidirectional hydration telemetry. +func enrichBidirectionalHydrationTelemetry( + telemetry *TraversalExecutionTelemetry, + pathCount, edgeCells *int64, + observedRows []string, + metrics PostgresPlanMetrics, +) { + if telemetry == nil || telemetry.Diagnostic == nil || !slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) || pathCount == nil || edgeCells == nil { + return + } + bytes := int64(0) + for _, row := range observedRows { + bytes += int64(len(row)) + } + nodeLookups := *edgeCells + *pathCount + var hydrationTimeNS int64 + for _, node := range metrics.PlanNodes { + identity := strings.ToLower(strings.Join([]string{node.CTEName, node.Alias, node.SubplanName}, " ")) + if strings.Contains(identity, "hydrat") || strings.Contains(identity, "materializ") || node.RelationName == "node" { + hydrationTimeNS += int64(node.ActualTotalMS * float64(time.Millisecond)) + } + } + rows := metrics.HydrationRows + if rows == 0 { + rows = nodeLookups + *edgeCells + } + loops := metrics.HydrationLoops + telemetry.Diagnostic.Counters.Hydration = &TraversalHydrationCounters{ + PathCount: pathCount, + NodeLookups: traversalTelemetryPointer(nodeLookups), + EdgeLookups: edgeCells, + Loops: traversalTelemetryPointer(loops), + Rows: traversalTelemetryPointer(rows), + TimeNS: traversalTelemetryPointer(hydrationTimeNS), + Bytes: traversalTelemetryPointer(bytes), + } + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + telemetry.Diagnostic.Provenance["hydration."+name] = "invocation_local_path_counts+untimed_timing_on_plan+exact_public_observation" + } + if telemetry.Summary.FallbackExecuted != nil && !*telemetry.Summary.FallbackExecuted { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil + } +} + +// replayBidirectionalTraversalDiagnostic executes the exact statement in a +// separate repeatable-read transaction on the runner's single physical +// connection. Its duration and counters are never added to latency samples. +func (s *postgresSQLRunner) replayBidirectionalTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, +) (*postgresBidirectionalDiagnosticDocument, string, error) { + rawDocument, workspaceBytes, unavailableReason, err := s.replayInvocationLocalTraversalDiagnostic( + ctx, invocationID, sqlQuery, parameters, expectedRows, + "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", + "select coalesce(public.read_bidirectional_shortest_path_diagnostic_v1($1)::text, '')", + "select public.clear_bidirectional_shortest_path_diagnostic_v1($1)", + []string{"spb_%", "asb_%"}, + ) + if err != nil || unavailableReason != "" { + return nil, unavailableReason, err + } + document := &postgresBidirectionalDiagnosticDocument{} + if err := json.Unmarshal([]byte(rawDocument), document); err != nil { + return nil, "diagnostic reader returned malformed JSON: " + err.Error(), nil + } + document.WorkspaceBytes = workspaceBytes + return document, "", nil +} + +// replayBidirectionalAllShortestTraversalDiagnostic supports benchmark evidence processing for replay bidirectional all shortest traversal diagnostic. +func (s *postgresSQLRunner) replayBidirectionalAllShortestTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, +) (*postgresBidirectionalAllShortestDiagnosticDocument, string, error) { + rawDocument, workspaceBytes, unavailableReason, err := s.replayInvocationLocalTraversalDiagnostic( + ctx, invocationID, sqlQuery, parameters, expectedRows, + "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", + "select coalesce(public.read_bidirectional_all_shortest_path_diagnostic_v1($1)::text, '')", + "select public.clear_bidirectional_all_shortest_path_diagnostic_v1($1)", + []string{"spb_%", "asb_%"}, + ) + if err != nil || unavailableReason != "" { + return nil, unavailableReason, err + } + document := &postgresBidirectionalAllShortestDiagnosticDocument{} + if err := json.Unmarshal([]byte(rawDocument), document); err != nil { + return nil, "all-shortest diagnostic reader returned malformed JSON: " + err.Error(), nil + } + document.WorkspaceBytes = workspaceBytes + return document, "", nil +} + +// replayInvocationLocalTraversalDiagnostic supports benchmark evidence processing for replay invocation local traversal diagnostic. +func (s *postgresSQLRunner) replayInvocationLocalTraversalDiagnostic( + ctx context.Context, + invocationID string, + sqlQuery string, + parameters map[string]any, + expectedRows int64, + beginSQL string, + readSQL string, + clearSQL string, + workspacePatterns []string, +) (string, int64, string, error) { + connection, err := s.pool.Acquire(ctx) + if err != nil { + return "", 0, "", fmt.Errorf("acquire diagnostic connection: %w", err) + } + defer connection.Release() + + var backendPID int32 + if err := connection.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + return "", 0, "", fmt.Errorf("read diagnostic connection identity: %w", err) + } + connectionID := strconv.FormatInt(int64(backendPID), 10) + if connectionID != s.backendPID { + return "", 0, "", fmt.Errorf("diagnostic connection identity %s differs from timed-sample connection %s", connectionID, s.backendPID) + } + + tx, err := connection.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + if err != nil { + return "", 0, "", fmt.Errorf("begin repeatable-read diagnostic transaction: %w", err) + } + initialized := false + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if initialized { + _, _ = tx.Exec(cleanupCtx, clearSQL, invocationID) + } + _ = tx.Rollback(cleanupCtx) + }() + + if _, err := tx.Exec(ctx, beginSQL, invocationID); err != nil { + return "", 0, "", fmt.Errorf("begin invocation-local diagnostic: %w", err) + } + initialized = true + + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}} + if len(parameters) > 0 { + queryArgs = append(queryArgs, pgx.NamedArgs(parameters)) + } + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + if err != nil { + return "", 0, "", fmt.Errorf("execute untimed diagnostic replay: %w", err) + } + var rowCount int64 + for rows.Next() { + rowCount++ + if _, err := rows.Values(); err != nil { + rows.Close() + return "", 0, "", fmt.Errorf("decode untimed diagnostic replay: %w", err) + } + } + rows.Close() + if err := rows.Err(); err != nil { + return "", 0, "", fmt.Errorf("drain untimed diagnostic replay: %w", err) + } + if rowCount != expectedRows { + return "", 0, "", fmt.Errorf("untimed diagnostic replay row count %d differs from measured row count %d", rowCount, expectedRows) + } + var workspaceBytes int64 + if len(workspacePatterns) == 0 { + return "", 0, "", fmt.Errorf("diagnostic workspace patterns are required") + } + if err := tx.QueryRow(ctx, ` + select coalesce(sum(pg_total_relation_size(c.oid)), 0)::int8 + from pg_class c + where c.relnamespace = pg_my_temp_schema() + and c.relname like any($1::text[]) + and c.relname not like '%telemetry%' + `, workspacePatterns).Scan(&workspaceBytes); err != nil { + return "", 0, "", fmt.Errorf("measure diagnostic workspace high-water bytes: %w", err) + } + + var replayBackendPID int32 + if err := tx.QueryRow(ctx, "select pg_backend_pid()").Scan(&replayBackendPID); err != nil { + return "", 0, "", fmt.Errorf("verify diagnostic transaction connection identity: %w", err) + } + if replayBackendPID != backendPID { + return "", 0, "", fmt.Errorf("diagnostic transaction changed physical connection from %d to %d", backendPID, replayBackendPID) + } + + var rawDocument string + if err := tx.QueryRow(ctx, readSQL, invocationID).Scan(&rawDocument); err != nil { + return "", 0, "", fmt.Errorf("read invocation-local diagnostic: %w", err) + } + if _, err := tx.Exec(ctx, clearSQL, invocationID); err != nil { + return "", 0, "", fmt.Errorf("clear invocation-local diagnostic: %w", err) + } + initialized = false + if err := tx.Commit(ctx); err != nil { + return "", 0, "", fmt.Errorf("commit cleared diagnostic transaction: %w", err) + } + + if strings.TrimSpace(rawDocument) == "" { + return "", workspaceBytes, "diagnostic reader returned no document for this invocation", nil + } + return rawDocument, workspaceBytes, "", nil +} + +// applyBidirectionalTraversalDiagnostic applies bidirectional traversal diagnostic. +func applyBidirectionalTraversalDiagnostic( + telemetry *TraversalExecutionTelemetry, + document *postgresBidirectionalDiagnosticDocument, + expectedInvocationID string, + expectedConnectionID string, +) error { + if telemetry == nil || document == nil { + return fmt.Errorf("bidirectional diagnostic document is missing") + } + if document.SchemaVersion != 1 { + return fmt.Errorf("bidirectional diagnostic schema_version must be 1") + } + if document.InvocationID != expectedInvocationID { + return fmt.Errorf("bidirectional diagnostic invocation identity %q differs from requested %q", document.InvocationID, expectedInvocationID) + } + if telemetry.Diagnostic != nil && telemetry.Diagnostic.ConnectionID != expectedConnectionID { + return fmt.Errorf("attached diagnostic connection identity %q differs from replay connection %q", telemetry.Diagnostic.ConnectionID, expectedConnectionID) + } + if document.SearchCalls == nil || *document.SearchCalls != 1 { + return fmt.Errorf("instrumented singleton SP-B1/B2 replay must invoke exactly one search call") + } + if int64(len(document.Calls)) != *document.SearchCalls { + return fmt.Errorf("bidirectional diagnostic call count %d differs from search_calls %d", len(document.Calls), *document.SearchCalls) + } + if document.RuntimeBranch == "" || document.RuntimeBranch == "missing" || document.RuntimeBranch == "mixed" { + return fmt.Errorf("bidirectional diagnostic runtime branch is not singular") + } + if document.Overflowed == nil || document.FallbackExecuted == nil { + return fmt.Errorf("bidirectional diagnostic runtime outcome flags are missing") + } + if err := validateDiagnosticRuntimeOutcome(document.RuntimeBranch, *document.Overflowed, *document.FallbackExecuted, "exact_s4_fallback", []string{ + "zero_hop_preflight", "one_hop_preflight", "two_hop_preflight", "preflight_no_path", "search_no_path", "bidirectional_search", + }); err != nil { + return fmt.Errorf("bidirectional diagnostic: %w", err) + } + if err := validateBidirectionalDiagnosticCalls(document.Calls, document.Overflowed, document.FallbackExecuted); err != nil { + return err + } + if document.Calls[0].RuntimeBranch != document.RuntimeBranch { + return fmt.Errorf("bidirectional diagnostic aggregate runtime branch differs from its call") + } + if strings.TrimSpace(document.Scheduler) == "" || document.Scheduler != telemetry.Summary.SchedulerVersion { + return fmt.Errorf("bidirectional diagnostic scheduler %q differs from planned scheduler %q", document.Scheduler, telemetry.Summary.SchedulerVersion) + } + for name, observed := range map[string]*int64{ + "state_rows": document.StateLimit, + "frontier_rows": document.FrontierLimit, + "queue_rows": document.FrontierLimit, + "predecessor_rows": document.PredecessorLimit, + } { + planned, ok := telemetry.Summary.Caps[name] + if !ok || observed == nil || *observed != planned { + return fmt.Errorf("bidirectional diagnostic cap %s does not match the planned value", name) + } + } + if document.Counters == nil { + return fmt.Errorf("bidirectional diagnostic counters are missing") + } + if err := validateBidirectionalDiagnosticCounts(document.Counters); err != nil { + return err + } + if err := validateBidirectionalSingleCallAggregate(document.Counters, document.Calls[0]); err != nil { + return err + } + + fallbackIdentity := bidirectionalFallbackIdentity(bidirectionalTelemetryIdentity(telemetry.Summary)) + if *document.FallbackExecuted { + if fallbackIdentity == "" { + return fmt.Errorf("bidirectional diagnostic reports fallback without a declared exact control") + } + if !slices.Contains(telemetry.Summary.PlannedIdentities, fallbackIdentity) { + telemetry.Summary.PlannedIdentities = append(telemetry.Summary.PlannedIdentities, fallbackIdentity) + } + telemetry.Summary.RuntimeIdentity = fallbackIdentity + telemetry.Summary.AppliedIdentity = fallbackIdentity + telemetry.Summary.FallbackIdentity = fallbackIdentity + telemetry.Summary.Provenance["fallback_identity"] = postgresBidirectionalDiagnosticSource + ".fallback_executed" + } else { + identity := bidirectionalTelemetryIdentity(telemetry.Summary) + telemetry.Summary.RuntimeIdentity = identity + telemetry.Summary.AppliedIdentity = identity + telemetry.Summary.FallbackIdentity = "" + } + telemetry.Summary.RuntimeBranch = document.RuntimeBranch + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(true) + telemetry.Summary.Overflow = traversalTelemetryPointer(*document.Overflowed) + telemetry.Summary.FallbackExecuted = traversalTelemetryPointer(*document.FallbackExecuted) + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "overflow", "fallback_executed", "scheduler_version"} { + telemetry.Summary.Provenance[name] = postgresBidirectionalDiagnosticSource + } + telemetry.Summary.Provenance["runtime_outcome_available"] = postgresBidirectionalDiagnosticSource + + if telemetry.Diagnostic == nil { + return nil + } + levels := make([]ShortestPathLevelCounters, len(document.Counters.Levels)) + for idx, level := range document.Counters.Levels { + levels[idx] = ShortestPathLevelCounters{ + SearchID: *level.SearchID, + ActionIndex: *level.ActionIndex, + Side: level.Side, + Action: level.Action, + Depth: level.Depth, + FrontierRows: level.FrontierRows, + CandidateEdges: level.CandidateEdges, + DistinctNewNodes: level.DistinctNewNodes, + SeenRows: level.SeenRows, + QueueRows: level.QueueRows, + PredecessorRows: level.PredecessorRows, + MeetingCandidates: level.MeetingCandidates, + Provenance: fmt.Sprintf("%s.counters.levels[%d]", postgresBidirectionalDiagnosticSource, idx), + } + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.IncompleteReasons = nil + telemetry.Diagnostic.RequiredFamilies = traversalRequiredFamilies(telemetry.Summary, TraversalTelemetryFamilySP) + telemetry.Diagnostic.Counters = TraversalDiagnosticCounters{ShortestPath: &ShortestPathTraversalCounters{ + SchedulerActions: document.Counters.SchedulerActions, + Levels: levels, + CandidateEdges: document.Counters.CandidateEdges, + DistinctNewNodes: document.Counters.DistinctNewNodes, + SeenPeak: document.Counters.SeenPeak, + FrontierPeak: document.Counters.FrontierPeak, + QueuePeak: document.Counters.QueuePeak, + PredecessorPeak: document.Counters.PredecessorPeak, + MeetingCandidates: document.Counters.MeetingCandidates, + FrozenDistance: document.Counters.FrozenDistance, + WitnessRows: document.Counters.WitnessRows, + FallbackExecuted: document.FallbackExecuted, + }} + telemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + PoolPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + } + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{ + "scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", "frontier_peak", "queue_peak", + "predecessor_peak", "meeting_candidates", "frozen_distance", "witness_rows", "fallback_executed", + } { + telemetry.Diagnostic.Provenance["shortest_path."+name] = postgresBidirectionalDiagnosticSource + ".counters." + name + } + telemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "pg_total_relation_size(pg_temp.spb_*)" + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "single_connection_diagnostic_pool.session_peak_bytes" + if *document.FallbackExecuted { + // The document completely describes bounded B-candidate work and the + // exact-fallback decision, but the nested S4 executor does not yet emit + // its own edge/state counters. Keep the measured candidate evidence and + // fail total-work qualification closed. + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"nested exact S4 fallback traversal work counters are unavailable"} + } + if slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = append(telemetry.Diagnostic.IncompleteReasons, "complete invocation-local path hydration counters are unavailable") + } + return nil +} + +// validateBidirectionalDiagnosticCounts validates bidirectional diagnostic counts. +func validateBidirectionalDiagnosticCounts(counters *postgresBidirectionalDiagnosticCounts) error { + for name, value := range map[string]*int64{ + "scheduler_actions": counters.SchedulerActions, "candidate_edges": counters.CandidateEdges, + "distinct_new_nodes": counters.DistinctNewNodes, "seen_peak": counters.SeenPeak, + "frontier_peak": counters.FrontierPeak, "queue_peak": counters.QueuePeak, + "predecessor_peak": counters.PredecessorPeak, "meeting_candidates": counters.MeetingCandidates, + "witness_rows": counters.WitnessRows, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional diagnostic counter %s is missing or negative", name) + } + } + if counters.FrozenDistance == nil || *counters.FrozenDistance < -1 { + return fmt.Errorf("bidirectional diagnostic frozen_distance is missing or invalid") + } + if len(counters.Levels) == 0 { + return fmt.Errorf("bidirectional diagnostic level counters are missing") + } + for idx, level := range counters.Levels { + if level.SearchID == nil || level.ActionIndex == nil || *level.SearchID < 1 || *level.ActionIndex < 1 || + strings.TrimSpace(level.Side) == "" || strings.TrimSpace(level.Action) == "" { + return fmt.Errorf("bidirectional diagnostic level %d has incomplete identity", idx) + } + for name, value := range map[string]*int64{ + "depth": level.Depth, "frontier_rows": level.FrontierRows, "candidate_edges": level.CandidateEdges, + "distinct_new_nodes": level.DistinctNewNodes, "seen_rows": level.SeenRows, "queue_rows": level.QueueRows, + "predecessor_rows": level.PredecessorRows, "meeting_candidates": level.MeetingCandidates, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional diagnostic level %d counter %s is missing or negative", idx, name) + } + } + } + return nil +} + +// validateBidirectionalDiagnosticCalls validates bidirectional diagnostic calls. +func validateBidirectionalDiagnosticCalls(calls []postgresBidirectionalDiagnosticCall, overflowed, fallbackExecuted *bool) error { + return validateBidirectionalDiagnosticCallsFor(calls, overflowed, fallbackExecuted, "exact_s4_fallback") +} + +// validateBidirectionalDiagnosticCallsFor validates bidirectional diagnostic calls for. +func validateBidirectionalDiagnosticCallsFor(calls []postgresBidirectionalDiagnosticCall, overflowed, fallbackExecuted *bool, exactFallback string) error { + seen := map[int64]struct{}{} + anyOverflow, anyFallback := false, false + for idx, call := range calls { + if call.SearchID == nil || *call.SearchID < 1 || call.SourceID == nil || call.TargetID == nil { + return fmt.Errorf("bidirectional diagnostic call %d has incomplete identity", idx) + } + if _, duplicate := seen[*call.SearchID]; duplicate { + return fmt.Errorf("bidirectional diagnostic call %d repeats search_id %d", idx, *call.SearchID) + } + seen[*call.SearchID] = struct{}{} + if call.RuntimeBranch == "" || call.RuntimeBranch == "started" { + return fmt.Errorf("bidirectional diagnostic call %d did not finish", idx) + } + for name, value := range map[string]*int64{ + "scheduler_actions": call.SchedulerActions, "candidate_edges": call.CandidateEdges, + "distinct_new_nodes": call.DistinctNewNodes, "seen_peak": call.SeenPeak, + "frontier_peak": call.FrontierPeak, "queue_peak": call.QueuePeak, + "predecessor_peak": call.PredecessorPeak, "meeting_candidates": call.MeetingCandidates, + "witness_rows": call.WitnessRows, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional diagnostic call %d counter %s is missing or negative", idx, name) + } + } + if call.Overflowed == nil || call.FallbackExecuted == nil { + return fmt.Errorf("bidirectional diagnostic call %d outcome flags are missing", idx) + } + if err := validateDiagnosticRuntimeOutcome(call.RuntimeBranch, *call.Overflowed, *call.FallbackExecuted, exactFallback, []string{ + "zero_hop_preflight", "one_hop_preflight", "two_hop_preflight", "preflight_no_path", "search_no_path", "bidirectional_search", + }); err != nil { + return fmt.Errorf("bidirectional diagnostic call %d: %w", idx, err) + } + anyOverflow = anyOverflow || *call.Overflowed + anyFallback = anyFallback || *call.FallbackExecuted + } + if overflowed == nil || fallbackExecuted == nil || anyOverflow != *overflowed || anyFallback != *fallbackExecuted { + return fmt.Errorf("bidirectional diagnostic aggregate outcome differs from its calls") + } + return nil +} + +// validateDiagnosticRuntimeOutcome validates diagnostic runtime outcome. +func validateDiagnosticRuntimeOutcome(branch string, overflowed, fallbackExecuted bool, exactFallback string, nonFallback []string) error { + allowed := slices.Contains(nonFallback, branch) || branch == exactFallback + if !allowed { + return fmt.Errorf("runtime branch %q is unsupported", branch) + } + if fallbackExecuted != (branch == exactFallback) { + return fmt.Errorf("runtime branch %q contradicts fallback_executed=%t", branch, fallbackExecuted) + } + if overflowed != fallbackExecuted { + return fmt.Errorf("overflowed=%t contradicts fallback_executed=%t", overflowed, fallbackExecuted) + } + return nil +} + +// validateBidirectionalSingleCallAggregate validates bidirectional single call aggregate. +func validateBidirectionalSingleCallAggregate(counters *postgresBidirectionalDiagnosticCounts, call postgresBidirectionalDiagnosticCall) error { + for name, values := range map[string][2]*int64{ + "scheduler_actions": {counters.SchedulerActions, call.SchedulerActions}, + "candidate_edges": {counters.CandidateEdges, call.CandidateEdges}, + "distinct_new_nodes": {counters.DistinctNewNodes, call.DistinctNewNodes}, + "seen_peak": {counters.SeenPeak, call.SeenPeak}, "frontier_peak": {counters.FrontierPeak, call.FrontierPeak}, + "queue_peak": {counters.QueuePeak, call.QueuePeak}, "predecessor_peak": {counters.PredecessorPeak, call.PredecessorPeak}, + "meeting_candidates": {counters.MeetingCandidates, call.MeetingCandidates}, + "witness_rows": {counters.WitnessRows, call.WitnessRows}, + } { + if values[0] == nil || values[1] == nil || *values[0] != *values[1] { + return fmt.Errorf("bidirectional diagnostic aggregate counter %s differs from its single call", name) + } + } + // The SQL reader serializes an absent call-level meeting distance as the + // explicit aggregate -1 sentinel. A completed meeting is always a concrete + // non-negative value, so accepting only this nil/-1 pairing preserves the + // distinction instead of manufacturing a distance for a no-path result. + if counters.FrozenDistance == nil || + (call.FrozenDistance == nil && *counters.FrozenDistance != -1) || + (call.FrozenDistance != nil && *counters.FrozenDistance != *call.FrozenDistance) { + return fmt.Errorf("bidirectional diagnostic aggregate counter frozen_distance differs from its single call") + } + for idx, level := range counters.Levels { + if level.SearchID == nil || call.SearchID == nil || *level.SearchID != *call.SearchID { + return fmt.Errorf("bidirectional diagnostic level %d is not attributed to its single call", idx) + } + } + return nil +} + +// applyBidirectionalAllShortestTraversalDiagnostic applies bidirectional all shortest traversal diagnostic. +func applyBidirectionalAllShortestTraversalDiagnostic( + telemetry *TraversalExecutionTelemetry, + document *postgresBidirectionalAllShortestDiagnosticDocument, + expectedInvocationID string, + expectedConnectionID string, +) error { + if telemetry == nil || document == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic document is missing") + } + if document.SchemaVersion != 1 { + return fmt.Errorf("bidirectional all-shortest diagnostic schema_version must be 1") + } + if document.InvocationID != expectedInvocationID { + return fmt.Errorf("bidirectional all-shortest diagnostic invocation identity %q differs from requested %q", document.InvocationID, expectedInvocationID) + } + if telemetry.Diagnostic != nil && telemetry.Diagnostic.ConnectionID != expectedConnectionID { + return fmt.Errorf("attached diagnostic connection identity %q differs from replay connection %q", telemetry.Diagnostic.ConnectionID, expectedConnectionID) + } + if document.SearchCalls == nil || *document.SearchCalls != 1 { + return fmt.Errorf("instrumented singleton ASP-B1/B2 replay must invoke exactly one search call") + } + if int64(len(document.Calls)) != *document.SearchCalls { + return fmt.Errorf("bidirectional all-shortest diagnostic call count %d differs from search_calls %d", len(document.Calls), *document.SearchCalls) + } + if document.RuntimeBranch == "" || document.RuntimeBranch == "missing" || document.RuntimeBranch == "mixed" { + return fmt.Errorf("bidirectional all-shortest diagnostic runtime branch is not singular") + } + if document.Overflowed == nil || document.FallbackExecuted == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic runtime outcome flags are missing") + } + if err := validateDiagnosticRuntimeOutcome(document.RuntimeBranch, *document.Overflowed, *document.FallbackExecuted, "exact_a1_fallback", []string{ + "preflight_one_hop", "preflight_two_hop", "preflight_no_path", "search_no_path", "bidirectional_search", + }); err != nil { + return fmt.Errorf("bidirectional all-shortest diagnostic: %w", err) + } + if strings.TrimSpace(document.Scheduler) == "" || document.Scheduler != telemetry.Summary.SchedulerVersion { + return fmt.Errorf("bidirectional all-shortest diagnostic scheduler %q differs from planned scheduler %q", document.Scheduler, telemetry.Summary.SchedulerVersion) + } + for name, observed := range map[string]*int64{ + "state_rows": document.StateLimit, + "frontier_rows": document.FrontierLimit, + "queue_rows": document.FrontierLimit, + "predecessor_rows": document.PredecessorLimit, + "output_rows": document.EnumerationLimit, + "output_bytes": document.OutputBytesLimit, + } { + planned, ok := telemetry.Summary.Caps[name] + if !ok || observed == nil || *observed != planned { + return fmt.Errorf("bidirectional all-shortest diagnostic cap %s does not match the planned value", name) + } + } + if document.Counters == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic counters are missing") + } + if err := validateBidirectionalAllShortestDiagnosticCounts(document.Counters); err != nil { + return err + } + if err := validateBidirectionalAllShortestDiagnosticCalls(document.Calls, document.Overflowed, document.FallbackExecuted); err != nil { + return err + } + if document.Calls[0].RuntimeBranch != document.RuntimeBranch { + return fmt.Errorf("bidirectional all-shortest diagnostic aggregate runtime branch differs from its call") + } + if err := validateBidirectionalAllShortestSingleCallAggregate(document.Counters, document.Calls[0]); err != nil { + return err + } + + fallbackIdentity := bidirectionalFallbackIdentity(bidirectionalTelemetryIdentity(telemetry.Summary)) + if *document.FallbackExecuted { + if fallbackIdentity == "" { + return fmt.Errorf("bidirectional all-shortest diagnostic reports fallback without a declared exact control") + } + if !slices.Contains(telemetry.Summary.PlannedIdentities, fallbackIdentity) { + telemetry.Summary.PlannedIdentities = append(telemetry.Summary.PlannedIdentities, fallbackIdentity) + } + telemetry.Summary.RuntimeIdentity = fallbackIdentity + telemetry.Summary.AppliedIdentity = fallbackIdentity + telemetry.Summary.FallbackIdentity = fallbackIdentity + telemetry.Summary.Provenance["fallback_identity"] = postgresBidirectionalAllShortestDiagnosticSource + ".fallback_executed" + } else { + identity := bidirectionalTelemetryIdentity(telemetry.Summary) + telemetry.Summary.RuntimeIdentity = identity + telemetry.Summary.AppliedIdentity = identity + telemetry.Summary.FallbackIdentity = "" + } + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(true) + telemetry.Summary.RuntimeBranch = document.RuntimeBranch + telemetry.Summary.Overflow = traversalTelemetryPointer(*document.Overflowed) + telemetry.Summary.FallbackExecuted = traversalTelemetryPointer(*document.FallbackExecuted) + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "overflow", "fallback_executed", "scheduler_version", "runtime_outcome_available"} { + telemetry.Summary.Provenance[name] = postgresBidirectionalAllShortestDiagnosticSource + } + + if telemetry.Diagnostic == nil { + return nil + } + search := shortestPathCountersFromAllShortest(document.Counters, document.FallbackExecuted) + telemetry.Diagnostic.RequiredFamilies = traversalRequiredFamilies(telemetry.Summary, TraversalTelemetryFamilyASP) + telemetry.Diagnostic.Counters = TraversalDiagnosticCounters{AllShortestPaths: &AllShortestPathsTraversalCounters{ + Search: search, + SameDepthPredecessorAdditions: document.Counters.SameDepthPredecessorAdditions, + PredecessorPeak: document.Counters.PredecessorPeak, + MeetingNodes: document.Counters.MeetingNodes, + CutDepth: document.Counters.CutDepth, + PathCountEstimate: document.Counters.PathCountEstimate, + PathCountSaturated: document.Counters.PathCountSaturated, + EnumeratedCandidates: document.Counters.EnumeratedCandidates, + DuplicateRejects: document.Counters.DuplicateRejects, + OutputPaths: document.Counters.OutputPaths, + OutputEdgeCells: document.Counters.OutputEdgeCells, + OutputBytes: document.Counters.OutputBytes, + }} + telemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + PoolPeakBytes: traversalTelemetryPointer(document.WorkspaceBytes), + } + telemetry.Diagnostic.Provenance = map[string]string{} + for _, name := range []string{ + "scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", "frontier_peak", "queue_peak", + "predecessor_peak", "meeting_candidates", "frozen_distance", "witness_rows", "fallback_executed", + } { + telemetry.Diagnostic.Provenance["all_shortest_paths.search."+name] = postgresBidirectionalAllShortestDiagnosticSource + ".counters." + name + } + telemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "pg_total_relation_size(pg_temp.asb_*)" + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "single_connection_diagnostic_pool.session_peak_bytes" + for _, name := range []string{ + "same_depth_predecessor_additions", "predecessor_peak", "meeting_nodes", "cut_depth", "path_count_estimate", + "path_count_saturated", "enumerated_candidates", "duplicate_rejects", "output_paths", "output_edge_cells", "output_bytes", + } { + telemetry.Diagnostic.Provenance["all_shortest_paths."+name] = postgresBidirectionalAllShortestDiagnosticSource + ".counters." + name + } + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{ + "complete invocation-local path hydration counters are unavailable", + } + if *document.FallbackExecuted { + telemetry.Diagnostic.IncompleteReasons = append(telemetry.Diagnostic.IncompleteReasons, "nested exact ASP-A1 fallback traversal work counters are unavailable") + } + return nil +} + +// shortestPathCountersFromAllShortest supports benchmark evidence processing for shortest path counters from all shortest. +func shortestPathCountersFromAllShortest(counters *postgresBidirectionalAllShortestDiagnosticCounts, fallbackExecuted *bool) ShortestPathTraversalCounters { + levels := make([]ShortestPathLevelCounters, len(counters.Levels)) + for idx, level := range counters.Levels { + levels[idx] = ShortestPathLevelCounters{ + SearchID: *level.SearchID, + ActionIndex: *level.ActionIndex, + Side: level.Side, + Action: level.Action, + Depth: level.Depth, + FrontierRows: level.FrontierRows, + CandidateEdges: level.CandidateEdges, + DistinctNewNodes: level.DistinctNewNodes, + SeenRows: level.SeenRows, + QueueRows: level.QueueRows, + PredecessorRows: level.PredecessorRows, + MeetingCandidates: level.MeetingCandidates, + Provenance: fmt.Sprintf("%s.counters.levels[%d]", postgresBidirectionalAllShortestDiagnosticSource, idx), + } + } + return ShortestPathTraversalCounters{ + SchedulerActions: counters.SchedulerActions, + Levels: levels, + CandidateEdges: counters.CandidateEdges, + DistinctNewNodes: counters.DistinctNewNodes, + SeenPeak: counters.SeenPeak, + FrontierPeak: counters.FrontierPeak, + QueuePeak: counters.QueuePeak, + PredecessorPeak: counters.PredecessorPeak, + MeetingCandidates: counters.MeetingCandidates, + FrozenDistance: counters.FrozenDistance, + WitnessRows: counters.WitnessRows, + FallbackExecuted: fallbackExecuted, + } +} + +// validateBidirectionalAllShortestDiagnosticCounts validates bidirectional all shortest diagnostic counts. +func validateBidirectionalAllShortestDiagnosticCounts(counters *postgresBidirectionalAllShortestDiagnosticCounts) error { + if counters == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic counters are missing") + } + if err := validateBidirectionalDiagnosticCounts(&postgresBidirectionalDiagnosticCounts{ + SchedulerActions: counters.SchedulerActions, + CandidateEdges: counters.CandidateEdges, + DistinctNewNodes: counters.DistinctNewNodes, + SeenPeak: counters.SeenPeak, + FrontierPeak: counters.FrontierPeak, + QueuePeak: counters.QueuePeak, + PredecessorPeak: counters.PredecessorPeak, + MeetingCandidates: counters.MeetingCandidates, + FrozenDistance: counters.FrozenDistance, + WitnessRows: counters.WitnessRows, + Levels: counters.Levels, + }); err != nil { + return err + } + for name, value := range map[string]*int64{ + "same_depth_predecessor_additions": counters.SameDepthPredecessorAdditions, + "meeting_nodes": counters.MeetingNodes, "path_count_estimate": counters.PathCountEstimate, + "enumerated_candidates": counters.EnumeratedCandidates, "duplicate_rejects": counters.DuplicateRejects, + "output_paths": counters.OutputPaths, "output_edge_cells": counters.OutputEdgeCells, "output_bytes": counters.OutputBytes, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional all-shortest diagnostic counter %s is missing or negative", name) + } + } + if counters.CutDepth == nil || *counters.CutDepth < -1 { + return fmt.Errorf("bidirectional all-shortest diagnostic cut_depth is missing or invalid") + } + if counters.PathCountSaturated == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic path_count_saturated is missing") + } + return nil +} + +// validateBidirectionalAllShortestDiagnosticCalls validates bidirectional all shortest diagnostic calls. +func validateBidirectionalAllShortestDiagnosticCalls(calls []postgresBidirectionalAllShortestDiagnosticCall, overflowed, fallbackExecuted *bool) error { + baseCalls := make([]postgresBidirectionalDiagnosticCall, len(calls)) + for idx, call := range calls { + baseCalls[idx] = postgresBidirectionalDiagnosticCall{ + SearchID: call.SearchID, + SourceID: call.SourceID, + TargetID: call.TargetID, + RuntimeBranch: call.RuntimeBranch, + SchedulerActions: call.SchedulerActions, + CandidateEdges: call.CandidateEdges, + DistinctNewNodes: call.DistinctNewNodes, + SeenPeak: call.SeenPeak, + FrontierPeak: call.FrontierPeak, + QueuePeak: call.QueuePeak, + PredecessorPeak: call.PredecessorPeak, + MeetingCandidates: call.MeetingCandidates, + FrozenDistance: call.FrozenDistance, + WitnessRows: call.WitnessRows, + Overflowed: call.Overflowed, + FallbackExecuted: call.FallbackExecuted, + } + } + if err := validateBidirectionalDiagnosticCallsFor(baseCalls, overflowed, fallbackExecuted, "exact_a1_fallback"); err != nil { + return err + } + for idx, call := range calls { + for name, value := range map[string]*int64{ + "same_depth_predecessor_additions": call.SameDepthPredecessorAdditions, + "meeting_nodes": call.MeetingNodes, "path_count_estimate": call.PathCountEstimate, + "enumerated_candidates": call.EnumeratedCandidates, "duplicate_rejects": call.DuplicateRejects, + "output_paths": call.OutputPaths, "output_edge_cells": call.OutputEdgeCells, "output_bytes": call.OutputBytes, + } { + if value == nil || *value < 0 { + return fmt.Errorf("bidirectional all-shortest diagnostic call %d counter %s is missing or negative", idx, name) + } + } + if call.CutDepth == nil || *call.CutDepth < -1 || call.PathCountSaturated == nil { + return fmt.Errorf("bidirectional all-shortest diagnostic call %d has incomplete cut/count state", idx) + } + } + return nil +} + +// validateBidirectionalAllShortestSingleCallAggregate validates bidirectional all shortest single call aggregate. +func validateBidirectionalAllShortestSingleCallAggregate(counters *postgresBidirectionalAllShortestDiagnosticCounts, call postgresBidirectionalAllShortestDiagnosticCall) error { + if err := validateBidirectionalSingleCallAggregate(&postgresBidirectionalDiagnosticCounts{ + SchedulerActions: counters.SchedulerActions, + CandidateEdges: counters.CandidateEdges, + DistinctNewNodes: counters.DistinctNewNodes, + SeenPeak: counters.SeenPeak, + FrontierPeak: counters.FrontierPeak, + QueuePeak: counters.QueuePeak, + PredecessorPeak: counters.PredecessorPeak, + MeetingCandidates: counters.MeetingCandidates, + FrozenDistance: counters.FrozenDistance, + WitnessRows: counters.WitnessRows, + Levels: counters.Levels, + }, postgresBidirectionalDiagnosticCall{ + SearchID: call.SearchID, + SchedulerActions: call.SchedulerActions, + CandidateEdges: call.CandidateEdges, + DistinctNewNodes: call.DistinctNewNodes, + SeenPeak: call.SeenPeak, + FrontierPeak: call.FrontierPeak, + QueuePeak: call.QueuePeak, + PredecessorPeak: call.PredecessorPeak, + MeetingCandidates: call.MeetingCandidates, + FrozenDistance: call.FrozenDistance, + WitnessRows: call.WitnessRows, + }); err != nil { + return err + } + for name, values := range map[string][2]*int64{ + "same_depth_predecessor_additions": {counters.SameDepthPredecessorAdditions, call.SameDepthPredecessorAdditions}, + "meeting_nodes": {counters.MeetingNodes, call.MeetingNodes}, "cut_depth": {counters.CutDepth, call.CutDepth}, + "path_count_estimate": {counters.PathCountEstimate, call.PathCountEstimate}, + "enumerated_candidates": {counters.EnumeratedCandidates, call.EnumeratedCandidates}, + "duplicate_rejects": {counters.DuplicateRejects, call.DuplicateRejects}, + "output_paths": {counters.OutputPaths, call.OutputPaths}, "output_edge_cells": {counters.OutputEdgeCells, call.OutputEdgeCells}, + "output_bytes": {counters.OutputBytes, call.OutputBytes}, + } { + if values[0] == nil || values[1] == nil || *values[0] != *values[1] { + return fmt.Errorf("bidirectional all-shortest diagnostic aggregate counter %s differs from its single call", name) + } + } + if counters.PathCountSaturated == nil || call.PathCountSaturated == nil || *counters.PathCountSaturated != *call.PathCountSaturated { + return fmt.Errorf("bidirectional all-shortest diagnostic aggregate path_count_saturated differs from its single call") + } + return nil +} + +// markTraversalCountersUnavailable supports benchmark evidence processing for mark traversal counters unavailable. +func markTraversalCountersUnavailable(diagnostic *TraversalExecutionDiagnostic, reason string) { + if diagnostic == nil { + return + } + diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + diagnostic.IncompleteReasons = []string{reason} + diagnostic.Counters = TraversalDiagnosticCounters{} + diagnostic.Provenance = map[string]string{} +} + +// markTraversalSummaryUnavailable supports benchmark evidence processing for mark traversal summary unavailable. +func markTraversalSummaryUnavailable(telemetry *TraversalExecutionTelemetry, reason string) { + if telemetry == nil { + return + } + telemetry.Summary.RuntimeOutcomeAvailable = traversalTelemetryPointer(false) + telemetry.Summary.RuntimeIdentity = "" + telemetry.Summary.AppliedIdentity = "" + telemetry.Summary.RuntimeBranch = "runtime_outcome_unavailable" + telemetry.Summary.Overflow = nil + telemetry.Summary.FallbackExecuted = nil + telemetry.Summary.FallbackIdentity = "" + for _, name := range []string{"runtime_identity", "applied_identity", "runtime_branch", "runtime_outcome_available"} { + telemetry.Summary.Provenance[name] = "runtime_outcome_unavailable:" + reason + } + delete(telemetry.Summary.Provenance, "overflow") + delete(telemetry.Summary.Provenance, "fallback_executed") + delete(telemetry.Summary.Provenance, "fallback_identity") +} + +// isBidirectionalSPIdentity reports whether is bidirectional sp identity. +func isBidirectionalSPIdentity(identity string) bool { + return strings.HasPrefix(identity, "SP-B1-") || strings.HasPrefix(identity, "SP-B2-") +} + +// bidirectionalFallbackIdentity derives the stable identity used to compare bidirectional fallback. +func bidirectionalFallbackIdentity(identity string) string { + if isBidirectionalASPIdentity(identity) { + return "ASP-A1-DAG" + } + if isBidirectionalSPIdentity(identity) { + if strings.Contains(identity, "WE+") { + return "SP-S4-C-WE+MAT-M0" + } + return "SP-S4-C-D" + } + return "" +} + +// isBidirectionalASPIdentity reports whether is bidirectional asp identity. +func isBidirectionalASPIdentity(identity string) bool { + return strings.HasPrefix(identity, "ASP-B1-") || strings.HasPrefix(identity, "ASP-B2-") +} + +// traversalTelemetryPointer returns an addressable representation of traversal telemetry. +func traversalTelemetryPointer[T any](value T) *T { + return &value +} diff --git a/cmd/graphbench/postgres_traversal_telemetry_test.go b/cmd/graphbench/postgres_traversal_telemetry_test.go new file mode 100644 index 00000000..0544dc72 --- /dev/null +++ b/cmd/graphbench/postgres_traversal_telemetry_test.go @@ -0,0 +1,2041 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestPostgresTraversalTelemetryCompletesBidirectionalCandidateIdentityChain verifies postgres traversal telemetry completes bidirectional candidate identity chain behavior. +func TestPostgresTraversalTelemetryCompletesBidirectionalCandidateIdentityChain(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + + require.Equal(t, "SP-B2-C-MIN-LEVEL-D", telemetry.Summary.RequestedIdentity) + require.Equal(t, "SP-B2-C-MIN-LEVEL-D", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "SP-B2-C-MIN-LEVEL-D", telemetry.Summary.AppliedIdentity) + require.Equal(t, "bidirectional_search", telemetry.Summary.RuntimeBranch) + require.False(t, *telemetry.Summary.FallbackExecuted) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) + require.False(t, *telemetry.Diagnostic.TimedSample) + require.Equal(t, int64(7), *telemetry.Diagnostic.Counters.ShortestPath.CandidateEdges) + require.Equal(t, int64(4), *telemetry.Diagnostic.Counters.ShortestPath.PredecessorPeak) + require.Equal(t, int64(4), *telemetry.Diagnostic.Counters.ShortestPath.Levels[0].PredecessorRows) + require.NotNil(t, telemetry.Diagnostic.Counters.Workspace) + observed := traversalNumericObservations(telemetry.Diagnostic.Counters) + require.Equal(t, int64(6), observed["state_rows"]) + require.Equal(t, int64(3), observed["frontier_rows"]) + require.Equal(t, int64(3), observed["queue_rows"]) + require.Equal(t, int64(4), observed["predecessor_rows"]) +} + +// TestPostgresTraversalTelemetryRebindsRuntimeIdentityOnExactFallback verifies postgres traversal telemetry rebinds runtime identity on exact fallback behavior. +func TestPostgresTraversalTelemetryRebindsRuntimeIdentityOnExactFallback(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.RuntimeBranch = "exact_s4_fallback" + document.Overflowed = traversalTelemetryPointer(true) + document.FallbackExecuted = traversalTelemetryPointer(true) + document.Calls[0].RuntimeBranch = "exact_s4_fallback" + document.Calls[0].Overflowed = traversalTelemetryPointer(true) + document.Calls[0].FallbackExecuted = traversalTelemetryPointer(true) + + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + + require.Equal(t, "SP-S4-C-D", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "SP-S4-C-D", telemetry.Summary.AppliedIdentity) + require.Equal(t, "SP-S4-C-D", telemetry.Summary.FallbackIdentity) + require.True(t, *telemetry.Summary.Overflow) + require.True(t, *telemetry.Summary.FallbackExecuted) + require.Contains(t, telemetry.Summary.PlannedIdentities, "SP-S4-C-D") + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons[0], "S4 fallback") +} + +// TestPostgresTraversalTelemetryRejectsInvocationConnectionAndCapMismatch verifies postgres traversal telemetry rejects invocation connection and cap mismatch behavior. +func TestPostgresTraversalTelemetryRejectsInvocationConnectionAndCapMismatch(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + + err := applyBidirectionalTraversalDiagnostic(telemetry, document, "another-invocation", "9123") + require.ErrorContains(t, err, "invocation identity") + + telemetry = bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document = validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + err = applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "different-backend") + require.ErrorContains(t, err, "connection identity") + + telemetry = bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document = validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.FrontierLimit = traversalTelemetryPointer(int64(99)) + err = applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "cap") + + telemetry = bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document = validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.Counters = nil + err = applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "counters are missing") +} + +// TestPostgresTraversalTelemetryRequiresExactlyOneSingletonSearchCall verifies postgres traversal telemetry requires exactly one singleton search call behavior. +func TestPostgresTraversalTelemetryRequiresExactlyOneSingletonSearchCall(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.SearchCalls = traversalTelemetryPointer(int64(2)) + document.Calls = append(document.Calls, document.Calls[0]) + document.Calls[1].SearchID = traversalTelemetryPointer(int64(2)) + + err := applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "exactly one search call") +} + +// TestPostgresTraversalTelemetryAcceptsSQLPreflightBranchesAndNoPathSentinel +// verifies the Go reader accepts the exact runtime branch spellings emitted by +// the compact SQL kernel, including its nil call/-1 aggregate no-path +// distance representation. +func TestPostgresTraversalTelemetryAcceptsSQLPreflightBranchesAndNoPathSentinel(t *testing.T) { + for _, branch := range []string{ + "zero_hop_preflight", + "one_hop_preflight", + "two_hop_preflight", + "preflight_no_path", + "search_no_path", + } { + t.Run(branch, func(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.RuntimeBranch = branch + document.Calls[0].RuntimeBranch = branch + if branch == "preflight_no_path" || branch == "search_no_path" { + document.Counters.FrozenDistance = traversalTelemetryPointer(int64(-1)) + document.Calls[0].FrozenDistance = nil + } + + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Equal(t, branch, telemetry.Summary.RuntimeBranch) + }) + } +} + +// TestPostgresTraversalTelemetryCapturesASPWorkAndWorkspaceButFailsClosedWithoutHydration verifies postgres traversal telemetry captures asp work and workspace but fails closed without hydration behavior. +func TestPostgresTraversalTelemetryCapturesASPWorkAndWorkspaceButFailsClosedWithoutHydration(t *testing.T) { + telemetry := bidirectionalASPCaseTelemetry(t) + document := validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + + require.NoError(t, applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "ASP-B2-DAG-MIN-LEVEL", telemetry.Summary.RuntimeIdentity) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Equal(t, []TraversalTelemetryFamily{ + TraversalTelemetryFamilyASP, + TraversalTelemetryFamilyHydration, + TraversalTelemetryFamilyWorkspace, + }, telemetry.Diagnostic.RequiredFamilies) + require.Equal(t, int64(13), *telemetry.Diagnostic.Counters.AllShortestPaths.EnumeratedCandidates) + require.Equal(t, int64(384), *telemetry.Diagnostic.Counters.AllShortestPaths.OutputBytes) + require.Nil(t, telemetry.Diagnostic.Counters.Hydration) + require.NotNil(t, telemetry.Diagnostic.Counters.Workspace) +} + +// TestPostgresTraversalTelemetryCompletesASPHydrationFromInvocationAndPlanEvidence verifies postgres traversal telemetry completes asp hydration from invocation and plan evidence behavior. +func TestPostgresTraversalTelemetryCompletesASPHydrationFromInvocationAndPlanEvidence(t *testing.T) { + telemetry := bidirectionalASPCaseTelemetry(t) + document := validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + require.NoError(t, applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + metrics := PostgresPlanMetrics{ + HydrationRows: 48, + HydrationLoops: 12, + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Index Scan", + RelationName: "node", + Alias: "hydrated_nodes", + ActualRows: 4, + ActualLoops: 12, + ActualTotalMS: .25, + }}, + } + enrichBidirectionalHydrationTelemetry(telemetry, document.Counters.OutputPaths, document.Counters.OutputEdgeCells, []string{`["p1"]`, `["p2"]`}, metrics) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(12), *telemetry.Diagnostic.Counters.Hydration.PathCount) + require.Equal(t, int64(36), *telemetry.Diagnostic.Counters.Hydration.EdgeLookups) + require.Equal(t, int64(48), *telemetry.Diagnostic.Counters.Hydration.NodeLookups) +} + +// TestPostgresTraversalTelemetryRebindsASPExactFallbackAndRejectsMissingCounters verifies postgres traversal telemetry rebinds asp exact fallback and rejects missing counters behavior. +func TestPostgresTraversalTelemetryRebindsASPExactFallbackAndRejectsMissingCounters(t *testing.T) { + telemetry := bidirectionalASPCaseTelemetry(t) + document := validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.RuntimeBranch = "exact_a1_fallback" + document.Overflowed = traversalTelemetryPointer(true) + document.FallbackExecuted = traversalTelemetryPointer(true) + document.Calls[0].RuntimeBranch = "exact_a1_fallback" + document.Calls[0].Overflowed = traversalTelemetryPointer(true) + document.Calls[0].FallbackExecuted = traversalTelemetryPointer(true) + + require.NoError(t, applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "ASP-A1-DAG", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "ASP-A1-DAG", telemetry.Summary.FallbackIdentity) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons, "nested exact ASP-A1 fallback traversal work counters are unavailable") + + telemetry = bidirectionalASPCaseTelemetry(t) + document = validBidirectionalAllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.Counters.OutputBytes = nil + err := applyBidirectionalAllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123") + require.ErrorContains(t, err, "output_bytes") +} + +// TestPostgresTraversalTelemetryWitnessRequiresSeparateHydrationEvidence verifies postgres traversal telemetry witness requires separate hydration evidence behavior. +func TestPostgresTraversalTelemetryWitnessRequiresSeparateHydrationEvidence(t *testing.T) { + telemetry := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelDiagnostic) + telemetry.Summary.RequestedIdentity = "SP-B2-C-MIN-LEVEL-WE+MAT-M0" + telemetry.Summary.PlannedIdentities = []string{"SP-B2-C-MIN-LEVEL-WE+MAT-M0", "SP-S4-C-WE+MAT-M0"} + telemetry.Summary.EmittedIdentity = "SP-B2-C-MIN-LEVEL-WE+MAT-M0" + document := validBidirectionalDiagnosticDocument(telemetry.Diagnostic.InvocationID) + + require.NoError(t, applyBidirectionalTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123")) + require.NoError(t, telemetry.Validate()) + require.Contains(t, telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons, "complete invocation-local path hydration counters are unavailable") +} + +// TestPostgresTraversalTelemetryLeavesNonBidirectionalHiddenFunctionsUnavailable verifies postgres traversal telemetry leaves non bidirectional hidden functions unavailable behavior. +func TestPostgresTraversalTelemetryLeavesNonBidirectionalHiddenFunctionsUnavailable(t *testing.T) { + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Function Scan", + FunctionName: "all_shortest_paths_dag", + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + reference := PostgresReferenceResult{ + Architecture: "ASP-A1-DAG", + ImplementationID: "typed_predecessor_dag_v1", + PostgresMetrics: &metrics, + } + + telemetry, err := buildPostgresReferenceTraversalTelemetry(reference, nil, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Nil(t, telemetry.Diagnostic.Counters.AllShortestPaths) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons[0], "Function Scan") +} + +// TestPostgresTraversalTelemetryCompletesA1AllShortestWorkspaceReceipt verifies +// the A1 reader maps its own single-ended workspace receipt into complete +// all-shortest, hydration, and workspace telemetry. +func TestPostgresTraversalTelemetryCompletesA1AllShortestWorkspaceReceipt(t *testing.T) { + telemetry := a1AllShortestCaseTelemetry(t) + document := validA1AllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + observed := []string{`["path-1"]`, `["path-2"]`} + metrics := PostgresPlanMetrics{ + HydrationRows: 8, + HydrationLoops: 4, + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Index Scan", RelationName: "node", ActualRows: 2, ActualLoops: 4, ActualTotalMS: .25, + }}, + } + + require.NoError(t, applyA1AllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123", observed, metrics)) + require.NoError(t, telemetry.Validate()) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), telemetry.Summary.RuntimeIdentity) + require.Equal(t, "single_ended_search", telemetry.Summary.RuntimeBranch) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(6), *telemetry.Diagnostic.Counters.AllShortestPaths.Search.CandidateEdges) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.AllShortestPaths.OutputPaths) + require.Equal(t, int64(6), *telemetry.Diagnostic.Counters.AllShortestPaths.OutputEdgeCells) + require.NotNil(t, telemetry.Diagnostic.Counters.Hydration) + require.NotNil(t, telemetry.Diagnostic.Counters.Workspace) + require.Equal(t, int64(4096), *telemetry.Diagnostic.Counters.Workspace.SessionPeakBytes) +} + +// TestPostgresTraversalTelemetryRejectsA1AllShortestStaleOrContradictoryReceipt +// keeps the A1 diagnostic fail-closed when a prior session workspace is reused. +func TestPostgresTraversalTelemetryRejectsA1AllShortestStaleOrContradictoryReceipt(t *testing.T) { + telemetry := a1AllShortestCaseTelemetry(t) + document := validA1AllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + document.SearchCalls = traversalTelemetryPointer(int64(2)) + err := applyA1AllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123", []string{`["path-1"]`, `["path-2"]`}, PostgresPlanMetrics{}) + require.ErrorContains(t, err, "invocation state is incomplete") + + telemetry = a1AllShortestCaseTelemetry(t) + document = validA1AllShortestDiagnosticDocument(telemetry.Diagnostic.InvocationID) + err = applyA1AllShortestTraversalDiagnostic(telemetry, document, telemetry.Diagnostic.InvocationID, "9123", []string{`["path-1"]`}, PostgresPlanMetrics{}) + require.ErrorContains(t, err, "output count") +} + +// TestPostgresTraversalTelemetryUsesPlanReplayForSQLVisibleOrientation verifies postgres traversal telemetry uses plan replay for sql visible orientation behavior. +func TestPostgresTraversalTelemetryUsesPlanReplayForSQLVisibleOrientation(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: "EXPANSION-SUFFIX-SEEDED-REVERSE", + Selected: "EXPANSION-STEPWISE-FORWARD", + Applied: "EXPANSION-STEPWISE-FORWARD", + Fallback: "EXPANSION-STEPWISE-FORWARD", + PlannedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedPolicy: "orientation-probe-v1", + SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", + StateLimit: 4096, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_candidate", + ActualRows: 1, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + metrics, + "9123", + TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryFamilyOrientation, telemetry.Diagnostic.RequiredFamilies[0]) + require.Equal(t, TraversalTelemetryCounterStatusPlanPartial, telemetry.Diagnostic.CounterStatus) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", telemetry.Summary.RuntimeIdentity) + require.Equal(t, telemetry.Summary.RuntimeIdentity, telemetry.Summary.AppliedIdentity) + require.Equal(t, "guarded_dual_arm", telemetry.Summary.ExecutionBoundary) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["orientation_executed_candidate_rows"]) +} + +// TestPostgresTraversalTelemetryKeepsEndpointGuardInOrientationFamily verifies postgres traversal telemetry keeps endpoint guard in orientation family behavior. +func TestPostgresTraversalTelemetryKeepsEndpointGuardInOrientationFamily(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_prefix_terminal_expansion", + Candidate: string(optimize.ExpansionSearchEndpointSeededReverse), + Selected: string(optimize.ExpansionSearchEndpointSeededReverse), + Applied: string(optimize.ExpansionSearchEndpointSeededReverse), + Fallback: string(optimize.ExpansionSearchStepwiseForward), + PlannedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchEndpointSeededReverse)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchEndpointSeededReverse)}, + EmittedPolicy: string(optimize.ExpansionSearchPolicyEndpointGuardV1), + ExecutionBoundary: "guarded_dual_arm", + } + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_candidate", + ActualRows: 1, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_incumbent", + ActualRows: 0, + ActualLoops: 1, + }, + }, + } + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, []TraversalTelemetryFamily{TraversalTelemetryFamilyOrientation}, telemetry.Diagnostic.RequiredFamilies) + require.Equal(t, string(optimize.ExpansionSearchEndpointSeededReverse), telemetry.Summary.RuntimeIdentity) +} + +// TestPostgresTraversalTelemetryCompletesGuardedInlineASPCounters verifies postgres traversal telemetry completes guarded inline asp counters behavior. +func TestPostgresTraversalTelemetryCompletesGuardedInlineASPCounters(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "ASP", + Candidate: "ASP-I1-U-DAG+MAT-M0", + Selected: "ASP-I1-U-DAG+MAT-M0", + Applied: "ASP-I1-U-DAG+MAT-M0", + Fallback: "ASP-A1-DAG", + PlannedCandidates: []string{"ASP-A1-DAG", "ASP-I1-U-DAG+MAT-M0"}, + EmittedCandidates: []string{"ASP-I1-U-DAG+MAT-M0", "ASP-A1-DAG"}, + EmittedPolicy: "asp-i1-guarded-v1", + SelectionMode: "production_canary", + SelectorVersion: "asp-i1-canary-v1", + ExecutionBoundary: "guarded_dual_arm", + ObservationMode: "all_paths", + StateLimit: 10, + PredecessorLimit: 20, + EnumerationLimit: 30, + OutputBytesLimit: 1000, + } + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + HydrationRows: 4, + HydrationLoops: 2, + PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 3, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 2, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", 4, 1), + inlinePredecessorPlanNode("asp_i1_shortest", 2, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", 2, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + }, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichInlineASPTraversalTelemetry(telemetry, metrics, 2, []string{`["p1"]`, `["p2"]`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "ASP-I1-U-DAG+MAT-M0", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "inline_predecessor_dag", telemetry.Summary.RuntimeBranch) + require.False(t, *telemetry.Summary.FallbackExecuted) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(3), *telemetry.Diagnostic.Counters.InlineASP.DistanceRows) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.InlineASP.PredecessorRows) + require.Equal(t, int64(4), *telemetry.Diagnostic.Counters.InlineASP.EnumerationRows) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.InlineASP.CandidateMarkerRows) + require.Equal(t, int64(0), *telemetry.Diagnostic.Counters.InlineASP.FallbackMarkerRows) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.InlineASP.CandidateExecutorLoops) + require.Equal(t, int64(0), *telemetry.Diagnostic.Counters.InlineASP.FallbackExecutorLoops) +} + +// TestPostgresTraversalTelemetryCompletesGuardedInlineCanonicalSPCounters verifies postgres traversal telemetry completes guarded inline canonical sp counters behavior. +func TestPostgresTraversalTelemetryCompletesGuardedInlineCanonicalSPCounters(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "SP", + Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Selected: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Applied: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Fallback: string(optimize.ShortestPathExecutorS4CanonicalWitness), + PlannedCandidates: []string{ + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + }, + EmittedCandidates: []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + }, + EmittedPolicy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + SelectionMode: "production_canary", + SelectorVersion: "sp-i1-canary-v1", + ExecutionBoundary: "guarded_dual_arm", + ObservationMode: "one_path", + StateLimit: 10, + PredecessorLimit: 20, + EnumerationLimit: 30, + OutputBytesLimit: 1000, + } + + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // candidateMarker retains the candidate marker while anonymous record is assembled or evaluated. + candidateMarker int64 + // fallbackMarker retains the fallback marker while anonymous record is assembled or evaluated. + fallbackMarker int64 + // outputRows records the number of output rows. + outputRows int64 + // distanceRows records the number of distance rows. + distanceRows int64 + // expectedIdentity identifies the expected identity. + expectedIdentity string + // expectedBranch retains the expected branch while anonymous record is assembled or evaluated. + expectedBranch string + // expectedFallback indicates whether expected fallback applies. + expectedFallback bool + }{ + { + name: "candidate witness", + candidateMarker: 1, + outputRows: 1, + distanceRows: 3, + expectedIdentity: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + expectedBranch: "inline_canonical_witness", + }, + { + name: "candidate no path", + candidateMarker: 1, + distanceRows: 3, + expectedIdentity: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + expectedBranch: "inline_canonical_no_path", + }, + { + name: "exact S4 fallback", + fallbackMarker: 1, + outputRows: 1, + distanceRows: 11, + expectedIdentity: string(optimize.ShortestPathExecutorS4CanonicalWitness), + expectedBranch: "exact_s4_fallback", + expectedFallback: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + HydrationRows: test.outputRows, + HydrationLoops: test.outputRows, + PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", test.distanceRows, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 2, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", 4, 1), + inlinePredecessorPlanNode("asp_i1_shortest", test.outputRows, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", test.candidateMarker, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", test.fallbackMarker, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", test.candidateMarker*test.outputRows, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", test.fallbackMarker*test.outputRows, 1), + inlinePredecessorMarkerGateNode("candidate", test.candidateMarker, 1), + inlinePredecessorMarkerGateNode("fallback", test.fallbackMarker, 1), + inlinePredecessorExecutorNode("candidate", test.candidateMarker), + inlinePredecessorExecutorNode("fallback", test.fallbackMarker), + }, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichInlinePredecessorTraversalTelemetry(telemetry, metrics, test.outputRows, []string{`["p1"]`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, test.expectedIdentity, telemetry.Summary.RuntimeIdentity) + require.Equal(t, test.expectedBranch, telemetry.Summary.RuntimeBranch) + require.Equal(t, test.expectedFallback, *telemetry.Summary.FallbackExecuted) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.NotNil(t, telemetry.Diagnostic.Counters.InlineShortestPath) + require.Nil(t, telemetry.Diagnostic.Counters.InlineASP) + require.Equal(t, test.candidateMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.CandidateMarkerRows) + require.Equal(t, test.fallbackMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.FallbackMarkerRows) + require.Equal(t, test.candidateMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.CandidateExecutorLoops) + require.Equal(t, test.fallbackMarker, *telemetry.Diagnostic.Counters.InlineShortestPath.FallbackExecutorLoops) + }) + } +} + +func TestPostgresTraversalTelemetryCompletesGuardedInlineDistanceCounters(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "SP", Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), Selected: string(optimize.ShortestPathExecutorI2GuardedDistance), + Applied: string(optimize.ShortestPathExecutorI2GuardedDistance), Fallback: string(optimize.ShortestPathExecutorS4CanonicalDistance), + EmittedPolicy: optimize.ShortestPathPolicyI2DistanceGuardedV1, SelectionMode: "production_canary", + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", ObservationMode: "distance", + StateLimit: 10, FrontierLimit: 10, + } + ids := map[string]int64{"sp_i2_distance_bounded": 1, "sp_i2_target": 2, "sp_i2_candidate_marker": 3, "sp_i2_fallback_marker": 4, "sp_i2_candidate_rows": 5, "sp_i2_fallback_rows": 6} + node := func(name string, rows int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{PlanNodeID: ids[name], NodeType: "Result", SubplanName: "CTE " + name, ActualRows: rows, ActualLoops: 1} + } + gate := func(branch string, markerRows int64) []PostgresPlanNodeMetric { + body := ids["sp_i2_"+branch+"_rows"] + return []PostgresPlanNodeMetric{ + {PlanNodeID: body + 100, ParentPlanNodeID: body, ParentRelationship: "Outer", NodeType: "CTE Scan", CTEName: "sp_i2_" + branch + "_marker", ActualRows: markerRows, ActualLoops: 1}, + {PlanNodeID: body + 200, ParentPlanNodeID: body, ParentRelationship: "Inner", NodeType: "Result", ActualLoops: markerRows}, + } + } + metrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + node("sp_i2_distance_bounded", 3), node("sp_i2_target", 1), node("sp_i2_candidate_marker", 1), node("sp_i2_fallback_marker", 0), + node("sp_i2_candidate_rows", 1), node("sp_i2_fallback_rows", 0), + }} + metrics.PlanNodes = append(metrics.PlanNodes, gate("candidate", 1)...) + metrics.PlanNodes = append(metrics.PlanNodes, gate("fallback", 0)...) + telemetry, err := buildPostgresCaseTraversalTelemetry(translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichInlineDistanceTraversalTelemetry(telemetry, 1) + require.NoError(t, telemetry.Validate()) + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistance), telemetry.Summary.RuntimeIdentity) + require.Equal(t, "inline_canonical_distance", telemetry.Summary.RuntimeBranch) + require.NotNil(t, telemetry.Diagnostic.Counters.InlineShortestDistance) + require.Equal(t, int64(3), *telemetry.Diagnostic.Counters.InlineShortestDistance.StateRows) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.InlineShortestDistance.CandidateExecutorLoops) + require.Equal(t, int64(0), *telemetry.Diagnostic.Counters.InlineShortestDistance.FallbackExecutorLoops) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["sp_i2_target_rows"]) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["sp_i2_output_rows"]) + + mismatched, err := buildPostgresCaseTraversalTelemetry(translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichInlineDistanceTraversalTelemetry(mismatched, 2) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, mismatched.Diagnostic.CounterStatus) + require.Contains(t, mismatched.Diagnostic.IncompleteReasons, "inline distance plan output does not match the exact public observation") + require.Nil(t, mismatched.Diagnostic.Counters.InlineShortestDistance) +} + +// TestResourceGateRequiresSingularInlineDistanceBranchAndInactiveArm verifies +// SP-I2 qualification binds complementary markers, branch rows, executor loops, +// typed counters, and the runtime receipt for both possible guarded arms. +func TestResourceGateRequiresSingularInlineDistanceBranchAndInactiveArm(t *testing.T) { + newTelemetry := func(fallback bool) *TraversalExecutionTelemetry { + const limit int64 = 3 + candidateMarker, fallbackMarker := int64(1), int64(0) + candidateRows, fallbackRows := int64(1), int64(0) + candidateLoops, fallbackLoops := int64(1), int64(0) + stateRows := limit + runtimeIdentity := string(optimize.ShortestPathExecutorI2GuardedDistance) + runtimeBranch := "inline_canonical_distance" + if fallback { + candidateMarker, fallbackMarker = 0, 1 + candidateRows, fallbackRows = 0, 1 + candidateLoops, fallbackLoops = 0, 1 + stateRows = limit + 1 + runtimeIdentity = string(optimize.ShortestPathExecutorS4CanonicalDistance) + runtimeBranch = "exact_s4_distance_fallback" + } + plan := map[string]int64{ + "sp_i2_distance_rows": stateRows, "sp_i2_target_rows": candidateRows, "sp_i2_output_rows": candidateRows + fallbackRows, + "sp_i2_candidate_marker_rows": candidateMarker, "sp_i2_fallback_marker_rows": fallbackMarker, + "sp_i2_candidate_branch_rows": candidateRows, "sp_i2_fallback_branch_rows": fallbackRows, + "sp_i2_candidate_executor_loops": candidateLoops, "sp_i2_fallback_executor_loops": fallbackLoops, + } + return &TraversalExecutionTelemetry{ + Summary: TraversalExecutionSummary{ + EmittedIdentity: optimize.ShortestPathPolicyI2DistanceGuardedV1, RuntimeIdentity: runtimeIdentity, RuntimeBranch: runtimeBranch, + Caps: map[string]int64{"state_rows": limit, "frontier_rows": limit}, + RuntimeOutcomeAvailable: telemetryBool(true), FallbackExecuted: telemetryBool(fallback), Overflow: telemetryBool(fallback), + }, + Diagnostic: &TraversalExecutionDiagnostic{ + RequiredFamilies: []TraversalTelemetryFamily{TraversalTelemetryFamilySP}, + Counters: TraversalDiagnosticCounters{InlineShortestDistance: &InlineDistanceTraversalCounters{ + StateRows: telemetryInt64(stateRows), FrontierRows: telemetryInt64(stateRows), OutputRows: telemetryInt64(candidateRows + fallbackRows), + CandidateMarkerRows: telemetryInt64(candidateMarker), FallbackMarkerRows: telemetryInt64(fallbackMarker), + CandidateBranchRows: telemetryInt64(candidateRows), FallbackBranchRows: telemetryInt64(fallbackRows), + CandidateExecutorLoops: telemetryInt64(candidateLoops), FallbackExecutorLoops: telemetryInt64(fallbackLoops), + }}, + PlanReplay: &TraversalPlanReplayEvidence{Counters: plan}, + }, + } + } + + t.Run("candidate exact cap boundary", func(t *testing.T) { + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, newTelemetry(false)) + require.Empty(t, gateCase.Reasons) + }) + t.Run("fallback exact cap plus one sentinel", func(t *testing.T) { + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, newTelemetry(true)) + require.Empty(t, gateCase.Reasons) + }) + + tests := map[string]struct { + mutate func(*TraversalExecutionTelemetry) + reason string + }{ + "inactive fallback executor": {func(telemetry *TraversalExecutionTelemetry) { + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_fallback_executor_loops"] = 1 + }, "candidate selection did not suppress the fallback executor"}, + "dual markers": {func(telemetry *TraversalExecutionTelemetry) { + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_fallback_marker_rows"] = 1 + }, "must attribute exactly one candidate or fallback marker"}, + "branch output mismatch": {func(telemetry *TraversalExecutionTelemetry) { + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_output_rows"] = 0 + }, "output does not equal its complementary branch rows"}, + "typed counter drift": {func(telemetry *TraversalExecutionTelemetry) { + telemetry.Diagnostic.Counters.InlineShortestDistance.CandidateExecutorLoops = telemetryInt64(0) + }, "typed counter does not match plan counter sp_i2_candidate_executor_loops"}, + "runtime receipt drift": {func(telemetry *TraversalExecutionTelemetry) { + telemetry.Summary.RuntimeBranch = "exact_s4_distance_fallback" + }, "candidate marker contradicts the runtime receipt"}, + "candidate cap plus one": {func(telemetry *TraversalExecutionTelemetry) { + value := telemetry.Summary.Caps["state_rows"] + 1 + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_distance_rows"] = value + telemetry.Diagnostic.Counters.InlineShortestDistance.StateRows = telemetryInt64(value) + telemetry.Diagnostic.Counters.InlineShortestDistance.FrontierRows = telemetryInt64(value) + }, "candidate selection exceeds its state or conservative frontier cap"}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + telemetry := newTelemetry(false) + test.mutate(telemetry) + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, telemetry) + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), test.reason) + }) + } + + for name, stateRows := range map[string]int64{ + "fallback without sentinel": 3, + "fallback beyond cap plus one": 5, + } { + t.Run(name, func(t *testing.T) { + telemetry := newTelemetry(true) + telemetry.Diagnostic.PlanReplay.Counters["sp_i2_distance_rows"] = stateRows + telemetry.Diagnostic.Counters.InlineShortestDistance.StateRows = telemetryInt64(stateRows) + telemetry.Diagnostic.Counters.InlineShortestDistance.FrontierRows = telemetryInt64(stateRows) + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, telemetry) + require.Contains(t, gateCase.Reasons, "inline SP distance fallback selection lacks an exact state or conservative frontier cap+1 sentinel") + }) + } + + telemetry := newTelemetry(false) + record := CaseResult{ + RowCount: 1, TraversalTelemetry: telemetry, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", Applied: string(optimize.ShortestPathExecutorI2GuardedDistance), EmittedPolicy: optimize.ShortestPathPolicyI2DistanceGuardedV1, + }}}, + } + contract, found := guardedInlineResourceContractForArchitecture(string(optimize.ShortestPathExecutorI2GuardedDistance)) + require.True(t, found) + gateCase := &ResourceGateCase{} + appendGuardedInlineResourceBindingReasons(gateCase, record, contract) + require.Empty(t, gateCase.Reasons) + record.RowCount = 2 + appendGuardedInlineResourceBindingReasons(gateCase, record, contract) + require.Contains(t, gateCase.Reasons, "inline SP distance typed output does not match the exact public observation") + require.Contains(t, gateCase.Reasons, "inline SP distance plan output does not match the exact public observation") +} + +// TestPostgresTraversalTelemetryRejectsEveryMissingInlinePredecessorCounter verifies postgres traversal telemetry rejects every missing inline predecessor counter behavior. +func TestPostgresTraversalTelemetryRejectsEveryMissingInlinePredecessorCounter(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "SP", + Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Selected: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Applied: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Fallback: string(optimize.ShortestPathExecutorS4CanonicalWitness), + PlannedCandidates: []string{ + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + }, + EmittedCandidates: []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + }, + EmittedPolicy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + ObservationMode: "one_path", + StateLimit: 10, + PredecessorLimit: 20, + EnumerationLimit: 30, + OutputBytesLimit: 1000, + } + fullPlan := []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 3, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 2, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", 4, 1), + inlinePredecessorPlanNode("asp_i1_shortest", 1, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + } + expectedCounter := map[string]string{ + "asp_i1_distance_bounded": "asp_i1_distance_rows", + "asp_i1_predecessor_bounded": "asp_i1_predecessor_rows", + "asp_i1_paths_bounded": "asp_i1_enumeration_rows", + "asp_i1_shortest": "asp_i1_output_rows", + "asp_i1_candidate_marker": "asp_i1_candidate_marker_rows", + "asp_i1_fallback_marker": "asp_i1_fallback_marker_rows", + "asp_i1_candidate_rows": "asp_i1_candidate_branch_rows", + "asp_i1_fallback_rows": "asp_i1_fallback_branch_rows", + "test_candidate_executor": "asp_i1_candidate_executor_loops", + "test_fallback_executor": "asp_i1_fallback_executor_loops", + } + + for omitted, counter := range expectedCounter { + t.Run(omitted, func(t *testing.T) { + metrics := PostgresPlanMetrics{Provenance: map[string]string{}} + for _, node := range fullPlan { + if node.SubplanName != "CTE "+omitted && node.Alias != omitted { + metrics.PlanNodes = append(metrics.PlanNodes, node) + } + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichInlinePredecessorTraversalTelemetry(telemetry, metrics, 1, []string{`["p1"]`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Nil(t, telemetry.Diagnostic.Counters.InlineShortestPath) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons[0], counter) + }) + } +} + +// TestPostgresTraversalPlanReplayUsesExactInlinePredecessorCTEBodies verifies postgres traversal plan replay uses exact inline predecessor cte bodies behavior. +func TestPostgresTraversalPlanReplayUsesExactInlinePredecessorCTEBodies(t *testing.T) { + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 3, 1), + { + PlanNodeID: 500, + NodeType: "Limit", + SubplanName: "CTE prefix_asp_i1_distance_bounded", + ActualRows: 77, + ActualLoops: 1, + }, + { + NodeType: "CTE Scan", + CTEName: "asp_i1_distance_bounded", + Alias: "asp_i1_distance_bounded", + ActualRows: 99, + ActualLoops: 7, + }, + inlinePredecessorPlanNode("asp_i1_candidate_rows", 0, 1), + { + NodeType: "CTE Scan", + CTEName: "asp_i1_candidate_rows", + Alias: "asp_i1_candidate_rows", + ActualRows: 10, + ActualLoops: 5, + }, + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + { + NodeType: "CTE Scan", + CTEName: "asp_i1_fallback_rows", + Alias: "asp_i1_fallback_rows", + ActualRows: 8, + ActualLoops: 3, + }, + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + }, + } + + replay := postgresTraversalPlanReplay(metrics) + require.Equal(t, int64(3), replay.Counters["asp_i1_distance_rows"]) + require.Equal(t, int64(0), replay.Counters["asp_i1_candidate_branch_rows"]) + require.Equal(t, int64(0), replay.Counters["asp_i1_fallback_branch_rows"]) + require.Equal(t, int64(1), replay.Counters["asp_i1_candidate_executor_loops"]) + require.Equal(t, int64(0), replay.Counters["asp_i1_fallback_executor_loops"]) +} + +// TestPostgresTraversalPlanReplayRejectsAmbiguousInlineBranchShape verifies postgres traversal plan replay rejects ambiguous inline branch shape behavior. +func TestPostgresTraversalPlanReplayRejectsAmbiguousInlineBranchShape(t *testing.T) { + t.Run("duplicate exact body", func(t *testing.T) { + body := inlinePredecessorPlanNode("asp_i1_candidate_rows", 1, 1) + duplicate := body + duplicate.PlanNodeID = 99 + replay := postgresTraversalPlanReplay(PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + body, duplicate, inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorExecutorNode("candidate", 1), + }, + }) + _, branchPresent := replay.Counters["asp_i1_candidate_branch_rows"] + _, executorPresent := replay.Counters["asp_i1_candidate_executor_loops"] + require.False(t, branchPresent) + require.False(t, executorPresent) + }) + + t.Run("wrong direct outer marker", func(t *testing.T) { + body := inlinePredecessorPlanNode("asp_i1_candidate_rows", 1, 1) + wrongMarker := inlinePredecessorMarkerGateNode("candidate", 1, 1) + wrongMarker.CTEName = "asp_i1_fallback_marker" + replay := postgresTraversalPlanReplay(PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + body, inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), wrongMarker, inlinePredecessorExecutorNode("candidate", 1), + }, + }) + require.Equal(t, int64(1), replay.Counters["asp_i1_candidate_branch_rows"]) + _, executorPresent := replay.Counters["asp_i1_candidate_executor_loops"] + require.False(t, executorPresent) + }) +} + +// inlinePredecessorPlanNode prepares or inspects test evidence for inline predecessor plan node. +func inlinePredecessorPlanNode(name string, rows, loops int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{ + PlanNodeID: inlinePredecessorPlanNodeID(name), + NodeType: "Result", + SubplanName: "CTE " + name, + ActualRows: rows, + ActualLoops: loops, + } +} + +// inlinePredecessorExecutorNode prepares or inspects test evidence for inline predecessor executor node. +func inlinePredecessorExecutorNode(branch string, loops int64) PostgresPlanNodeMetric { + bodyID := inlinePredecessorPlanNodeID("asp_i1_" + branch + "_rows") + return PostgresPlanNodeMetric{ + PlanNodeID: bodyID + 100, + ParentPlanNodeID: bodyID, + ParentRelationship: "Inner", + NodeType: "Result", + Alias: "test_" + branch + "_executor", + ActualLoops: loops, + } +} + +// inlinePredecessorMarkerGateNode prepares or inspects test evidence for inline predecessor marker gate node. +func inlinePredecessorMarkerGateNode(branch string, rows, loops int64) PostgresPlanNodeMetric { + bodyID := inlinePredecessorPlanNodeID("asp_i1_" + branch + "_rows") + return PostgresPlanNodeMetric{ + PlanNodeID: bodyID + 200, + ParentPlanNodeID: bodyID, + ParentRelationship: "Outer", + NodeType: "CTE Scan", + CTEName: "asp_i1_" + branch + "_marker", + Alias: "test_" + branch + "_marker_gate", + ActualRows: rows, + ActualLoops: loops, + } +} + +// inlinePredecessorPlanNodeID prepares or inspects test evidence for inline predecessor plan node id. +func inlinePredecessorPlanNodeID(name string) int64 { + ids := map[string]int64{ + "asp_i1_distance_bounded": 1, "asp_i1_predecessor_bounded": 2, + "asp_i1_paths_bounded": 3, "asp_i1_shortest": 4, + "asp_i1_candidate_marker": 5, "asp_i1_fallback_marker": 6, + "asp_i1_candidate_rows": 7, "asp_i1_fallback_rows": 8, + } + return ids[name] +} + +// TestPostgresTraversalTelemetryCompletesSuffixReverseGuardCounters verifies +// that the reverse-first guard uses its own counter family and reports both +// mutually exclusive runtime outcomes without orientation-only score fields. +func TestPostgresTraversalTelemetryCompletesSuffixReverseGuardCounters(t *testing.T) { + tests := []struct { + name string + candidateMarker int64 + fallbackMarker int64 + candidateLoops int64 + fallbackLoops int64 + suffixRows int64 + stateRows int64 + expectedIdentity string + expectedBranch string + expectedFallback bool + }{ + {name: "candidate", candidateMarker: 1, candidateLoops: 1, suffixRows: 2, stateRows: 9, expectedIdentity: string(optimize.ExpansionSearchSuffixSeededReverse), expectedBranch: "suffix_seeded_reverse"}, + {name: "suffix overflow fallback", fallbackMarker: 1, fallbackLoops: 1, suffixRows: 513, stateRows: 0, expectedIdentity: string(optimize.ExpansionSearchStepwiseForward), expectedBranch: "exact_forward_suffix_overflow", expectedFallback: true}, + {name: "state overflow fallback", fallbackMarker: 1, fallbackLoops: 1, suffixRows: 2, stateRows: 513, expectedIdentity: string(optimize.ExpansionSearchStepwiseForward), expectedBranch: "exact_forward_state_overflow", expectedFallback: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + outcome := suffixGuardTestOutcome() + metrics := suffixGuardTestMetrics(test.candidateMarker, test.fallbackMarker, test.candidateLoops, test.fallbackLoops, test.suffixRows, test.stateRows) + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichSuffixGuardTraversalTelemetry(telemetry, metrics, 1, []string{`{"path":"p"}`}) + require.NoError(t, telemetry.Validate()) + require.Equal(t, test.expectedIdentity, telemetry.Summary.RuntimeIdentity) + require.Equal(t, test.expectedBranch, telemetry.Summary.RuntimeBranch) + require.Equal(t, test.expectedFallback, *telemetry.Summary.FallbackExecuted) + require.Contains(t, telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilySuffixGuard) + require.NotNil(t, telemetry.Diagnostic.Counters.SuffixGuard) + require.Nil(t, telemetry.Diagnostic.Counters.Orientation) + require.Equal(t, test.stateRows, *telemetry.Diagnostic.Counters.SuffixGuard.StateRows) + require.Equal(t, test.candidateLoops, *telemetry.Diagnostic.Counters.SuffixGuard.CandidateExecutorLoops) + require.Equal(t, test.fallbackLoops, *telemetry.Diagnostic.Counters.SuffixGuard.FallbackExecutorLoops) + }) + } +} + +// TestSuffixReverseGuardRuntimeOutcomeFailsClosedWithoutBothSentinels verifies +// that marker rows alone cannot invent an admitted branch when either cap+1 +// relation is absent from the exact plan replay. +func TestSuffixReverseGuardRuntimeOutcomeFailsClosedWithoutBothSentinels(t *testing.T) { + metrics := suffixGuardTestMetrics(1, 0, 1, 0, 2, 9) + filtered := metrics.PlanNodes[:0] + for _, node := range metrics.PlanNodes { + if !strings.HasSuffix(strings.ToLower(node.SubplanName), "suffix_guard_states") { + filtered = append(filtered, node) + } + } + metrics.PlanNodes = filtered + identity, branch, fallback, overflow := runtimeTraversalIdentity( + suffixGuardTestOutcome(), metrics, string(optimize.ExpansionSearchSuffixSeededReverse), string(optimize.ExpansionSearchSuffixSeededReverse), + ) + require.Empty(t, identity) + require.Equal(t, "runtime_outcome_unavailable", branch) + require.False(t, fallback) + require.False(t, overflow) +} + +// TestSuffixReverseGuardDiagnosticRejectsPlanOutputMismatch binds typed output +// telemetry to both the JSON plan and the exact public observation. +func TestSuffixReverseGuardDiagnosticRejectsPlanOutputMismatch(t *testing.T) { + metrics := suffixGuardTestMetrics(1, 0, 1, 0, 2, 9) + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{suffixGuardTestOutcome()}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichSuffixGuardTraversalTelemetry(telemetry, metrics, 2, []string{`{"path":"p1"}`, `{"path":"p2"}`}) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons, "suffix-reverse guard plan output does not match the exact public observation") +} + +// TestPostgresTraversalTelemetryCompletesSuffixRouteComponentCounters verifies +// the direct component has its own complete one-arm counter contract rather +// than borrowing guarded-selector or generic recursive counters. +func TestPostgresTraversalTelemetryCompletesSuffixRouteComponentCounters(t *testing.T) { + planningMS, executionMS := 1.25, 2.5 + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchSuffixSeededReverse), + Applied: string(optimize.ExpansionSearchSuffixSeededReverse), + PlannedCandidates: []string{string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchSuffixSeededReverse)}, + SelectionMode: "component_tool", + SelectorVersion: optimize.ExpansionSearchSelectorSuffixRouteComponentV1, + ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryInlineStatement, + ObservationMode: string(optimize.ExpansionSearchObservationFullPath), + } + metrics := PostgresPlanMetrics{ + PlanningMS: &planningMS, + ExecutionMS: &executionMS, + Provenance: map[string]string{ + "planning_ms": "measured_plan_json", + "execution_ms": "measured_plan_json", + }, + PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Nested Loop", SubplanName: "CTE s5_suffix_seeded_suffix", ActualRows: 3, ActualLoops: 1}, + {NodeType: "Aggregate", SubplanName: "CTE s5_suffix_seeded_boundaries", ActualRows: 2, ActualLoops: 1}, + {NodeType: "Recursive Union", SubplanName: "CTE s5_suffix_seeded_reverse", ActualRows: 9, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_suffix_seeded_component_receipt", ActualRows: 1, ActualLoops: 1}, + {NodeType: "Index Scan", RelationName: "node_3", Alias: "_ordered_path_node", ActualRows: 1, ActualLoops: 12}, + {NodeType: "Index Scan", RelationName: "edge_3", Alias: "_ordered_path_edge", ActualRows: 1, ActualLoops: 11}, + }, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichSuffixRouteComponentTraversalTelemetry(telemetry, metrics, 2) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, []TraversalTelemetryFamily{TraversalTelemetryFamilySuffixComponent}, telemetry.Diagnostic.RequiredFamilies) + require.NotNil(t, telemetry.Diagnostic.Counters.SuffixComponent) + require.Nil(t, telemetry.Diagnostic.Counters.SuffixGuard) + require.Equal(t, int64(3), *telemetry.Diagnostic.Counters.SuffixComponent.SuffixRows) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.SuffixComponent.BoundaryRows) + require.Equal(t, int64(9), *telemetry.Diagnostic.Counters.SuffixComponent.ReverseStateRows) + require.Equal(t, int64(12), *telemetry.Diagnostic.Counters.SuffixComponent.OrderedNodeHydrationLoops) + require.Equal(t, int64(12), *telemetry.Diagnostic.Counters.SuffixComponent.OrderedNodeHydrationRows) + require.Equal(t, int64(11), *telemetry.Diagnostic.Counters.SuffixComponent.OrderedEdgeHydrationLoops) + require.Equal(t, int64(11), *telemetry.Diagnostic.Counters.SuffixComponent.OrderedEdgeHydrationRows) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.SuffixComponent.OutputRows) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.SuffixComponent.ReceiptRows) + require.Equal(t, int64(1_250_000), *telemetry.Diagnostic.Counters.SuffixComponent.PlanningTimeNS) + require.Equal(t, int64(2_500_000), *telemetry.Diagnostic.Counters.SuffixComponent.ExecutionTimeNS) + observed := traversalNumericObservations(telemetry.Diagnostic.Counters) + require.Equal(t, int64(3), observed["suffix_rows"]) + require.Equal(t, int64(9), observed["state_rows"]) + require.Equal(t, int64(23), observed["hydration_rows"]) +} + +// TestPostgresTraversalTelemetryAddsClosureWorkspace verifies direct-component +// workspace telemetry becomes complete only when an explicit boundary closure +// supplies measured size-one session and pool high-water values. +func TestPostgresTraversalTelemetryAddsSuffixRouteComponentClosureWorkspace(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilySuffixComponent} + telemetry.Diagnostic.Counters = TraversalDiagnosticCounters{SuffixComponent: &SuffixComponentTraversalCounters{ + SuffixRows: traversalTelemetryPointer(int64(1)), BoundaryRows: traversalTelemetryPointer(int64(1)), ReverseStateRows: traversalTelemetryPointer(int64(1)), + OrderedNodeHydrationLoops: traversalTelemetryPointer(int64(0)), OrderedNodeHydrationRows: traversalTelemetryPointer(int64(0)), + OrderedEdgeHydrationLoops: traversalTelemetryPointer(int64(0)), OrderedEdgeHydrationRows: traversalTelemetryPointer(int64(0)), + OutputRows: traversalTelemetryPointer(int64(1)), ReceiptRows: traversalTelemetryPointer(int64(1)), + PlanningTimeNS: traversalTelemetryPointer(int64(1)), ExecutionTimeNS: traversalTelemetryPointer(int64(1)), + }} + telemetry.Diagnostic.Provenance = map[string]string{} + telemetry.Summary.SelectorVersion = optimize.ExpansionSearchSelectorSuffixRouteComponentV1 + for _, name := range []string{ + "suffix_component.suffix_rows", "suffix_component.boundary_rows", "suffix_component.reverse_state_rows", "suffix_component.ordered_node_hydration_loops", + "suffix_component.ordered_node_hydration_rows", "suffix_component.ordered_edge_hydration_loops", "suffix_component.ordered_edge_hydration_rows", + "suffix_component.output_rows", "suffix_component.receipt_rows", "suffix_component.planning_time_ns", "suffix_component.execution_time_ns", + } { + telemetry.Diagnostic.Provenance[name] = "test" + } + + enrichSuffixRouteComponentClosureWorkspaceTelemetry(&telemetry, &PostgresBoundaryClosure{ + Workspace: PostgresBoundaryWorkspaceHighWater{SessionPeakBytes: 0, PoolPeakBytes: 0}, + }) + + require.NoError(t, telemetry.Validate()) + require.Equal(t, []TraversalTelemetryFamily{TraversalTelemetryFamilySuffixComponent, TraversalTelemetryFamilyWorkspace}, telemetry.Diagnostic.RequiredFamilies) + require.Zero(t, *telemetry.Diagnostic.Counters.Workspace.SessionPeakBytes) + require.Zero(t, *telemetry.Diagnostic.Counters.Workspace.PoolPeakBytes) +} + +// TestSuffixRouteComponentCountersFailClosedWithoutReceipt ensures a statement +// that lacks the one-row runtime receipt never becomes qualifying evidence. +func TestSuffixRouteComponentCountersFailClosedWithoutReceipt(t *testing.T) { + planningMS, executionMS := 1.0, 2.0 + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchSuffixSeededReverse), Applied: string(optimize.ExpansionSearchSuffixSeededReverse), + SelectorVersion: optimize.ExpansionSearchSelectorSuffixRouteComponentV1, + } + metrics := PostgresPlanMetrics{ + PlanningMS: &planningMS, ExecutionMS: &executionMS, + Provenance: map[string]string{"planning_ms": "measured_plan_json", "execution_ms": "measured_plan_json"}, + PlanNodes: []PostgresPlanNodeMetric{ + {NodeType: "Result", SubplanName: "CTE s5_suffix_seeded_suffix", ActualRows: 2, ActualLoops: 1}, + {NodeType: "Result", SubplanName: "CTE s5_suffix_seeded_boundaries", ActualRows: 1, ActualLoops: 1}, + {NodeType: "Recursive Union", SubplanName: "CTE s5_suffix_seeded_reverse", ActualRows: 8, ActualLoops: 1}, + }, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichSuffixRouteComponentTraversalTelemetry(telemetry, metrics, 1) + require.Equal(t, TraversalTelemetryCounterStatusHiddenUnavailable, telemetry.Diagnostic.CounterStatus) + require.Contains(t, telemetry.Diagnostic.IncompleteReasons, "suffix-route component is missing exact plan counter suffix_component_receipt_rows") + require.Nil(t, telemetry.Diagnostic.Counters.SuffixComponent) +} + +func suffixGuardTestOutcome() translate.TargetLoweringOutcome { + return translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchSuffixSeededReverse), Applied: string(optimize.ExpansionSearchSuffixSeededReverse), + Fallback: string(optimize.ExpansionSearchStepwiseForward), + PlannedCandidates: []string{string(optimize.ExpansionSearchSuffixSeededReverse), string(optimize.ExpansionSearchStepwiseForward)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchSuffixSeededReverse), string(optimize.ExpansionSearchStepwiseForward)}, + EmittedPolicy: string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), + SelectorVersion: optimize.ExpansionSearchSelectorFixedSuffixPathV1, + ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, ObservationMode: string(optimize.ExpansionSearchObservationFullPath), + StateLimit: 512, ProbeCaps: &optimize.ExpansionSearchProbeCaps{ReverseSeedRowLimit: 512}, + } +} + +// suffixGuardTestMetrics builds the exact marker-outer plan shape required by +// suffix guard qualification. +func suffixGuardTestMetrics(candidateMarker, fallbackMarker, candidateLoops, fallbackLoops, suffixRows, stateRows int64) PostgresPlanMetrics { + stage := "s5_" + planNode := func(id int64, suffix string, rows int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{PlanNodeID: id, NodeType: "Result", SubplanName: "CTE " + stage + suffix, ActualRows: rows, ActualLoops: 1} + } + markerGate := func(id, bodyID int64, branch string, rows int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{PlanNodeID: id, ParentPlanNodeID: bodyID, ParentRelationship: "Outer", NodeType: "CTE Scan", CTEName: stage + "suffix_guard_" + branch + "_marker", ActualRows: rows, ActualLoops: 1} + } + executor := func(id, bodyID int64, branch string, loops int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{PlanNodeID: id, ParentPlanNodeID: bodyID, ParentRelationship: "Inner", NodeType: "Result", Alias: "suffix_guard_" + branch + "_executor", ActualLoops: loops} + } + return PostgresPlanMetrics{Provenance: map[string]string{}, RecursiveRows: stateRows, PlanNodes: []PostgresPlanNodeMetric{ + planNode(1, "suffix_guard_root_presence", 1), planNode(2, "suffix_guard_suffix_probe", suffixRows), + planNode(3, "suffix_guard_boundaries", 1), planNode(4, "suffix_guard_states", stateRows), + planNode(5, "suffix_guard_candidate_marker", candidateMarker), planNode(6, "suffix_guard_fallback_marker", fallbackMarker), + planNode(7, "suffix_guard_candidate_body", candidateMarker), planNode(8, "suffix_guard_fallback_body", fallbackMarker), + markerGate(70, 7, "candidate", candidateMarker), executor(71, 7, "candidate", candidateLoops), + markerGate(80, 8, "fallback", fallbackMarker), executor(81, 8, "fallback", fallbackLoops), + }} +} + +// TestPostgresTraversalTelemetryPrefersShortestExecutorOverAnalysisOutcomes verifies postgres traversal telemetry prefers shortest executor over analysis outcomes behavior. +func TestPostgresTraversalTelemetryPrefersShortestExecutorOverAnalysisOutcomes(t *testing.T) { + shortest := translate.TargetLoweringOutcome{ + TargetKind: "traversal", + Family: "SP", + Applied: "SP-B1-C-ALT-NODE-D", + } + outcome, found := singleTraversalOutcome([]translate.TargetLoweringOutcome{ + { + TargetKind: "endpoint_resolution", + Family: "endpoint_resolution", + TraversalFamily: "SP", + Applied: "ENDPOINT-RESOLUTION-INCUMBENT", + }, + { + TargetKind: "traversal_predicate", + Family: "traversal_predicate", + Applied: "TRAVERSAL-PREDICATE-INCUMBENT", + }, + { + TargetKind: "traversal", + Family: "fixed_suffix_expansion", + Applied: "EXPANSION-STEPWISE-FORWARD", + }, + shortest, + }) + require.True(t, found) + require.Equal(t, shortest, outcome) +} + +// TestPostgresTraversalTelemetrySeparatesShadowChoiceFromExecutedIncumbent verifies postgres traversal telemetry separates shadow choice from executed incumbent behavior. +func TestPostgresTraversalTelemetrySeparatesShadowChoiceFromExecutedIncumbent(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: "EXPANSION-SUFFIX-SEEDED-REVERSE", + Selected: "EXPANSION-STEPWISE-FORWARD", + Applied: "EXPANSION-STEPWISE-FORWARD", + Fallback: "EXPANSION-STEPWISE-FORWARD", + PlannedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedCandidates: []string{"EXPANSION-STEPWISE-FORWARD"}, + EmittedPolicy: "orientation-probe-v1", + SelectionMode: "shadow_tool", + SelectorVersion: "orientation-probe-v1", + StateLimit: 4096, + ProbeCaps: &optimize.ExpansionSearchProbeCaps{ + ReverseSeedRowLimit: 512, + }, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{ + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_shadow_reverse", + ActualRows: 1, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_shadow_forward", + ActualRows: 0, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_incumbent", + ActualRows: 1, + ActualLoops: 1, + }, + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_suffix_probe", + ActualRows: 513, + ActualLoops: 1, + }, + }, + Provenance: map[string]string{}, + } + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + metrics, + "9123", + TraversalTelemetryLevelSummary, + ) + require.NoError(t, err) + require.NoError(t, telemetry.Validate()) + require.Equal(t, "EXPANSION-STEPWISE-FORWARD", telemetry.Summary.RuntimeIdentity) + require.Equal(t, "EXPANSION-STEPWISE-FORWARD", telemetry.Summary.AppliedIdentity) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", telemetry.Summary.WouldSelectIdentity) + require.Equal(t, "shadow_incumbent", telemetry.Summary.RuntimeBranch) + require.False(t, *telemetry.Summary.FallbackExecuted) + require.True(t, *telemetry.Summary.Overflow) +} + +// TestPostgresTraversalTelemetryUsesExactGuardedOrientationReceiptBranches verifies postgres traversal telemetry uses exact guarded orientation receipt branches behavior. +func TestPostgresTraversalTelemetryUsesExactGuardedOrientationReceiptBranches(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // candidateRows records the number of candidate rows. + candidateRows int64 + // incumbentRows records the number of incumbent rows. + incumbentRows int64 + // rootProbeRows records the number of root probe rows. + rootProbeRows int64 + // runtimeIdentity identifies the runtime identity. + runtimeIdentity string + // runtimeBranch retains the runtime branch while anonymous record is assembled or evaluated. + runtimeBranch string + // fallbackExecuted indicates whether fallback executed applies. + fallbackExecuted bool + // overflow indicates whether overflow applies. + overflow bool + }{ + { + name: "reverse candidate", + candidateRows: 1, + runtimeIdentity: string(optimize.ExpansionSearchSuffixSeededReverse), + runtimeBranch: "suffix_seeded_reverse", + }, + { + name: "forward selection", + incumbentRows: 1, + runtimeIdentity: string(optimize.ExpansionSearchStepwiseForward), + runtimeBranch: "exact_forward_incumbent", + }, + { + name: "overflow fallback", + incumbentRows: 1, + rootProbeRows: optimize.ExpansionSearchOrientationRootRowLimit + 1, + runtimeIdentity: string(optimize.ExpansionSearchStepwiseForward), + runtimeBranch: "exact_forward_incumbent", + fallbackExecuted: true, + overflow: true, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchStepwiseForward), + Applied: string(optimize.ExpansionSearchStepwiseForward), + Fallback: string(optimize.ExpansionSearchStepwiseForward), + PlannedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedCandidates: []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse)}, + EmittedPolicy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + ProbeCaps: &optimize.ExpansionSearchProbeCaps{RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit}, + } + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_candidate", + ActualRows: testCase.candidateRows, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_incumbent", + ActualRows: testCase.incumbentRows, + ActualLoops: 1, + }, + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_root_probe", + ActualRows: testCase.rootProbeRows, + ActualLoops: 1, + }, + }, + } + + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelSummary, + ) + require.NoError(t, err) + require.Equal(t, testCase.runtimeIdentity, telemetry.Summary.RuntimeIdentity) + require.Equal(t, testCase.runtimeBranch, telemetry.Summary.RuntimeBranch) + require.Equal(t, testCase.fallbackExecuted, *telemetry.Summary.FallbackExecuted) + require.Equal(t, testCase.overflow, *telemetry.Summary.Overflow) + require.NoError(t, validateRuntimeReceiptEvents([]RuntimeReceiptEvent{{ + Ordinal: 1, + RuntimeIdentity: testCase.runtimeIdentity, + RuntimeBranch: testCase.runtimeBranch, + FallbackExecuted: testCase.fallbackExecuted, + }}, telemetry.Summary.RuntimeIdentity, telemetry.Summary.RuntimeBranch, telemetry.Summary.FallbackExecuted)) + }) + } +} + +// TestPostgresTraversalTelemetryUsesV2DepthWeightedDiagnosticScore verifies postgres traversal telemetry uses v2 depth weighted diagnostic score behavior. +func TestPostgresTraversalTelemetryUsesV2DepthWeightedDiagnosticScore(t *testing.T) { + maximumDepth := int64(16) + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: string(optimize.ExpansionSearchSuffixSeededReverse), + Selected: string(optimize.ExpansionSearchStepwiseForward), + Applied: string(optimize.ExpansionSearchStepwiseForward), + Fallback: string(optimize.ExpansionSearchStepwiseForward), + EmittedPolicy: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + MaximumDepth: &maximumDepth, + } + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_root_probe", + ActualRows: 2, + ActualLoops: 1, + }, + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_forward_degree_probe", + ActualRows: 8, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_incumbent", + ActualRows: 1, + ActualLoops: 1, + }, + }, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + enrichOrientationTraversalTelemetry(telemetry, metrics, 1, []string{`["path"]`}, maximumDepth) + require.NoError(t, telemetry.Validate()) + require.Equal(t, float64(130), *telemetry.Diagnostic.Counters.Orientation.ForwardScore) + require.Equal(t, maximumDepth, orientationPolicyMaximumDepth(translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{outcome}, + }, string(optimize.ExpansionSearchPolicyOrientationProbeV2))) +} + +// TestPostgresTraversalTelemetryCompletesOrientationCountersFromNamedPlanNodes verifies postgres traversal telemetry completes orientation counters from named plan nodes behavior. +func TestPostgresTraversalTelemetryCompletesOrientationCountersFromNamedPlanNodes(t *testing.T) { + outcome := translate.TargetLoweringOutcome{ + Family: "fixed_suffix_expansion", + Candidate: "EXPANSION-SUFFIX-SEEDED-REVERSE", + Selected: "EXPANSION-STEPWISE-FORWARD", + Applied: "EXPANSION-STEPWISE-FORWARD", + Fallback: "EXPANSION-STEPWISE-FORWARD", + PlannedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedCandidates: []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"}, + EmittedPolicy: "orientation-probe-v1", + SelectionMode: "production_canary", + SelectorVersion: "orientation-probe-v1", + StateLimit: 4096, + } + metrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + PlanNodes: []PostgresPlanNodeMetric{ + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_root_probe", + ActualRows: 2, + ActualLoops: 1, + ActualTotalMS: .01, + Buffers: Buffers{SharedHit: 1}, + }, + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_suffix_probe", + ActualRows: 5, + ActualLoops: 1, + ActualTotalMS: .02, + }, + { + NodeType: "Aggregate", + SubplanName: "CTE s5_orientation_boundaries", + ActualRows: 3, + ActualLoops: 1, + ActualTotalMS: .01, + }, + { + PlanNodeID: 40, + NodeType: "Aggregate", + SubplanName: "CTE s5_orientation_forward_degree_probe", + ActualRows: 1, + ActualLoops: 1, + ActualTotalMS: .01, + }, + { + PlanNodeID: 41, + ParentPlanNodeID: 40, + ParentRelationship: "Outer", + NodeType: "Limit", + ActualRows: 8, + ActualLoops: 1, + }, + { + PlanNodeID: 50, + NodeType: "Aggregate", + SubplanName: "CTE s5_orientation_reverse_degree_probe", + ActualRows: 1, + ActualLoops: 1, + ActualTotalMS: .01, + }, + { + PlanNodeID: 51, + ParentPlanNodeID: 50, + ParentRelationship: "Outer", + NodeType: "Limit", + ActualRows: 1, + ActualLoops: 1, + }, + { + NodeType: "Limit", + SubplanName: "CTE s5_orientation_states", + ActualRows: 4, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_candidate", + ActualRows: 1, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_executed_incumbent", + ActualRows: 0, + ActualLoops: 1, + }, + { + NodeType: "Recursive Union", + SubplanName: "CTE s5_orientation_reverse", + ActualRows: 4, + ActualLoops: 1, + }, + { + NodeType: "Result", + SubplanName: "CTE s5_orientation_decision", + ActualRows: 1, + ActualLoops: 1, + }, + // Consumer scans are deliberately repeated and must not inflate the + // single materialization's row, loop, or branch attribution. + { + NodeType: "CTE Scan", + CTEName: "s5_orientation_root_probe", + Alias: "s5_orientation_root_probe", + ActualRows: 2, + ActualLoops: 3, + }, + { + NodeType: "CTE Scan", + CTEName: "s5_orientation_reverse_degree_probe", + Alias: "s5_orientation_reverse_degree_probe", + ActualRows: 1, + ActualLoops: 7, + }, + }, + } + telemetry, err := buildPostgresCaseTraversalTelemetry(translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichOrientationTraversalTelemetry(telemetry, metrics, 1, []string{`["path"]`}, 0) + require.NoError(t, telemetry.Validate()) + require.Equal(t, TraversalTelemetryCounterStatusComplete, telemetry.Diagnostic.CounterStatus) + require.Equal(t, int64(5), *telemetry.Diagnostic.Counters.Orientation.ReverseSeeds) + require.Equal(t, int64(2), *telemetry.Diagnostic.Counters.Orientation.DuplicateSeeds) + require.Equal(t, int64(8), *telemetry.Diagnostic.Counters.Orientation.ForwardDegreeSamples) + require.Equal(t, int64(1), *telemetry.Diagnostic.Counters.Orientation.ReverseDegreeSamples) + require.Equal(t, "reverse", telemetry.Diagnostic.Counters.Orientation.SelectedSide) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["orientation_root_probe_loops"]) + require.Equal(t, int64(1), telemetry.Diagnostic.PlanReplay.Counters["orientation_candidate_branch_loops"]) + require.Equal(t, int64(0), telemetry.Diagnostic.PlanReplay.Counters["orientation_incumbent_branch_loops"]) +} + +// TestPostgresTraversalTelemetrySummaryAndDisabledModesDoNotAttachDiagnosticCounters verifies postgres traversal telemetry summary and disabled modes do not attach diagnostic counters behavior. +func TestPostgresTraversalTelemetrySummaryAndDisabledModesDoNotAttachDiagnosticCounters(t *testing.T) { + summary := bidirectionalCaseTelemetry(t, TraversalTelemetryLevelSummary) + require.NoError(t, summary.Validate()) + require.Nil(t, summary.Diagnostic) + require.False(t, *summary.Summary.RuntimeOutcomeAvailable) + require.Empty(t, summary.Summary.RuntimeIdentity) + require.Empty(t, summary.Summary.AppliedIdentity) + require.Nil(t, summary.Summary.FallbackExecuted) + + record := CaseResult{ + PostgresReferences: []PostgresReferenceResult{{ + traversalTelemetryParameters: map[string]any{"state_limit": int64(1)}, + }}, + } + runner := postgresSQLRunner{traversalTelemetry: postgresTraversalTelemetryOff} + require.NoError(t, runner.attachPostgresTraversalTelemetry(t.Context(), &record, nil)) + require.Nil(t, record.TraversalTelemetry) + require.Nil(t, record.PostgresReferences[0].TraversalTelemetry) + require.Nil(t, record.PostgresReferences[0].traversalTelemetryParameters) +} + +// TestPostgresTraversalTelemetryAttachesToEveryTraversalReference verifies postgres traversal telemetry attaches to every traversal reference behavior. +func TestPostgresTraversalTelemetryAttachesToEveryTraversalReference(t *testing.T) { + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Recursive Union", + ActualRows: 2, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + record := CaseResult{PostgresReferences: []PostgresReferenceResult{ + { + Name: "forward", + Architecture: "EXPANSION-STEPWISE-FORWARD-SQL", + ImplementationID: "forward_v1", + PostgresMetrics: &metrics, + traversalTelemetryParameters: map[string]any{}, + }, + { + Name: "reverse", + Architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + ImplementationID: "reverse_v1", + PostgresMetrics: &metrics, + traversalTelemetryParameters: map[string]any{}, + }, + }} + runner := postgresSQLRunner{ + traversalTelemetry: postgresTraversalTelemetrySummary, + backendPID: "9123", + } + + require.NoError(t, runner.attachPostgresTraversalTelemetry(t.Context(), &record, nil)) + require.Len(t, record.PostgresReferences, 2) + for _, reference := range record.PostgresReferences { + require.NotNil(t, reference.TraversalTelemetry) + require.Equal(t, TraversalTelemetryLevelSummary, reference.TraversalTelemetry.Level) + require.Equal(t, reference.Architecture, reference.TraversalTelemetry.Summary.RuntimeIdentity) + require.Nil(t, reference.TraversalTelemetry.Diagnostic) + require.NoError(t, reference.TraversalTelemetry.Validate()) + } +} + +// TestPostgresTraversalTelemetrySkipsNonTraversalReferenceBoundaries verifies postgres traversal telemetry skips non traversal reference boundaries behavior. +func TestPostgresTraversalTelemetrySkipsNonTraversalReferenceBoundaries(t *testing.T) { + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Result", + ActualRows: 1, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + for _, architecture := range []string{"component_probe", "protocol", "root_validation", "root_adjacency", "factored_suffix"} { + reference := PostgresReferenceResult{ + Architecture: architecture, + ImplementationID: architecture + "_v1", + PostgresMetrics: &metrics, + } + telemetry, err := buildPostgresReferenceTraversalTelemetry(reference, nil, "9123", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + require.Nil(t, telemetry, architecture) + } +} + +// TestParseConfigValidatesPostgresTraversalTelemetryMode verifies parse config validates postgres traversal telemetry mode behavior. +func TestParseConfigValidatesPostgresTraversalTelemetryMode(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-traversal-telemetry", "summary"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, postgresTraversalTelemetrySummary, cfg.PostgresTraversalTelemetry) + + cfg, err = parseConfig([]string{"-postgres-traversal-telemetry", "diagnostic"}, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, postgresTraversalTelemetryDiagnostic, cfg.PostgresTraversalTelemetry) + + _, err = parseConfig([]string{"-postgres-traversal-telemetry", "unknown"}, func(string) string { return "" }) + require.ErrorContains(t, err, "must be off, summary, or diagnostic") + + _, err = parseConfig([]string{"-postgres-traversal-telemetry", "diagnostic", "-pool-size", "2"}, func(string) string { return "" }) + require.ErrorContains(t, err, "requires pool-size 1") + + cfg, err = parseConfig([]string{"-postgres-expansion-orientation-shadow"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresExpansionOrientationShadow) + + _, err = parseConfig([]string{"-postgres-expansion-orientation-shadow", "-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE"}, func(string) string { return "" }) + require.ErrorContains(t, err, "mutually exclusive") +} + +// TestParseConfigAcceptsExplicitOrientationProbeV2MeasurementModes verifies parse config accepts explicit orientation probe v2 measurement modes behavior. +func TestParseConfigAcceptsExplicitOrientationProbeV2MeasurementModes(t *testing.T) { + for _, mode := range [][]string{ + {"-postgres-expansion-orientation-shadow"}, + {"-postgres-expansion-orientation-tournament"}, + } { + args := append(append([]string(nil), mode...), + "-postgres-expansion-orientation-policy", "orientation-probe-v2", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", "summary", + ) + cfg, err := parseConfig(args, func(string) string { return "" }) + require.NoError(t, err, mode) + require.Equal(t, "orientation-probe-v2", cfg.PostgresExpansionOrientationPolicy) + require.True(t, cfg.PostgresRepeatableRead) + require.Equal(t, postgresTraversalTelemetrySummary, cfg.PostgresTraversalTelemetry) + } + + for _, args := range [][]string{ + {"-postgres-expansion-orientation-policy", "orientation-probe-v2", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-policy", "orientation-probe-v3", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-tournament", "-postgres-repeatable-read"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-policy", "orientation-probe-v2", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-orientation-shadow", "-postgres-expansion-orientation-policy", "orientation-probe-v2", "-postgres-repeatable-read"}, + {"-postgres-expansion-orientation-tournament"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigValidatesSuffixReverseGuardMeasurementMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-postgres-expansion-suffix-reverse-guard", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", "diagnostic", + "-postgres-suffix-guard-suffix-limit", "64", + "-postgres-suffix-guard-state-limit", "128", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresExpansionSuffixReverseGuard) + require.Equal(t, int64(64), cfg.PostgresSuffixGuardSuffixLimit) + require.Equal(t, int64(128), cfg.PostgresSuffixGuardStateLimit) + + for _, args := range [][]string{ + {"-postgres-expansion-suffix-reverse-guard"}, + {"-postgres-expansion-suffix-reverse-guard", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-suffix-guard-state-limit", "1"}, + {"-postgres-expansion-suffix-reverse-guard", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-postgres-force-expansion-search", "EXPANSION-SUFFIX-SEEDED-REVERSE"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigValidatesSuffixReverseRetryMeasurementMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-postgres-expansion-suffix-reverse-retry", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", "diagnostic", + "-postgres-suffix-guard-suffix-limit", "64", + "-postgres-suffix-guard-state-limit", "128", + "-postgres-suffix-retry-output-row-limit", "256", + "-postgres-suffix-retry-output-bytes-limit", "1048576", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresExpansionSuffixReverseRetry) + require.Equal(t, int64(256), cfg.PostgresSuffixRetryOutputRowLimit) + require.Equal(t, int64(1048576), cfg.PostgresSuffixRetryOutputBytesLimit) + + for _, args := range [][]string{ + {"-postgres-expansion-suffix-reverse-retry"}, + {"-postgres-expansion-suffix-reverse-retry", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-suffix-reverse-retry", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-pool-size", "2"}, + {"-postgres-suffix-retry-output-row-limit", "1"}, + {"-postgres-expansion-suffix-reverse-retry", "-postgres-expansion-suffix-reverse-guard", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestParseConfigValidatesSuffixRouteComponentMeasurementMode(t *testing.T) { + cfg, err := parseConfig([]string{ + "-postgres-expansion-suffix-route-component", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", "diagnostic", + "-require-clean-source", + }, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresExpansionSuffixRouteComponent) + require.True(t, cfg.RequireCleanSource) + + for _, args := range [][]string{ + {"-postgres-expansion-suffix-route-component"}, + {"-postgres-expansion-suffix-route-component", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "summary"}, + {"-postgres-expansion-suffix-route-component", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-pool-size", "2"}, + {"-postgres-expansion-suffix-route-component", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-postgres-suffix-guard-state-limit", "1"}, + {"-postgres-expansion-suffix-route-component", "-postgres-expansion-suffix-reverse-retry", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// TestParseConfigValidatesSuffixRouteComponentClosureMode verifies the +// closure cannot accidentally acquire selectors, reference arms, or unbounded +// workspace while collecting diagnostic boundary evidence. +func TestParseConfigValidatesSuffixRouteComponentClosureMode(t *testing.T) { + base := []string{ + "-postgres-suffix-route-component-closure", + "-postgres-repeatable-read", + "-postgres-traversal-telemetry", "diagnostic", + "-session-memory-ceiling-bytes", "1048576", + "-pool-memory-ceiling-bytes", "1048576", + } + cfg, err := parseConfig(base, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresSuffixRouteComponentClosure) + + componentCfg, err := parseConfig(append(append([]string{}, base...), "-postgres-expansion-suffix-route-component"), func(string) string { return "" }) + require.NoError(t, err) + require.True(t, componentCfg.PostgresExpansionSuffixRouteComponent) + + for _, args := range [][]string{ + {"-postgres-suffix-route-component-closure"}, + {"-postgres-suffix-route-component-closure", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-session-memory-ceiling-bytes", "1", "-pool-memory-ceiling-bytes", "1", "-pool-size", "2"}, + {"-postgres-suffix-route-component-closure", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-session-memory-ceiling-bytes", "1", "-pool-memory-ceiling-bytes", "1", "-postgres-references"}, + {"-postgres-suffix-route-component-closure", "-postgres-repeatable-read", "-postgres-traversal-telemetry", "diagnostic", "-session-memory-ceiling-bytes", "1", "-pool-memory-ceiling-bytes", "1", "-postgres-expansion-suffix-reverse-retry"}, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +// bidirectionalCaseTelemetry prepares or inspects test evidence for bidirectional case telemetry. +func bidirectionalCaseTelemetry(t *testing.T, level TraversalTelemetryLevel) *TraversalExecutionTelemetry { + t.Helper() + outcome := translate.TargetLoweringOutcome{ + Family: "SP", + Candidate: "SP-B2-C-MIN-LEVEL-D", + Selected: "SP-B2-C-MIN-LEVEL-D", + Applied: "SP-B2-C-MIN-LEVEL-D", + Fallback: "SP-S4-C-D", + PlannedCandidates: []string{"SP-B2-C-MIN-LEVEL-D", "SP-S4-C-D"}, + Scheduler: "smaller_current_level", + SelectorVersion: "sp-tool-v1", + StateLimit: 100, + FrontierLimit: 50, + PredecessorLimit: 25, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Function Scan", + FunctionName: "shortest_path_b2_smaller_current_level", + ActualRows: 1, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, + metrics, + "9123", + level, + ) + require.NoError(t, err) + require.NotNil(t, telemetry) + return telemetry +} + +// validBidirectionalDiagnosticDocument returns a self-consistent one-path bidirectional runtime receipt. +func validBidirectionalDiagnosticDocument(invocationID string) *postgresBidirectionalDiagnosticDocument { + return &postgresBidirectionalDiagnosticDocument{ + SchemaVersion: 1, + InvocationID: invocationID, + Scheduler: "smaller_current_level", + StateLimit: traversalTelemetryPointer(int64(100)), + FrontierLimit: traversalTelemetryPointer(int64(50)), + PredecessorLimit: traversalTelemetryPointer(int64(25)), + SearchCalls: traversalTelemetryPointer(int64(1)), + RuntimeBranch: "bidirectional_search", + Overflowed: traversalTelemetryPointer(false), + FallbackExecuted: traversalTelemetryPointer(false), + Counters: &postgresBidirectionalDiagnosticCounts{ + SchedulerActions: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(7)), + DistinctNewNodes: traversalTelemetryPointer(int64(5)), + SeenPeak: traversalTelemetryPointer(int64(6)), + FrontierPeak: traversalTelemetryPointer(int64(3)), + QueuePeak: traversalTelemetryPointer(int64(3)), + PredecessorPeak: traversalTelemetryPointer(int64(4)), + MeetingCandidates: traversalTelemetryPointer(int64(1)), + FrozenDistance: traversalTelemetryPointer(int64(3)), + WitnessRows: traversalTelemetryPointer(int64(1)), + Levels: []postgresBidirectionalDiagnosticLevel{{ + SearchID: traversalTelemetryPointer(int64(1)), + ActionIndex: traversalTelemetryPointer(int64(1)), + Side: "forward", + Action: "expand_level", + Depth: traversalTelemetryPointer(int64(1)), + FrontierRows: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(7)), + DistinctNewNodes: traversalTelemetryPointer(int64(5)), + SeenRows: traversalTelemetryPointer(int64(6)), + QueueRows: traversalTelemetryPointer(int64(3)), + PredecessorRows: traversalTelemetryPointer(int64(4)), + MeetingCandidates: traversalTelemetryPointer(int64(1)), + }}, + }, + Calls: []postgresBidirectionalDiagnosticCall{{ + SearchID: traversalTelemetryPointer(int64(1)), + SourceID: traversalTelemetryPointer(int64(10)), + TargetID: traversalTelemetryPointer(int64(20)), + RuntimeBranch: "bidirectional_search", + SchedulerActions: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(7)), + DistinctNewNodes: traversalTelemetryPointer(int64(5)), + SeenPeak: traversalTelemetryPointer(int64(6)), + FrontierPeak: traversalTelemetryPointer(int64(3)), + QueuePeak: traversalTelemetryPointer(int64(3)), + PredecessorPeak: traversalTelemetryPointer(int64(4)), + MeetingCandidates: traversalTelemetryPointer(int64(1)), + FrozenDistance: traversalTelemetryPointer(int64(3)), + WitnessRows: traversalTelemetryPointer(int64(1)), + Overflowed: traversalTelemetryPointer(false), + FallbackExecuted: traversalTelemetryPointer(false), + }}, + } +} + +// bidirectionalASPCaseTelemetry prepares or inspects test evidence for bidirectional asp case telemetry. +func bidirectionalASPCaseTelemetry(t *testing.T) *TraversalExecutionTelemetry { + t.Helper() + outcome := translate.TargetLoweringOutcome{ + Family: "ASP", + Candidate: "ASP-B2-DAG-MIN-LEVEL", + Selected: "ASP-B2-DAG-MIN-LEVEL", + Applied: "ASP-B2-DAG-MIN-LEVEL", + Fallback: "ASP-A1-DAG", + PlannedCandidates: []string{"ASP-B2-DAG-MIN-LEVEL", "ASP-A1-DAG"}, + Scheduler: "smaller_current_level", + SelectorVersion: "asp-tool-v1", + StateLimit: 100, + FrontierLimit: 50, + PredecessorLimit: 25, + EnumerationLimit: 1000, + OutputBytesLimit: 4096, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Function Scan", + FunctionName: "all_shortest_paths_b2_smaller_current_level", + ActualRows: 1, + ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NotNil(t, telemetry) + return telemetry +} + +// a1AllShortestCaseTelemetry prepares the opaque function-scan baseline that +// the A1 invocation-local workspace receipt must complete. +func a1AllShortestCaseTelemetry(t *testing.T) *TraversalExecutionTelemetry { + t.Helper() + identity := string(optimize.ShortestPathExecutorASPA1DAG) + outcome := translate.TargetLoweringOutcome{ + Family: "ASP", + Candidate: identity, + Selected: identity, + Applied: identity, + Fallback: "SP-S0", + PlannedCandidates: []string{identity}, + Scheduler: "single_ended_level", + SelectorVersion: "asp-tool-v1", + StateLimit: 100, + FrontierLimit: 50, + PredecessorLimit: 25, + EnumerationLimit: 1000, + OutputBytesLimit: 4096, + } + metrics := PostgresPlanMetrics{ + PlanNodes: []PostgresPlanNodeMetric{{ + NodeType: "Function Scan", FunctionName: "all_shortest_paths_dag", ActualRows: 2, ActualLoops: 1, + }}, + Provenance: map[string]string{}, + } + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{outcome}}, metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + require.NoError(t, err) + require.NotNil(t, telemetry) + return telemetry +} + +// validA1AllShortestDiagnosticDocument returns a complete recursive A1 +// workspace receipt with two exact output paths at depth three. +func validA1AllShortestDiagnosticDocument(invocationID string) *postgresA1AllShortestDiagnosticDocument { + return &postgresA1AllShortestDiagnosticDocument{ + SchemaVersion: 1, + InvocationID: invocationID, + Scheduler: "single_ended_level", + SearchCalls: traversalTelemetryPointer(int64(1)), + SourceID: traversalTelemetryPointer(int64(10)), + TargetID: traversalTelemetryPointer(int64(20)), + RuntimeBranch: "single_ended_search", + TargetDepth: traversalTelemetryPointer(int64(3)), + OutputPaths: traversalTelemetryPointer(int64(2)), + FallbackExecuted: traversalTelemetryPointer(false), + WorkspaceBytes: 4096, + Levels: []postgresA1AllShortestDiagnosticLevel{ + { + ActionIndex: traversalTelemetryPointer(int64(1)), Depth: traversalTelemetryPointer(int64(1)), + CandidateEdges: traversalTelemetryPointer(int64(2)), DistinctNewNodes: traversalTelemetryPointer(int64(2)), + SeenRows: traversalTelemetryPointer(int64(3)), PredecessorRows: traversalTelemetryPointer(int64(2)), + }, + { + ActionIndex: traversalTelemetryPointer(int64(2)), Depth: traversalTelemetryPointer(int64(2)), + CandidateEdges: traversalTelemetryPointer(int64(4)), DistinctNewNodes: traversalTelemetryPointer(int64(2)), + SeenRows: traversalTelemetryPointer(int64(5)), PredecessorRows: traversalTelemetryPointer(int64(4)), + }, + }, + } +} + +// validBidirectionalAllShortestDiagnosticDocument returns a self-consistent all-shortest runtime receipt. +func validBidirectionalAllShortestDiagnosticDocument(invocationID string) *postgresBidirectionalAllShortestDiagnosticDocument { + base := validBidirectionalDiagnosticDocument(invocationID) + counts := &postgresBidirectionalAllShortestDiagnosticCounts{ + SchedulerActions: base.Counters.SchedulerActions, + CandidateEdges: base.Counters.CandidateEdges, + DistinctNewNodes: base.Counters.DistinctNewNodes, + SeenPeak: base.Counters.SeenPeak, + FrontierPeak: base.Counters.FrontierPeak, + QueuePeak: base.Counters.QueuePeak, + PredecessorPeak: base.Counters.PredecessorPeak, + MeetingCandidates: base.Counters.MeetingCandidates, + FrozenDistance: base.Counters.FrozenDistance, + WitnessRows: base.Counters.WitnessRows, + Levels: base.Counters.Levels, + SameDepthPredecessorAdditions: traversalTelemetryPointer(int64(5)), + MeetingNodes: traversalTelemetryPointer(int64(2)), + CutDepth: traversalTelemetryPointer(int64(3)), + PathCountEstimate: traversalTelemetryPointer(int64(12)), + PathCountSaturated: traversalTelemetryPointer(false), + EnumeratedCandidates: traversalTelemetryPointer(int64(13)), + DuplicateRejects: traversalTelemetryPointer(int64(1)), + OutputPaths: traversalTelemetryPointer(int64(12)), + OutputEdgeCells: traversalTelemetryPointer(int64(36)), + OutputBytes: traversalTelemetryPointer(int64(384)), + } + call := postgresBidirectionalAllShortestDiagnosticCall{ + SearchID: base.Calls[0].SearchID, + SourceID: base.Calls[0].SourceID, + TargetID: base.Calls[0].TargetID, + RuntimeBranch: base.Calls[0].RuntimeBranch, + SchedulerActions: base.Calls[0].SchedulerActions, + CandidateEdges: base.Calls[0].CandidateEdges, + DistinctNewNodes: base.Calls[0].DistinctNewNodes, + SeenPeak: base.Calls[0].SeenPeak, + FrontierPeak: base.Calls[0].FrontierPeak, + QueuePeak: base.Calls[0].QueuePeak, + PredecessorPeak: base.Calls[0].PredecessorPeak, + MeetingCandidates: base.Calls[0].MeetingCandidates, + FrozenDistance: base.Calls[0].FrozenDistance, + WitnessRows: base.Calls[0].WitnessRows, + SameDepthPredecessorAdditions: counts.SameDepthPredecessorAdditions, + MeetingNodes: counts.MeetingNodes, + CutDepth: counts.CutDepth, + PathCountEstimate: counts.PathCountEstimate, + PathCountSaturated: counts.PathCountSaturated, + EnumeratedCandidates: counts.EnumeratedCandidates, + DuplicateRejects: counts.DuplicateRejects, + OutputPaths: counts.OutputPaths, + OutputEdgeCells: counts.OutputEdgeCells, + OutputBytes: counts.OutputBytes, + Overflowed: base.Calls[0].Overflowed, + FallbackExecuted: base.Calls[0].FallbackExecuted, + } + return &postgresBidirectionalAllShortestDiagnosticDocument{ + SchemaVersion: 1, + InvocationID: invocationID, + Scheduler: "smaller_current_level", + StateLimit: traversalTelemetryPointer(int64(100)), + FrontierLimit: traversalTelemetryPointer(int64(50)), + PredecessorLimit: traversalTelemetryPointer(int64(25)), + EnumerationLimit: traversalTelemetryPointer(int64(1000)), + OutputBytesLimit: traversalTelemetryPointer(int64(4096)), + SearchCalls: traversalTelemetryPointer(int64(1)), + RuntimeBranch: "bidirectional_search", + Overflowed: traversalTelemetryPointer(false), + FallbackExecuted: traversalTelemetryPointer(false), + Counters: counts, + Calls: []postgresBidirectionalAllShortestDiagnosticCall{call}, + } +} diff --git a/cmd/graphbench/postgresql_plan_invariants_integration_test.go b/cmd/graphbench/postgresql_plan_invariants_integration_test.go new file mode 100644 index 00000000..108bc9f1 --- /dev/null +++ b/cmd/graphbench/postgresql_plan_invariants_integration_test.go @@ -0,0 +1,1657 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestPostgreSQLBidirectionalOperationalPoolMatrix exercises the required +// pool-size/concurrency cross-product with an exact B2 distance candidate. +// It is intentionally a smoke matrix; latency qualification uses GraphBench's +// separately balanced discovery and confirmation protocols. +func TestPostgreSQLBidirectionalOperationalPoolMatrix(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D16-F016_distance"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + for _, poolSize := range []int{1, 2, 8} { + t.Run(fmt.Sprintf("pool-%d", poolSize), func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, poolSize, 1, []int{1, 8, 16}, false, nil, "SP-B2-C-MIN-LEVEL-D", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + require.Contains(t, records[0].SQL, "shortest_path_b2_smaller_current_level") + require.Len(t, records[0].Concurrency, 3) + for idx, concurrency := range []int{1, 8, 16} { + block := records[0].Concurrency[idx] + require.Equal(t, poolSize, block.PoolSize) + require.Equal(t, concurrency, block.Concurrency) + require.Equal(t, concurrency, block.Operations) + require.Len(t, block.Samples, concurrency) + } + if poolSize == 1 { + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + for _, planMode := range []string{"auto", "force_custom_plan", "force_generic_plan"} { + tx, err := connectionHandle.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local work_mem = '64kB'") + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local plan_cache_mode = "+planMode) + require.NoError(t, err) + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + var rowCount int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, records[0].RowCount, rowCount, planMode) + require.NoError(t, tx.Rollback(ctx)) + } + } + if poolSize == 2 { + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + reader, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer reader.Release() + writer, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer writer.Release() + const snapshotTable = "public.graphbench_traversal_snapshot_probe" + _, err = writer.Exec(ctx, "drop table if exists "+snapshotTable) + require.NoError(t, err) + _, err = writer.Exec(ctx, "create table "+snapshotTable+" (value int primary key)") + require.NoError(t, err) + t.Cleanup(func() { _, _ = runner.pool.Exec(context.Background(), "drop table if exists "+snapshotTable) }) + _, err = writer.Exec(ctx, "insert into "+snapshotTable+" values (1)") + require.NoError(t, err) + + readerTx, err := reader.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + var before, during int + require.NoError(t, readerTx.QueryRow(ctx, "select count(*) from "+snapshotTable).Scan(&before)) + _, err = writer.Exec(ctx, "insert into "+snapshotTable+" values (2)") + require.NoError(t, err) + rows, err := readerTx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + var rowCount int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, records[0].RowCount, rowCount) + require.NoError(t, readerTx.QueryRow(ctx, "select count(*) from "+snapshotTable).Scan(&during)) + require.Equal(t, 1, before) + require.Equal(t, before, during, "candidate internal statements must retain the reader snapshot across a concurrent commit") + require.NoError(t, readerTx.Commit(ctx)) + var after int + require.NoError(t, reader.QueryRow(ctx, "select count(*) from "+snapshotTable).Scan(&after)) + require.Equal(t, 2, after) + _, err = writer.Exec(ctx, "drop table "+snapshotTable) + require.NoError(t, err) + } + }) + } +} + +// postgresPlanNodeLoops extracts Actual Loops for every EXPLAIN node with the requested alias, allowing integration assertions to detect repeated execution. +func postgresPlanNodeLoops(t *testing.T, raw json.RawMessage, alias string) []int64 { + t.Helper() + var document []map[string]any + require.NoError(t, json.Unmarshal(raw, &document)) + require.NotEmpty(t, document) + root, ok := document[0]["Plan"].(map[string]any) + require.True(t, ok) + + var ( + loops []int64 + walk func(map[string]any) + ) + + walk = func(node map[string]any) { + nodeAlias, _ := node["Alias"].(string) + functionName, _ := node["Function Name"].(string) + if nodeAlias == alias || functionName == alias { + if actualLoops, ok := node["Actual Loops"].(float64); ok { + loops = append(loops, int64(actualLoops)) + } + } + children, _ := node["Plans"].([]any) + for _, child := range children { + if childNode, ok := child.(map[string]any); ok { + walk(childNode) + } + } + } + walk(root) + return loops +} + +// TestPostgreSQLA1AllShortestDiagnosticTelemetry verifies the baseline A1 +// helper exposes a complete session-local receipt for its shallow, recursive, +// inbound, and no-path branches without borrowing B1/B2 telemetry. +func TestPostgreSQLA1AllShortestDiagnosticTelemetry(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{ + "GSPV2-TRAINING-early-depth1-all-shortest-max16", + "GSPV2-TRAINING-early-depth2-all-shortest-max64", + "GSPV2-TRAINING-inbound-early-depth3-all-shortest-max64", + "GSPV2-TRAINING-reconvergent-all-shortest-max16", + "GSPV2-TRAINING-disconnected-all-shortest-max64", + }}) + require.NoError(t, err) + require.Len(t, selected.Cases, 5) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "ASP-A1-DAG", "") + require.NoError(t, err) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 5) + expectedBranches := map[string]string{ + "GSPV2-TRAINING-early-depth1-all-shortest-max16": "one_hop_preflight", + "GSPV2-TRAINING-early-depth2-all-shortest-max64": "two_hop_preflight", + "GSPV2-TRAINING-inbound-early-depth3-all-shortest-max64": "single_ended_search", + "GSPV2-TRAINING-reconvergent-all-shortest-max16": "two_hop_preflight", + "GSPV2-TRAINING-disconnected-all-shortest-max64": "search_no_path", + } + for _, record := range records { + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotNil(t, record.TraversalTelemetry) + require.Equal(t, "ASP-A1-DAG", record.TraversalTelemetry.Summary.RuntimeIdentity) + require.Equal(t, expectedBranches[record.Name], record.TraversalTelemetry.Summary.RuntimeBranch) + require.NotNil(t, record.TraversalTelemetry.Diagnostic) + require.Equal(t, TraversalTelemetryCounterStatusComplete, record.TraversalTelemetry.Diagnostic.CounterStatus) + require.NotNil(t, record.TraversalTelemetry.Diagnostic.Counters.AllShortestPaths) + require.NotNil(t, record.TraversalTelemetry.Diagnostic.Counters.Hydration) + require.NotNil(t, record.TraversalTelemetry.Diagnostic.Counters.Workspace) + } +} + +// TestPostgreSQLA1AllShortestDiagnosticCancellationAndSessionIsolation verifies +// A1's own diagnostic rows remain session-local and disappear after a cancelled +// invocation rolls back, leaving the same physical connection reusable. +func TestPostgreSQLA1AllShortestDiagnosticCancellationAndSessionIsolation(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{ + "GSPV2-NORMAL-outbound-all-shortest-depth3", + }}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, nil, false, nil, "ASP-A1-DAG", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + first, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer first.Release() + second, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer second.Release() + firstPID, secondPID := first.Conn().PgConn().PID(), second.Conn().PgConn().PID() + require.NotEqual(t, firstPID, secondPID) + // Materialize the session-local diagnostic tables outside rollback checks so + // the test can inspect an empty receipt after a cancelled transaction. + _, err = first.Exec(ctx, "select public.ensure_all_shortest_paths_a1_diagnostic_workspace_v1()") + require.NoError(t, err) + _, err = second.Exec(ctx, "select public.ensure_all_shortest_paths_a1_diagnostic_workspace_v1()") + require.NoError(t, err) + + drain := func(tx pgx.Tx) int64 { + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + defer rows.Close() + var count int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + count++ + } + require.NoError(t, rows.Err()) + return count + } + readCalls := func(tx pgx.Tx, invocationID string) (int64, bool) { + var raw string + err := tx.QueryRow(ctx, "select coalesce(public.read_all_shortest_paths_a1_diagnostic_v1($1)::text, '')", invocationID).Scan(&raw) + require.NoError(t, err) + if raw == "" { + return 0, false + } + var document struct { + SearchCalls int64 `json:"search_calls"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &document)) + return document.SearchCalls, true + } + + firstTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + secondTx, err := second.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + const sharedInvocation = "a1-same-key-different-sessions" + _, err = firstTx.Exec(ctx, "select public.begin_all_shortest_paths_a1_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + _, err = secondTx.Exec(ctx, "select public.begin_all_shortest_paths_a1_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(firstTx)) + firstCalls, found := readCalls(firstTx, sharedInvocation) + require.True(t, found) + require.Equal(t, int64(1), firstCalls) + secondCalls, found := readCalls(secondTx, sharedInvocation) + require.True(t, found) + require.Zero(t, secondCalls) + _, err = firstTx.Exec(ctx, "select public.clear_all_shortest_paths_a1_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + _, found = readCalls(firstTx, sharedInvocation) + require.False(t, found) + _, found = readCalls(secondTx, sharedInvocation) + require.True(t, found) + require.NoError(t, firstTx.Rollback(ctx)) + require.NoError(t, secondTx.Rollback(ctx)) + + cancelTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + const cancelledInvocation = "a1-cancelled-replay" + _, err = cancelTx.Exec(ctx, "select public.begin_all_shortest_paths_a1_diagnostic_v1($1)", cancelledInvocation) + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + _, queryErr := cancelTx.Exec(ctx, "select pg_sleep(0.05)") + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, cancelTx.Rollback(ctx)) + require.Equal(t, firstPID, first.Conn().PgConn().PID()) + + reuseTx, err := first.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead, AccessMode: pgx.ReadWrite}) + require.NoError(t, err) + _, found = readCalls(reuseTx, cancelledInvocation) + require.False(t, found, "rolled-back A1 diagnostic state must not survive") + const reuseInvocation = "a1-successful-reuse" + _, err = reuseTx.Exec(ctx, "select public.begin_all_shortest_paths_a1_diagnostic_v1($1)", reuseInvocation) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(reuseTx)) + reuseCalls, found := readCalls(reuseTx, reuseInvocation) + require.True(t, found) + require.Equal(t, int64(1), reuseCalls) + _, err = reuseTx.Exec(ctx, "select public.clear_all_shortest_paths_a1_diagnostic_v1($1)", reuseInvocation) + require.NoError(t, err) + require.NoError(t, reuseTx.Commit(ctx)) + t.Logf("cancelled A1 diagnostic in %s and reused backend PID %d without cross-session state from PID %d", cancellationLatency, firstPID, secondPID) +} + +// TestPostgreSQLScalePlanInvariants verifies analyzed-plan capture, indexed anchors, correct mutation targets, and preserved branch-local predicates across required scale representatives. +func TestPostgreSQLScalePlanInvariants(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + required := scaleCorpusRequiredIDSet() + filtered := ScaleCorpus{} + for _, testCase := range corpus.Cases { + id := scaleCorpusCaseID(testCase.Name) + _, isRequired := required[id] + if isRequired || id == "TRUST-03" { + filtered.Cases = append(filtered.Cases, testCase) + } + } + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, filtered, 1, 1, nil, true, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, runner.Close(ctx)) + }) + + records, err := runner.Run(ctx, 1, 1, filtered) + require.NoError(t, err) + require.Len(t, records, len(filtered.Cases)) + + byID := map[string][]CaseResult{} + for _, record := range records { + record := record + id := scaleCorpusCaseID(record.Name) + byID[id] = append(byID[id], record) + + t.Run(record.Name, func(t *testing.T) { + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotEmpty(t, record.SQL) + require.NotEmpty(t, record.PostgresPlan) + require.NotNil(t, record.PostgresMetrics) + require.NotNil(t, record.PostgresMetrics.PlanningMS) + require.NotNil(t, record.PostgresMetrics.ExecutionMS) + require.NotNil(t, record.Optimization) + + plan := strings.Join(record.PostgresPlan, "\n") + require.Contains(t, plan, "actual rows=", "plan must come from EXPLAIN ANALYZE") + assertMutationPlanTarget(t, id, plan) + assertAnchorPlanIndex(t, id, plan) + }) + } + + for _, id := range scaleCorpusRequiredIDs { + require.NotEmpty(t, byID[id], "missing PostgreSQL plan-invariant execution for %s", id) + } + + t.Run("LOGIC-01 branch-local direction and kind plan", func(t *testing.T) { + record := requireSingleScaleRecord(t, byID, "TRUST-03") + normalizedSQL := strings.ToLower(record.SQL) + require.Contains(t, normalizedSQL, " or ") + require.GreaterOrEqual(t, strings.Count(normalizedSQL, "kind_id"), 2) + require.Contains(t, normalizedSQL, "start_id") + require.Contains(t, normalizedSQL, "end_id") + }) + + t.Run("LOGIC-02 cross-binding temporal plan", func(t *testing.T) { + record := requireSingleScaleRecord(t, byID, "TRUST-01") + normalizedSQL := strings.ToLower(record.SQL) + require.Contains(t, normalizedSQL, " or ") + require.GreaterOrEqual(t, strings.Count(normalizedSQL, "lastcollected"), 2) + require.GreaterOrEqual(t, strings.Count(normalizedSQL, " < "), 2) + }) + + t.Run("LOGIC-04 filtered mutation targets", func(t *testing.T) { + edgeDelete := requireSingleScaleRecord(t, byID, "REC-01") + nodeDelete := requireSingleScaleRecord(t, byID, "REC-08") + require.Contains(t, strings.Join(edgeDelete.PostgresPlan, "\n"), "Delete on edge") + require.Contains(t, strings.Join(nodeDelete.PostgresPlan, "\n"), "Delete on node") + }) +} + +// TestPostgreSQLZeroLengthShortestMaterializersAreExact verifies that all search-and-hydration references reproduce a singleton zero-edge path and hydration-only arms avoid recursive search. +func TestPostgreSQLZeroLengthShortestMaterializersAreExact(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + var ( + zeroDepth = 0 + oneDepth = 1 + oneRow = int64(1) + ) + testCase := ScaleCase{ + Name: "GSP-D00-F001_path", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*0..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + NodeParams: map[string]string{ + "start_id": "sp-start", + "end_id": "sp-start", + }, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-start"}, + RelationshipKinds: []string{}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, + MinDepth: &zeroDepth, + MaxDepth: &oneDepth, + PathMaterializationRequired: true, + }, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + } + corpus := ScaleCorpus{Cases: []ScaleCase{testCase}} + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, corpus, 1, 1, nil, true, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, runner.Close(ctx)) + }) + + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Equal(t, oneRow, record.RowCount) + + for _, name := range []string{ + "m0_directed_hydration_only", + "m1_ordered_ids_hydration_only", + "s3_unidirectional_cte_m0_directed", + "s3_unidirectional_cte_m1_ordered_ids", + } { + reference := requirePostgresReference(t, record.PostgresReferences, name) + require.Equal(t, oneRow, reference.RowCount) + require.Equal(t, record.ObservedRows, reference.ObservedRows) + if strings.Contains(name, "hydration_only") { + require.NotContains(t, reference.SQL, "with recursive") + } + } +} + +// TestPostgreSQLForcedShortestDistanceEndpointSemantics verifies zero-depth identity, missing-root emptiness, and the minimum-depth self-endpoint error under forced distance execution. +func TestPostgreSQLForcedShortestDistanceEndpointSemantics(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + var ( + zeroDepth = 0 + oneDepth = 1 + oneRow = int64(1) + zeroRows = int64(0) + zeroScalar = int64(0) + maxDepth = 1 + ) + baseShape := WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, + MaxDepth: &maxDepth, + PathMaterializationRequired: false, + } + zeroShape := baseShape + zeroShape.MinDepth = &zeroDepth + oneShape := baseShape + oneShape.MinDepth = &oneDepth + + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "forced-shortest-zero-depth", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*0..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + NodeParams: map[string]string{"start_id": "sp-start", "end_id": "sp-start"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ScalarInt: &zeroScalar, + ResultKind: "scalar", + }, + Shape: zeroShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "forced-shortest-missing-root", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + Params: testutil.Params{"start_id": int64(9223372036854775807)}, + NodeParams: map[string]string{"end_id": "sp-end"}, + Expected: ExpectedResult{ + RowCount: &zeroRows, + }, + Shape: oneShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "forced-shortest-min-one-same-endpoint", + Dataset: "generated_shortest_paths_d1_f1", + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((s)-[:Traverse*1..1]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)", + NodeParams: map[string]string{"start_id": "sp-start", "end_id": "sp-start"}, + Expected: ExpectedResult{ + RowCount: &zeroRows, + }, + Shape: oneShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, corpus, 1, 1, nil, false, nil, "SP-S3-U-D", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + require.Len(t, records, 3) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + require.Equal(t, []string{"[0]"}, records[0].ObservedRows) + require.Equal(t, StatusOK, records[1].Status, records[1].Error) + require.Equal(t, zeroRows, records[1].RowCount) + require.Equal(t, StatusError, records[2].Status) + require.Contains(t, records[2].Error, "shortest path") +} + +// TestPostgreSQLForcedShortestDirectPreflightSkipsAndFallsBackExactly verifies that one-hop direct hits bypass the recursive harness while longer paths invoke it and preserve exact ordered path output. +func TestPostgreSQLForcedShortestDirectPreflightSkipsAndFallsBackExactly(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + oneRow := int64(1) + minDepth, maxDepth := 1, 3 + dataset := "generated_shortest_paths_v2_d3_o2_r1_fo2_fi128_l2_k7_t16_w2_x16_p0_c1_s1" + shape := WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"Traverse"}, + Direction: "inbound", + RelationshipKindCount: 1, + MinDepth: &minDepth, + MaxDepth: &maxDepth, + PathMaterializationRequired: true, + } + multiKindMaxDepth := 2 + multiKindShape := WorkloadShape{ + RootPredicate: "bound_id", + TerminalPredicate: "bound_id", + EdgeKinds: []string{"ParallelKind00", "ParallelKind01", "ParallelKind02", "ParallelKind03", "ParallelKind04", "ParallelKind05", "ParallelKind06"}, + Direction: "outbound", + RelationshipKindCount: 7, + MinDepth: &minDepth, + MaxDepth: &multiKindMaxDepth, + PathMaterializationRequired: true, + } + corpus := ScaleCorpus{ + Cases: []ScaleCase{ + { + Name: "direct-hit", + Dataset: dataset, + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((root)<-[:Traverse*1..3]-(terminal)) WHERE id(root) = $root_id AND id(terminal) = $end_id RETURN p", + NodeParams: map[string]string{"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-linear-01"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-v2-inbound-root", "sp-v2-inbound-linear-01"}, + RelationshipKinds: []string{"Traverse"}, + RelationshipKeys: []string{"inbound-primary-03"}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: shape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "fallback-hit", + Dataset: dataset, + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((root)<-[:Traverse*1..3]-(terminal)) WHERE id(root) = $root_id AND id(terminal) = $end_id RETURN p", + NodeParams: map[string]string{"root_id": "sp-v2-inbound-root", "end_id": "sp-v2-inbound-end"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-v2-inbound-root", "sp-v2-inbound-linear-01", "sp-v2-inbound-linear-02", "sp-v2-inbound-end"}, + RelationshipKinds: []string{"Traverse", "Traverse", "Traverse"}, + RelationshipKeys: []string{"inbound-primary-03", "inbound-primary-02", "inbound-primary-01"}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: shape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + { + Name: "direct-multi-kind", + Dataset: dataset, + Category: "generated_shortest_path", + Cypher: "MATCH p = shortestPath((root)-[:ParallelKind00|ParallelKind01|ParallelKind02|ParallelKind03|ParallelKind04|ParallelKind05|ParallelKind06*1..2]->(terminal)) WHERE id(root) = $root_id AND id(terminal) = $end_id RETURN p", + NodeParams: map[string]string{"root_id": "sp-v2-parallel-start", "end_id": "sp-v2-parallel-target-000000"}, + Expected: ExpectedResult{ + RowCount: &oneRow, + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"sp-v2-parallel-start", "sp-v2-parallel-target-000000"}, + RelationshipKinds: []string{"ParallelKind00"}, + RelationshipKeys: []string{"parallel-k00-t000000"}, + }}, + }, + Observes: ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, + Shape: multiKindShape, + CandidateModes: []ExecutionMode{ModePostgresSQL}, + }, + }, + } + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, corpus, 1, 1, nil, false, nil, "SP-S0-DIRECT", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + require.Len(t, records, 3) + for _, record := range records { + require.Equal(t, StatusOK, record.Status, "%s: %s", record.Name, record.Error) + require.Equal(t, oneRow, record.RowCount) + require.NotEmpty(t, record.PostgresPlanJSON) + } + + directLoops := postgresPlanNodeLoops(t, records[0].PostgresPlanJSON, "bidirectional_sp_harness") + require.NotEmpty(t, directLoops) + require.Equal(t, int64(0), directLoops[0], records[0].PostgresPlan) + fallbackLoops := postgresPlanNodeLoops(t, records[1].PostgresPlanJSON, "bidirectional_sp_harness") + require.NotEmpty(t, fallbackLoops) + require.Positive(t, fallbackLoops[0], records[1].PostgresPlan) + multiKindLoops := postgresPlanNodeLoops(t, records[2].PostgresPlanJSON, "bidirectional_sp_harness") + require.NotEmpty(t, multiKindLoops) + require.Equal(t, int64(0), multiKindLoops[0], records[2].PostgresPlan) +} + +// TestPostgreSQLForcedShortestDistanceCancellationReusesSession verifies prompt timeout cancellation, rollback recovery on the same backend PID, and successful replay of forced distance SQL. +func TestPostgreSQLForcedShortestDistanceCancellationReusesSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_distance"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "SP-S3-U-D", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + backendPID := connectionHandle.Conn().PgConn().PID() + + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + + var reusedPID uint32 + require.NoError(t, connectionHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + + rows, err = connectionHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := 0 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, 1, rowCount) + t.Logf("cancelled exact SP-S3-U-D SQL in %s and reused backend PID %d", cancellationLatency, backendPID) +} + +// TestPostgreSQLForcedShortestPathEdgeM0PlanResourcesAndConcurrency verifies direct edge-array hydration, zero local/temp/WAL usage, concurrency sample counts, and no edge work for a missing endpoint. +func TestPostgreSQLForcedShortestPathEdgeM0PlanResourcesAndConcurrency(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D16-F016_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + zeroRows := int64(0) + missingEndpoint := selected.Cases[0] + missingEndpoint.Name = "forced-m0-missing-start-endpoint" + missingEndpoint.Params = testutil.Params{"start_id": int64(9223372036854775807)} + missingEndpoint.NodeParams = map[string]string{"end_id": "sp-end"} + missingEndpoint.Expected = ExpectedResult{ + RowCount: &zeroRows, + ResultKind: "path_set", + } + selected.Cases = append(selected.Cases, missingEndpoint) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, []int{1, 2, 4}, true, nil, "SP-S3-U-E+MAT-M0", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 25, selected) + require.NoError(t, err) + require.Len(t, records, 2) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, "s1(next_id, depth, path)") + require.Equal(t, 1, strings.Count(record.SQL, "generate_subscripts(s1.path, 1)"), record.SQL) + require.NotContains(t, record.SQL, "ordered_edge_ids_to_path") + require.NotContains(t, record.SQL, "sp_harness") + + require.NotNil(t, record.PostgresMetrics) + metrics := record.PostgresMetrics + require.Greater(t, metrics.RecursiveRows, int64(0)) + require.Greater(t, metrics.HydrationLoops, int64(0)) + require.Zero(t, metrics.Buffers.LocalHit) + require.Zero(t, metrics.Buffers.LocalRead) + require.Zero(t, metrics.Buffers.LocalDirtied) + require.Zero(t, metrics.Buffers.LocalWritten) + require.Zero(t, metrics.Buffers.TempRead) + require.Zero(t, metrics.Buffers.TempWritten) + require.Zero(t, metrics.TempFiles) + require.Zero(t, metrics.TempBytes) + require.Zero(t, metrics.WALRecords) + require.Zero(t, metrics.WALBytes) + + require.Len(t, record.Concurrency, 3) + for index, level := range []int{1, 2, 4} { + block := record.Concurrency[index] + require.Equal(t, level, block.Concurrency) + require.Equal(t, 2, block.PoolSize) + require.Equal(t, level*25, block.Operations) + require.Len(t, block.Samples, level*25) + } + + missingRecord := records[1] + require.Equal(t, StatusOK, missingRecord.Status, missingRecord.Error) + require.Zero(t, missingRecord.RowCount) + require.NotNil(t, missingRecord.PostgresMetrics) + require.Zero(t, missingRecord.PostgresMetrics.RecursiveRows) + var missingEdgeLoops int64 + for _, node := range missingRecord.PostgresMetrics.PlanNodes { + if node.RelationName == "edge" || strings.HasPrefix(node.RelationName, "edge_") { + missingEdgeLoops += node.ActualLoops + } + } + require.Zero(t, missingEdgeLoops, "missing endpoint must execute zero edge-search loops") +} + +// TestPostgreSQLForcedShortestPathEdgeM0CancellationReusesSession verifies prompt timeout cancellation, rollback recovery on the same backend PID, and successful replay of M0 path SQL. +func TestPostgreSQLForcedShortestPathEdgeM0CancellationReusesSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "SP-S3-U-E+MAT-M0", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + backendPID := connectionHandle.Conn().PgConn().PID() + + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + + var reusedPID uint32 + require.NoError(t, connectionHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + + rows, err = connectionHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := 0 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, 1, rowCount) + t.Logf("cancelled exact SP-S3-U-E+MAT-M0 SQL in %s and reused backend PID %d", cancellationLatency, backendPID) +} + +// TestPostgreSQLForcedSuffixSeededReversePlanResourcesAndConcurrency verifies compact reverse-search SQL, relationship uniqueness, zero local/temp/WAL usage, and complete samples at each concurrency level. +func TestPostgreSQLForcedSuffixSeededReversePlanResourcesAndConcurrency(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{ + Cases: []string{"GFSE-V2-D16-F1000-R1-X1-M1-sparse_path"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, []int{1, 2, 4}, false, nil, "", "EXPANSION-SUFFIX-SEEDED-REVERSE") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + + records, err := runner.Run(ctx, 0, 25, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, "_suffix_seeded_suffix as materialized") + require.Contains(t, record.SQL, "_suffix_seeded_reverse(boundary_id, next_id, depth, path, node_path)") + require.Contains(t, record.SQL, "array_prepend") + require.Contains(t, record.SQL, "generate_subscripts(") + require.NotContains(t, record.SQL, "ordered_edge_ids_to_path") + require.Contains(t, record.SQL, "!= all (") + require.NotContains(t, record.SQL, "satisfied, is_cycle") + + require.NotNil(t, record.PostgresMetrics) + metrics := record.PostgresMetrics + require.Greater(t, metrics.RecursiveRows, int64(0)) + require.Zero(t, metrics.Buffers.LocalHit) + require.Zero(t, metrics.Buffers.LocalRead) + require.Zero(t, metrics.Buffers.LocalDirtied) + require.Zero(t, metrics.Buffers.LocalWritten) + require.Zero(t, metrics.Buffers.TempRead) + require.Zero(t, metrics.Buffers.TempWritten) + require.Zero(t, metrics.TempFiles) + require.Zero(t, metrics.TempBytes) + require.Zero(t, metrics.WALRecords) + require.Zero(t, metrics.WALBytes) + + require.Len(t, record.Concurrency, 3) + for index, level := range []int{1, 2, 4} { + block := record.Concurrency[index] + require.Equal(t, level, block.Concurrency) + require.Equal(t, 2, block.PoolSize) + require.Equal(t, level*25, block.Operations) + require.Len(t, block.Samples, level*25) + } +} + +// TestPostgreSQLForcedSuffixSeededReverseCancellationReusesSession verifies prompt timeout cancellation, rollback recovery on the same backend PID, and cardinality-preserving replay of reverse expansion SQL. +func TestPostgreSQLForcedSuffixSeededReverseCancellationReusesSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{ + Cases: []string{"GFSE-V2-D08-F016-R1-I1000-high_reverse_fanin"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx := context.Background() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "EXPANSION-SUFFIX-SEEDED-REVERSE") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + backendPID := connectionHandle.Conn().PgConn().PID() + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + + var reusedPID uint32 + require.NoError(t, connectionHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + rows, err = connectionHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := 0 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, records[0].RowCount, int64(rowCount)) + t.Logf("cancelled exact EXPANSION-SUFFIX-SEEDED-REVERSE SQL in %s and reused backend PID %d", cancellationLatency, backendPID) +} + +// TestPostgreSQLForcedBidirectionalShortestCandidatesPreservePublicResults +// verifies both scheduler wrappers at the distance, one-witness, and complete +// all-shortest public boundaries. ASP production selection remains A1; these +// identities are reachable only through explicit tool forcing. +func TestPostgreSQLForcedBidirectionalShortestCandidatesPreservePublicResults(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // caseName identifies the case name. + caseName string + // executor retains the executor while anonymous record is assembled or evaluated. + executor string + // functionName identifies the function name. + functionName string + }{ + { + name: "SP B1 distance", + caseName: "GSP-D16-F016_distance", + executor: "SP-B1-C-ALT-NODE-D", + functionName: "shortest_path_b1_strict_alternating", + }, + { + name: "SP B2 witness", + caseName: "GSP-D16-F016_path", + executor: "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + functionName: "shortest_path_b2_smaller_current_level", + }, + { + name: "ASP B1 complete multiset", + caseName: "GSPV2-NORMAL-outbound-all-shortest-depth3", + executor: "ASP-B1-DAG-ALT-NODE", + functionName: "all_shortest_paths_b1_strict_alternating", + }, + { + name: "ASP B2 complete multiset", + caseName: "GSPV2-NORMAL-outbound-all-shortest-depth3", + executor: "ASP-B2-DAG-MIN-LEVEL", + functionName: "all_shortest_paths_b2_smaller_current_level", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{test.caseName}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, test.executor, "") + require.NoError(t, err) + defer func() { require.NoError(t, runner.Close(context.Background())) }() + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, test.functionName) + require.NotNil(t, record.Optimization) + found := false + for _, outcome := range record.Optimization.TargetOutcomes { + if outcome.Applied == test.executor { + found = true + break + } + } + require.True(t, found, "forced traversal outcome missing from %+v", record.Optimization.TargetOutcomes) + }) + } +} + +// TestPostgreSQLBidirectionalASPCancellationAndSessionIsolation verifies an +// aborted B1 replay rolls back cleanly, the same backend PID can immediately +// execute again, and identical invocation keys on two pooled sessions never +// share workspace or diagnostic rows. +func TestPostgreSQLBidirectionalASPCancellationAndSessionIsolation(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + selected.Cases[0].Name = "forced-asp-b1-operational-depth64" + selected.Cases[0].Cypher = strings.Replace(selected.Cases[0].Cypher, "shortestPath", "allShortestPaths", 1) + selected.Cases[0].Category = "generated_all_shortest_paths" + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, nil, false, nil, "ASP-B1-DAG-ALT-NODE", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + first, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer first.Release() + second, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer second.Release() + firstPID, secondPID := first.Conn().PgConn().PID(), second.Conn().PgConn().PID() + require.NotEqual(t, firstPID, secondPID) + // Materialize session-local telemetry tables outside the rollback checks so + // both sessions have the same schema but independent contents. + _, err = first.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_workspace()") + require.NoError(t, err) + _, err = second.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_workspace()") + require.NoError(t, err) + _, err = first.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_telemetry_workspace()") + require.NoError(t, err) + _, err = second.Exec(ctx, "select public.ensure_bidirectional_all_shortest_path_telemetry_workspace()") + require.NoError(t, err) + + drain := func(tx pgx.Tx) int64 { + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + defer rows.Close() + var count int64 + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + count++ + } + require.NoError(t, rows.Err()) + return count + } + readCalls := func(tx pgx.Tx, invocationID string) (int64, bool) { + var raw string + err := tx.QueryRow(ctx, "select coalesce(public.read_bidirectional_all_shortest_path_diagnostic_v1($1)::text, '')", invocationID).Scan(&raw) + require.NoError(t, err) + if raw == "" { + return 0, false + } + var document struct { + // SearchCalls supplies the search calls input to the anonymous record contract. + SearchCalls int64 `json:"search_calls"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &document)) + return document.SearchCalls, true + } + + firstTx, err := first.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + secondTx, err := second.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + const sharedInvocation = "same-key-different-sessions" + _, err = firstTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + _, err = secondTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(firstTx)) + firstCalls, found := readCalls(firstTx, sharedInvocation) + require.True(t, found) + require.Equal(t, int64(1), firstCalls) + secondCalls, found := readCalls(secondTx, sharedInvocation) + require.True(t, found) + require.Zero(t, secondCalls) + _, err = firstTx.Exec(ctx, "select public.clear_bidirectional_all_shortest_path_diagnostic_v1($1)", sharedInvocation) + require.NoError(t, err) + _, found = readCalls(firstTx, sharedInvocation) + require.False(t, found) + _, found = readCalls(secondTx, sharedInvocation) + require.True(t, found) + require.NoError(t, firstTx.Rollback(ctx)) + require.NoError(t, secondTx.Rollback(ctx)) + + cancelTx, err := first.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1('cancelled-replay')") + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := cancelTx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, cancelTx.Rollback(ctx)) + require.Equal(t, firstPID, first.Conn().PgConn().PID()) + + reuseTx, err := first.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + _, found = readCalls(reuseTx, "cancelled-replay") + require.False(t, found, "rolled-back invocation state must not survive") + _, err = reuseTx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1('successful-reuse')") + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(reuseTx)) + reuseCalls, found := readCalls(reuseTx, "successful-reuse") + require.True(t, found) + require.Equal(t, int64(1), reuseCalls) + _, err = reuseTx.Exec(ctx, "select public.clear_bidirectional_all_shortest_path_diagnostic_v1('successful-reuse')") + require.NoError(t, err) + require.NoError(t, reuseTx.Commit(ctx)) + t.Logf("cancelled ASP-B1 in %s and reused backend PID %d without cross-session state from PID %d", cancellationLatency, firstPID, secondPID) +} + +// TestPostgreSQLBidirectionalSPCancellationAndSessionIsolation applies the +// cancellation, rollback/reuse, and session-local telemetry contract to both +// compact SP schedulers. Candidate and telemetry workspaces are materialized +// before the timed query so the timeout interrupts search rather than DDL. +func TestPostgreSQLBidirectionalSPCancellationAndSessionIsolation(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + for _, scheduler := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // executor retains the executor while anonymous record is assembled or evaluated. + executor string + // functionName identifies the function name. + functionName string + }{ + { + name: "B1 strict alternating", + executor: "SP-B1-C-ALT-NODE-WE+MAT-M0", + functionName: "shortest_path_b1_strict_alternating", + }, + { + name: "B2 smaller level", + executor: "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + functionName: "shortest_path_b2_smaller_current_level", + }, + } { + scheduler := scheduler + t.Run(scheduler.name, func(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GSP-D64-F1000_path"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 2, 1, nil, false, nil, scheduler.executor, "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + require.Equal(t, StatusOK, records[0].Status, records[0].Error) + require.Contains(t, records[0].SQL, scheduler.functionName) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, records[0].Params) + require.NoError(t, err) + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + first, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer first.Release() + second, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer second.Release() + firstPID, secondPID := first.Conn().PgConn().PID(), second.Conn().PgConn().PID() + require.NotEqual(t, firstPID, secondPID) + for _, session := range []*pgxpool.Conn{first, second} { + _, err = session.Exec(ctx, "select public.ensure_bidirectional_shortest_path_workspace()") + require.NoError(t, err) + _, err = session.Exec(ctx, "select public.ensure_bidirectional_shortest_path_telemetry_workspace()") + require.NoError(t, err) + } + + drain := func(tx pgx.Tx) int64 { + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, queryErr) + defer rows.Close() + var count int64 + for rows.Next() { + _, queryErr = rows.Values() + require.NoError(t, queryErr) + count++ + } + require.NoError(t, rows.Err()) + return count + } + readCalls := func(tx pgx.Tx, invocationID string) (int64, bool) { + var raw string + err := tx.QueryRow(ctx, "select coalesce(public.read_bidirectional_shortest_path_diagnostic_v1($1)::text, '')", invocationID).Scan(&raw) + require.NoError(t, err) + if raw == "" { + return 0, false + } + var document struct { + // SearchCalls supplies the search calls input to the anonymous record contract. + SearchCalls int64 `json:"search_calls"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &document)) + return document.SearchCalls, true + } + + firstTx, err := first.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + secondTx, err := second.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + invocationID := "sp-same-key-" + scheduler.executor + _, err = firstTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, err) + _, err = secondTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(firstTx)) + firstCalls, found := readCalls(firstTx, invocationID) + require.True(t, found) + require.Equal(t, int64(1), firstCalls) + secondCalls, found := readCalls(secondTx, invocationID) + require.True(t, found) + require.Zero(t, secondCalls) + _, err = firstTx.Exec(ctx, "select public.clear_bidirectional_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, err) + _, found = readCalls(firstTx, invocationID) + require.False(t, found) + _, found = readCalls(secondTx, invocationID) + require.True(t, found) + require.NoError(t, firstTx.Rollback(ctx)) + require.NoError(t, secondTx.Rollback(ctx)) + + cancelTx, err := first.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + cancelInvocation := "sp-cancelled-" + scheduler.executor + _, err = cancelTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", cancelInvocation) + require.NoError(t, err) + _, err = cancelTx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := cancelTx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, cancelTx.Rollback(ctx)) + require.Equal(t, firstPID, first.Conn().PgConn().PID()) + + reuseTx, err := first.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + _, found = readCalls(reuseTx, cancelInvocation) + require.False(t, found, "rolled-back invocation state must not survive") + reuseInvocation := "sp-successful-" + scheduler.executor + _, err = reuseTx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1($1)", reuseInvocation) + require.NoError(t, err) + require.Equal(t, records[0].RowCount, drain(reuseTx)) + reuseCalls, found := readCalls(reuseTx, reuseInvocation) + require.True(t, found) + require.Equal(t, int64(1), reuseCalls) + _, err = reuseTx.Exec(ctx, "select public.clear_bidirectional_shortest_path_diagnostic_v1($1)", reuseInvocation) + require.NoError(t, err) + require.NoError(t, reuseTx.Commit(ctx)) + t.Logf("cancelled %s in %s and reused backend PID %d without cross-session state from PID %d", scheduler.executor, cancellationLatency, firstPID, secondPID) + }) + } +} + +// requirePostgresReference returns the named comparator result or fails when the runner omitted that reference arm. +func requirePostgresReference(t *testing.T, references []PostgresReferenceResult, name string) PostgresReferenceResult { + t.Helper() + for _, reference := range references { + if reference.Name == name { + return reference + } + } + t.Fatalf("missing PostgreSQL reference %s", name) + return PostgresReferenceResult{} +} + +// requireSingleScaleRecord returns the sole result for a corpus ID and rejects missing or duplicate representatives. +func requireSingleScaleRecord(t *testing.T, byID map[string][]CaseResult, id string) CaseResult { + t.Helper() + require.Len(t, byID[id], 1, "%s must have one representative", id) + return byID[id][0] +} + +// assertMutationPlanTarget verifies that delete representatives modify the physical entity table implied by their corpus ID. +func assertMutationPlanTarget(t *testing.T, id, plan string) { + t.Helper() + + switch id { + case "REC-01", "REC-02", "REC-04", "REC-06": + require.Contains(t, plan, "Delete on edge") + case "REC-08": + require.Contains(t, plan, "Delete on node") + } +} + +// assertAnchorPlanIndex verifies that each indexed representative anchors through an endpoint or selective graph-partition index rather than a heap-wide scan. +func assertAnchorPlanIndex(t *testing.T, id, plan string) { + t.Helper() + + switch id { + case "HOP-01", "HOP-03", "HOP-04", "HOP-05": + // PostgreSQL may prefer the covering kind index when the edge kind is + // more selective than the bound endpoint. Both choices remain scoped + // to the graph partition and avoid a heap-wide edge scan. + require.Regexp(t, `(Bitmap Index Scan on|Index Scan using) edge_[0-9]+_(start_id|kind_id)`, plan) + require.Contains(t, plan, "start_id =") + case "HOP-02": + require.Regexp(t, `(Bitmap Index Scan on|Index Scan using) edge_[0-9]+_end_id`, plan) + case "HOP-07": + // The selective terminal predicate can legitimately reverse the join + // order, but either endpoint orientation must stay indexed. + require.Regexp(t, `(Bitmap Index Scan on|Index Scan using) edge_[0-9]+_(start|end)_id`, plan) + case "REC-01", "REC-02", "REC-04", "REC-06", "REC-08", "SCAN-05", + "LOOKUP-02", "LOOKUP-04", "LOOKUP-05", "LOOKUP-09", "LOOKUP-11", "LOOKUP-13", "LOOKUP-16", + "TRUST-01", "TRUST-02", "PRUNE-02", "PRUNE-03": + require.Contains(t, plan, "Index Scan") + } +} diff --git a/cmd/graphbench/promotion_evidence_validation.go b/cmd/graphbench/promotion_evidence_validation.go new file mode 100644 index 00000000..0fc2b871 --- /dev/null +++ b/cmd/graphbench/promotion_evidence_validation.go @@ -0,0 +1,1063 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "reflect" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// Bound promotion reports embed the producer's native schema and add only the +// authorization identity installed by bindPromotionEvidenceReport. Keeping +// these wrappers concrete makes DisallowUnknownFields effective; decoding into +// map[string]any would silently accept misspelled or invented proof fields. +type promotionAAResolutionReport struct { + AAResolutionReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + // NativeReportSHA256 binds the wrapped document to the exact producer report + // before promotion_identity is attached. Other roles use this digest when + // they declare which A/A report supplied their statistical floor. + NativeReportSHA256 string `json:"native_report_sha256"` + // NativeReportBase64 preserves the producer's exact bytes so verification can + // recompute the digest rather than trusting a digest-shaped assertion. + NativeReportBase64 string `json:"native_report_base64"` +} + +type promotionConfirmationReport struct { + ConfirmationReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` +} + +type promotionPerfGateReport struct { + PerfGateReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` +} + +type promotionSPI1QualificationReport struct { + SPI1QualificationReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` +} + +type promotionSPI2QualificationReport struct { + SPI2QualificationReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` +} + +type promotionOrientationSelectorReport struct { + OrientationSelectorReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` +} + +type promotionOrientationSelectorV2Report struct { + OrientationSelectorV2Report + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` +} + +func validatePromotionAAReport(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionAAResolutionReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("A/A report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("A/A report promotion identity does not match manifest") + } + report := bound.AAResolutionReport + nativeRaw, err := base64.StdEncoding.DecodeString(bound.NativeReportBase64) + if err != nil || len(nativeRaw) == 0 { + return fmt.Errorf("A/A report does not contain decodable native producer bytes") + } + nativeDigest := sha256.Sum256(nativeRaw) + if hex.EncodeToString(nativeDigest[:]) != bound.NativeReportSHA256 { + return fmt.Errorf("A/A native producer report SHA-256 does not match its embedded bytes") + } + var native AAResolutionReport + if err := decodePromotionEvidence(nativeRaw, &native); err != nil { + return fmt.Errorf("A/A native producer report: %w", err) + } + if !reflect.DeepEqual(native, report) { + return fmt.Errorf("A/A bound projection differs from its native producer report") + } + if report.Version != aaReportVersion { + return fmt.Errorf("A/A report version must be %d", aaReportVersion) + } + if !lowercaseSHA256(report.ArtifactSHA256) || !lowercaseSHA256(report.HostFingerprint) { + return fmt.Errorf("A/A report lacks canonical artifact and host digests") + } + if report.Confidence != defaultConfidenceLevel || math.IsNaN(report.Confidence) || math.IsInf(report.Confidence, 0) || !lowercaseSHA256(bound.NativeReportSHA256) { + return fmt.Errorf("A/A report has invalid frozen confidence or native report digest") + } + if !report.OrderBalanced || report.MinimumRounds != minimumGateRounds || report.MinimumSamplesPerArmPerRound != 10 || report.MinimumP99SamplesPerArm != 10_000 { + return fmt.Errorf("A/A report lacks the promotion-grade balanced sampling contract") + } + chronology := report.PhysicalChronology + if chronology == nil || chronology.Version != aaPhysicalChronologyVersion || !chronology.Validated || + chronology.ArtifactSHA256 != report.ArtifactSHA256 || chronology.Rounds < report.MinimumRounds || + len(chronology.Arms) != 2 || strings.TrimSpace(chronology.Arms[0]) == "" || strings.TrimSpace(chronology.Arms[1]) == "" || chronology.Arms[0] == chronology.Arms[1] { + return fmt.Errorf("A/A report lacks artifact-bound physical chronology") + } + if len(report.Cases) == 0 { + return fmt.Errorf("A/A report has no cases") + } + seen := map[string]struct{}{} + for _, gateCase := range report.Cases { + key := gateCase.Dataset + "\x00" + gateCase.Name + if strings.TrimSpace(gateCase.Dataset) == "" || strings.TrimSpace(gateCase.Name) == "" || gateCase.Backend != ModePostgresSQL || + !lowercaseSHA256(gateCase.WorkloadSHA256) || !lowercaseSHA256(gateCase.PostgresEnvironmentSHA256) || !lowercaseSHA256(gateCase.FixtureSHA256) { + return fmt.Errorf("A/A report contains an incomplete case identity") + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("A/A report duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seen[key] = struct{}{} + if gateCase.Rounds != chronology.Rounds || gateCase.Rounds < report.MinimumRounds || + gateCase.SamplesPerArm < gateCase.Rounds*report.MinimumSamplesPerArmPerRound { + return fmt.Errorf("A/A case %s/%s lacks the declared rounds or samples", gateCase.Dataset, gateCase.Name) + } + if err := validatePromotionAAMetric(gateCase.P50); err != nil { + return fmt.Errorf("A/A case %s/%s p50: %w", gateCase.Dataset, gateCase.Name, err) + } + if err := validatePromotionAAMetric(gateCase.P95); err != nil { + return fmt.Errorf("A/A case %s/%s p95: %w", gateCase.Dataset, gateCase.Name, err) + } + expectedP99 := gateCase.SamplesPerArm >= report.MinimumP99SamplesPerArm + if report.MinimumP99SamplesPerArm <= 0 || gateCase.P99Gated != expectedP99 || expectedP99 && gateCase.P99Reason != "" || !expectedP99 && strings.TrimSpace(gateCase.P99Reason) == "" { + return fmt.Errorf("A/A case %s/%s has contradictory p99 gating", gateCase.Dataset, gateCase.Name) + } + } + return nil +} + +func validatePromotionAAMetric(metric AAMetricResolution) error { + if !validRatioInterval(metric.Ratio) || !validDurationInterval(metric.AbsoluteChange) || + math.IsNaN(metric.RatioResolution) || math.IsInf(metric.RatioResolution, 0) || metric.RatioResolution < 0 || metric.AbsoluteResolution < 0 { + return fmt.Errorf("invalid statistical interval") + } + expectedRatio := math.Max(math.Abs(1-metric.Ratio.Lower), math.Abs(metric.Ratio.Upper-1)) + expectedAbsolute := max(absDuration(metric.AbsoluteChange.Lower), absDuration(metric.AbsoluteChange.Upper)) + if metric.RatioResolution != expectedRatio || metric.AbsoluteResolution != expectedAbsolute { + return fmt.Errorf("derived resolution contradicts its interval") + } + return nil +} + +func validatePromotionConfirmationReport(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + switch expectedIdentity.Candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return validatePromotionSPI1Confirmation(raw, expectedIdentity) + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return validatePromotionSPI2Confirmation(raw, expectedIdentity) + case string(optimize.ExpansionSearchPolicyOrientationProbeV1): + return fmt.Errorf("orientation-probe-v1 promotion is disabled because its report schema cannot bind source, corpus, and frozen cohort identity") + case string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return validatePromotionOrientationV2Confirmation(raw, expectedIdentity) + case string(optimize.ShortestPathExecutorASPI1DAG): + return validatePromotionGenericConfirmation(raw, expectedIdentity) + default: + return fmt.Errorf("candidate %q has no registered confirmation-report schema", expectedIdentity.Candidate) + } +} + +func validatePromotionGenericConfirmation(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionConfirmationReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("confirmation report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("confirmation report promotion identity does not match manifest") + } + report := bound.ConfirmationReport + if report.Version != confirmationReportVersion || report.Kind != "causal_confirmation" { + return fmt.Errorf("confirmation report is not schema-v%d causal confirmation", confirmationReportVersion) + } + if report.Seed != 1 || report.Confidence != defaultConfidenceLevel || math.IsNaN(report.Confidence) || math.IsInf(report.Confidence, 0) || + strings.TrimSpace(report.LeftArm) == "" || strings.TrimSpace(report.RightArm) == "" || report.LeftArm == report.RightArm || + !lowercaseSHA256(report.LeftSHA256) || !lowercaseSHA256(report.RightSHA256) || !lowercaseSHA256(report.AAReportSHA256) || strings.TrimSpace(report.AAReport) == "" { + return fmt.Errorf("confirmation report lacks immutable arm and A/A identity") + } + if !report.PromotionEligible || !report.QualificationRequired || !report.TrainingPassed || !report.HoldoutPassed || !report.QualificationPassed || + report.TrainingCases <= 0 || report.HoldoutCases <= 0 || len(report.Cases) == 0 { + return fmt.Errorf("confirmation report did not pass complete training and holdout qualification") + } + if err := validatePromotionQualificationFamilies(report.QualificationFamilies, expectedIdentity.Candidate, report.TrainingCases, report.HoldoutCases); err != nil { + return fmt.Errorf("confirmation report: %w", err) + } + seenCases := map[string]struct{}{} + seenInvocations := map[string]struct{}{} + trainingCases, holdoutCases := 0, 0 + for _, gateCase := range report.Cases { + key := fmt.Sprintf("%s\x00%s\x00%s", gateCase.Dataset, gateCase.Name, gateCase.Backend) + if strings.TrimSpace(gateCase.Dataset) == "" || strings.TrimSpace(gateCase.Name) == "" { + return fmt.Errorf("confirmation report contains an incomplete case identity") + } + if _, duplicate := seenCases[key]; duplicate { + return fmt.Errorf("confirmation report duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seenCases[key] = struct{}{} + if !gateCase.TimingGated { + continue + } + if gateCase.Backend != ModePostgresSQL || (gateCase.QualificationSplit != "training" && gateCase.QualificationSplit != "holdout") || + gateCase.MatchedRounds < 10 || gateCase.MatchedRounds > 20 || gateCase.LeftSamples < gateCase.MatchedRounds*50 || gateCase.RightSamples < gateCase.MatchedRounds*50 || + !gateCase.Comparable || len(gateCase.Comparability) != 0 || gateCase.P95.Classification != "cleared_non_inferior" || gateCase.Disposition != gateCase.P95.Classification { + return fmt.Errorf("confirmation case %s/%s lacks passing promotion evidence", gateCase.Dataset, gateCase.Name) + } + if err := validatePromotionConfirmationMetric(gateCase.P50); err != nil { + return fmt.Errorf("confirmation case %s/%s p50: %w", gateCase.Dataset, gateCase.Name, err) + } + if err := validatePromotionConfirmationMetric(gateCase.P95); err != nil { + return fmt.Errorf("confirmation case %s/%s p95: %w", gateCase.Dataset, gateCase.Name, err) + } + if len(gateCase.RightRuntimeReceiptChains) != gateCase.RightSamples { + return fmt.Errorf("confirmation case %s/%s runtime receipt count differs from right-arm samples", gateCase.Dataset, gateCase.Name) + } + if err := validatePromotionReceiptChains(gateCase.RightRuntimeReceiptChains, expectedIdentity.Candidate, seenInvocations); err != nil { + return fmt.Errorf("confirmation case %s/%s: %w", gateCase.Dataset, gateCase.Name, err) + } + if gateCase.QualificationSplit == "training" { + trainingCases++ + } else { + holdoutCases++ + } + } + if trainingCases != report.TrainingCases || holdoutCases != report.HoldoutCases { + return fmt.Errorf("confirmation report split counts contradict its cases") + } + return nil +} + +func validatePromotionSPI1Confirmation(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionSPI1QualificationReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("SP-I1 confirmation report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("SP-I1 confirmation promotion identity does not match manifest") + } + report := bound.SPI1QualificationReport + cohort, err := canonicalSPI1Cohort() + if err != nil { + return fmt.Errorf("SP-I1 confirmation cohort: %w", err) + } + if report.Version != spI1QualificationVersion || report.Protocol != referencePairProtocolConfirmation || + report.Baseline != string(optimize.ShortestPathExecutorS4CanonicalWitness) || report.Candidate != expectedIdentity.Candidate || + report.Policy != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || report.QuerySHA256 != spI1QuerySHA256 { + return fmt.Errorf("SP-I1 confirmation report has the wrong version, protocol, or candidate contract") + } + if expectedIdentity.SelectorVersion != optimize.ShortestPathSelectorStaticV6 || expectedIdentity.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + expectedIdentity.FallbackExecutor != report.Baseline || promotionIdentityQueryCount(expectedIdentity, report.QuerySHA256) != 1 { + return fmt.Errorf("SP-I1 confirmation report does not match the manifest selector, fallback, or exact query cohort") + } + if report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.BootstrapCount != defaultBootstrapCount || + report.MaterialityRatio != 0.95 || report.MaterialityAbsolute != 100*time.Microsecond || report.P95RatioLimit != 1.05 || + !exactPromotionCaps(report.Caps, spI1QualificationCaps()) || !exactPromotionCaps(report.Caps, expectedIdentity.Caps) { + return fmt.Errorf("SP-I1 confirmation report changes frozen statistical or cap settings") + } + if report.SourceCommit != expectedIdentity.SourceCommit || report.SourceArchiveSHA256 != expectedIdentity.SourceSHA256 || + report.BinarySHA256 != expectedIdentity.BinarySHA256 || report.CorpusSHA256 != expectedIdentity.CorpusSHA256 || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || report.CorpusSHA256 != spI1FullCorpusSHA256 || + report.CohortDeclarationSHA256 != cohort.declarationSHA256 || report.ResolvedSelectionSHA256 != cohort.fullResolvedSHA256 || + report.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || report.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + report.FullDeclarationSHA256 != cohort.declarationSHA256 || report.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || report.FullCorpusSHA256 != cohort.fullCorpusSHA256 { + return fmt.Errorf("SP-I1 confirmation report source, corpus, or cohort identity differs from the manifest and frozen protocol") + } + if !lowercaseSHA256(report.BaselineArtifactSHA256) || !lowercaseSHA256(report.CandidateArtifactSHA256) || + !lowercaseSHA256(report.ResourceReportSHA256) || !lowercaseSHA256(report.FreezeManifestSHA256) { + return fmt.Errorf("SP-I1 confirmation report lacks checksummed artifacts and freeze") + } + if !report.EvidencePassed || !report.TrainingPassed || !report.HoldoutPassed || !report.QualificationPassed || + report.TrainingCases != len(cohort.trainingKeys) || report.HoldoutCases != len(cohort.holdoutKeys) || len(report.Cases) != len(cohort.keys) { + return fmt.Errorf("SP-I1 confirmation report did not pass the complete frozen cohort") + } + seen := map[string]struct{}{} + for _, gateCase := range report.Cases { + key := performanceKey{dataset: gateCase.Dataset, name: gateCase.Name, backend: ModePostgresSQL} + if _, expected := cohort.keys[key]; !expected { + return fmt.Errorf("SP-I1 confirmation report contains unexpected case %s/%s", gateCase.Dataset, gateCase.Name) + } + caseKey := promotionCaseKey(gateCase.Dataset, gateCase.Name) + if _, duplicate := seen[caseKey]; duplicate { + return fmt.Errorf("SP-I1 confirmation report duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seen[caseKey] = struct{}{} + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + material := validRatioInterval(gateCase.MedianRatio) && validDurationInterval(gateCase.MedianSaving) && + (gateCase.MedianRatio.Upper <= report.MaterialityRatio || gateCase.MedianSaving.Lower >= report.MaterialityAbsolute) + p95Contained := validRatioInterval(gateCase.P95Ratio) && gateCase.P95Ratio.Upper <= report.P95RatioLimit + if gateCase.QualificationSplit != expectedSplit || gateCase.Rounds < 10 || gateCase.Rounds > 20 || + gateCase.BaselineSamples < gateCase.Rounds*50 || gateCase.CandidateSamples < gateCase.Rounds*50 || + !material || gateCase.Material != material || !p95Contained || gateCase.P95Contained != p95Contained || + !gateCase.ResourcePassed || strings.TrimSpace(gateCase.RuntimeBranch) == "" || !gateCase.Passed || len(gateCase.Reasons) != 0 { + return fmt.Errorf("SP-I1 confirmation case %s/%s has incomplete or contradictory evidence", gateCase.Dataset, gateCase.Name) + } + } + return nil +} + +func validatePromotionSPI2Confirmation(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionSPI2QualificationReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("SP-I2 confirmation report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("SP-I2 confirmation promotion identity does not match manifest") + } + report := bound.SPI2QualificationReport + cohort, err := canonicalSPI2Cohort() + if err != nil { + return fmt.Errorf("SP-I2 confirmation cohort: %w", err) + } + if report.Version != spI2QualificationVersion || report.Protocol != referencePairProtocolConfirmation || + report.Baseline != string(optimize.ShortestPathExecutorS4CanonicalDistance) || report.Candidate != expectedIdentity.Candidate || + report.Policy != optimize.ShortestPathPolicyI2DistanceGuardedV1 || report.QuerySHA256 != spI2QuerySHA256 { + return fmt.Errorf("SP-I2 confirmation report has the wrong version, protocol, or candidate contract") + } + if expectedIdentity.SelectorVersion != optimize.ShortestPathSelectorStaticV8HiddenFanIn || expectedIdentity.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + expectedIdentity.FallbackExecutor != report.Baseline || promotionIdentityQueryCount(expectedIdentity, report.QuerySHA256) != 1 { + return fmt.Errorf("SP-I2 confirmation report does not match the manifest selector, fallback, or exact query cohort") + } + if report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.BootstrapCount != defaultBootstrapCount || + report.MaterialityRatio != 0.95 || report.MaterialityAbsolute != 100*time.Microsecond || report.P95RatioLimit != 1.05 || + report.AdverseRatioLimit != 1.10 || report.AdverseAbsoluteLimit != 100*time.Microsecond || + !exactPromotionCaps(report.Caps, spI2QualificationCaps()) || !exactPromotionCaps(report.Caps, expectedIdentity.Caps) { + return fmt.Errorf("SP-I2 confirmation report changes frozen statistical or cap settings") + } + if report.SourceCommit != expectedIdentity.SourceCommit || report.SourceArchiveSHA256 != expectedIdentity.SourceSHA256 || + report.BinarySHA256 != expectedIdentity.BinarySHA256 || report.CorpusSHA256 != expectedIdentity.CorpusSHA256 || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || report.CorpusSHA256 != spI2FullCorpusSHA256 || + report.CohortDeclarationSHA256 != cohort.declarationSHA256 || report.ResolvedSelectionSHA256 != cohort.fullResolvedSHA256 || + report.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || report.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + report.FullDeclarationSHA256 != cohort.declarationSHA256 || report.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || report.FullCorpusSHA256 != cohort.fullCorpusSHA256 { + return fmt.Errorf("SP-I2 confirmation report source, corpus, or cohort identity differs from the manifest and frozen protocol") + } + if !lowercaseSHA256(report.BaselineArtifactSHA256) || !lowercaseSHA256(report.CandidateArtifactSHA256) || + !lowercaseSHA256(report.ResourceReportSHA256) || !lowercaseSHA256(report.FreezeManifestSHA256) { + return fmt.Errorf("SP-I2 confirmation report lacks checksummed artifacts and freeze") + } + if !report.EvidencePassed || !report.TrainingPassed || !report.HoldoutPassed || !report.QualificationPassed || + report.TrainingCases != len(cohort.trainingKeys) || report.HoldoutCases != len(cohort.holdoutKeys) || len(report.Cases) != len(cohort.keys) { + return fmt.Errorf("SP-I2 confirmation report did not pass the complete frozen cohort") + } + seen := map[string]struct{}{} + for _, gateCase := range report.Cases { + key := performanceKey{dataset: gateCase.Dataset, name: gateCase.Name, backend: ModePostgresSQL} + if _, expected := cohort.keys[key]; !expected { + return fmt.Errorf("SP-I2 confirmation report contains unexpected case %s/%s", gateCase.Dataset, gateCase.Name) + } + caseKey := promotionCaseKey(gateCase.Dataset, gateCase.Name) + if _, duplicate := seen[caseKey]; duplicate { + return fmt.Errorf("SP-I2 confirmation report duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seen[caseKey] = struct{}{} + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + expectedRole := "target" + material := validRatioInterval(gateCase.MedianRatio) && validDurationInterval(gateCase.MedianSaving) && + (gateCase.MedianRatio.Upper <= report.MaterialityRatio || gateCase.MedianSaving.Lower >= report.MaterialityAbsolute) + if strings.Contains(gateCase.Name, "cycle-control") { + expectedRole = "adverse_control" + material = validRatioInterval(gateCase.MedianRatio) && validDurationInterval(gateCase.MedianSaving) && + (gateCase.MedianRatio.Upper <= report.AdverseRatioLimit || gateCase.MedianSaving.Lower >= -report.AdverseAbsoluteLimit) + } + p95Contained := validRatioInterval(gateCase.P95Ratio) && gateCase.P95Ratio.Upper <= report.P95RatioLimit + if gateCase.QualificationSplit != expectedSplit || gateCase.QualificationRole != expectedRole || + gateCase.Rounds < 10 || gateCase.Rounds > 20 || gateCase.BaselineSamples < gateCase.Rounds*50 || gateCase.CandidateSamples < gateCase.Rounds*50 || + !material || gateCase.Material != material || !p95Contained || gateCase.P95Contained != p95Contained || + !gateCase.ResourcePassed || strings.TrimSpace(gateCase.RuntimeBranch) == "" || !gateCase.Passed || len(gateCase.Reasons) != 0 { + return fmt.Errorf("SP-I2 confirmation case %s/%s has incomplete or contradictory evidence", gateCase.Dataset, gateCase.Name) + } + } + return nil +} + +func validatePromotionOrientationConfirmation(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionOrientationSelectorReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("orientation confirmation report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("orientation confirmation promotion identity does not match manifest") + } + report := bound.OrientationSelectorReport + if report.Version != orientationSelectorReportVersion || report.Policy != expectedIdentity.Candidate || report.Protocol != referencePairProtocolConfirmation || + report.Confidence <= 0 || report.Confidence >= 1 || math.IsNaN(report.Confidence) || math.IsInf(report.Confidence, 0) || + report.SelectorRegretRatioLimit != 1.10 || report.ProbeOverheadRatioLimit != 1.10 || report.ProbeOverheadAbsoluteLimit != 100*time.Microsecond { + return fmt.Errorf("orientation confirmation report changes its version, protocol, policy, or frozen thresholds") + } + if expectedIdentity.SelectorVersion != expectedIdentity.Candidate || expectedIdentity.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + expectedIdentity.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) || !exactPromotionCaps(expectedIdentity.Caps, orientationPromotionCaps()) { + return fmt.Errorf("orientation confirmation report does not match the manifest selector, fallback, or cap contract") + } + if !lowercaseSHA256(report.ShadowArtifactSHA256) || !lowercaseSHA256(report.IncumbentArtifactSHA256) || + !lowercaseSHA256(report.ReverseArtifactSHA256) || !lowercaseSHA256(report.AAReportSHA256) { + return fmt.Errorf("orientation confirmation report lacks checksummed arm and A/A artifacts") + } + if !report.EvidencePassed || !report.TrainingPassed || !report.HoldoutPassed || !report.QualificationPassed || + report.TrainingCases <= 0 || report.HoldoutCases <= 0 || len(report.Cases) != report.TrainingCases+report.HoldoutCases { + return fmt.Errorf("orientation confirmation report did not pass complete training and holdout evidence") + } + seen := map[string]struct{}{} + trainingCases, holdoutCases := 0, 0 + for _, gateCase := range report.Cases { + key := promotionCaseKey(gateCase.Dataset, gateCase.Name) + if strings.TrimSpace(gateCase.Dataset) == "" || strings.TrimSpace(gateCase.Name) == "" { + return fmt.Errorf("orientation confirmation contains an incomplete case identity") + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("orientation confirmation duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seen[key] = struct{}{} + if (gateCase.QualificationSplit != "training" && gateCase.QualificationSplit != "holdout") || !gateCase.QualificationEligible || + gateCase.Rounds < 10 || gateCase.Rounds > 20 || !gateCase.ExactObservationsMatched || + strings.TrimSpace(gateCase.WouldSelectIdentity) == "" || strings.TrimSpace(gateCase.FastestExactIdentity) == "" { + return fmt.Errorf("orientation confirmation case %s/%s lacks frozen qualification evidence", gateCase.Dataset, gateCase.Name) + } + if err := validatePromotionOrientationLatencyGate(gateCase.SelectorRegret, gateCase.Rounds, 50, gateCase.SelectorRegret.BaselineIdentity, gateCase.SelectorRegret.ObservedIdentity, report.SelectorRegretRatioLimit, 0, false); err != nil { + return fmt.Errorf("orientation confirmation case %s/%s selector regret: %w", gateCase.Dataset, gateCase.Name, err) + } + if err := validatePromotionOrientationLatencyGate(gateCase.ProbeOverhead, gateCase.Rounds, 50, gateCase.ProbeOverhead.BaselineIdentity, gateCase.ProbeOverhead.ObservedIdentity, report.ProbeOverheadRatioLimit, report.ProbeOverheadAbsoluteLimit, true); err != nil { + return fmt.Errorf("orientation confirmation case %s/%s probe overhead: %w", gateCase.Dataset, gateCase.Name, err) + } + expectedPassed := gateCase.SelectorRegret.Passed && gateCase.ProbeOverhead.Passed + if !expectedPassed || gateCase.Passed != expectedPassed || len(gateCase.Reasons) != 0 { + return fmt.Errorf("orientation confirmation case %s/%s has contradictory passing disposition", gateCase.Dataset, gateCase.Name) + } + if gateCase.QualificationSplit == "training" { + trainingCases++ + } else { + holdoutCases++ + } + } + if trainingCases != report.TrainingCases || holdoutCases != report.HoldoutCases { + return fmt.Errorf("orientation confirmation split counts contradict its cases") + } + return nil +} + +func validatePromotionOrientationV2Confirmation(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionOrientationSelectorV2Report + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("orientation-v2 confirmation report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("orientation-v2 confirmation promotion identity does not match manifest") + } + report := bound.OrientationSelectorV2Report + canonical, err := canonicalOrientationV2Cohort() + if err != nil { + return fmt.Errorf("orientation-v2 canonical cohort: %w", err) + } + const formula = "F2=root_rows+maximum_depth*forward_degree_rows;R2=suffix_rows+boundary_rows+reverse_degree_rows;reverse=complete&&4*R2<3*F2" + if report.Version != orientationSelectorReportV2Version || report.Policy != expectedIdentity.Candidate || report.Protocol != referencePairProtocolConfirmation || + report.Seed != 1 || report.Confidence != defaultConfidenceLevel || math.IsNaN(report.Confidence) || math.IsInf(report.Confidence, 0) || report.Formula != formula || + report.ShadowForwardRatioLimit != 1.10 || report.GuardedSelectedRatioLimit != 1.10 || report.GuardedFastestRatioLimit != 1.10 || report.OverheadAbsoluteLimit != 100*time.Microsecond { + return fmt.Errorf("orientation-v2 confirmation changes its version, protocol, policy, formula, or frozen thresholds") + } + if expectedIdentity.SelectorVersion != expectedIdentity.Candidate || expectedIdentity.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + expectedIdentity.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) || + !exactPromotionCaps(report.Caps, orientationPromotionCaps()) || !exactPromotionCaps(report.Caps, expectedIdentity.Caps) { + return fmt.Errorf("orientation-v2 confirmation does not match the manifest selector, fallback, or cap contract") + } + if report.SourceCommit != expectedIdentity.SourceCommit || report.BinarySHA256 != expectedIdentity.BinarySHA256 || report.CorpusSHA256 != expectedIdentity.CorpusSHA256 || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || report.CohortDeclarationSHA256 != canonical.declarationSHA256 || + !lowercaseSHA256(report.ShadowArtifactSHA256) || !lowercaseSHA256(report.IncumbentArtifactSHA256) || + !lowercaseSHA256(report.ReverseArtifactSHA256) || !lowercaseSHA256(report.GuardedArtifactSHA256) || + !lowercaseSHA256(report.AAReportSHA256) || !lowercaseSHA256(report.FreezeManifestSHA256) { + return fmt.Errorf("orientation-v2 confirmation lacks manifest-bound clean source and checksummed artifacts") + } + if !report.EvidencePassed || !report.TrainingPassed || !report.HoldoutPassed || !report.QualificationPassed || + report.TrainingCases != 8 || report.HoldoutCases != 4 || len(report.Cases) != 12 { + return fmt.Errorf("orientation-v2 confirmation did not pass its exact frozen cohort") + } + seen := map[string]struct{}{} + trainingCases, holdoutCases := 0, 0 + for _, gateCase := range report.Cases { + key := promotionCaseKey(gateCase.Dataset, gateCase.Name) + performanceCaseKey := performanceKey{dataset: gateCase.Dataset, name: gateCase.Name, backend: ModePostgresSQL} + if _, expected := canonical.keys[performanceCaseKey]; !expected { + return fmt.Errorf("orientation-v2 confirmation contains a case outside the frozen V3 corpus") + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("orientation-v2 confirmation duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seen[key] = struct{}{} + expectedSplit, expectedRole, expectedTuning := "holdout", "frozen_evaluation", false + if _, training := canonical.trainingKeys[performanceCaseKey]; training { + expectedSplit, expectedRole, expectedTuning = "training", "selector_training", true + } + forward := string(optimize.ExpansionSearchStepwiseForward) + expectedRuntimeBranch := "exact_forward_incumbent" + if gateCase.GuardedRuntimeIdentity == string(optimize.ExpansionSearchSuffixSeededReverse) { + expectedRuntimeBranch = "suffix_seeded_reverse" + } + if gateCase.QualificationSplit != expectedSplit || gateCase.QualificationRole != expectedRole || gateCase.ThresholdTuningEligible != expectedTuning || !gateCase.QualificationEligible || + gateCase.Rounds < 10 || gateCase.Rounds > 20 || !gateCase.ExactObservationsMatched || gateCase.Overflow || gateCase.FallbackExecuted || + !promotionOrientationRuntimeArm(gateCase.WouldSelectIdentity) || !promotionOrientationRuntimeArm(gateCase.FastestExactIdentity) || + !promotionOrientationRuntimeArm(gateCase.GuardedRuntimeIdentity) || gateCase.GuardedRuntimeIdentity != gateCase.WouldSelectIdentity || gateCase.GuardedRuntimeBranch != expectedRuntimeBranch || + gateCase.ShadowForwardOverhead.Applicable != (gateCase.WouldSelectIdentity == forward) { + return fmt.Errorf("orientation-v2 confirmation case %s/%s lacks frozen runtime qualification evidence", gateCase.Dataset, gateCase.Name) + } + guardedObserved := expectedIdentity.Candidate + ":" + gateCase.GuardedRuntimeIdentity + if err := validatePromotionOrientationLatencyGate(gateCase.ShadowForwardOverhead.OrientationLatencyGate, gateCase.Rounds, 50, forward, expectedIdentity.Candidate+":shadow", report.ShadowForwardRatioLimit, report.OverheadAbsoluteLimit, true); err != nil { + return fmt.Errorf("orientation-v2 confirmation case %s/%s shadow overhead: %w", gateCase.Dataset, gateCase.Name, err) + } + if err := validatePromotionOrientationLatencyGate(gateCase.GuardedSelectedOverhead, gateCase.Rounds, 50, gateCase.WouldSelectIdentity, guardedObserved, report.GuardedSelectedRatioLimit, report.OverheadAbsoluteLimit, true); err != nil { + return fmt.Errorf("orientation-v2 confirmation case %s/%s selected overhead: %w", gateCase.Dataset, gateCase.Name, err) + } + if err := validatePromotionOrientationLatencyGate(gateCase.GuardedFastestRegret, gateCase.Rounds, 50, gateCase.FastestExactIdentity, guardedObserved, report.GuardedFastestRatioLimit, report.OverheadAbsoluteLimit, false); err != nil { + return fmt.Errorf("orientation-v2 confirmation case %s/%s fastest regret: %w", gateCase.Dataset, gateCase.Name, err) + } + expectedPassed := (!gateCase.ShadowForwardOverhead.Applicable || gateCase.ShadowForwardOverhead.Passed) && + gateCase.GuardedSelectedOverhead.Passed && gateCase.GuardedFastestRegret.Passed + if !expectedPassed || gateCase.Passed != expectedPassed || len(gateCase.Reasons) != 0 { + return fmt.Errorf("orientation-v2 confirmation case %s/%s has contradictory passing disposition", gateCase.Dataset, gateCase.Name) + } + if gateCase.QualificationSplit == "training" { + trainingCases++ + } else { + holdoutCases++ + } + } + if trainingCases != report.TrainingCases || holdoutCases != report.HoldoutCases { + return fmt.Errorf("orientation-v2 confirmation split counts contradict its cases") + } + return nil +} + +func validatePromotionOrientationLatencyGate(gate OrientationLatencyGate, rounds, minimumSamplesPerRound int, baselineIdentity, observedIdentity string, ratioLimit float64, absoluteFloor time.Duration, exactAbsoluteFloor bool) error { + if gate.BaselineIdentity != baselineIdentity || gate.ObservedIdentity != observedIdentity || gate.RatioUpperLimit != ratioLimit || + gate.BaselineSamples < rounds*minimumSamplesPerRound || gate.ObservedSamples < rounds*minimumSamplesPerRound || + !validRatioInterval(gate.Ratio) || !validDurationInterval(gate.AbsoluteChange) || gate.RatioUpperLimit <= 0 || gate.AbsoluteFloor < absoluteFloor || exactAbsoluteFloor && gate.AbsoluteFloor != absoluteFloor { + return fmt.Errorf("incomplete statistical evidence") + } + expectedGap := max(time.Duration(0), gate.AbsoluteChange.Upper) + expectedPassed := gate.Ratio.Upper <= gate.RatioUpperLimit || gate.AbsoluteGapUpper <= gate.AbsoluteFloor + if gate.AbsoluteGapUpper != expectedGap || gate.Passed != expectedPassed { + return fmt.Errorf("derived evidence contradicts the reported gate decision") + } + return nil +} + +func promotionOrientationRuntimeArm(identity string) bool { + return identity == string(optimize.ExpansionSearchStepwiseForward) || identity == string(optimize.ExpansionSearchSuffixSeededReverse) +} + +func validatePromotionConfirmationMetric(metric ConfirmationMetric) error { + if !validRatioInterval(metric.Ratio) || !validDurationInterval(metric.AbsoluteChange) || + !validPromotionNoiseFloor(metric.NoiseRatio, metric.NoiseAbsolute) || strings.TrimSpace(metric.Classification) == "" { + return fmt.Errorf("invalid statistical evidence") + } + if expected := classifyConfirmationMetric(metric.Ratio, metric.AbsoluteChange, metric.NoiseRatio, metric.NoiseAbsolute).Classification; metric.Classification != expected { + return fmt.Errorf("classification contradicts ratio, change, and noise floors") + } + return nil +} + +func validatePromotionPerformanceReport(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionPerfGateReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("performance report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("performance report promotion identity does not match manifest") + } + report := bound.PerfGateReport + if report.Version != perfGateVersion { + return fmt.Errorf("performance report version must be %d", perfGateVersion) + } + if report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.RegressionThreshold != minimumTimingNoiseRatio || + math.IsNaN(report.Confidence) || math.IsInf(report.Confidence, 0) || + math.IsNaN(report.RegressionThreshold) || math.IsInf(report.RegressionThreshold, 0) || + !lowercaseSHA256(report.BaselineSHA256) || !lowercaseSHA256(report.CandidateSHA256) || !lowercaseSHA256(report.AAReportSHA256) || !lowercaseSHA256(report.DeclarationSHA256) { + return fmt.Errorf("performance report lacks immutable artifacts and frozen settings") + } + if !report.Passed || !report.PromotionEligible || !report.MaterialityRequired || !report.MaterialityPassed || report.MaterialityTargets <= 0 || + !report.QualificationRequired || !report.TrainingPassed || !report.HoldoutPassed || !report.QualificationPassed || + report.TrainingCases <= 0 || report.HoldoutCases <= 0 || len(report.Cases) == 0 { + return fmt.Errorf("performance report is not complete promotion-eligible evidence") + } + if err := validatePromotionQualificationFamilies(report.QualificationFamilies, expectedIdentity.Candidate, report.TrainingCases, report.HoldoutCases); err != nil { + return fmt.Errorf("performance report: %w", err) + } + seenCases := map[string]struct{}{} + seenInvocations := map[string]struct{}{} + trainingCases, holdoutCases, materialityTargets := 0, 0, 0 + for _, gateCase := range report.Cases { + key := fmt.Sprintf("%s\x00%s\x00%s", gateCase.Dataset, gateCase.Name, gateCase.Backend) + if strings.TrimSpace(gateCase.Dataset) == "" || strings.TrimSpace(gateCase.Name) == "" || strings.TrimSpace(gateCase.Tier) == "" { + return fmt.Errorf("performance report contains an incomplete case identity") + } + if _, duplicate := seenCases[key]; duplicate { + return fmt.Errorf("performance report duplicates case %s/%s", gateCase.Dataset, gateCase.Name) + } + seenCases[key] = struct{}{} + if !gateCase.Passed { + return fmt.Errorf("performance report passing disposition contradicts case %s/%s", gateCase.Dataset, gateCase.Name) + } + if !gateCase.TimingGated { + continue + } + if gateCase.Backend != ModePostgresSQL || gateCase.OracleOnly || (gateCase.QualificationSplit != "training" && gateCase.QualificationSplit != "holdout") || + gateCase.BaselineStatus != string(StatusOK) || gateCase.CandidateStatus != string(StatusOK) || len(gateCase.Reasons) != 0 || + gateCase.Rounds < minimumGateRounds || gateCase.BaselineSamples < minimumP95Samples || gateCase.CandidateSamples < minimumP95Samples || + !validRatioInterval(gateCase.MedianRatio) || gateCase.P95Ratio == nil || !validRatioInterval(*gateCase.P95Ratio) || + gateCase.MedianSaving == nil || !validDurationInterval(*gateCase.MedianSaving) || gateCase.MedianChange == nil || !validDurationInterval(*gateCase.MedianChange) || + gateCase.P95Change == nil || !validDurationInterval(*gateCase.P95Change) { + return fmt.Errorf("performance case %s/%s lacks complete passing timing evidence", gateCase.Dataset, gateCase.Name) + } + if gateCase.MedianChange.Estimate != -gateCase.MedianSaving.Estimate || gateCase.MedianChange.Lower != -gateCase.MedianSaving.Upper || gateCase.MedianChange.Upper != -gateCase.MedianSaving.Lower { + return fmt.Errorf("performance case %s/%s has contradictory median change and saving", gateCase.Dataset, gateCase.Name) + } + if !validPromotionNoiseFloor(gateCase.P50NoiseRatio, gateCase.P50NoiseAbsolute) || + !validPromotionNoiseFloor(gateCase.P95NoiseRatio, gateCase.P95NoiseAbsolute) { + return fmt.Errorf("performance case %s/%s changes or omits the minimum finite noise floors", gateCase.Dataset, gateCase.Name) + } + if gateCase.MedianRatio.Lower > 1+gateCase.P50NoiseRatio && gateCase.MedianChange.Lower > gateCase.P50NoiseAbsolute { + return fmt.Errorf("performance case %s/%s contains a noise-adjusted p50 regression", gateCase.Dataset, gateCase.Name) + } + if gateCase.P95Ratio.Lower > 1+gateCase.P95NoiseRatio && gateCase.P95Change.Lower > gateCase.P95NoiseAbsolute { + return fmt.Errorf("performance case %s/%s contains a noise-adjusted p95 regression", gateCase.Dataset, gateCase.Name) + } + if len(gateCase.CandidateRuntimeReceiptChains) != gateCase.CandidateSamples { + return fmt.Errorf("performance case %s/%s runtime receipt count differs from candidate samples", gateCase.Dataset, gateCase.Name) + } + if err := validatePromotionReceiptChains(gateCase.CandidateRuntimeReceiptChains, expectedIdentity.Candidate, seenInvocations); err != nil { + return fmt.Errorf("performance case %s/%s: %w", gateCase.Dataset, gateCase.Name, err) + } + if gateCase.MaterialityRatio != nil || gateCase.MaterialityAbsolute != nil { + expectedRatio := min(0.95, 1-gateCase.P50NoiseRatio) + expectedAbsolute := max(100*time.Microsecond, gateCase.P50NoiseAbsolute) + if gateCase.MaterialityRatio == nil || gateCase.MaterialityAbsolute == nil || *gateCase.MaterialityRatio <= 0 || + *gateCase.MaterialityRatio != expectedRatio || *gateCase.MaterialityAbsolute != expectedAbsolute || + gateCase.MedianRatio.Upper > *gateCase.MaterialityRatio && gateCase.MedianSaving.Lower < *gateCase.MaterialityAbsolute { + return fmt.Errorf("performance case %s/%s has contradictory materiality evidence", gateCase.Dataset, gateCase.Name) + } + materialityTargets++ + } + if gateCase.QualificationSplit == "training" { + trainingCases++ + } else { + holdoutCases++ + } + } + if trainingCases != report.TrainingCases || holdoutCases != report.HoldoutCases || materialityTargets != report.MaterialityTargets { + return fmt.Errorf("performance report aggregate counts contradict its cases") + } + return nil +} + +// validatePromotionEvidenceClosure verifies relationships that no report can +// prove in isolation. In particular, the performance and candidate-specific +// confirmation reports must use the exact native A/A document that was wrapped +// into the manifest, and fixed-cohort candidates must share one declaration. +func validatePromotionEvidenceClosure(base string, evidence map[string]PromotionEvidenceReference, identity PromotionEvidenceIdentity) error { + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure"} { + if _, found := evidence[role]; !found { + return nil // The ordinary required-role check reports this more directly. + } + } + read := func(role string) ([]byte, error) { + reference := evidence[role] + raw, err := readContainedPromotionEvidence(base, reference.Path) + if err != nil { + return nil, fmt.Errorf("read %s report: %w", role, err) + } + digest := sha256.Sum256(raw) + if hex.EncodeToString(digest[:]) != reference.SHA256 { + return nil, fmt.Errorf("%s report changed while evidence closure was being verified", role) + } + return raw, nil + } + aaRaw, err := read("aa") + if err != nil { + return err + } + performanceRaw, err := read("performance") + if err != nil { + return err + } + confirmationRaw, err := read("confirmation") + if err != nil { + return err + } + resourceRaw, err := read("resource") + if err != nil { + return err + } + referenceRaw, err := read("reference_closure") + if err != nil { + return err + } + var aa promotionAAResolutionReport + if err := decodePromotionEvidence(aaRaw, &aa); err != nil { + return fmt.Errorf("decode A/A closure: %w", err) + } + var performance promotionPerfGateReport + if err := decodePromotionEvidence(performanceRaw, &performance); err != nil { + return fmt.Errorf("decode performance closure: %w", err) + } + if performance.AAReportSHA256 != aa.NativeReportSHA256 { + return fmt.Errorf("performance report does not use the manifest's exact native A/A report") + } + var resource promotionResourceReport + if err := decodePromotionEvidence(resourceRaw, &resource); err != nil { + return fmt.Errorf("decode resource closure: %w", err) + } + var reference promotionReferenceClosureReport + if err := decodePromotionEvidence(referenceRaw, &reference); err != nil { + return fmt.Errorf("decode reference closure: %w", err) + } + + expectedDeclaration := "" + expectedCandidateArtifact := "" + expectedCases := map[promotionCohortCase]struct{}{} + switch identity.Candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + var confirmation promotionSPI1QualificationReport + if err := decodePromotionEvidence(confirmationRaw, &confirmation); err != nil { + return fmt.Errorf("decode SP-I1 confirmation closure: %w", err) + } + cohort, cohortErr := canonicalSPI1Cohort() + if cohortErr != nil { + return cohortErr + } + expectedDeclaration = cohort.declarationSHA256 + for _, gateCase := range spI1CanonicalCases { + expectedCases[promotionCohortCase{dataset: gateCase.dataset, name: gateCase.name, split: gateCase.split}] = struct{}{} + } + expectedCandidateArtifact = confirmation.CandidateArtifactSHA256 + if confirmation.ResourceReportSHA256 != resource.NativeReportSHA256 { + return fmt.Errorf("SP-I1 confirmation does not use the manifest's exact native resource report") + } + case string(optimize.ShortestPathExecutorI2GuardedDistance): + var confirmation promotionSPI2QualificationReport + if err := decodePromotionEvidence(confirmationRaw, &confirmation); err != nil { + return fmt.Errorf("decode SP-I2 confirmation closure: %w", err) + } + cohort, cohortErr := canonicalSPI2Cohort() + if cohortErr != nil { + return cohortErr + } + expectedDeclaration = cohort.declarationSHA256 + for _, gateCase := range spI2CanonicalCases { + expectedCases[promotionCohortCase{dataset: gateCase.dataset, name: gateCase.name, split: gateCase.split}] = struct{}{} + } + expectedCandidateArtifact = confirmation.CandidateArtifactSHA256 + if confirmation.ResourceReportSHA256 != resource.NativeReportSHA256 { + return fmt.Errorf("SP-I2 confirmation does not use the manifest's exact native resource report") + } + case string(optimize.ShortestPathExecutorASPI1DAG): + var confirmation promotionConfirmationReport + if err := decodePromotionEvidence(confirmationRaw, &confirmation); err != nil { + return fmt.Errorf("decode confirmation closure: %w", err) + } + if confirmation.AAReportSHA256 != aa.NativeReportSHA256 { + return fmt.Errorf("confirmation report does not use the manifest's exact native A/A report") + } + expectedCases = genericPromotionConfirmationCohort(confirmation.ConfirmationReport) + expectedCandidateArtifact = confirmation.RightSHA256 + case string(optimize.ExpansionSearchPolicyOrientationProbeV1): + var confirmation promotionOrientationSelectorReport + if err := decodePromotionEvidence(confirmationRaw, &confirmation); err != nil { + return fmt.Errorf("decode orientation confirmation closure: %w", err) + } + if confirmation.AAReportSHA256 != aa.NativeReportSHA256 { + return fmt.Errorf("orientation confirmation does not use the manifest's exact native A/A report") + } + for _, gateCase := range confirmation.Cases { + expectedCases[promotionCohortCase{dataset: gateCase.Dataset, name: gateCase.Name, split: gateCase.QualificationSplit}] = struct{}{} + } + case string(optimize.ExpansionSearchPolicyOrientationProbeV2): + var confirmation promotionOrientationSelectorV2Report + if err := decodePromotionEvidence(confirmationRaw, &confirmation); err != nil { + return fmt.Errorf("decode orientation-v2 confirmation closure: %w", err) + } + if confirmation.AAReportSHA256 != aa.NativeReportSHA256 { + return fmt.Errorf("orientation-v2 confirmation does not use the manifest's exact native A/A report") + } + expectedDeclaration = confirmation.CohortDeclarationSHA256 + expectedCandidateArtifact = confirmation.GuardedArtifactSHA256 + for _, gateCase := range confirmation.Cases { + expectedCases[promotionCohortCase{dataset: gateCase.Dataset, name: gateCase.Name, split: gateCase.QualificationSplit}] = struct{}{} + } + } + if !lowercaseSHA256(expectedCandidateArtifact) || performance.CandidateSHA256 != expectedCandidateArtifact || + resource.ArtifactSHA256 != expectedCandidateArtifact { + return fmt.Errorf("confirmation, performance, and resource reports do not bind the same exact candidate artifact") + } + if expectedDeclaration != "" && performance.DeclarationSHA256 != expectedDeclaration { + return fmt.Errorf("performance and confirmation reports do not bind the same frozen cohort declaration") + } + performanceCases := map[promotionCohortCase]struct{}{} + performanceCasesByKey := map[promotionCohortCase][]PerfGateCase{} + for _, gateCase := range performance.Cases { + if gateCase.TimingGated { + key := promotionCohortCase{dataset: gateCase.Dataset, name: gateCase.Name, split: gateCase.QualificationSplit} + performanceCases[key] = struct{}{} + performanceCasesByKey[key] = append(performanceCasesByKey[key], gateCase) + } + } + if !reflect.DeepEqual(performanceCases, expectedCases) { + return fmt.Errorf("performance and confirmation reports do not contain the same exact promotion cohort") + } + resourceCases := map[promotionCohortCase]struct{}{} + resourceCasesByKey := map[promotionCohortCase][]ResourceGateCase{} + for _, gateCase := range resource.Cases { + key := promotionCohortCase{dataset: gateCase.Dataset, name: gateCase.Name, split: gateCase.QualificationSplit} + resourceCases[key] = struct{}{} + resourceCasesByKey[key] = append(resourceCasesByKey[key], gateCase) + } + if !reflect.DeepEqual(resourceCases, expectedCases) { + return fmt.Errorf("resource and confirmation reports do not contain the same exact promotion cohort") + } + referenceCases := map[promotionCohortCase]struct{}{} + referenceCasesByKey := map[promotionCohortCase][]ReferenceClosureCase{} + for _, gateCase := range reference.Cases { + key := promotionCohortCase{dataset: gateCase.Dataset, name: gateCase.Name, split: gateCase.QualificationSplit} + referenceCases[key] = struct{}{} + referenceCasesByKey[key] = append(referenceCasesByKey[key], gateCase) + } + if !reflect.DeepEqual(referenceCases, expectedCases) { + return fmt.Errorf("reference-closure and confirmation reports do not contain the same exact promotion cohort") + } + + // Resource evidence is produced once per round, while performance evidence + // aggregates the same candidate invocations. Set equality at the case level + // is insufficient: omitted rounds or a substituted receipt subset would + // otherwise retain the same dataset/name/split keys. + for key := range expectedCases { + performanceForCase := performanceCasesByKey[key] + if len(performanceForCase) != 1 { + return fmt.Errorf("performance report must contain exactly one timing-gated case for %s/%s (%s)", key.dataset, key.name, key.split) + } + performanceCase := performanceForCase[0] + resourcesForCase := resourceCasesByKey[key] + if len(resourcesForCase) != performanceCase.Rounds { + return fmt.Errorf("resource report must contain exactly %d rounds for %s/%s (%s)", performanceCase.Rounds, key.dataset, key.name, key.split) + } + rounds := make(map[int]struct{}, performanceCase.Rounds) + resourceReceipts := make([][]RuntimeReceiptEvent, 0, performanceCase.CandidateSamples) + for _, resourceCase := range resourcesForCase { + if resourceCase.Round < 1 || resourceCase.Round > performanceCase.Rounds { + return fmt.Errorf("resource report round %d is outside 1..%d for %s/%s (%s)", resourceCase.Round, performanceCase.Rounds, key.dataset, key.name, key.split) + } + if _, duplicate := rounds[resourceCase.Round]; duplicate { + return fmt.Errorf("resource report duplicates round %d for %s/%s (%s)", resourceCase.Round, key.dataset, key.name, key.split) + } + rounds[resourceCase.Round] = struct{}{} + resourceReceipts = append(resourceReceipts, resourceCase.RuntimeReceiptChains...) + } + performanceReceiptSet, setErr := promotionReceiptChainSet(performanceCase.CandidateRuntimeReceiptChains) + if setErr != nil { + return fmt.Errorf("performance report receipts for %s/%s (%s): %w", key.dataset, key.name, key.split, setErr) + } + resourceReceiptSet, setErr := promotionReceiptChainSet(resourceReceipts) + if setErr != nil { + return fmt.Errorf("resource report receipts for %s/%s (%s): %w", key.dataset, key.name, key.split, setErr) + } + if !reflect.DeepEqual(resourceReceiptSet, performanceReceiptSet) { + return fmt.Errorf("resource and performance reports do not bind the same exact candidate receipt chains for %s/%s (%s)", key.dataset, key.name, key.split) + } + } + + // Reference closure and PostgreSQL A/A must name the same logical workload, + // not merely cases with matching human-readable labels. Extra A/A cases are + // harmless, but every promotion cohort workload must resolve exactly once. + aaCasesByKey := map[promotionWorkloadCase][]AAResolutionCase{} + for _, aaCase := range aa.Cases { + if aaCase.Backend == ModePostgresSQL { + key := promotionWorkloadCase{dataset: aaCase.Dataset, name: aaCase.Name} + aaCasesByKey[key] = append(aaCasesByKey[key], aaCase) + } + } + for key := range expectedCases { + referenceForCase := referenceCasesByKey[key] + if len(referenceForCase) != 1 { + return fmt.Errorf("reference-closure report must contain exactly one workload identity for %s/%s (%s)", key.dataset, key.name, key.split) + } + workloadKey := promotionWorkloadCase{dataset: key.dataset, name: key.name} + aaForCase := aaCasesByKey[workloadKey] + if len(aaForCase) != 1 { + return fmt.Errorf("native A/A report must contain exactly one PostgreSQL workload identity for %s/%s", key.dataset, key.name) + } + if referenceForCase[0].WorkloadSHA256 != aaForCase[0].WorkloadSHA256 { + return fmt.Errorf("reference-closure workload identity differs from native A/A for %s/%s", key.dataset, key.name) + } + } + return nil +} + +type promotionCohortCase struct { + dataset string + name string + split string +} + +type promotionWorkloadCase struct { + dataset string + name string +} + +func promotionReceiptChainSet(chains [][]RuntimeReceiptEvent) (map[string]struct{}, error) { + set := make(map[string]struct{}, len(chains)) + for _, chain := range chains { + raw, err := json.Marshal(chain) + if err != nil { + return nil, err + } + key := string(raw) + if _, duplicate := set[key]; duplicate { + return nil, fmt.Errorf("contains a duplicate candidate receipt chain") + } + set[key] = struct{}{} + } + return set, nil +} + +func genericPromotionConfirmationCohort(report ConfirmationReport) map[promotionCohortCase]struct{} { + cohort := map[promotionCohortCase]struct{}{} + for _, gateCase := range report.Cases { + if gateCase.TimingGated { + cohort[promotionCohortCase{dataset: gateCase.Dataset, name: gateCase.Name, split: gateCase.QualificationSplit}] = struct{}{} + } + } + return cohort +} + +func validatePromotionQualificationFamilies(statuses []TraversalQualificationStatus, candidate string, trainingCases, holdoutCases int) error { + if len(statuses) == 0 { + return fmt.Errorf("qualification family evidence is missing") + } + seen := map[string]struct{}{} + totalTraining, totalHoldout := 0, 0 + candidateFound := false + for _, status := range statuses { + if strings.TrimSpace(status.Family) == "" || status.TrainingCases <= 0 || status.HoldoutCases <= 0 || + !status.TrainingPassed || !status.HoldoutPassed || !status.Passed { + return fmt.Errorf("qualification family %q is incomplete or failing", status.Family) + } + if _, duplicate := seen[status.Family]; duplicate { + return fmt.Errorf("qualification family %q is duplicated", status.Family) + } + seen[status.Family] = struct{}{} + candidateFound = candidateFound || promotionFamilyMatches(status.Family, candidate) + totalTraining += status.TrainingCases + totalHoldout += status.HoldoutCases + } + if !candidateFound { + return fmt.Errorf("qualification families do not identify candidate %q", candidate) + } + if totalTraining != trainingCases || totalHoldout != holdoutCases { + return fmt.Errorf("qualification family counts contradict report aggregates") + } + return nil +} + +func promotionFamilyMatches(family, candidate string) bool { + return family == candidate || strings.HasPrefix(family, candidate+"@") +} + +func validatePromotionReceiptChains(chains [][]RuntimeReceiptEvent, candidate string, seenInvocations map[string]struct{}) error { + if len(chains) == 0 { + return fmt.Errorf("candidate runtime receipt chains are missing") + } + for _, chain := range chains { + if len(chain) == 0 { + return fmt.Errorf("candidate runtime receipt chain is empty") + } + invocationID := strings.TrimSpace(chain[0].InvocationID) + if invocationID == "" { + return fmt.Errorf("candidate runtime receipt invocation is missing") + } + if _, duplicate := seenInvocations[invocationID]; duplicate { + return fmt.Errorf("candidate runtime receipt invocation %q is reused", invocationID) + } + seenInvocations[invocationID] = struct{}{} + for index, event := range chain { + if event.Ordinal != index+1 || event.InvocationID != invocationID || strings.TrimSpace(event.RuntimeIdentity) == "" || strings.TrimSpace(event.RuntimeBranch) == "" || event.FallbackExecuted { + return fmt.Errorf("candidate runtime receipt chain is non-canonical") + } + } + if !promotionReceiptTerminalAllowed(candidate, chain[len(chain)-1].RuntimeIdentity) { + return fmt.Errorf("candidate runtime receipt terminal identity differs from promotion candidate") + } + for _, event := range chain { + if !promotionReceiptBranchAllowed(candidate, event.RuntimeIdentity, event.RuntimeBranch) { + return fmt.Errorf("candidate runtime receipt branch %q is not authorized for %s", event.RuntimeBranch, candidate) + } + } + } + return nil +} + +// promotionReceiptBranchAllowed freezes the successful non-fallback runtime +// tuples emitted by production candidates. Nested receipt chains remain +// supported, but every transition must itself be a recognized candidate arm. +func promotionReceiptBranchAllowed(candidate, runtimeIdentity, runtimeBranch string) bool { + switch candidate { + case string(optimize.ShortestPathExecutorASPI1DAG): + return runtimeIdentity == candidate && (runtimeBranch == "inline_predecessor_dag" || runtimeBranch == "inline_no_path") + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return runtimeIdentity == candidate && (runtimeBranch == "inline_canonical_witness" || runtimeBranch == "inline_canonical_no_path") + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return runtimeIdentity == candidate && (runtimeBranch == "inline_canonical_distance" || runtimeBranch == "inline_canonical_distance_no_path") + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return runtimeIdentity == string(optimize.ExpansionSearchSuffixSeededReverse) && runtimeBranch == "suffix_seeded_reverse" || + runtimeIdentity == string(optimize.ExpansionSearchStepwiseForward) && runtimeBranch == "exact_forward_incumbent" + default: + return false + } +} + +func promotionReceiptTerminalAllowed(candidate, runtimeIdentity string) bool { + switch candidate { + case string(optimize.ShortestPathExecutorASPI1DAG), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorI2GuardedDistance): + return runtimeIdentity == candidate + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), + string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return runtimeIdentity == string(optimize.ExpansionSearchSuffixSeededReverse) || + runtimeIdentity == string(optimize.ExpansionSearchStepwiseForward) + default: + return false + } +} + +func validPromotionNoiseFloor(ratio float64, absolute time.Duration) bool { + return !math.IsNaN(ratio) && !math.IsInf(ratio, 0) && ratio >= minimumTimingNoiseRatio && absolute >= minimumTimingNoiseAbsolute +} + +func promotionIdentityQueryCount(identity PromotionEvidenceIdentity, querySHA256 string) int { + count := 0 + for _, bucket := range identity.Buckets { + for _, query := range bucket.QuerySHA256 { + if query == querySHA256 { + count++ + } + } + } + return count +} + +func promotionCaseKey(dataset, name string) string { + return dataset + "\x00" + name +} + +func exactPromotionCaps(actual, expected map[string]int64) bool { + return reflect.DeepEqual(actual, expected) +} diff --git a/cmd/graphbench/promotion_manifest.go b/cmd/graphbench/promotion_manifest.go new file mode 100644 index 00000000..e6271fb9 --- /dev/null +++ b/cmd/graphbench/promotion_manifest.go @@ -0,0 +1,1081 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "math" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + pgdriver "github.com/specterops/dawgs/drivers/pg" +) + +// promotionManifestVersion reserves the stable protocol value used to recognize promotion manifest version across artifacts and executions. +const promotionManifestVersion = 2 + +// structuralPromotionManifestVersion adds reusable structural bucket bindings +// while retaining v2's exact-query SQL-anchor protocol. +const structuralPromotionManifestVersion = 3 + +const topologyPromotionManifestVersion = 4 + +// topologyFirstUsePromotionManifestVersion freezes the separately qualified +// first-use topology-routing protocol. +const topologyFirstUsePromotionManifestVersion = 5 + +// requiredPromotionEvidenceRoles contains the frozen required promotion evidence roles declaration consulted by package validation. +var requiredPromotionEvidenceRoles = []string{ + "aa", "confirmation", "performance", "resource", "reference_closure", "operational", +} + +// orientationPromotionCaps returns the resource limits enforced for orientation promotion. +func orientationPromotionCaps() map[string]int64 { + return map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + } +} + +// spI2PromotionCaps returns the exact cap contract preregistered for SP-I2 +// evidence, provisional measurement, and any future production admission. +func spI2PromotionCaps() map[string]int64 { + return map[string]int64{ + "state_limit": optimize.ShortestPathI2QualifiedStateLimit, + "frontier_limit": optimize.ShortestPathI2QualifiedFrontierLimit, + } +} + +// validateStaticV6CanonicalInboundBucket validates static v6 canonical inbound bucket. +func validateStaticV6CanonicalInboundBucket(bucket PromotionBucket) error { + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth != 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("SP-I1 canonical witness bucket %s must be the qualified inbound typed single-kind one-path depth 1..64 envelope", bucket.Name) + } + return nil +} + +func validateStaticV8HiddenFanInBucket(bucket PromotionBucket) error { + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationDistance) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("SP-I2 distance bucket %s must be inbound, typed single-kind, distance-only, and depth-bounded", bucket.Name) + } + return nil +} + +// validateTopologyFixedSuffixBucket binds every v4 bucket to the classifier +// and SQL-template protocol the PostgreSQL driver will later enforce. This +// prevents a qualification artifact from authorizing a different shape than +// the live route selector. +func validateTopologyFixedSuffixBucket(manifest PromotionManifest, bucket PromotionBucket) error { + shape := pgdriver.TraversalShape{ + Version: bucket.StructuralShapeVersion, + Family: bucket.StructuralFamily, + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: int64(bucket.MinimumDepth), + MaximumDepth: int64(bucket.MaximumDepth), + SuffixLength: bucket.SuffixLength, + CandidateStrategy: bucket.CandidateStrategy, + Fingerprint: bucket.StructuralShapeSHA256, + } + if shape.Version != pgdriver.TraversalFixedSuffixShapeVersion || shape.Family != "fixed_suffix_expansion" || + shape.Direction != "outbound" || shape.ObservationMode != string(optimize.ExpansionSearchObservationFullPath) || + shape.MinimumDepth != 0 || shape.MaximumDepth != 16 || shape.SuffixLength != 3 || + shape.CandidateStrategy != string(optimize.ExpansionSearchSuffixSeededReverse) || + !isLowerHexSHA256(shape.Fingerprint) || shape.Fingerprint != pgdriver.TraversalShapeFingerprint(shape) { + return fmt.Errorf("topology fixed-suffix bucket %s must match the qualified outbound full-path fixed-suffix classifier envelope", bucket.Name) + } + if !isLowerHexSHA256(bucket.SQLTemplateSHA256) || bucket.SQLTemplateSHA256 != pgdriver.TraversalSQLTemplateSHA256(manifest.Candidate, manifest.SelectorVersion, manifest.ExecutionBoundary, shape) { + return fmt.Errorf("topology fixed-suffix bucket %s must bind the driver's v4 SQL template digest", bucket.Name) + } + return nil +} + +// PromotionEvidenceReference groups state that must remain consistent while processing promotion evidence reference. +type PromotionEvidenceReference struct { + // Path identifies the filesystem path. + Path string `json:"path"` + // SHA256 binds the referenced content by SHA-256 digest. + SHA256 string `json:"sha256"` +} + +// PromotionBucket groups state that must remain consistent while processing promotion bucket. +type PromotionBucket struct { + // Name identifies the name. + Name string `json:"name"` + // QuerySHA256 binds the referenced query content by SHA-256 digest. + QuerySHA256 []string `json:"query_sha256"` + // Direction selects the traversal orientation covered by the contract. + Direction string `json:"direction,omitempty"` + // ObservationMode identifies the observation mode. + ObservationMode string `json:"observation_mode,omitempty"` + // MinimumDepth sets the inclusive lower traversal-depth bound. + MinimumDepth int `json:"minimum_depth,omitempty"` + // MaximumDepth sets the inclusive upper traversal-depth bound. + MaximumDepth int `json:"maximum_depth,omitempty"` + // RelationshipKindCount records the number of relationship kind count. + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + // UntypedRelationship indicates whether untyped relationship applies. + UntypedRelationship bool `json:"untyped_relationship,omitempty"` + // SuffixLength binds the terminal fixed suffix width for a v4 bucket. + SuffixLength int `json:"suffix_length,omitempty"` + // CandidateStrategy binds the fixed-suffix optimizer candidate used to + // derive this v4 bucket. + CandidateStrategy string `json:"candidate_strategy,omitempty"` + // StructuralShapeVersion identifies the shared structural classifier for a + // v3 production-wide bucket. + StructuralShapeVersion string `json:"structural_shape_version,omitempty"` + // StructuralFamily binds the SP or ASP classifier family. + StructuralFamily string `json:"structural_family,omitempty"` + // StructuralShapeSHA256 binds the query-text-free structural identity. + StructuralShapeSHA256 string `json:"structural_shape_sha256,omitempty"` + // SQLTemplateSHA256 binds the reusable candidate SQL template contract. + SQLTemplateSHA256 string `json:"sql_template_sha256,omitempty"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit []string `json:"qualification_split"` +} + +// PromotionManifest is the sole authorization record consumed by a rollout. +// It binds one immutable candidate and selector to source, binary, corpus, +// caps, exact query cohorts, and every required passing report. +type PromotionManifest struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // Policy binds the emitted policy generation independently of executor and selector. + Policy string `json:"policy,omitempty"` + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string `json:"selector_version"` + // ExecutionBoundary supplies the execution boundary input to the PromotionManifest contract. + ExecutionBoundary string `json:"execution_boundary"` + // FallbackExecutor supplies the fallback executor input to the PromotionManifest contract. + FallbackExecutor string `json:"fallback_executor,omitempty"` + // SourceCommit supplies the source commit input to the PromotionManifest contract. + SourceCommit string `json:"source_commit"` + // SourceSHA256 binds the referenced source content by SHA-256 digest. + SourceSHA256 string `json:"source_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds the referenced corpus content by SHA-256 digest. + CorpusSHA256 string `json:"corpus_sha256"` + // OperationalCandidateSQLSHA256 binds the exact rendered SQL emitted for + // the candidate at the operational timing boundary. + OperationalCandidateSQLSHA256 string `json:"operational_candidate_sql_sha256"` + // TopologyEstimatorVersion binds the frozen estimator used by topology + // selected manifest v4 buckets. + TopologyEstimatorVersion string `json:"topology_estimator_version,omitempty"` + // SynopsisSchemaVersion binds the compatible published synopsis schema. + SynopsisSchemaVersion string `json:"synopsis_schema_version,omitempty"` + // RouteCacheProtocol binds the transaction-owned route-decision contract. + RouteCacheProtocol string `json:"route_cache_protocol,omitempty"` + // TopologyThresholds bind the immutable estimator thresholds for manifest v4. + TopologyThresholds map[string]int64 `json:"topology_thresholds,omitempty"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // Buckets supplies the buckets input to the PromotionManifest contract. + Buckets []PromotionBucket `json:"buckets"` + // Evidence supplies the evidence input to the PromotionManifest contract. + Evidence map[string]PromotionEvidenceReference `json:"evidence"` +} + +// PromotionEvidenceIdentity is repeated verbatim by every evidence report. +// It deliberately excludes evidence paths and digests, avoiding a circular +// dependency while binding the report to every authorization-relevant field. +type PromotionEvidenceIdentity struct { + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // Policy binds the emitted policy generation independently of executor and selector. + Policy string `json:"policy,omitempty"` + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string `json:"selector_version"` + // ExecutionBoundary supplies the execution boundary input to the PromotionEvidenceIdentity contract. + ExecutionBoundary string `json:"execution_boundary"` + // FallbackExecutor supplies the fallback executor input to the PromotionEvidenceIdentity contract. + FallbackExecutor string `json:"fallback_executor,omitempty"` + // SourceCommit supplies the source commit input to the PromotionEvidenceIdentity contract. + SourceCommit string `json:"source_commit"` + // SourceSHA256 binds the referenced source content by SHA-256 digest. + SourceSHA256 string `json:"source_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds the referenced corpus content by SHA-256 digest. + CorpusSHA256 string `json:"corpus_sha256"` + // OperationalCandidateSQLSHA256 binds the exact rendered SQL emitted for + // the candidate at the operational timing boundary. + OperationalCandidateSQLSHA256 string `json:"operational_candidate_sql_sha256"` + TopologyEstimatorVersion string `json:"topology_estimator_version,omitempty"` + SynopsisSchemaVersion string `json:"synopsis_schema_version,omitempty"` + RouteCacheProtocol string `json:"route_cache_protocol,omitempty"` + TopologyThresholds map[string]int64 `json:"topology_thresholds,omitempty"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // Buckets supplies the buckets input to the PromotionEvidenceIdentity contract. + Buckets []PromotionBucket `json:"buckets"` +} + +// promotionEvidenceIdentity derives the stable identity used to compare promotion evidence. +func promotionEvidenceIdentity(manifest PromotionManifest) PromotionEvidenceIdentity { + return PromotionEvidenceIdentity{ + Candidate: manifest.Candidate, + Policy: manifest.Policy, + SelectorVersion: manifest.SelectorVersion, + ExecutionBoundary: manifest.ExecutionBoundary, + FallbackExecutor: manifest.FallbackExecutor, + SourceCommit: manifest.SourceCommit, + SourceSHA256: manifest.SourceSHA256, + BinarySHA256: manifest.BinarySHA256, + CorpusSHA256: manifest.CorpusSHA256, + OperationalCandidateSQLSHA256: manifest.OperationalCandidateSQLSHA256, + TopologyEstimatorVersion: manifest.TopologyEstimatorVersion, + SynopsisSchemaVersion: manifest.SynopsisSchemaVersion, + RouteCacheProtocol: manifest.RouteCacheProtocol, + TopologyThresholds: clonePromotionCaps(manifest.TopologyThresholds), + Caps: clonePromotionCaps(manifest.Caps), + Buckets: clonePromotionBuckets(manifest.Buckets), + } +} + +// clonePromotionCaps returns an independent copy of promotion caps. +func clonePromotionCaps(input map[string]int64) map[string]int64 { + if input == nil { + return nil + } + result := make(map[string]int64, len(input)) + for name, value := range input { + result[name] = value + } + return result +} + +// clonePromotionBuckets returns an independent copy of promotion buckets. +func clonePromotionBuckets(input []PromotionBucket) []PromotionBucket { + result := append([]PromotionBucket(nil), input...) + for idx := range result { + result[idx].QuerySHA256 = append([]string(nil), result[idx].QuerySHA256...) + result[idx].QualificationSplit = append([]string(nil), result[idx].QualificationSplit...) + } + return result +} + +// validatePromotionBucketSets enforces the set-valued manifest fields used by +// both final verification and provisional capture. A single SQL anchor is +// meaningful only for one unique query identity, and qualification evidence +// must close the exact training/holdout split rather than a superset. +func validatePromotionBucketSets(version int, buckets []PromotionBucket) []string { + var reasons []string + seenBuckets := map[string]struct{}{} + seenQueries := map[string]string{} + + for _, bucket := range buckets { + if strings.TrimSpace(bucket.Name) == "" || len(bucket.QuerySHA256) == 0 { + reasons = append(reasons, "every bucket requires a name and query allowlist") + continue + } + if _, found := seenBuckets[bucket.Name]; found { + reasons = append(reasons, "bucket "+bucket.Name+" is duplicated") + } + seenBuckets[bucket.Name] = struct{}{} + + seenBucketQueries := map[string]struct{}{} + for _, query := range bucket.QuerySHA256 { + if !isLowerHexSHA256(query) { + reasons = append(reasons, "bucket "+bucket.Name+" contains an invalid query digest") + continue + } + if _, duplicate := seenBucketQueries[query]; duplicate { + reasons = append(reasons, "bucket "+bucket.Name+" duplicates query digest "+query) + } + seenBucketQueries[query] = struct{}{} + if owner, duplicate := seenQueries[query]; duplicate && owner != bucket.Name { + reasons = append(reasons, "query digest "+query+" is authorized by more than one bucket") + } else { + seenQueries[query] = bucket.Name + } + } + + if !reflect.DeepEqual(bucket.QualificationSplit, []string{"training", "holdout"}) { + reasons = append(reasons, "bucket "+bucket.Name+" must bind exactly one training and one holdout qualification split in canonical order") + } + if (version == structuralPromotionManifestVersion || version == topologyPromotionManifestVersion || version == topologyFirstUsePromotionManifestVersion) && (bucket.StructuralShapeVersion == "" || bucket.StructuralFamily == "" || !isLowerHexSHA256(bucket.StructuralShapeSHA256) || !isLowerHexSHA256(bucket.SQLTemplateSHA256)) { + reasons = append(reasons, "structural bucket "+bucket.Name+" requires classifier version, family, shape digest, and SQL template digest") + } + } + if version == promotionManifestVersion && len(seenQueries) != 1 { + reasons = append(reasons, "operational SQL anchor requires exactly one authorized query digest") + } + if (version == structuralPromotionManifestVersion || version == topologyPromotionManifestVersion || version == topologyFirstUsePromotionManifestVersion) && len(seenQueries) == 0 { + reasons = append(reasons, "structural promotion requires at least one evidence query digest") + } + return reasons +} + +// validatePromotionEvidenceRoleSet rejects missing and invented evidence +// roles. JSON object keys are unique after strict duplicate-key validation, so +// exact cardinality here closes the role set rather than checking a subset. +func validatePromotionEvidenceRoleSet(evidence map[string]PromotionEvidenceReference) []string { + var reasons []string + required := make(map[string]struct{}, len(requiredPromotionEvidenceRoles)) + for _, role := range requiredPromotionEvidenceRoles { + required[role] = struct{}{} + if _, found := evidence[role]; !found { + reasons = append(reasons, "required evidence role "+role+" is missing") + } + } + for role := range evidence { + if _, found := required[role]; !found { + reasons = append(reasons, "unsupported evidence role "+role+" is present") + } + } + return reasons +} + +// PromotionManifestVerification groups state that must remain consistent while processing promotion manifest verification. +type PromotionManifestVerification struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // ManifestSHA256 binds the referenced manifest content by SHA-256 digest. + ManifestSHA256 string `json:"manifest_sha256"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate,omitempty"` + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string `json:"selector_version,omitempty"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` +} + +// verifyPromotionManifest verifies promotion manifest. +func verifyPromotionManifest(path string) (PromotionManifestVerification, error) { + raw, err := os.ReadFile(path) + if err != nil { + return PromotionManifestVerification{}, err + } + digest := sha256.Sum256(raw) + verification := PromotionManifestVerification{ + Version: promotionManifestVersion, + ManifestSHA256: hex.EncodeToString(digest[:]), + Passed: true, + } + var manifest PromotionManifest + if err := decodePromotionEvidence(raw, &manifest); err != nil { + return PromotionManifestVerification{}, fmt.Errorf("decode promotion manifest: %w", err) + } + verification.Candidate = manifest.Candidate + verification.SelectorVersion = manifest.SelectorVersion + addReason := func(reason string) { + verification.Passed = false + verification.Reasons = append(verification.Reasons, reason) + } + if manifest.Version != promotionManifestVersion && manifest.Version != structuralPromotionManifestVersion && manifest.Version != topologyPromotionManifestVersion && manifest.Version != topologyFirstUsePromotionManifestVersion { + addReason("manifest version must be 2, 3, 4, or 5") + } + if strings.TrimSpace(manifest.Candidate) == "" || strings.TrimSpace(manifest.SelectorVersion) == "" { + addReason("candidate and selector_version are required") + } + if manifest.ExecutionBoundary != "inline_statement" && manifest.ExecutionBoundary != "stored_helper" && manifest.ExecutionBoundary != "guarded_dual_arm" && manifest.ExecutionBoundary != "transaction_retry" && manifest.ExecutionBoundary != "first_use_transaction_retry" { + addReason("execution_boundary must identify the measured production boundary") + } + for name, value := range map[string]string{"source_sha256": manifest.SourceSHA256, "binary_sha256": manifest.BinarySHA256, "corpus_sha256": manifest.CorpusSHA256} { + if !isLowerHexSHA256(value) { + addReason(name + " must be a lowercase SHA-256 digest") + } + } + if !isLowerHexSHA256(manifest.OperationalCandidateSQLSHA256) { + addReason("operational_candidate_sql_sha256 must be a lowercase SHA-256 digest") + } + if strings.TrimSpace(manifest.SourceCommit) == "" { + addReason("source_commit is required") + } + if len(manifest.Caps) == 0 { + addReason("at least one immutable candidate cap is required") + } + for name, limit := range manifest.Caps { + if strings.TrimSpace(name) == "" || limit <= 0 { + addReason("candidate caps must have nonempty names and positive limits") + } + } + if manifest.Candidate == "ASP-I1-U-DAG+MAT-M0" { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("ASP-I1 requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != "ASP-A1-DAG" { + addReason("ASP-I1 requires ASP-A1-DAG as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason("ASP-I1 requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + addReason("ASP-I1 cap " + name + " must be positive") + } + } + } + if manifest.Candidate == "SP-I1-C-WE+MAT-M0" { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("SP-I1 canonical witness requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != "SP-S4-C-WE+MAT-M0" { + addReason("SP-I1 canonical witness requires SP-S4-C-WE+MAT-M0 as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason("SP-I1 canonical witness requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + addReason("SP-I1 canonical witness cap " + name + " must be positive") + } + } + if manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + addReason("SP-I1 canonical witness requires selector " + optimize.ShortestPathSelectorStaticV6) + } + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) { + expectedCaps := spI2PromotionCaps() + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason("SP-I2 distance requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != string(optimize.ShortestPathExecutorS4CanonicalDistance) { + addReason("SP-I2 distance requires SP-S4-C-D as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason("SP-I2 distance requires exactly state and frontier caps") + } + for name, expected := range expectedCaps { + if actual, found := manifest.Caps[name]; !found || actual != expected { + addReason(fmt.Sprintf("SP-I2 distance cap %s must equal %d", name, expected)) + } + } + if manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV8HiddenFanIn { + addReason("SP-I2 distance requires selector " + optimize.ShortestPathSelectorStaticV8HiddenFanIn) + } + } + if isOrientationProbePolicy(manifest.Candidate) { + expectedCaps := orientationPromotionCaps() + if manifest.SelectorVersion != manifest.Candidate { + addReason(fmt.Sprintf("%s requires the same selector version", manifest.Candidate)) + } + if manifest.ExecutionBoundary != "guarded_dual_arm" { + addReason(manifest.Candidate + " requires the guarded_dual_arm production boundary") + } + if manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) { + addReason(manifest.Candidate + " requires EXPANSION-STEPWISE-FORWARD as its exact fallback") + } + if len(manifest.Caps) != len(expectedCaps) { + addReason(manifest.Candidate + " requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps") + } + for name, expected := range expectedCaps { + if manifest.Caps[name] != expected { + addReason(fmt.Sprintf("%s cap %s must equal %d", manifest.Candidate, name, expected)) + } + } + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyOrientationProbeV2) { + addReason("orientation-probe-v2 is terminally rejected because its immutable training overhead gate failed; authorization requires a new policy generation") + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) { + expectedCaps := map[string]int64{ + "suffix_row_limit": optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, + "state_limit": optimize.ExpansionSearchSuffixReverseGuardStateLimit, + "output_row_limit": optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, + "output_bytes_limit": optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit, + } + if manifest.Version != topologyPromotionManifestVersion || manifest.ExecutionBoundary != "transaction_retry" || manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) { + addReason("topology fixed-suffix requires manifest v4, transaction_retry, and EXPANSION-STEPWISE-FORWARD fallback") + } + if !reflect.DeepEqual(manifest.Caps, expectedCaps) { + addReason("topology fixed-suffix requires the exact frozen suffix, state, output-row, and output-byte caps") + } + if manifest.SelectorVersion != string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) || manifest.TopologyEstimatorVersion != "topology-fixed-suffix-counts-v1" || manifest.SynopsisSchemaVersion != "topology-synopsis-schema-v2" || manifest.RouteCacheProtocol != "topology-selected-routing-v1" || !reflect.DeepEqual(manifest.TopologyThresholds, map[string]int64{"maximum_edge_to_node_ratio_per_mille": 1000}) { + addReason("topology fixed-suffix requires its selector, estimator, synopsis schema, and route-cache protocol identities") + } + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) { + expectedCaps := map[string]int64{ + "suffix_row_limit": optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, + "state_limit": optimize.ExpansionSearchSuffixReverseGuardStateLimit, + "output_row_limit": optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, + "output_bytes_limit": optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit, + } + if manifest.Version != topologyFirstUsePromotionManifestVersion || manifest.ExecutionBoundary != "first_use_transaction_retry" || manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) { + addReason("topology fixed-suffix first-use requires manifest v5, first_use_transaction_retry, and EXPANSION-STEPWISE-FORWARD fallback") + } + if !reflect.DeepEqual(manifest.Caps, expectedCaps) { + addReason("topology fixed-suffix first-use requires the exact frozen suffix, state, output-row, and output-byte caps") + } + if manifest.SelectorVersion != string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) || manifest.TopologyEstimatorVersion != "topology-fixed-suffix-counts-v1" || manifest.SynopsisSchemaVersion != "topology-synopsis-schema-v2" || manifest.RouteCacheProtocol != "topology-selected-first-use-routing-v1" || !reflect.DeepEqual(manifest.TopologyThresholds, map[string]int64{"maximum_edge_to_node_ratio_per_mille": 1000}) { + addReason("topology fixed-suffix first-use requires its selector, estimator, synopsis schema, and route-cache protocol identities") + } + } + if len(manifest.Buckets) == 0 { + addReason("at least one authorized bucket is required") + } + for _, reason := range validatePromotionBucketSets(manifest.Version, manifest.Buckets) { + addReason(reason) + } + for _, bucket := range manifest.Buckets { + if manifest.Candidate == "ASP-I1-U-DAG+MAT-M0" { + if (bucket.Direction != "outbound" && bucket.Direction != "inbound") || bucket.ObservationMode != "all_paths" || bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 { + addReason("ASP-I1 bucket " + bucket.Name + " is outside the directed all-paths depth envelope") + } + if bucket.RelationshipKindCount < 0 || bucket.UntypedRelationship != (bucket.RelationshipKindCount == 0) { + addReason("ASP-I1 bucket " + bucket.Name + " has inconsistent relationship-kind metadata") + } + } + if manifest.Candidate == "SP-I1-C-WE+MAT-M0" { + if err := validateStaticV6CanonicalInboundBucket(bucket); err != nil { + addReason(err.Error()) + } + } + if manifest.Candidate == string(optimize.ShortestPathExecutorI2GuardedDistance) { + if err := validateStaticV8HiddenFanInBucket(bucket); err != nil { + addReason(err.Error()) + } + } + if manifest.Candidate == string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) || manifest.Candidate == string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) { + if err := validateTopologyFixedSuffixBucket(manifest, bucket); err != nil { + addReason(err.Error()) + } + } + } + for _, reason := range validatePromotionEvidenceRoleSet(manifest.Evidence) { + addReason(reason) + } + base := filepath.Dir(path) + for _, role := range requiredPromotionEvidenceRoles { + reference, found := manifest.Evidence[role] + if !found { + continue + } + if err := verifyPromotionEvidence(base, role, reference, promotionEvidenceIdentity(manifest)); err != nil { + addReason(role + ": " + err.Error()) + } + } + if err := validatePromotionEvidenceClosure(base, manifest.Evidence, promotionEvidenceIdentity(manifest)); err != nil { + addReason("evidence closure: " + err.Error()) + } + sort.Strings(verification.Reasons) + return verification, nil +} + +// writePromotionManifestVerification writes promotion manifest verification. +func writePromotionManifestVerification(path, output string) (bool, error) { + verification, err := verifyPromotionManifest(path) + if err != nil { + return false, err + } + raw, err := json.MarshalIndent(verification, "", " ") + if err != nil { + return false, err + } + if output == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + } else { + err = os.WriteFile(output, append(raw, '\n'), 0o644) + } + return verification.Passed, err +} + +// verifyPromotionEvidence verifies promotion evidence. +func verifyPromotionEvidence(base, role string, reference PromotionEvidenceReference, expectedIdentity PromotionEvidenceIdentity) error { + raw, err := readContainedPromotionEvidence(base, reference.Path) + if err != nil { + return err + } + digest := sha256.Sum256(raw) + if hex.EncodeToString(digest[:]) != reference.SHA256 { + return fmt.Errorf("SHA-256 mismatch") + } + var document map[string]any + if err := json.Unmarshal(raw, &document); err != nil { + return fmt.Errorf("decode report: %w", err) + } + identityRaw, found := document["promotion_identity"] + if !found { + return fmt.Errorf("report has no promotion_identity") + } + encodedIdentity, err := json.Marshal(identityRaw) + if err != nil { + return fmt.Errorf("encode promotion identity: %w", err) + } + var actualIdentity PromotionEvidenceIdentity + if err := json.Unmarshal(encodedIdentity, &actualIdentity); err != nil { + return fmt.Errorf("decode promotion identity: %w", err) + } + if !reflect.DeepEqual(actualIdentity, expectedIdentity) { + return fmt.Errorf("promotion identity does not match manifest") + } + if err := validatePromotionEvidenceDocument(role, raw, expectedIdentity); err != nil { + return err + } + return nil +} + +// readContainedPromotionEvidence resolves both the manifest directory and the +// referenced evidence through symlinks, then proves the resolved report is +// still beneath that directory. Lexical path cleaning alone does not stop an +// in-tree symlink from redirecting final authorization to an external file. +func readContainedPromotionEvidence(base, referencePath string) ([]byte, error) { + if filepath.IsAbs(referencePath) || referencePath == "" { + return nil, fmt.Errorf("path must be a nonempty relative path") + } + clean := filepath.Clean(referencePath) + if clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("path escapes the manifest directory") + } + resolvedBase, err := filepath.EvalSymlinks(base) + if err != nil { + return nil, fmt.Errorf("resolve manifest directory: %w", err) + } + resolvedPath, err := filepath.EvalSymlinks(filepath.Join(base, clean)) + if err != nil { + return nil, err + } + relative, err := filepath.Rel(resolvedBase, resolvedPath) + if err != nil { + return nil, fmt.Errorf("compare evidence path with manifest directory: %w", err) + } + if relative == ".." || filepath.IsAbs(relative) || strings.HasPrefix(relative, ".."+string(filepath.Separator)) { + return nil, fmt.Errorf("path escapes the manifest directory through a symlink") + } + info, err := os.Stat(resolvedPath) + if err != nil { + return nil, err + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("path must resolve to a regular file") + } + return os.ReadFile(resolvedPath) +} + +// validatePromotionEvidenceDocument strictly decodes the schema associated +// with a required evidence role. The identity and checksum are checked by the +// caller first; these checks prevent a correctly bound but structurally empty +// or internally contradictory JSON object from satisfying promotion. +func validatePromotionEvidenceDocument(role string, raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + switch role { + case "aa": + return validatePromotionAAReport(raw, expectedIdentity) + case "confirmation": + return validatePromotionConfirmationReport(raw, expectedIdentity) + case "performance": + return validatePromotionPerformanceReport(raw, expectedIdentity) + case "resource": + return validatePromotionResourceReport(raw, expectedIdentity) + case "reference_closure": + return validatePromotionReferenceClosureReport(raw, expectedIdentity) + case "operational": + return validatePromotionOperationalReport(raw, expectedIdentity) + default: + return fmt.Errorf("unsupported promotion evidence role %q", role) + } +} + +// decodePromotionEvidence rejects unknown fields and concatenated documents. +// Bound report schemas embed their promotion identity separately because the +// native report producers deliberately avoid a manifest digest cycle. +func decodePromotionEvidence(raw []byte, destination any) error { + if err := rejectDuplicateJSONObjectKeys(raw); err != nil { + return err + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { + return fmt.Errorf("decode report schema: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return fmt.Errorf("report contains trailing JSON data") + } + return fmt.Errorf("decode trailing report data: %w", err) + } + return nil +} + +// rejectDuplicateJSONObjectKeys walks the complete JSON token stream and +// rejects duplicate keys at every nesting level. encoding/json otherwise lets +// a later duplicate silently overwrite an authorization-relevant field. +func rejectDuplicateJSONObjectKeys(raw []byte) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := rejectDuplicateJSONValue(decoder); err != nil { + return fmt.Errorf("decode report schema: %w", err) + } + if token, err := decoder.Token(); err != io.EOF { + if err == nil { + return fmt.Errorf("report contains trailing JSON data after %v", token) + } + return fmt.Errorf("decode trailing report data: %w", err) + } + return nil +} + +func rejectDuplicateJSONValue(decoder *json.Decoder) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, composite := token.(json.Delim) + if !composite { + return nil + } + switch delimiter { + case '{': + seen := map[string]struct{}{} + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + key, ok := keyToken.(string) + if !ok { + return fmt.Errorf("object key is not a string") + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("duplicate JSON object key %q", key) + } + seen[key] = struct{}{} + if err := rejectDuplicateJSONValue(decoder); err != nil { + return err + } + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim('}') { + return fmt.Errorf("object has invalid closing delimiter") + } + case '[': + for decoder.More() { + if err := rejectDuplicateJSONValue(decoder); err != nil { + return err + } + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim(']') { + return fmt.Errorf("array has invalid closing delimiter") + } + default: + return fmt.Errorf("unexpected JSON delimiter %q", delimiter) + } + return nil +} + +type promotionResourceReport struct { + ResourceGateReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + NativeReportSHA256 string `json:"native_report_sha256"` + NativeReportBase64 string `json:"native_report_base64"` +} + +func validatePromotionResourceReport(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionResourceReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("resource report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("resource report promotion identity does not match manifest") + } + report := bound.ResourceGateReport + nativeRaw, err := promotionEmbeddedNativeReport("resource", bound.NativeReportSHA256, bound.NativeReportBase64) + if err != nil { + return err + } + var native ResourceGateReport + if err := decodePromotionEvidence(nativeRaw, &native); err != nil { + return fmt.Errorf("resource native producer report: %w", err) + } + if !reflect.DeepEqual(native, report) { + return fmt.Errorf("resource bound projection differs from its native producer report") + } + if report.Version != resourceGateVersion { + return fmt.Errorf("resource report version must be %d", resourceGateVersion) + } + if !lowercaseSHA256(report.ArtifactSHA256) { + return fmt.Errorf("resource report artifact_sha256 is not a canonical SHA-256 digest") + } + if !report.Passed { + return fmt.Errorf("resource report did not pass") + } + if len(report.Cases) == 0 { + return fmt.Errorf("resource report has no cases") + } + expectedLimits, supported := promotionResourceNumericLimits(expectedIdentity) + if !supported { + return fmt.Errorf("resource report candidate %q has no registered numeric cap contract", expectedIdentity.Candidate) + } + seen := map[string]struct{}{} + seenInvocations := map[string]struct{}{} + for _, gateCase := range report.Cases { + if strings.TrimSpace(gateCase.Dataset) == "" || strings.TrimSpace(gateCase.Name) == "" || strings.TrimSpace(gateCase.Tier) == "" || + (gateCase.QualificationSplit != "training" && gateCase.QualificationSplit != "holdout") || + !promotionResourceArchitectureAllowed(expectedIdentity.Candidate, gateCase.Architecture) || gateCase.Reference != "" || gateCase.FallbackArchitecture != "" || + gateCase.Round < 1 || gateCase.Round > 20 || gateCase.Block != gateCase.Round || strings.TrimSpace(gateCase.RunUUID) == "" || + strings.TrimSpace(gateCase.Arm) == "" || gateCase.ArmOrder < 1 || gateCase.ArmOrder > 2 { + return fmt.Errorf("resource report contains an incomplete case identity") + } + if !gateCase.Passed || len(gateCase.Reasons) != 0 { + return fmt.Errorf("resource report passing disposition contradicts case %s/%s", gateCase.Dataset, gateCase.Name) + } + if !reflect.DeepEqual(gateCase.NumericLimits, expectedLimits) || len(gateCase.NumericObserved) != len(expectedLimits) { + return fmt.Errorf("resource case %s/%s does not use the manifest candidate's exact numeric limits", gateCase.Dataset, gateCase.Name) + } + for name, limit := range expectedLimits { + observed, found := gateCase.NumericObserved[name] + if !found || observed < 0 || observed > limit { + return fmt.Errorf("resource case %s/%s observation %s=%d is absent, negative, or exceeds limit %d", gateCase.Dataset, gateCase.Name, name, observed, limit) + } + } + if len(gateCase.RuntimeReceiptChains) < 50 { + return fmt.Errorf("resource case %s/%s lacks at least 50 candidate runtime receipts", gateCase.Dataset, gateCase.Name) + } + if err := validatePromotionReceiptChains(gateCase.RuntimeReceiptChains, expectedIdentity.Candidate, seenInvocations); err != nil { + return fmt.Errorf("resource case %s/%s: %w", gateCase.Dataset, gateCase.Name, err) + } + key := fmt.Sprintf("%s\x00%s\x00%d\x00%d\x00%s\x00%s\x00%s", gateCase.Dataset, gateCase.Name, gateCase.Round, gateCase.Block, gateCase.RunUUID, gateCase.Arm, gateCase.Reference) + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("resource report duplicates a case decision") + } + seen[key] = struct{}{} + } + return nil +} + +type promotionReferenceClosureReport struct { + ReferenceClosureReport + PromotionIdentity PromotionEvidenceIdentity `json:"promotion_identity"` + NativeReportSHA256 string `json:"native_report_sha256"` + NativeReportBase64 string `json:"native_report_base64"` +} + +func validatePromotionReferenceClosureReport(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var bound promotionReferenceClosureReport + if err := decodePromotionEvidence(raw, &bound); err != nil { + return fmt.Errorf("reference-closure report: %w", err) + } + if !reflect.DeepEqual(bound.PromotionIdentity, expectedIdentity) { + return fmt.Errorf("reference-closure report promotion identity does not match manifest") + } + report := bound.ReferenceClosureReport + nativeRaw, err := promotionEmbeddedNativeReport("reference-closure", bound.NativeReportSHA256, bound.NativeReportBase64) + if err != nil { + return err + } + var native ReferenceClosureReport + if err := decodePromotionEvidence(nativeRaw, &native); err != nil { + return fmt.Errorf("reference-closure native producer report: %w", err) + } + if !reflect.DeepEqual(native, report) { + return fmt.Errorf("reference-closure bound projection differs from its native producer report") + } + if report.Version != referenceClosureReportVersion { + return fmt.Errorf("reference-closure report version must be %d", referenceClosureReportVersion) + } + if !lowercaseSHA256(report.ArtifactSHA256) { + return fmt.Errorf("reference-closure report artifact_sha256 is not a canonical SHA-256 digest") + } + if report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.BootstrapCount != defaultBootstrapCount || + math.IsNaN(report.Confidence) || math.IsInf(report.Confidence, 0) || strings.TrimSpace(report.ReferenceName) == "" { + return fmt.Errorf("reference-closure report has invalid frozen settings") + } + if report.Candidate != expectedIdentity.Candidate || report.SourceCommit != expectedIdentity.SourceCommit || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || report.BinarySHA256 != expectedIdentity.BinarySHA256 || + report.CorpusSHA256 != expectedIdentity.CorpusSHA256 { + return fmt.Errorf("reference-closure report source, binary, corpus, or candidate identity differs from the manifest") + } + if !report.Passed { + return fmt.Errorf("reference-closure report did not pass") + } + if len(report.Cases) == 0 { + return fmt.Errorf("reference-closure report has no cases") + } + seen := map[string]struct{}{} + seenInvocations := map[string]struct{}{} + for _, closureCase := range report.Cases { + if strings.TrimSpace(closureCase.Dataset) == "" || strings.TrimSpace(closureCase.Name) == "" || + (closureCase.QualificationSplit != "training" && closureCase.QualificationSplit != "holdout") || + !lowercaseSHA256(closureCase.WorkloadSHA256) || promotionIdentityQueryCount(expectedIdentity, closureCase.QuerySHA256) != 1 || + closureCase.ReferenceName != report.ReferenceName || strings.TrimSpace(closureCase.ReferenceArchitecture) == "" { + return fmt.Errorf("reference-closure report contains an incomplete case identity") + } + if closureCase.Rounds < 10 || closureCase.Rounds > 20 || closureCase.ProductionSamples < closureCase.Rounds*50 || closureCase.ReferenceSamples < closureCase.Rounds*50 { + return fmt.Errorf("reference-closure case %s/%s lacks the required rounds or samples", closureCase.Dataset, closureCase.Name) + } + if !validRatioInterval(closureCase.MedianRatio) || !validDurationInterval(closureCase.MedianChange) || + closureCase.RatioUpperLimit != 1.10 || closureCase.AbsoluteFloor != 100*time.Microsecond || + closureCase.ProductionAAResolution < 0 || closureCase.ReferenceAAResolution < 0 || + closureCase.AbsoluteResolution < closureCase.AbsoluteFloor || closureCase.AbsoluteGapUpper < 0 { + return fmt.Errorf("reference-closure case %s/%s has invalid statistical evidence", closureCase.Dataset, closureCase.Name) + } + expectedGap := max(absDuration(closureCase.MedianChange.Lower), absDuration(closureCase.MedianChange.Upper)) + if closureCase.AbsoluteGapUpper != expectedGap || closureCase.AbsoluteResolution != max(closureCase.AbsoluteFloor, closureCase.ProductionAAResolution, closureCase.ReferenceAAResolution) { + return fmt.Errorf("reference-closure case %s/%s has inconsistent derived evidence", closureCase.Dataset, closureCase.Name) + } + if !closureCase.Passed || len(closureCase.Reasons) != 0 || closureCase.MedianRatio.Upper > closureCase.RatioUpperLimit && closureCase.AbsoluteGapUpper > closureCase.AbsoluteResolution { + return fmt.Errorf("reference-closure report passing disposition contradicts case %s/%s", closureCase.Dataset, closureCase.Name) + } + if len(closureCase.ProductionRuntimeReceiptChains) != closureCase.ProductionSamples { + return fmt.Errorf("reference-closure case %s/%s runtime receipt count differs from production samples", closureCase.Dataset, closureCase.Name) + } + if err := validatePromotionReceiptChains(closureCase.ProductionRuntimeReceiptChains, expectedIdentity.Candidate, seenInvocations); err != nil { + return fmt.Errorf("reference-closure case %s/%s: %w", closureCase.Dataset, closureCase.Name, err) + } + key := closureCase.Dataset + "\x00" + closureCase.Name + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("reference-closure report duplicates a case decision") + } + seen[key] = struct{}{} + } + return nil +} + +func promotionEmbeddedNativeReport(role, expectedSHA256, encoded string) ([]byte, error) { + if !lowercaseSHA256(expectedSHA256) { + return nil, fmt.Errorf("%s native producer report SHA-256 is not canonical", role) + } + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(raw) == 0 { + return nil, fmt.Errorf("%s report does not contain decodable native producer bytes", role) + } + digest := sha256.Sum256(raw) + if hex.EncodeToString(digest[:]) != expectedSHA256 { + return nil, fmt.Errorf("%s native producer report SHA-256 does not match its embedded bytes", role) + } + return raw, nil +} + +func promotionResourceNumericLimits(identity PromotionEvidenceIdentity) (map[string]int64, bool) { + switch identity.Candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), string(optimize.ShortestPathExecutorASPI1DAG): + return map[string]int64{ + "state_rows": identity.Caps["state_limit"], + "predecessor_rows": identity.Caps["predecessor_limit"], + "output_rows": identity.Caps["enumeration_limit"], + "output_bytes": identity.Caps["output_bytes_limit"], + }, true + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return spI2TelemetryCaps(), true + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return map[string]int64{ + "forward_seed_rows": identity.Caps["root_row_limit"], + "reverse_seed_rows": identity.Caps["reverse_seed_row_limit"], + "directional_degree_rows": identity.Caps["directional_degree_row_limit"], + "state_rows": identity.Caps["state_limit"], + }, true + default: + return nil, false + } +} + +func promotionResourceArchitectureAllowed(candidate, architecture string) bool { + if candidate == string(optimize.ExpansionSearchPolicyOrientationProbeV1) || candidate == string(optimize.ExpansionSearchPolicyOrientationProbeV2) { + return architecture == string(optimize.ExpansionSearchStepwiseForward) || architecture == string(optimize.ExpansionSearchSuffixSeededReverse) + } + return architecture == candidate +} + +func validRatioInterval(interval RatioInterval) bool { + return !math.IsNaN(interval.Estimate) && !math.IsNaN(interval.Lower) && !math.IsNaN(interval.Upper) && + !math.IsInf(interval.Estimate, 0) && !math.IsInf(interval.Lower, 0) && !math.IsInf(interval.Upper, 0) && + interval.Lower > 0 && interval.Lower <= interval.Estimate && interval.Estimate <= interval.Upper +} + +func validDurationInterval(interval DurationInterval) bool { + return interval.Lower <= interval.Estimate && interval.Estimate <= interval.Upper +} + +func validatePromotionOperationalReport(raw []byte, expectedIdentity PromotionEvidenceIdentity) error { + var report OperationalGateReport + if err := decodePromotionEvidence(raw, &report); err != nil { + return fmt.Errorf("operational report: %w", err) + } + return validateRecomputedOperationalGateReport(report, expectedIdentity) +} + +// bindPromotionEvidenceReport attaches the manifest's authorization identity +// to an already generated role-specific report. The final manifest may then +// checksum the bound report without creating an identity/digest cycle. +func bindPromotionEvidenceReport(manifestPath, role, inputPath, outputPath string) error { + if !containsString(requiredPromotionEvidenceRoles, role) { + return fmt.Errorf("unsupported promotion evidence role %q", role) + } + manifestRaw, err := os.ReadFile(manifestPath) + if err != nil { + return err + } + var manifest PromotionManifest + if err := decodePromotionEvidence(manifestRaw, &manifest); err != nil { + return fmt.Errorf("decode promotion manifest: %w", err) + } + if reasons := validatePromotionBucketSets(manifest.Version, manifest.Buckets); len(reasons) != 0 { + return fmt.Errorf("promotion manifest has an invalid query/split set: %s", strings.Join(reasons, "; ")) + } + if manifest.OperationalCandidateSQLSHA256 != "" && !isLowerHexSHA256(manifest.OperationalCandidateSQLSHA256) { + return fmt.Errorf("promotion manifest has an invalid operational candidate SQL SHA-256") + } + reportRaw, err := os.ReadFile(inputPath) + if err != nil { + return err + } + if err := rejectDuplicateJSONObjectKeys(reportRaw); err != nil { + return fmt.Errorf("decode evidence report: %w", err) + } + var report map[string]any + decoder := json.NewDecoder(bytes.NewReader(reportRaw)) + decoder.UseNumber() + if err := decoder.Decode(&report); err != nil { + return fmt.Errorf("decode evidence report: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return fmt.Errorf("decode evidence report: trailing JSON data") + } + return fmt.Errorf("decode evidence report trailing data: %w", err) + } + if _, exists := report["promotion_identity"]; exists { + return fmt.Errorf("evidence report is already promotion-bound") + } + if _, exists := report["native_report_sha256"]; exists { + return fmt.Errorf("evidence report contains reserved native_report_sha256") + } + if _, exists := report["native_report_base64"]; exists { + return fmt.Errorf("evidence report contains reserved native_report_base64") + } + if role == "aa" || role == "resource" || role == "reference_closure" { + digest := sha256.Sum256(reportRaw) + report["native_report_sha256"] = hex.EncodeToString(digest[:]) + report["native_report_base64"] = base64.StdEncoding.EncodeToString(reportRaw) + } + report["promotion_identity"] = promotionEvidenceIdentity(manifest) + bound, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + return os.WriteFile(outputPath, append(bound, '\n'), 0o644) +} diff --git a/cmd/graphbench/promotion_manifest_test.go b/cmd/graphbench/promotion_manifest_test.go new file mode 100644 index 00000000..a8b701f6 --- /dev/null +++ b/cmd/graphbench/promotion_manifest_test.go @@ -0,0 +1,1806 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + pgdriver "github.com/specterops/dawgs/drivers/pg" + "github.com/stretchr/testify/require" +) + +// writePromotionManifestWithPassingEvidence writes promotion manifest with passing evidence. +func writePromotionManifestWithPassingEvidence(t *testing.T, manifest PromotionManifest) string { + t.Helper() + if manifest.OperationalCandidateSQLSHA256 == "" { + manifest.OperationalCandidateSQLSHA256 = sqlFingerprint(operationalTestSQL) + } + directory := t.TempDir() + manifest.Evidence = map[string]PromotionEvidenceReference{} + for _, role := range requiredPromotionEvidenceRoles { + document := passingPromotionEvidenceDocument(t, manifest, role) + raw, err := json.Marshal(document) + require.NoError(t, err) + path := role + ".json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + digest := sha256.Sum256(raw) + manifest.Evidence[role] = PromotionEvidenceReference{ + Path: path, + SHA256: hex.EncodeToString(digest[:]), + } + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(directory, "promotion.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + return path +} + +func passingPromotionEvidenceDocument(t *testing.T, manifest PromotionManifest, role string) any { + t.Helper() + identity := promotionEvidenceIdentity(manifest) + switch role { + case "aa": + return passingPromotionAAReport(identity) + case "confirmation": + return passingPromotionConfirmationReport(t, identity) + case "performance": + return passingPromotionPerformanceReport(t, identity) + case "resource": + return passingPromotionResourceReport(identity) + case "reference_closure": + return passingPromotionReferenceClosureReport(identity) + case "operational": + return passingPromotionOperationalReport(t, identity) + default: + t.Fatalf("unknown promotion evidence role %q", role) + return nil + } +} + +func passingPromotionResourceReport(identity PromotionEvidenceIdentity) promotionResourceReport { + report := promotionTestResourceReport(identity) + nativeRaw, err := json.Marshal(report) + if err != nil { + panic(err) + } + digest := sha256.Sum256(nativeRaw) + return promotionResourceReport{ + ResourceGateReport: report, PromotionIdentity: identity, + NativeReportSHA256: hex.EncodeToString(digest[:]), NativeReportBase64: base64.StdEncoding.EncodeToString(nativeRaw), + } +} + +func promotionTestResourceReport(identity PromotionEvidenceIdentity) ResourceGateReport { + limits, supported := promotionResourceNumericLimits(identity) + if !supported { + panic("unsupported promotion test resource candidate") + } + report := ResourceGateReport{Version: resourceGateVersion, ArtifactSHA256: strings.Repeat("2", 64), Passed: true} + architecture := identity.Candidate + if isOrientationProbePolicy(identity.Candidate) { + architecture = string(optimize.ExpansionSearchSuffixSeededReverse) + } + for index, cohortCase := range promotionTestPerformanceCohort(identity.Candidate) { + observed := make(map[string]int64, len(limits)) + for name := range limits { + observed[name] = 1 + } + chains := promotionTestReceiptChains(identity.Candidate, fmt.Sprintf("candidate-%d", index), 500) + for round := 1; round <= 10; round++ { + report.Cases = append(report.Cases, ResourceGateCase{ + Dataset: cohortCase.dataset, Name: cohortCase.name, Round: round, Block: round, + RunUUID: fmt.Sprintf("resource-run-%d", index), Arm: "candidate", ArmOrder: 1 + (round+1)%2, + Tier: "normal", QualificationSplit: cohortCase.split, Architecture: architecture, Passed: true, + NumericLimits: limits, NumericObserved: observed, + RuntimeReceiptChains: chains[(round-1)*50 : round*50], + }) + } + } + return report +} + +func promotionTestNativeResourceSHA256(identity PromotionEvidenceIdentity) string { + raw, err := json.Marshal(promotionTestResourceReport(identity)) + if err != nil { + panic(err) + } + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +func passingPromotionReferenceClosureReport(identity PromotionEvidenceIdentity) promotionReferenceClosureReport { + report := promotionTestReferenceClosureReport(identity) + nativeRaw, err := json.Marshal(report) + if err != nil { + panic(err) + } + digest := sha256.Sum256(nativeRaw) + return promotionReferenceClosureReport{ + ReferenceClosureReport: report, PromotionIdentity: identity, + NativeReportSHA256: hex.EncodeToString(digest[:]), NativeReportBase64: base64.StdEncoding.EncodeToString(nativeRaw), + } +} + +func promotionTestReferenceClosureReport(identity PromotionEvidenceIdentity) ReferenceClosureReport { + query := "" + for _, bucket := range identity.Buckets { + if len(bucket.QuerySHA256) > 0 { + query = bucket.QuerySHA256[0] + break + } + } + report := ReferenceClosureReport{ + Version: referenceClosureReportVersion, Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + ArtifactSHA256: strings.Repeat("2", 64), Candidate: identity.Candidate, SourceCommit: identity.SourceCommit, + DirtyDiffSHA256: cleanWorkingTreeSHA256(), BinarySHA256: identity.BinarySHA256, CorpusSHA256: identity.CorpusSHA256, + ReferenceName: "s3_unidirectional_trail_cte", Passed: true, + } + for index, cohortCase := range promotionTestPerformanceCohort(identity.Candidate) { + report.Cases = append(report.Cases, ReferenceClosureCase{ + Dataset: cohortCase.dataset, Name: cohortCase.name, QualificationSplit: cohortCase.split, + WorkloadSHA256: promotionTestWorkloadSHA256(cohortCase), QuerySHA256: query, + ReferenceName: report.ReferenceName, ReferenceArchitecture: "SP-S3-U-D", + Rounds: 10, ProductionSamples: 500, ReferenceSamples: 500, + MedianRatio: RatioInterval{Estimate: 1, Lower: 0.99, Upper: 1.01}, MedianChange: DurationInterval{Estimate: 0, Lower: -time.Microsecond, Upper: time.Microsecond}, + AbsoluteGapUpper: time.Microsecond, RatioUpperLimit: 1.10, AbsoluteFloor: 100 * time.Microsecond, + AbsoluteResolution: 100 * time.Microsecond, Passed: true, + ProductionRuntimeReceiptChains: promotionTestReceiptChains(identity.Candidate, fmt.Sprintf("closure-%d", index), 500), + }) + } + return report +} + +func passingPromotionAAReport(identity PromotionEvidenceIdentity) promotionAAResolutionReport { + report := promotionTestAAResolutionReport(identity) + nativeRaw, err := json.Marshal(report) + if err != nil { + panic(err) + } + digest := sha256.Sum256(nativeRaw) + return promotionAAResolutionReport{ + AAResolutionReport: report, + PromotionIdentity: identity, + NativeReportSHA256: hex.EncodeToString(digest[:]), + NativeReportBase64: base64.StdEncoding.EncodeToString(nativeRaw), + } +} + +func promotionTestAAResolutionReport(identity PromotionEvidenceIdentity) AAResolutionReport { + artifact := strings.Repeat("1", 64) + metric := AAMetricResolution{ + Ratio: RatioInterval{Estimate: 1, Lower: 1, Upper: 1}, + AbsoluteChange: DurationInterval{}, + } + report := AAResolutionReport{ + Version: aaReportVersion, Seed: 1, Confidence: defaultConfidenceLevel, + ArtifactSHA256: artifact, HostFingerprint: strings.Repeat("2", 64), + MinimumRounds: minimumGateRounds, MinimumSamplesPerArmPerRound: 10, OrderBalanced: true, + PhysicalChronology: &AAPhysicalChronology{ + Version: aaPhysicalChronologyVersion, Validated: true, ArtifactSHA256: artifact, + Rounds: minimumGateRounds, Arms: []string{"aa-a", "aa-b"}, + }, + MinimumP99SamplesPerArm: 10_000, + } + for _, cohortCase := range promotionTestPerformanceCohort(identity.Candidate) { + report.Cases = append(report.Cases, AAResolutionCase{ + Dataset: cohortCase.dataset, Name: cohortCase.name, Backend: ModePostgresSQL, + WorkloadSHA256: promotionTestWorkloadSHA256(cohortCase), PostgresEnvironmentSHA256: strings.Repeat("4", 64), FixtureSHA256: strings.Repeat("5", 64), + Rounds: minimumGateRounds, SamplesPerArm: minimumGateRounds * 10, P50: metric, P95: metric, + P99Reason: "diagnostic only: insufficient samples", + }) + } + return report +} + +func passingPromotionConfirmationReport(t *testing.T, identity PromotionEvidenceIdentity) any { + t.Helper() + switch identity.Candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return passingPromotionSPI1Confirmation(t, identity) + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return passingPromotionSPI2Confirmation(t, identity) + case string(optimize.ExpansionSearchPolicyOrientationProbeV1): + return passingPromotionOrientationConfirmation(identity) + case string(optimize.ExpansionSearchPolicyOrientationProbeV2): + return passingPromotionOrientationV2Confirmation(identity) + case string(optimize.ShortestPathExecutorASPI1DAG): + return passingPromotionGenericConfirmation(identity) + default: + t.Fatalf("no promotion confirmation fixture for candidate %q", identity.Candidate) + return nil + } +} + +func passingPromotionSPI1Confirmation(t *testing.T, identity PromotionEvidenceIdentity) promotionSPI1QualificationReport { + t.Helper() + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + report := SPI1QualificationReport{ + Version: spI1QualificationVersion, Protocol: referencePairProtocolConfirmation, + Baseline: string(optimize.ShortestPathExecutorS4CanonicalWitness), Candidate: identity.Candidate, Policy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + QuerySHA256: spI1QuerySHA256, Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + MaterialityRatio: 0.95, MaterialityAbsolute: 100 * time.Microsecond, P95RatioLimit: 1.05, Caps: spI1QualificationCaps(), + SourceCommit: identity.SourceCommit, SourceArchiveSHA256: identity.SourceSHA256, DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: identity.BinarySHA256, CorpusSHA256: identity.CorpusSHA256, + CohortDeclarationSHA256: cohort.declarationSHA256, ResolvedSelectionSHA256: cohort.fullResolvedSHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, TrainingCorpusSHA256: cohort.trainingCorpusSHA256, FullCorpusSHA256: cohort.fullCorpusSHA256, + BaselineArtifactSHA256: strings.Repeat("1", 64), CandidateArtifactSHA256: strings.Repeat("2", 64), + ResourceReportSHA256: promotionTestNativeResourceSHA256(identity), FreezeManifestSHA256: strings.Repeat("4", 64), + EvidencePassed: true, TrainingCases: len(cohort.trainingKeys), HoldoutCases: len(cohort.holdoutKeys), + TrainingPassed: true, HoldoutPassed: true, QualificationPassed: true, + } + for _, declaration := range spI1CanonicalCases { + report.Cases = append(report.Cases, SPI1QualificationCase{ + Dataset: declaration.dataset, Name: declaration.name, QualificationSplit: declaration.split, + Rounds: 10, BaselineSamples: 500, CandidateSamples: 500, + MedianRatio: RatioInterval{Estimate: 0.5, Lower: 0.4, Upper: 0.6}, + MedianSaving: DurationInterval{Estimate: 300 * time.Microsecond, Lower: 200 * time.Microsecond, Upper: 400 * time.Microsecond}, + P95Ratio: RatioInterval{Estimate: 0.7, Lower: 0.6, Upper: 0.8}, + Material: true, P95Contained: true, ResourcePassed: true, RuntimeBranch: "canonical_predecessor_witness", Passed: true, + }) + } + return promotionSPI1QualificationReport{SPI1QualificationReport: report, PromotionIdentity: identity} +} + +func passingPromotionSPI2Confirmation(t *testing.T, identity PromotionEvidenceIdentity) promotionSPI2QualificationReport { + t.Helper() + cohort, err := canonicalSPI2Cohort() + require.NoError(t, err) + report := SPI2QualificationReport{ + Version: spI2QualificationVersion, Protocol: referencePairProtocolConfirmation, + Baseline: string(optimize.ShortestPathExecutorS4CanonicalDistance), Candidate: identity.Candidate, Policy: optimize.ShortestPathPolicyI2DistanceGuardedV1, + QuerySHA256: spI2QuerySHA256, Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + MaterialityRatio: 0.95, MaterialityAbsolute: 100 * time.Microsecond, P95RatioLimit: 1.05, + AdverseRatioLimit: 1.10, AdverseAbsoluteLimit: 100 * time.Microsecond, Caps: spI2QualificationCaps(), + SourceCommit: identity.SourceCommit, SourceArchiveSHA256: identity.SourceSHA256, DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: identity.BinarySHA256, CorpusSHA256: identity.CorpusSHA256, + CohortDeclarationSHA256: cohort.declarationSHA256, ResolvedSelectionSHA256: cohort.fullResolvedSHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, TrainingCorpusSHA256: cohort.trainingCorpusSHA256, FullCorpusSHA256: cohort.fullCorpusSHA256, + BaselineArtifactSHA256: strings.Repeat("1", 64), CandidateArtifactSHA256: strings.Repeat("2", 64), + ResourceReportSHA256: promotionTestNativeResourceSHA256(identity), FreezeManifestSHA256: strings.Repeat("4", 64), + EvidencePassed: true, TrainingCases: len(cohort.trainingKeys), HoldoutCases: len(cohort.holdoutKeys), + TrainingPassed: true, HoldoutPassed: true, QualificationPassed: true, + } + for _, declaration := range spI2CanonicalCases { + role := "target" + if strings.Contains(declaration.name, "cycle-control") { + role = "adverse_control" + } + report.Cases = append(report.Cases, SPI2QualificationCase{ + Dataset: declaration.dataset, Name: declaration.name, QualificationSplit: declaration.split, QualificationRole: role, + Rounds: 10, BaselineSamples: 500, CandidateSamples: 500, + MedianRatio: RatioInterval{Estimate: 0.5, Lower: 0.4, Upper: 0.6}, + MedianSaving: DurationInterval{Estimate: 300 * time.Microsecond, Lower: 200 * time.Microsecond, Upper: 400 * time.Microsecond}, + P95Ratio: RatioInterval{Estimate: 0.7, Lower: 0.6, Upper: 0.8}, + Material: true, P95Contained: true, ResourcePassed: true, RuntimeBranch: "inline_canonical_distance", Passed: true, + }) + } + return promotionSPI2QualificationReport{SPI2QualificationReport: report, PromotionIdentity: identity} +} + +func passingPromotionOrientationConfirmation(identity PromotionEvidenceIdentity) promotionOrientationSelectorReport { + gate := promotionTestOrientationGate("baseline", "candidate", 500) + report := OrientationSelectorReport{ + Version: orientationSelectorReportVersion, Policy: identity.Candidate, Protocol: referencePairProtocolConfirmation, + Seed: 1, Confidence: defaultConfidenceLevel, + ShadowArtifactSHA256: strings.Repeat("1", 64), IncumbentArtifactSHA256: strings.Repeat("2", 64), + ReverseArtifactSHA256: strings.Repeat("3", 64), AAReportSHA256: promotionTestNativeAASHA256(identity), + SelectorRegretRatioLimit: 1.10, ProbeOverheadRatioLimit: 1.10, ProbeOverheadAbsoluteLimit: 100 * time.Microsecond, + EvidencePassed: true, TrainingCases: 1, HoldoutCases: 1, TrainingPassed: true, HoldoutPassed: true, QualificationPassed: true, + Cases: []OrientationSelectorCase{ + {Dataset: "orientation-training", Name: "training", QualificationSplit: "training", QualificationRole: "qualification", QualificationEligible: true, Rounds: 10, WouldSelectIdentity: "forward", FastestExactIdentity: "forward", ExactObservationsMatched: true, SelectorRegret: gate, ProbeOverhead: gate, Passed: true}, + {Dataset: "orientation-holdout", Name: "holdout", QualificationSplit: "holdout", QualificationRole: "qualification", QualificationEligible: true, Rounds: 10, WouldSelectIdentity: "reverse", FastestExactIdentity: "reverse", ExactObservationsMatched: true, SelectorRegret: gate, ProbeOverhead: gate, Passed: true}, + }, + } + return promotionOrientationSelectorReport{OrientationSelectorReport: report, PromotionIdentity: identity} +} + +func passingPromotionOrientationV2Confirmation(identity PromotionEvidenceIdentity) promotionOrientationSelectorV2Report { + cohort, err := canonicalOrientationV2Cohort() + if err != nil { + panic(err) + } + report := OrientationSelectorV2Report{ + Version: orientationSelectorReportV2Version, Policy: identity.Candidate, Protocol: referencePairProtocolConfirmation, + Seed: 1, Confidence: defaultConfidenceLevel, + SourceCommit: identity.SourceCommit, DirtyDiffSHA256: cleanWorkingTreeSHA256(), BinarySHA256: identity.BinarySHA256, CorpusSHA256: identity.CorpusSHA256, + CohortDeclarationSHA256: cohort.declarationSHA256, FreezeManifestSHA256: strings.Repeat("6", 64), + Formula: "F2=root_rows+maximum_depth*forward_degree_rows;R2=suffix_rows+boundary_rows+reverse_degree_rows;reverse=complete&&4*R2<3*F2", + Caps: orientationPromotionCaps(), ShadowArtifactSHA256: strings.Repeat("1", 64), IncumbentArtifactSHA256: strings.Repeat("3", 64), + ReverseArtifactSHA256: strings.Repeat("4", 64), GuardedArtifactSHA256: strings.Repeat("2", 64), AAReportSHA256: promotionTestNativeAASHA256(identity), + ShadowForwardRatioLimit: 1.10, GuardedSelectedRatioLimit: 1.10, GuardedFastestRatioLimit: 1.10, OverheadAbsoluteLimit: 100 * time.Microsecond, + EvidencePassed: true, TrainingCases: 8, HoldoutCases: 4, TrainingPassed: true, HoldoutPassed: true, QualificationPassed: true, + } + for _, declaration := range orientationV2CanonicalCases { + role, tuning, _ := orientationQualificationRole(declaration.split, referencePairProtocolConfirmation) + runtimeIdentity := string(optimize.ExpansionSearchSuffixSeededReverse) + observedIdentity := identity.Candidate + ":" + runtimeIdentity + report.Cases = append(report.Cases, OrientationSelectorV2Case{ + Dataset: declaration.dataset, Name: declaration.name, + QualificationSplit: declaration.split, QualificationRole: role, ThresholdTuningEligible: tuning, QualificationEligible: true, Rounds: 10, + WouldSelectIdentity: runtimeIdentity, FastestExactIdentity: runtimeIdentity, + GuardedRuntimeIdentity: runtimeIdentity, GuardedRuntimeBranch: "suffix_seeded_reverse", + ExactObservationsMatched: true, + ShadowForwardOverhead: OrientationLatencyGateV2{ + Applicable: false, + OrientationLatencyGate: promotionTestOrientationGate( + string(optimize.ExpansionSearchStepwiseForward), identity.Candidate+":shadow", 500, + ), + }, + GuardedSelectedOverhead: promotionTestOrientationGate(runtimeIdentity, observedIdentity, 500), + GuardedFastestRegret: promotionTestOrientationGate(runtimeIdentity, observedIdentity, 500), + Passed: true, + }) + } + return promotionOrientationSelectorV2Report{OrientationSelectorV2Report: report, PromotionIdentity: identity} +} + +func passingPromotionGenericConfirmation(identity PromotionEvidenceIdentity) promotionConfirmationReport { + report := ConfirmationReport{ + Version: confirmationReportVersion, Kind: "causal_confirmation", Seed: 1, Confidence: defaultConfidenceLevel, + LeftArm: "incumbent", RightArm: "candidate", LeftSHA256: strings.Repeat("1", 64), RightSHA256: strings.Repeat("2", 64), + AAReport: "aa.json", AAReportSHA256: promotionTestNativeAASHA256(identity), PromotionEligible: true, + QualificationRequired: true, TrainingCases: 1, HoldoutCases: 1, TrainingPassed: true, HoldoutPassed: true, QualificationPassed: true, + QualificationFamilies: []TraversalQualificationStatus{{ + Family: identity.Candidate, TrainingCases: 1, HoldoutCases: 1, TrainingPassed: true, HoldoutPassed: true, Passed: true, + }}, + } + for index, split := range []string{"training", "holdout"} { + report.Cases = append(report.Cases, ConfirmationCase{ + Dataset: "fixture", Name: split, Backend: ModePostgresSQL, Tier: "normal", QualificationSplit: split, TimingGated: true, + MatchedRounds: 10, LeftSamples: 500, RightSamples: 500, Comparable: true, + P50: promotionTestConfirmationMetric("cleared_non_inferior"), P95: promotionTestConfirmationMetric("cleared_non_inferior"), + Disposition: "cleared_non_inferior", + RightRuntimeReceiptChains: promotionTestReceiptChains(identity.Candidate, fmt.Sprintf("confirmation-%d", index), 500), + }) + } + return promotionConfirmationReport{ConfirmationReport: report, PromotionIdentity: identity} +} + +func passingPromotionPerformanceReport(t *testing.T, identity PromotionEvidenceIdentity) promotionPerfGateReport { + t.Helper() + materialityRatio := 0.95 + materialityAbsolute := 100 * time.Microsecond + cohort := promotionTestPerformanceCohort(identity.Candidate) + trainingCases, holdoutCases := 0, 0 + for _, gateCase := range cohort { + if gateCase.split == "training" { + trainingCases++ + } else { + holdoutCases++ + } + } + report := PerfGateReport{ + Version: perfGateVersion, Seed: 1, Confidence: defaultConfidenceLevel, RegressionThreshold: minimumTimingNoiseRatio, + BaselineSHA256: strings.Repeat("1", 64), CandidateSHA256: strings.Repeat("2", 64), AAReportSHA256: promotionTestNativeAASHA256(identity), + DeclarationSHA256: promotionTestDeclarationSHA256(identity.Candidate), Passed: true, PromotionEligible: true, + MaterialityRequired: true, MaterialityTargets: 1, MaterialityPassed: true, + QualificationRequired: true, TrainingCases: trainingCases, HoldoutCases: holdoutCases, TrainingPassed: true, HoldoutPassed: true, QualificationPassed: true, + QualificationFamilies: []TraversalQualificationStatus{{ + Family: identity.Candidate, TrainingCases: trainingCases, HoldoutCases: holdoutCases, TrainingPassed: true, HoldoutPassed: true, Passed: true, + }}, + } + for index, cohortCase := range cohort { + saving := DurationInterval{Estimate: 300 * time.Microsecond, Lower: 200 * time.Microsecond, Upper: 400 * time.Microsecond} + change := negateDurationInterval(saving) + p95Ratio := RatioInterval{Estimate: 0.7, Lower: 0.6, Upper: 0.8} + p95Change := DurationInterval{Estimate: -200 * time.Microsecond, Lower: -300 * time.Microsecond, Upper: -100 * time.Microsecond} + gateCase := PerfGateCase{ + Dataset: cohortCase.dataset, Name: cohortCase.name, Backend: ModePostgresSQL, Tier: "normal", QualificationSplit: cohortCase.split, TimingGated: true, + Rounds: 10, BaselineSamples: 500, CandidateSamples: 500, BaselineStatus: string(StatusOK), CandidateStatus: string(StatusOK), + MedianRatio: RatioInterval{Estimate: 0.5, Lower: 0.4, Upper: 0.6}, P95Ratio: &p95Ratio, + MedianSaving: &saving, MedianChange: &change, P95Change: &p95Change, + P50NoiseRatio: minimumTimingNoiseRatio, P50NoiseAbsolute: minimumTimingNoiseAbsolute, + P95NoiseRatio: minimumTimingNoiseRatio, P95NoiseAbsolute: minimumTimingNoiseAbsolute, Passed: true, + CandidateRuntimeReceiptChains: promotionTestReceiptChains(identity.Candidate, fmt.Sprintf("candidate-%d", index), 500), + } + if index == 0 { + gateCase.MaterialityRatio = &materialityRatio + gateCase.MaterialityAbsolute = &materialityAbsolute + } + report.Cases = append(report.Cases, gateCase) + } + return promotionPerfGateReport{PerfGateReport: report, PromotionIdentity: identity} +} + +type promotionTestCohortCase struct { + dataset string + name string + split string +} + +func promotionTestPerformanceCohort(candidate string) []promotionTestCohortCase { + switch candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + cohort := make([]promotionTestCohortCase, 0, len(spI1CanonicalCases)) + for _, gateCase := range spI1CanonicalCases { + cohort = append(cohort, promotionTestCohortCase{dataset: gateCase.dataset, name: gateCase.name, split: gateCase.split}) + } + return cohort + case string(optimize.ShortestPathExecutorI2GuardedDistance): + cohort := make([]promotionTestCohortCase, 0, len(spI2CanonicalCases)) + for _, gateCase := range spI2CanonicalCases { + cohort = append(cohort, promotionTestCohortCase{dataset: gateCase.dataset, name: gateCase.name, split: gateCase.split}) + } + return cohort + case string(optimize.ExpansionSearchPolicyOrientationProbeV1): + return []promotionTestCohortCase{ + {dataset: "orientation-training", name: "training", split: "training"}, + {dataset: "orientation-holdout", name: "holdout", split: "holdout"}, + } + case string(optimize.ExpansionSearchPolicyOrientationProbeV2): + cohort := make([]promotionTestCohortCase, 0, len(orientationV2CanonicalCases)) + for _, gateCase := range orientationV2CanonicalCases { + cohort = append(cohort, promotionTestCohortCase{ + dataset: gateCase.dataset, + name: gateCase.name, + split: gateCase.split, + }) + } + return cohort + case string(optimize.ShortestPathExecutorASPI1DAG): + return []promotionTestCohortCase{ + {dataset: "fixture", name: "training", split: "training"}, + {dataset: "fixture", name: "holdout", split: "holdout"}, + } + default: + return nil + } +} + +func promotionTestReceiptChains(candidate, prefix string, count int) [][]RuntimeReceiptEvent { + runtimeIdentity := candidate + if mapped, supported := operationalCandidateRuntimeIdentity(candidate); supported { + runtimeIdentity = mapped + } + chains := make([][]RuntimeReceiptEvent, 0, count) + for index := 0; index < count; index++ { + invocation := fmt.Sprintf("%s-%d", prefix, index) + chains = append(chains, []RuntimeReceiptEvent{{ + InvocationID: invocation, Ordinal: 1, RuntimeIdentity: runtimeIdentity, RuntimeBranch: promotionTestReceiptBranch(candidate, runtimeIdentity), + }}) + } + return chains +} + +func promotionTestReceiptBranch(candidate, runtimeIdentity string) string { + switch candidate { + case string(optimize.ShortestPathExecutorASPI1DAG): + return "inline_predecessor_dag" + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return "inline_canonical_witness" + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return "inline_canonical_distance" + case string(optimize.ExpansionSearchPolicyOrientationProbeV1), string(optimize.ExpansionSearchPolicyOrientationProbeV2): + if runtimeIdentity == string(optimize.ExpansionSearchStepwiseForward) { + return "exact_forward_incumbent" + } + return "suffix_seeded_reverse" + default: + return "selected" + } +} + +func promotionTestWorkloadSHA256(gateCase promotionTestCohortCase) string { + return sqlFingerprint(gateCase.dataset + "\x00" + gateCase.name) +} + +func promotionTestOrientationGate(baseline, observed string, samples int) OrientationLatencyGate { + return OrientationLatencyGate{ + BaselineIdentity: baseline, ObservedIdentity: observed, BaselineSamples: samples, ObservedSamples: samples, + Ratio: RatioInterval{Estimate: 1, Lower: 0.99, Upper: 1.01}, AbsoluteChange: DurationInterval{}, + RatioUpperLimit: 1.10, AbsoluteFloor: 100 * time.Microsecond, Passed: true, + } +} + +func promotionTestConfirmationMetric(classification string) ConfirmationMetric { + return ConfirmationMetric{ + Ratio: RatioInterval{Estimate: 1, Lower: 0.99, Upper: 1.01}, AbsoluteChange: DurationInterval{}, + NoiseRatio: 0.05, NoiseAbsolute: 100 * time.Microsecond, Classification: classification, + } +} + +func promotionTestNativeAASHA256(identity PromotionEvidenceIdentity) string { + raw, err := json.Marshal(promotionTestAAResolutionReport(identity)) + if err != nil { + panic(err) + } + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +func promotionTestDeclarationSHA256(candidate string) string { + switch candidate { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + cohort, _ := canonicalSPI1Cohort() + return cohort.declarationSHA256 + case string(optimize.ShortestPathExecutorI2GuardedDistance): + cohort, _ := canonicalSPI2Cohort() + return cohort.declarationSHA256 + case string(optimize.ExpansionSearchPolicyOrientationProbeV2): + cohort, _ := canonicalOrientationV2Cohort() + return cohort.declarationSHA256 + default: + return strings.Repeat("8", 64) + } +} + +func TestTopologyFixedSuffixBucketUsesDriverStructuralContract(t *testing.T) { + shape := pgdriver.TraversalShape{ + Version: pgdriver.TraversalFixedSuffixShapeVersion, + Family: "fixed_suffix_expansion", + Direction: "outbound", + ObservationMode: string(optimize.ExpansionSearchObservationFullPath), + MinimumDepth: 0, + MaximumDepth: 16, + SuffixLength: 3, + CandidateStrategy: string(optimize.ExpansionSearchSuffixSeededReverse), + } + shape.Fingerprint = pgdriver.TraversalShapeFingerprint(shape) + manifest := PromotionManifest{ + Version: topologyPromotionManifestVersion, + Candidate: string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1), + SelectorVersion: string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1), + ExecutionBoundary: "transaction_retry", + TopologyThresholds: map[string]int64{ + "maximum_edge_to_node_ratio_per_mille": 1000, + }, + } + bucket := PromotionBucket{ + Name: "fixed-suffix", + QuerySHA256: []string{strings.Repeat("a", 64)}, + QualificationSplit: []string{"training", "holdout"}, + Direction: shape.Direction, + ObservationMode: shape.ObservationMode, + MinimumDepth: int(shape.MinimumDepth), + MaximumDepth: int(shape.MaximumDepth), + SuffixLength: shape.SuffixLength, + CandidateStrategy: shape.CandidateStrategy, + StructuralShapeVersion: shape.Version, + StructuralFamily: shape.Family, + StructuralShapeSHA256: shape.Fingerprint, + } + bucket.SQLTemplateSHA256 = pgdriver.TraversalSQLTemplateSHA256(manifest.Candidate, manifest.SelectorVersion, manifest.ExecutionBoundary, shape) + require.NoError(t, validateTopologyFixedSuffixBucket(manifest, bucket)) + + bucket.SuffixLength = 2 + require.ErrorContains(t, validateTopologyFixedSuffixBucket(manifest, bucket), "classifier envelope") + bucket.SuffixLength = 3 + bucket.SQLTemplateSHA256 = strings.Repeat("b", 64) + require.ErrorContains(t, validateTopologyFixedSuffixBucket(manifest, bucket), "SQL template digest") +} + +// TestVerifyPromotionManifestRequiresExactOrientationProbeContract verifies verify promotion manifest requires exact orientation probe contract behavior. +func TestVerifyPromotionManifestRequiresExactOrientationProbeContract(t *testing.T) { + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV1), + SelectorVersion: "orientation-probe-v1", + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "deadbeef", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "fixed-suffix", + QuerySHA256: []string{pgdriver.TraversalPolicyQuerySHA256(operationalTestOrientationCypher)}, + Direction: "outbound", + ObservationMode: "endpoint_ids", + MinimumDepth: 0, + MaximumDepth: 16, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, base)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "confirmation: orientation-probe-v1 promotion is disabled because its report schema cannot bind source, corpus, and frozen cohort identity") + + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // mutate retains the mutate while anonymous record is assembled or evaluated. + mutate func(*PromotionManifest) + // reason retains the reason while anonymous record is assembled or evaluated. + reason string + }{ + { + name: "boundary", + mutate: func(manifest *PromotionManifest) { manifest.ExecutionBoundary = "inline_statement" }, + reason: "orientation-probe-v1 requires the guarded_dual_arm production boundary", + }, + { + name: "fallback", + mutate: func(manifest *PromotionManifest) { manifest.FallbackExecutor = "EXPANSION-SUFFIX-SEEDED-REVERSE" }, + reason: "orientation-probe-v1 requires EXPANSION-STEPWISE-FORWARD as its exact fallback", + }, + { + name: "extra cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["extra_limit"] = 1 }, + reason: "orientation-probe-v1 requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + { + name: "missing cap", + mutate: func(manifest *PromotionManifest) { delete(manifest.Caps, "root_row_limit") }, + reason: "orientation-probe-v1 requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + { + name: "root cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["root_row_limit"]-- }, + reason: "orientation-probe-v1 cap root_row_limit must equal 512", + }, + { + name: "reverse seed cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["reverse_seed_row_limit"]-- }, + reason: "orientation-probe-v1 cap reverse_seed_row_limit must equal 512", + }, + { + name: "directional degree cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["directional_degree_row_limit"]-- }, + reason: "orientation-probe-v1 cap directional_degree_row_limit must equal 16384", + }, + { + name: "state cap", + mutate: func(manifest *PromotionManifest) { manifest.Caps["state_limit"]-- }, + reason: "orientation-probe-v1 cap state_limit must equal 4096", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + test.mutate(&manifest) + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, test.reason) + }) + } +} + +// TestVerifyPromotionManifestRejectsTerminalOrientationProbeV2Contract verifies +// structurally valid v2 evidence remains readable but cannot authorize the +// terminal policy generation after its immutable training overhead gate failed. +func TestVerifyPromotionManifestRejectsTerminalOrientationProbeV2Contract(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "deadbeef", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "fixed-suffix-v2", + QuerySHA256: []string{pgdriver.TraversalPolicyQuerySHA256(operationalTestOrientationCypher)}, + Direction: "outbound", + ObservationMode: "endpoint_ids", + MinimumDepth: 0, + MaximumDepth: 16, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "orientation-probe-v2 is terminally rejected because its immutable training overhead gate failed; authorization requires a new policy generation") + + manifest.SelectorVersion = string(optimize.ExpansionSearchPolicyOrientationProbeV1) + verification, err = verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "orientation-probe-v2 requires the same selector version") +} + +// TestVerifyPromotionManifestRequiresStaticV6CanonicalInboundContract verifies verify promotion manifest requires static v6 canonical inbound contract behavior. +func TestVerifyPromotionManifestRequiresStaticV6CanonicalInboundContract(t *testing.T) { + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalWitness), + SourceCommit: "deadbeef", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: spI1FullCorpusSHA256, + Caps: map[string]int64{"state_limit": 100_000, "predecessor_limit": 100_000, "enumeration_limit": 100_000, "output_bytes_limit": 64 << 20}, + Buckets: []PromotionBucket{{ + Name: "canonical-inbound-depth64", + QuerySHA256: []string{spI1QuerySHA256}, + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, base)) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // mutate retains the mutate while anonymous record is assembled or evaluated. + mutate func(*PromotionManifest) + // reason retains the reason while anonymous record is assembled or evaluated. + reason string + }{ + { + name: "selector", + mutate: func(manifest *PromotionManifest) { manifest.SelectorVersion = "sp-static-v5-contained" }, + reason: "SP-I1 canonical witness requires selector sp-static-v6", + }, + { + name: "outbound", + mutate: func(manifest *PromotionManifest) { manifest.Buckets[0].Direction = "outbound" }, + reason: "SP-I1 canonical witness bucket canonical-inbound-depth64 must be the qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + { + name: "maximum", + mutate: func(manifest *PromotionManifest) { manifest.Buckets[0].MaximumDepth = 63 }, + reason: "SP-I1 canonical witness bucket canonical-inbound-depth64 must be the qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + { + name: "kinds", + mutate: func(manifest *PromotionManifest) { manifest.Buckets[0].RelationshipKindCount = 2 }, + reason: "SP-I1 canonical witness bucket canonical-inbound-depth64 must be the qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + manifest.Buckets = clonePromotionBuckets(base.Buckets) + test.mutate(&manifest) + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, test.reason) + }) + } +} + +// TestVerifyPromotionManifestRequiresCompleteImmutableEvidenceClosure verifies verify promotion manifest requires complete immutable evidence closure behavior. +func TestVerifyPromotionManifestRequiresCompleteImmutableEvidenceClosure(t *testing.T) { + directory := t.TempDir() + digest := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: "SP-S4-C-D", + SourceCommit: "deadbeef", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: spI2FullCorpusSHA256, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), + Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "deep-inbound-distance", + QuerySHA256: []string{spI2QuerySHA256}, + Direction: "inbound", + ObservationMode: "distance", + MinimumDepth: 1, + MaximumDepth: 16, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + evidence := map[string]PromotionEvidenceReference{} + for _, role := range requiredPromotionEvidenceRoles { + document := passingPromotionEvidenceDocument(t, manifest, role) + raw, err := json.Marshal(document) + require.NoError(t, err) + path := role + ".json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + digest := sha256.Sum256(raw) + evidence[role] = PromotionEvidenceReference{ + Path: path, + SHA256: hex.EncodeToString(digest[:]), + } + } + manifest.Evidence = evidence + raw, err := json.Marshal(manifest) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "promotion.json") + require.NoError(t, os.WriteFile(manifestPath, raw, 0o600)) + + verification, err := verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + require.NotEmpty(t, verification.ManifestSHA256) + + delete(manifest.Evidence, "operational") + raw, err = json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, raw, 0o600)) + verification, err = verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "required evidence role operational is missing") +} + +// TestVerifyPromotionEvidenceRejectsEveryCrossBindingMismatch verifies verify promotion evidence rejects every cross binding mismatch behavior. +func TestVerifyPromotionEvidenceRejectsEveryCrossBindingMismatch(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("0", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "bucket", + QuerySHA256: []string{digest}, + Direction: "outbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 4, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + tests := map[string]func(*PromotionEvidenceIdentity){ + "candidate": func(identity *PromotionEvidenceIdentity) { identity.Candidate = "candidate-b" }, + "selector": func(identity *PromotionEvidenceIdentity) { identity.SelectorVersion = "other-selector" }, + "boundary": func(identity *PromotionEvidenceIdentity) { identity.ExecutionBoundary = "stored_helper" }, + "fallback": func(identity *PromotionEvidenceIdentity) { identity.FallbackExecutor = "other-incumbent" }, + "source commit": func(identity *PromotionEvidenceIdentity) { identity.SourceCommit = "other-commit" }, + "source digest": func(identity *PromotionEvidenceIdentity) { identity.SourceSHA256 = strings.Repeat("1", 64) }, + "binary digest": func(identity *PromotionEvidenceIdentity) { identity.BinarySHA256 = strings.Repeat("2", 64) }, + "corpus digest": func(identity *PromotionEvidenceIdentity) { identity.CorpusSHA256 = strings.Repeat("3", 64) }, + "cap": func(identity *PromotionEvidenceIdentity) { identity.Caps["state_limit"]++ }, + "bucket envelope": func(identity *PromotionEvidenceIdentity) { identity.Buckets[0].MaximumDepth = 8 }, + "query cohort": func(identity *PromotionEvidenceIdentity) { + identity.Buckets[0].QuerySHA256[0] = strings.Repeat("4", 64) + }, + "qualification split": func(identity *PromotionEvidenceIdentity) { + identity.Buckets[0].QualificationSplit = []string{"training"} + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + wrong := promotionEvidenceIdentity(manifest) + mutate(&wrong) + document := passingPromotionEvidenceDocument(t, manifest, "resource").(promotionResourceReport) + document.PromotionIdentity = wrong + raw, err := json.Marshal(document) + require.NoError(t, err) + path := "resource.json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + sum := sha256.Sum256(raw) + reference := PromotionEvidenceReference{ + Path: path, + SHA256: hex.EncodeToString(sum[:]), + } + err = verifyPromotionEvidence(directory, "resource", reference, promotionEvidenceIdentity(manifest)) + require.EqualError(t, err, "promotion identity does not match manifest") + }) + } +} + +func TestVerifyPromotionEvidenceStrictlyValidatesVersionedGateRoles(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, Caps: spI2PromotionCaps(), + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), + Buckets: []PromotionBucket{{ + Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", + MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}, + }}, + } + identity := promotionEvidenceIdentity(manifest) + for _, role := range requiredPromotionEvidenceRoles { + t.Run(role+" valid", func(t *testing.T) { + reference, directory := writePromotionEvidenceReference(t, role, passingPromotionEvidenceDocument(t, manifest, role)) + require.NoError(t, verifyPromotionEvidence(directory, role, reference, identity)) + }) + t.Run(role+" minimal forgery", func(t *testing.T) { + reference, directory := writePromotionEvidenceReference(t, role, map[string]any{"passed": true, "promotion_identity": identity}) + err := verifyPromotionEvidence(directory, role, reference, identity) + require.Error(t, err) + require.NotErrorIs(t, err, os.ErrNotExist) + }) + t.Run(role+" unknown field", func(t *testing.T) { + raw, err := json.Marshal(passingPromotionEvidenceDocument(t, manifest, role)) + require.NoError(t, err) + var document map[string]any + require.NoError(t, json.Unmarshal(raw, &document)) + document["unrecognized_proof"] = true + reference, directory := writePromotionEvidenceReference(t, role, document) + require.ErrorContains(t, verifyPromotionEvidence(directory, role, reference, identity), "unknown field") + }) + } +} + +func TestVerifyPromotionEvidenceRejectsInternallyIncompleteGateReports(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, Caps: spI2PromotionCaps(), + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + } + identity := promotionEvidenceIdentity(manifest) + tests := []struct { + name string + role string + mutate func(any) any + reason string + }{ + {name: "resource version", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Version-- + return rebindPromotionTestResourceReport(report) + }, reason: "resource report version"}, + {name: "A/A embedded producer mismatch", role: "aa", mutate: func(value any) any { + report := value.(promotionAAResolutionReport) + report.NativeReportSHA256 = strings.Repeat("f", 64) + return report + }, reason: "does not match its embedded bytes"}, + {name: "A/A chronology artifact drift", role: "aa", mutate: func(value any) any { + report := value.(promotionAAResolutionReport) + report.PhysicalChronology.ArtifactSHA256 = strings.Repeat("f", 64) + return rebindPromotionTestAAReport(report) + }, reason: "artifact-bound physical chronology"}, + {name: "confirmation forged material pass", role: "confirmation", mutate: func(value any) any { + report := value.(promotionSPI2QualificationReport) + report.Cases[0].MedianRatio = RatioInterval{Estimate: 1, Lower: 1, Upper: 1} + report.Cases[0].MedianSaving = DurationInterval{} + return report + }, reason: "incomplete or contradictory evidence"}, + {name: "performance seed drift", role: "performance", mutate: func(value any) any { + report := value.(promotionPerfGateReport) + report.Seed++ + return report + }, reason: "frozen settings"}, + {name: "performance missing noise floor", role: "performance", mutate: func(value any) any { + report := value.(promotionPerfGateReport) + report.Cases[0].P50NoiseRatio = 0.01 + return report + }, reason: "minimum finite noise floors"}, + {name: "performance forged p50 pass", role: "performance", mutate: func(value any) any { + report := value.(promotionPerfGateReport) + report.Cases[0].MedianRatio = RatioInterval{Estimate: 1.2, Lower: 1.1, Upper: 1.3} + report.Cases[0].MedianChange = &DurationInterval{Estimate: 300 * time.Microsecond, Lower: 200 * time.Microsecond, Upper: 400 * time.Microsecond} + report.Cases[0].MedianSaving = &DurationInterval{Estimate: -300 * time.Microsecond, Lower: -400 * time.Microsecond, Upper: -200 * time.Microsecond} + return report + }, reason: "noise-adjusted p50 regression"}, + {name: "performance receipt terminal drift", role: "performance", mutate: func(value any) any { + report := value.(promotionPerfGateReport) + report.Cases[0].CandidateRuntimeReceiptChains[0][0].RuntimeIdentity = "forged-runtime" + return report + }, reason: "terminal identity differs"}, + {name: "performance receipt branch forgery", role: "performance", mutate: func(value any) any { + report := value.(promotionPerfGateReport) + report.Cases[0].CandidateRuntimeReceiptChains[0][0].RuntimeBranch = "invented-authorization-branch" + return report + }, reason: "is not authorized"}, + {name: "resource empty", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Cases = nil + return rebindPromotionTestResourceReport(report) + }, reason: "resource report has no cases"}, + {name: "resource contradictory case", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Cases[0].Passed = false + return rebindPromotionTestResourceReport(report) + }, reason: "contradicts case"}, + {name: "resource observed over limit", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Cases[0].NumericObserved["state_rows"] = report.Cases[0].NumericLimits["state_rows"] + 1 + return rebindPromotionTestResourceReport(report) + }, reason: "exceeds limit"}, + {name: "resource cap drift", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Cases[0].NumericLimits["state_rows"]-- + return rebindPromotionTestResourceReport(report) + }, reason: "exact numeric limits"}, + {name: "resource missing receipts", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Cases[0].RuntimeReceiptChains = report.Cases[0].RuntimeReceiptChains[:49] + return rebindPromotionTestResourceReport(report) + }, reason: "lacks at least 50"}, + {name: "resource receipt terminal drift", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.Cases[0].RuntimeReceiptChains[0][0].RuntimeIdentity = "forged-runtime" + return rebindPromotionTestResourceReport(report) + }, reason: "terminal identity differs"}, + {name: "resource native digest drift", role: "resource", mutate: func(value any) any { + report := value.(promotionResourceReport) + report.NativeReportSHA256 = strings.Repeat("f", 64) + return report + }, reason: "does not match its embedded bytes"}, + {name: "closure version", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Version++ + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "reference-closure report version"}, + {name: "closure empty", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases = nil + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "reference-closure report has no cases"}, + {name: "closure insufficient samples", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].ProductionSamples = 1 + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "lacks the required rounds or samples"}, + {name: "closure forged pass", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].MedianRatio.Upper = 2 + report.Cases[0].MedianRatio.Estimate = 1.5 + report.Cases[0].AbsoluteGapUpper = time.Second + report.Cases[0].MedianChange.Upper = time.Second + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "contradicts case"}, + {name: "closure seed drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Seed++ + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "invalid frozen settings"}, + {name: "closure bootstrap drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.BootstrapCount-- + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "invalid frozen settings"}, + {name: "closure candidate drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Candidate = "forged-candidate" + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "differs from the manifest"}, + {name: "closure source drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.SourceCommit = "forged-commit" + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "differs from the manifest"}, + {name: "closure query drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].QuerySHA256 = strings.Repeat("f", 64) + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "incomplete case identity"}, + {name: "closure ratio threshold drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].RatioUpperLimit = 2 + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "invalid statistical evidence"}, + {name: "closure absolute threshold drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].AbsoluteFloor++ + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "invalid statistical evidence"}, + {name: "closure receipt count drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].ProductionRuntimeReceiptChains = report.Cases[0].ProductionRuntimeReceiptChains[1:] + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "receipt count differs"}, + {name: "closure receipt terminal drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.Cases[0].ProductionRuntimeReceiptChains[0][0].RuntimeIdentity = "forged-runtime" + return rebindPromotionTestReferenceClosureReport(report) + }, reason: "terminal identity differs"}, + {name: "closure native digest drift", role: "reference_closure", mutate: func(value any) any { + report := value.(promotionReferenceClosureReport) + report.NativeReportSHA256 = strings.Repeat("f", 64) + return report + }, reason: "does not match its embedded bytes"}, + {name: "operational version", role: "operational", mutate: func(value any) any { report := value.(OperationalGateReport); report.Version++; return report }, reason: "operational report version"}, + {name: "operational incomplete coverage", role: "operational", mutate: func(value any) any { + report := value.(OperationalGateReport) + report.Coverage.CancellationReplay = false + return report + }, reason: "coverage differs from recomputed input"}, + {name: "operational missing matrix record", role: "operational", mutate: func(value any) any { + report := value.(OperationalGateReport) + report.Records = report.Records[1:] + return report + }, reason: "record decisions differ from recomputed input"}, + {name: "operational duplicate matrix cell", role: "operational", mutate: func(value any) any { + report := value.(OperationalGateReport) + report.Records[1].PoolSize = report.Records[0].PoolSize + report.Records[1].Concurrency = report.Records[0].Concurrency + report.Records[1].PlanCacheMode = report.Records[0].PlanCacheMode + return report + }, reason: "record decisions differ from recomputed input"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + document := test.mutate(passingPromotionEvidenceDocument(t, manifest, test.role)) + reference, directory := writePromotionEvidenceReference(t, test.role, document) + require.ErrorContains(t, verifyPromotionEvidence(directory, test.role, reference, identity), test.reason) + }) + } +} + +func TestVerifyPromotionGenericConfirmationRecomputesClassification(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorASPI1DAG), + SelectorVersion: "asp-static-v1", ExecutionBoundary: "guarded_dual_arm", FallbackExecutor: "ASP-A1-DAG", + SourceCommit: "deadbeef", SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), Caps: map[string]int64{"state_limit": 1}, + Buckets: []PromotionBucket{{Name: "asp", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}}}, + } + identity := promotionEvidenceIdentity(manifest) + report := passingPromotionGenericConfirmation(identity) + report.Cases[0].P95.Ratio = RatioInterval{Estimate: 1.2, Lower: 1.1, Upper: 1.3} + report.Cases[0].P95.AbsoluteChange = DurationInterval{Estimate: 300 * time.Microsecond, Lower: 200 * time.Microsecond, Upper: 400 * time.Microsecond} + + reference, directory := writePromotionEvidenceReference(t, "confirmation", report) + require.ErrorContains(t, verifyPromotionEvidence(directory, "confirmation", reference, identity), "classification contradicts") +} + +func TestVerifyPromotionOrientationV2RequiresCanonicalCohortAndRuntimeTuple(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), Caps: orientationPromotionCaps(), + Buckets: []PromotionBucket{{Name: "orientation", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}}}, + } + identity := promotionEvidenceIdentity(manifest) + + t.Run("case substitution", func(t *testing.T) { + report := passingPromotionOrientationV2Confirmation(identity) + report.Cases[0].Name = "invented-holdout" + reference, directory := writePromotionEvidenceReference(t, "confirmation", report) + require.ErrorContains(t, verifyPromotionEvidence(directory, "confirmation", reference, identity), "outside the frozen V3 corpus") + }) + t.Run("runtime branch forgery", func(t *testing.T) { + report := passingPromotionOrientationV2Confirmation(identity) + report.Cases[0].GuardedRuntimeBranch = "invented" + reference, directory := writePromotionEvidenceReference(t, "confirmation", report) + require.ErrorContains(t, verifyPromotionEvidence(directory, "confirmation", reference, identity), "frozen runtime qualification evidence") + }) +} + +func TestPromotionReceiptTerminalsAreCandidateSpecific(t *testing.T) { + forward := string(optimize.ExpansionSearchStepwiseForward) + reverse := string(optimize.ExpansionSearchSuffixSeededReverse) + policy := string(optimize.ExpansionSearchPolicyOrientationProbeV2) + require.True(t, promotionReceiptTerminalAllowed(policy, forward)) + require.True(t, promotionReceiptTerminalAllowed(policy, reverse)) + require.False(t, promotionReceiptTerminalAllowed(policy, policy)) + require.True(t, promotionReceiptTerminalAllowed(string(optimize.ShortestPathExecutorI2GuardedDistance), string(optimize.ShortestPathExecutorI2GuardedDistance))) + require.False(t, promotionReceiptTerminalAllowed(string(optimize.ShortestPathExecutorI2GuardedDistance), forward)) + require.True(t, promotionReceiptBranchAllowed(string(optimize.ShortestPathExecutorI2GuardedDistance), string(optimize.ShortestPathExecutorI2GuardedDistance), "inline_canonical_distance")) + require.False(t, promotionReceiptBranchAllowed(string(optimize.ShortestPathExecutorI2GuardedDistance), string(optimize.ShortestPathExecutorI2GuardedDistance), "invented")) + require.True(t, promotionReceiptBranchAllowed(string(optimize.ShortestPathExecutorASPI1DAG), string(optimize.ShortestPathExecutorASPI1DAG), "inline_no_path")) +} + +func TestVerifyPromotionEvidenceClosureRequiresExactCrossRoleCohort(t *testing.T) { + digest := strings.Repeat("a", 64) + manifestPath := writePromotionManifestWithPassingEvidence(t, PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + }) + manifestRaw, err := os.ReadFile(manifestPath) + require.NoError(t, err) + var manifest PromotionManifest + require.NoError(t, json.Unmarshal(manifestRaw, &manifest)) + performancePath := filepath.Join(filepath.Dir(manifestPath), manifest.Evidence["performance"].Path) + performanceRaw, err := os.ReadFile(performancePath) + require.NoError(t, err) + var performance promotionPerfGateReport + require.NoError(t, json.Unmarshal(performanceRaw, &performance)) + performance.Cases[0].Name += "-substituted" + performanceRaw, err = json.Marshal(performance) + require.NoError(t, err) + require.NoError(t, os.WriteFile(performancePath, performanceRaw, 0o600)) + performanceDigest := sha256.Sum256(performanceRaw) + manifest.Evidence["performance"] = PromotionEvidenceReference{Path: manifest.Evidence["performance"].Path, SHA256: hex.EncodeToString(performanceDigest[:])} + manifestRaw, err = json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) + + verification, err := verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "evidence closure: performance and confirmation reports do not contain the same exact promotion cohort") +} + +func TestVerifyPromotionEvidenceClosureBindsNativeResourceAndCandidateArtifacts(t *testing.T) { + newManifest := func(t *testing.T) (string, PromotionManifest) { + t.Helper() + digest := strings.Repeat("a", 64) + path := writePromotionManifestWithPassingEvidence(t, PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + }) + raw, err := os.ReadFile(path) + require.NoError(t, err) + var manifest PromotionManifest + require.NoError(t, json.Unmarshal(raw, &manifest)) + return path, manifest + } + + t.Run("confirmation native resource mismatch", func(t *testing.T) { + path, manifest := newManifest(t) + var report promotionSPI2QualificationReport + readPromotionTestEvidence(t, path, manifest, "confirmation", &report) + report.ResourceReportSHA256 = strings.Repeat("f", 64) + rewritePromotionTestEvidence(t, path, &manifest, "confirmation", report) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, verification.Reasons, "evidence closure: SP-I2 confirmation does not use the manifest's exact native resource report") + }) + + t.Run("resource candidate artifact mismatch", func(t *testing.T) { + path, manifest := newManifest(t) + var report promotionResourceReport + readPromotionTestEvidence(t, path, manifest, "resource", &report) + report.ArtifactSHA256 = strings.Repeat("f", 64) + report = rebindPromotionTestResourceReport(report) + rewritePromotionTestEvidence(t, path, &manifest, "resource", report) + + // Keep the confirmation-to-native-resource binding intact so this + // mutation reaches the independent candidate-artifact closure check. + var confirmation promotionSPI2QualificationReport + readPromotionTestEvidence(t, path, manifest, "confirmation", &confirmation) + confirmation.ResourceReportSHA256 = report.NativeReportSHA256 + rewritePromotionTestEvidence(t, path, &manifest, "confirmation", confirmation) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, verification.Reasons, "evidence closure: confirmation, performance, and resource reports do not bind the same exact candidate artifact") + }) + + t.Run("reference is an independently bound capture", func(t *testing.T) { + path, manifest := newManifest(t) + var report promotionReferenceClosureReport + readPromotionTestEvidence(t, path, manifest, "reference_closure", &report) + report.ArtifactSHA256 = strings.Repeat("9", 64) + report = rebindPromotionTestReferenceClosureReport(report) + rewritePromotionTestEvidence(t, path, &manifest, "reference_closure", report) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + }) + + t.Run("reference cohort substitution", func(t *testing.T) { + path, manifest := newManifest(t) + var report promotionReferenceClosureReport + readPromotionTestEvidence(t, path, manifest, "reference_closure", &report) + report.Cases[0].Name += "-substituted" + report = rebindPromotionTestReferenceClosureReport(report) + rewritePromotionTestEvidence(t, path, &manifest, "reference_closure", report) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, verification.Reasons, "evidence closure: reference-closure and confirmation reports do not contain the same exact promotion cohort") + }) + + t.Run("reference workload differs from native A/A", func(t *testing.T) { + path, manifest := newManifest(t) + var report promotionReferenceClosureReport + readPromotionTestEvidence(t, path, manifest, "reference_closure", &report) + report.Cases[0].WorkloadSHA256 = strings.Repeat("f", 64) + report = rebindPromotionTestReferenceClosureReport(report) + rewritePromotionTestEvidence(t, path, &manifest, "reference_closure", report) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, strings.Join(verification.Reasons, "\n"), "evidence closure: reference-closure workload identity differs from native A/A") + }) + + t.Run("native A/A omits a promotion workload", func(t *testing.T) { + path, manifest := newManifest(t) + var aa promotionAAResolutionReport + readPromotionTestEvidence(t, path, manifest, "aa", &aa) + aa.Cases = aa.Cases[1:] + aa = rebindPromotionTestAAReport(aa) + rewritePromotionTestEvidence(t, path, &manifest, "aa", aa) + + var performance promotionPerfGateReport + readPromotionTestEvidence(t, path, manifest, "performance", &performance) + performance.AAReportSHA256 = aa.NativeReportSHA256 + rewritePromotionTestEvidence(t, path, &manifest, "performance", performance) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, strings.Join(verification.Reasons, "\n"), "evidence closure: native A/A report must contain exactly one PostgreSQL workload identity") + }) + + t.Run("resource omits a measured round", func(t *testing.T) { + path, manifest := newManifest(t) + var resource promotionResourceReport + readPromotionTestEvidence(t, path, manifest, "resource", &resource) + resource.Cases = resource.Cases[1:] + resource = rebindPromotionTestResourceReport(resource) + rewritePromotionTestEvidence(t, path, &manifest, "resource", resource) + + var confirmation promotionSPI2QualificationReport + readPromotionTestEvidence(t, path, manifest, "confirmation", &confirmation) + confirmation.ResourceReportSHA256 = resource.NativeReportSHA256 + rewritePromotionTestEvidence(t, path, &manifest, "confirmation", confirmation) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, strings.Join(verification.Reasons, "\n"), "evidence closure: resource report must contain exactly 10 rounds") + }) + + t.Run("resource substitutes a candidate receipt", func(t *testing.T) { + path, manifest := newManifest(t) + var resource promotionResourceReport + readPromotionTestEvidence(t, path, manifest, "resource", &resource) + resource.Cases[0].RuntimeReceiptChains[0][0].InvocationID = "substituted-invocation" + resource = rebindPromotionTestResourceReport(resource) + rewritePromotionTestEvidence(t, path, &manifest, "resource", resource) + + var confirmation promotionSPI2QualificationReport + readPromotionTestEvidence(t, path, manifest, "confirmation", &confirmation) + confirmation.ResourceReportSHA256 = resource.NativeReportSHA256 + rewritePromotionTestEvidence(t, path, &manifest, "confirmation", confirmation) + + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, strings.Join(verification.Reasons, "\n"), "evidence closure: resource and performance reports do not bind the same exact candidate receipt chains") + }) +} + +func readPromotionTestEvidence(t *testing.T, manifestPath string, manifest PromotionManifest, role string, destination any) { + t.Helper() + raw, err := os.ReadFile(filepath.Join(filepath.Dir(manifestPath), manifest.Evidence[role].Path)) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, destination)) +} + +func rewritePromotionTestEvidence(t *testing.T, manifestPath string, manifest *PromotionManifest, role string, document any) { + t.Helper() + raw, err := json.Marshal(document) + require.NoError(t, err) + reference := manifest.Evidence[role] + require.NoError(t, os.WriteFile(filepath.Join(filepath.Dir(manifestPath), reference.Path), raw, 0o600)) + digest := sha256.Sum256(raw) + reference.SHA256 = hex.EncodeToString(digest[:]) + manifest.Evidence[role] = reference + manifestRaw, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) +} + +func rebindPromotionTestAAReport(report promotionAAResolutionReport) promotionAAResolutionReport { + nativeRaw, err := json.Marshal(report.AAResolutionReport) + if err != nil { + panic(err) + } + digest := sha256.Sum256(nativeRaw) + report.NativeReportSHA256 = hex.EncodeToString(digest[:]) + report.NativeReportBase64 = base64.StdEncoding.EncodeToString(nativeRaw) + return report +} + +func rebindPromotionTestResourceReport(report promotionResourceReport) promotionResourceReport { + nativeRaw, err := json.Marshal(report.ResourceGateReport) + if err != nil { + panic(err) + } + digest := sha256.Sum256(nativeRaw) + report.NativeReportSHA256 = hex.EncodeToString(digest[:]) + report.NativeReportBase64 = base64.StdEncoding.EncodeToString(nativeRaw) + return report +} + +func rebindPromotionTestReferenceClosureReport(report promotionReferenceClosureReport) promotionReferenceClosureReport { + nativeRaw, err := json.Marshal(report.ReferenceClosureReport) + if err != nil { + panic(err) + } + digest := sha256.Sum256(nativeRaw) + report.NativeReportSHA256 = hex.EncodeToString(digest[:]) + report.NativeReportBase64 = base64.StdEncoding.EncodeToString(nativeRaw) + return report +} + +func writePromotionEvidenceReference(t *testing.T, role string, document any) (PromotionEvidenceReference, string) { + t.Helper() + directory := t.TempDir() + raw, err := json.Marshal(document) + require.NoError(t, err) + path := role + ".json" + require.NoError(t, os.WriteFile(filepath.Join(directory, path), raw, 0o600)) + digest := sha256.Sum256(raw) + return PromotionEvidenceReference{Path: path, SHA256: hex.EncodeToString(digest[:])}, directory +} + +// TestBindPromotionEvidenceReportCopiesCompleteManifestIdentity verifies bind promotion evidence report copies complete manifest identity behavior. +func TestBindPromotionEvidenceReportCopiesCompleteManifestIdentity(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "bucket", + QuerySHA256: []string{digest}, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + manifestRaw, err := json.Marshal(manifest) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "manifest.json") + inputPath := filepath.Join(directory, "input.json") + outputPath := filepath.Join(directory, "output.json") + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) + inputRaw, err := json.Marshal(passingPromotionEvidenceDocument(t, manifest, "resource").(promotionResourceReport).ResourceGateReport) + require.NoError(t, err) + require.NoError(t, os.WriteFile(inputPath, inputRaw, 0o600)) + require.NoError(t, bindPromotionEvidenceReport(manifestPath, "resource", inputPath, outputPath)) + + boundRaw, err := os.ReadFile(outputPath) + require.NoError(t, err) + var bound promotionResourceReport + require.NoError(t, json.Unmarshal(boundRaw, &bound)) + require.True(t, bound.Passed) + require.Equal(t, promotionEvidenceIdentity(manifest), bound.PromotionIdentity) + nativeDigest := sha256.Sum256(inputRaw) + require.Equal(t, hex.EncodeToString(nativeDigest[:]), bound.NativeReportSHA256) + require.Equal(t, base64.StdEncoding.EncodeToString(inputRaw), bound.NativeReportBase64) +} + +func TestBindPromotionAAEmbedsExactNativeProducerBytes(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + } + manifestRaw, err := json.Marshal(manifest) + require.NoError(t, err) + nativeRaw, err := json.Marshal(promotionTestAAResolutionReport(promotionEvidenceIdentity(manifest))) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "manifest.json") + inputPath := filepath.Join(directory, "aa-native.json") + outputPath := filepath.Join(directory, "aa-bound.json") + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) + require.NoError(t, os.WriteFile(inputPath, nativeRaw, 0o600)) + require.NoError(t, bindPromotionEvidenceReport(manifestPath, "aa", inputPath, outputPath)) + + boundRaw, err := os.ReadFile(outputPath) + require.NoError(t, err) + var bound promotionAAResolutionReport + require.NoError(t, json.Unmarshal(boundRaw, &bound)) + require.Equal(t, base64.StdEncoding.EncodeToString(nativeRaw), bound.NativeReportBase64) + nativeDigest := sha256.Sum256(nativeRaw) + require.Equal(t, hex.EncodeToString(nativeDigest[:]), bound.NativeReportSHA256) + require.NoError(t, validatePromotionAAReport(boundRaw, promotionEvidenceIdentity(manifest))) +} + +func TestBindPromotionReferenceClosureEmbedsExactNativeProducerBytes(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "deadbeef", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + } + manifestRaw, err := json.Marshal(manifest) + require.NoError(t, err) + nativeRaw, err := json.Marshal(promotionTestReferenceClosureReport(promotionEvidenceIdentity(manifest))) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "manifest.json") + inputPath := filepath.Join(directory, "reference-native.json") + outputPath := filepath.Join(directory, "reference-bound.json") + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) + require.NoError(t, os.WriteFile(inputPath, nativeRaw, 0o600)) + require.NoError(t, bindPromotionEvidenceReport(manifestPath, "reference_closure", inputPath, outputPath)) + + boundRaw, err := os.ReadFile(outputPath) + require.NoError(t, err) + var bound promotionReferenceClosureReport + require.NoError(t, json.Unmarshal(boundRaw, &bound)) + nativeDigest := sha256.Sum256(nativeRaw) + require.Equal(t, hex.EncodeToString(nativeDigest[:]), bound.NativeReportSHA256) + require.Equal(t, base64.StdEncoding.EncodeToString(nativeRaw), bound.NativeReportBase64) + require.Equal(t, promotionEvidenceIdentity(manifest), bound.PromotionIdentity) +} + +// TestVerifyPromotionManifestRejectsVersionOne verifies verify promotion manifest rejects version one behavior. +func TestVerifyPromotionManifestRejectsVersionOne(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "manifest.json") + require.NoError(t, os.WriteFile(path, []byte(`{"version":1}`), 0o600)) + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "manifest version must be 2, 3, 4, or 5") +} + +func TestPromotionManifestDecodingRejectsUnknownAndTrailingJSON(t *testing.T) { + directory := t.TempDir() + for name, raw := range map[string][]byte{ + "unknown": []byte(`{"version":2,"invented_authorization":true}`), + "trailing": []byte(`{"version":2}{"version":2}`), + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(directory, name+".json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + _, err := verifyPromotionManifest(path) + require.Error(t, err) + }) + } +} + +func TestPromotionManifestDecodingRejectsDuplicateJSONKeys(t *testing.T) { + for name, raw := range map[string][]byte{ + "top level": []byte(`{"version":2,"version":2}`), + "nested": []byte(`{"version":2,"caps":{"state_limit":1,"state_limit":2}}`), + } { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + _, err := verifyPromotionManifest(path) + require.ErrorContains(t, err, "duplicate JSON object key") + }) + } +} + +func TestVerifyPromotionManifestRejectsNonExactAuthorizationSets(t *testing.T) { + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: spI2FullCorpusSHA256, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + } + + tests := map[string]struct { + mutate func(*PromotionManifest) + reason string + }{ + "extra evidence role": { + mutate: func(manifest *PromotionManifest) { + manifest.Evidence["invented"] = PromotionEvidenceReference{Path: "invented.json", SHA256: digest} + }, + reason: "unsupported evidence role invented is present", + }, + "duplicate split": { + mutate: func(manifest *PromotionManifest) { + manifest.Buckets[0].QualificationSplit = []string{"training", "training", "holdout"} + }, + reason: "must bind exactly one training and one holdout qualification split", + }, + "extra split": { + mutate: func(manifest *PromotionManifest) { + manifest.Buckets[0].QualificationSplit = []string{"training", "holdout", "diagnostic"} + }, + reason: "must bind exactly one training and one holdout qualification split", + }, + "reordered split": { + mutate: func(manifest *PromotionManifest) { + manifest.Buckets[0].QualificationSplit = []string{"holdout", "training"} + }, + reason: "canonical order", + }, + "duplicate query": { + mutate: func(manifest *PromotionManifest) { + manifest.Buckets[0].QuerySHA256 = append(manifest.Buckets[0].QuerySHA256, spI2QuerySHA256) + }, + reason: "duplicates query digest", + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + path := writePromotionManifestWithPassingEvidence(t, base) + raw, err := os.ReadFile(path) + require.NoError(t, err) + var manifest PromotionManifest + require.NoError(t, json.Unmarshal(raw, &manifest)) + manifest.Buckets = clonePromotionBuckets(manifest.Buckets) + test.mutate(&manifest) + raw, err = json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, raw, 0o600)) + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, strings.Join(verification.Reasons, "\n"), test.reason) + }) + } +} + +// TestVerifyPromotionManifestRequiresOperationalSQLAnchor verifies final +// authorization cannot delegate its SQL identity to the operational report. +func TestVerifyPromotionManifestRequiresOperationalSQLAnchor(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "candidate", SelectorVersion: "selector", + ExecutionBoundary: "inline_statement", SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + Caps: map[string]int64{"cap": 1}, + Buckets: []PromotionBucket{{Name: "bucket", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}}}, + } + directory := t.TempDir() + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(directory, "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "operational_candidate_sql_sha256 must be a lowercase SHA-256 digest") +} + +// TestVerifyPromotionManifestSQLAnchorIsUnambiguous verifies one scalar SQL +// digest cannot authorize a final manifest containing several query texts. +func TestVerifyPromotionManifestSQLAnchorIsUnambiguous(t *testing.T) { + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "candidate", SelectorVersion: "selector", + ExecutionBoundary: "inline_statement", SourceCommit: "commit", + SourceSHA256: digest, BinarySHA256: digest, CorpusSHA256: digest, + OperationalCandidateSQLSHA256: strings.Repeat("d", 64), Caps: map[string]int64{"cap": 1}, + Buckets: []PromotionBucket{{Name: "bucket", QuerySHA256: []string{digest, strings.Repeat("b", 64)}, QualificationSplit: []string{"training", "holdout"}}}, + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + path := filepath.Join(t.TempDir(), "manifest.json") + require.NoError(t, os.WriteFile(path, raw, 0o600)) + verification, err := verifyPromotionManifest(path) + require.NoError(t, err) + require.Contains(t, verification.Reasons, "operational SQL anchor requires exactly one authorized query digest") +} + +// TestVerifyPromotionManifestRejectsEscapingOrMutatedEvidence verifies verify promotion manifest rejects escaping or mutated evidence behavior. +func TestVerifyPromotionManifestRejectsEscapingOrMutatedEvidence(t *testing.T) { + directory := t.TempDir() + manifestPath := filepath.Join(directory, "promotion.json") + digest := "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + manifest := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: "candidate", + SelectorVersion: "selector", + ExecutionBoundary: "inline_statement", + SourceCommit: "commit", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: digest, + Caps: map[string]int64{"cap": 1}, + Buckets: []PromotionBucket{{ + Name: "bucket", + QuerySHA256: []string{digest}, + QualificationSplit: []string{"training", "holdout"}, + }}, + Evidence: map[string]PromotionEvidenceReference{}, + } + for _, role := range requiredPromotionEvidenceRoles { + manifest.Evidence[role] = PromotionEvidenceReference{ + Path: "../outside.json", + SHA256: digest, + } + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + require.NoError(t, os.WriteFile(manifestPath, raw, 0o600)) + + verification, err := verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.False(t, verification.Passed) + for _, role := range requiredPromotionEvidenceRoles { + require.Contains(t, verification.Reasons, role+": path escapes the manifest directory") + } +} + +func TestVerifyPromotionManifestRejectsEvidenceSymlinkEscape(t *testing.T) { + outside := t.TempDir() + manifestPath := writePromotionManifestWithPassingEvidence(t, PromotionManifest{ + Version: promotionManifestVersion, Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), SourceCommit: "commit", + SourceSHA256: strings.Repeat("a", 64), BinarySHA256: strings.Repeat("a", 64), CorpusSHA256: spI2FullCorpusSHA256, + OperationalCandidateSQLSHA256: sqlFingerprint(operationalTestSQL), Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{Name: "hidden-fan-in", QuerySHA256: []string{spI2QuerySHA256}, Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, QualificationSplit: []string{"training", "holdout"}}}, + }) + raw, err := os.ReadFile(manifestPath) + require.NoError(t, err) + var manifest PromotionManifest + require.NoError(t, json.Unmarshal(raw, &manifest)) + + role := "aa" + original := filepath.Join(filepath.Dir(manifestPath), manifest.Evidence[role].Path) + external := filepath.Join(outside, "external-aa.json") + externalRaw, err := os.ReadFile(original) + require.NoError(t, err) + require.NoError(t, os.WriteFile(external, externalRaw, 0o600)) + require.NoError(t, os.Remove(original)) + require.NoError(t, os.Symlink(external, original)) + + verification, err := verifyPromotionManifest(manifestPath) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, "aa: path escapes the manifest directory through a symlink") +} + +func TestBindPromotionEvidencePreservesLargeIntegersExactly(t *testing.T) { + directory := t.TempDir() + digest := strings.Repeat("a", 64) + manifest := PromotionManifest{ + Version: promotionManifestVersion, Candidate: "candidate", SelectorVersion: "selector", + ExecutionBoundary: "guarded_dual_arm", SourceCommit: "commit", SourceSHA256: digest, + BinarySHA256: digest, CorpusSHA256: digest, Caps: map[string]int64{"cap": 1}, + Buckets: []PromotionBucket{{Name: "bucket", QuerySHA256: []string{digest}, QualificationSplit: []string{"training", "holdout"}}}, + } + manifestRaw, err := json.Marshal(manifest) + require.NoError(t, err) + manifestPath := filepath.Join(directory, "manifest.json") + inputPath := filepath.Join(directory, "input.json") + outputPath := filepath.Join(directory, "output.json") + require.NoError(t, os.WriteFile(manifestPath, manifestRaw, 0o600)) + require.NoError(t, os.WriteFile(inputPath, []byte(`{"version":9007199254740993}`), 0o600)) + require.NoError(t, bindPromotionEvidenceReport(manifestPath, "resource", inputPath, outputPath)) + boundRaw, err := os.ReadFile(outputPath) + require.NoError(t, err) + require.Contains(t, string(boundRaw), `"version": 9007199254740993`) + require.NotContains(t, string(boundRaw), `9007199254740992`) + + require.NoError(t, os.WriteFile(inputPath, []byte(`{"version":1,"version":2}`), 0o600)) + require.ErrorContains(t, bindPromotionEvidenceReport(manifestPath, "resource", inputPath, outputPath), "duplicate JSON object key") +} diff --git a/cmd/graphbench/reference_closure_report.go b/cmd/graphbench/reference_closure_report.go new file mode 100644 index 00000000..2a4000ae --- /dev/null +++ b/cmd/graphbench/reference_closure_report.go @@ -0,0 +1,444 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "math" + "os" + "slices" + "sort" + "strings" + "time" + + pgdriver "github.com/specterops/dawgs/drivers/pg" +) + +// referenceClosureReportVersion identifies the serialized schema revision for reference closure report. +const referenceClosureReportVersion = 2 + +// ReferenceClosureOptions selects the reference arm and ratio and absolute limits used for closure analysis. +type ReferenceClosureOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // ReferenceName identifies the reference arm selected for closure analysis. + ReferenceName string + // RatioUpperLimit sets the largest production-to-reference median ratio accepted by closure analysis. + RatioUpperLimit float64 + // AbsoluteResolution supplies the absolute resolution input to the ReferenceClosureOptions contract. + AbsoluteResolution time.Duration +} + +// ReferenceClosureCase reports paired production/reference samples, A/A floors, and closure disposition for one case. +type ReferenceClosureCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // QualificationSplit binds the case to the frozen training or holdout cohort. + QualificationSplit string `json:"qualification_split"` + // WorkloadSHA256 binds the case to the exact benchmark workload declaration. + WorkloadSHA256 string `json:"workload_sha256"` + // QuerySHA256 binds the case to the normalized Cypher query authorized by the promotion manifest. + QuerySHA256 string `json:"query_sha256"` + // ReferenceName identifies the reference arm selected for closure analysis. + ReferenceName string `json:"reference_name"` + // ReferenceArchitecture supplies the reference architecture input to the ReferenceClosureCase contract. + ReferenceArchitecture string `json:"reference_architecture"` + // Rounds records the number of rounds. + Rounds int `json:"rounds"` + // ProductionSamples records warm timing samples available from production execution. + ProductionSamples int `json:"production_samples"` + // ReferenceSamples records warm timing samples available from the reference arm. + ReferenceSamples int `json:"reference_samples"` + // MedianRatio reports the candidate-to-baseline median latency ratio and confidence bounds. + MedianRatio RatioInterval `json:"median_ratio"` + // MedianChange reports the absolute median latency difference and confidence bounds. + MedianChange DurationInterval `json:"median_change"` + // AbsoluteGapUpper supplies the absolute gap upper input to the ReferenceClosureCase contract. + AbsoluteGapUpper time.Duration `json:"absolute_gap_upper"` + // RatioUpperLimit sets the largest production-to-reference median ratio accepted by closure analysis. + RatioUpperLimit float64 `json:"ratio_upper_limit"` + // AbsoluteFloor supplies the absolute floor input to the ReferenceClosureCase contract. + AbsoluteFloor time.Duration `json:"absolute_floor"` + // ProductionAAResolution records production-arm A/A noise used for closure materiality. + ProductionAAResolution time.Duration `json:"production_aa_resolution"` + // ReferenceAAResolution records reference-arm A/A noise used for closure materiality. + ReferenceAAResolution time.Duration `json:"reference_aa_resolution"` + // AbsoluteResolution supplies the absolute resolution input to the ReferenceClosureCase contract. + AbsoluteResolution time.Duration `json:"absolute_resolution"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Reasons lists explanations for the reported disposition. + Reasons []string `json:"reasons,omitempty"` + // ProductionRuntimeReceiptChains preserves the complete production branch + // chain for every measured invocation used by closure. + ProductionRuntimeReceiptChains [][]RuntimeReceiptEvent `json:"production_runtime_receipt_chains,omitempty"` +} + +// ReferenceClosureReport contains artifact identity, thresholds, and per-case production/reference closure results. +type ReferenceClosureReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // BootstrapCount records the frozen number of bootstrap resamples. + BootstrapCount int `json:"bootstrap_count"` + // ArtifactSHA256 identifies the exact input artifact summarized by the report. + ArtifactSHA256 string `json:"artifact_sha256"` + // Candidate identifies the production executor measured by the raw-pgx boundary. + Candidate string `json:"candidate"` + // SourceCommit identifies the exact source commit used by every input record. + SourceCommit string `json:"source_commit"` + // DirtyDiffSHA256 binds every input record to the same working-tree state. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds every input record to the exact benchmark executable. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds every input record to the exact workload corpus. + CorpusSHA256 string `json:"corpus_sha256"` + // ReferenceName identifies the reference arm selected for closure analysis. + ReferenceName string `json:"reference_name"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Cases contains production-to-reference closure evidence for each evaluated workload. + Cases []ReferenceClosureCase `json:"cases"` +} + +// buildReferenceClosureReport compares production and exact-reference samples under the closure protocol. +func buildReferenceClosureReport(records []CaseResult, options ReferenceClosureOptions) (ReferenceClosureReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return ReferenceClosureReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return ReferenceClosureReport{}, fmt.Errorf("bootstrap count must be positive") + } + if options.ReferenceName == "" { + options.ReferenceName = "s3_unidirectional_trail_cte" + } + if options.RatioUpperLimit == 0 { + options.RatioUpperLimit = 1.10 + } + if options.RatioUpperLimit <= 0 { + return ReferenceClosureReport{}, fmt.Errorf("reference ratio upper limit must be positive") + } + if options.AbsoluteResolution == 0 { + options.AbsoluteResolution = 100 * time.Microsecond + } + if options.AbsoluteResolution < 0 { + return ReferenceClosureReport{}, fmt.Errorf("reference absolute resolution must not be negative") + } + + // closureSeries groups production and reference samples with the architecture fixed across rounds. + type closureSeries struct { + // production groups production duration samples by measurement round. + production roundSamples + // reference groups reference-arm duration samples by measurement round. + reference roundSamples + // architecture retains the executor architecture that must remain stable across rounds. + architecture string + // qualificationSplit binds all rounds to one frozen cohort partition. + qualificationSplit string + // workloadSHA256 binds all rounds to one exact case declaration. + workloadSHA256 string + // querySHA256 binds all rounds to one exact normalized Cypher query. + querySHA256 string + } + series := map[performanceKey]*closureSeries{} + seenRounds := map[performanceKey]map[int]struct{}{} + var sourceIdentity *RunEnvironment + productionCandidate := "" + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + continue + } + if record.Status != StatusOK { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has non-ok status %s", record.Dataset, record.Name, record.Status) + } + if record.Environment == nil { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has no run environment", record.Dataset, record.Name) + } + if record.Environment.ArtifactSchemaVersion != 2 || strings.TrimSpace(record.Environment.SourceCommit) == "" || + !lowercaseSHA256(record.Environment.DirtyDiffSHA256) || !lowercaseSHA256(record.Environment.BinarySHA256) || + !lowercaseSHA256(record.Environment.CorpusSHA256) { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s lacks complete source, binary, and corpus identity", record.Dataset, record.Name) + } + if sourceIdentity == nil { + copy := *record.Environment + sourceIdentity = © + } else if record.Environment.SourceCommit != sourceIdentity.SourceCommit || + record.Environment.DirtyDiffSHA256 != sourceIdentity.DirtyDiffSHA256 || + record.Environment.BinarySHA256 != sourceIdentity.BinarySHA256 || + record.Environment.CorpusSHA256 != sourceIdentity.CorpusSHA256 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s source, binary, or corpus identity changed across the closure artifact", record.Dataset, record.Name) + } + candidate := promotionCandidateForReferenceClosure(record) + if strings.TrimSpace(candidate) == "" { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has no applied production executor identity", record.Dataset, record.Name) + } + if productionCandidate == "" { + productionCandidate = candidate + } else if candidate != productionCandidate { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s production executor identity changed across the closure artifact", record.Dataset, record.Name) + } + if !lowercaseSHA256(record.WorkloadSHA256) || strings.TrimSpace(record.Cypher) == "" || + !lowercaseSHA256(record.SQLFingerprint) || record.SQLFingerprint != sqlFingerprint(record.SQL) || + record.RawPGXWaterfall == nil || record.RawPGXWaterfall.SQLFingerprint != record.SQLFingerprint { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s lacks exact workload and translated-SQL identity", record.Dataset, record.Name) + } + querySHA256 := pgdriver.TraversalPolicyQuerySHA256(record.Cypher) + if !lowercaseSHA256(querySHA256) || strings.TrimSpace(record.Shape.QualificationSplit) == "" { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s lacks a normalized query digest or qualification split", record.Dataset, record.Name) + } + if record.Environment.WarmupIterations < 20 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d requires at least 20 warmups, got %d", record.Dataset, record.Name, record.Environment.Round, record.Environment.WarmupIterations) + } + if record.RawPGXWaterfall == nil || record.RawPGXWaterfall.WarmupIterations < 20 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d lacks a 20-warmup production raw-pgx boundary", record.Dataset, record.Name, record.Environment.Round) + } + var reference *PostgresReferenceResult + for idx := range record.PostgresReferences { + if record.PostgresReferences[idx].Name == options.ReferenceName { + reference = &record.PostgresReferences[idx] + break + } + } + if reference == nil { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d is missing reference %s", record.Dataset, record.Name, record.Environment.Round, options.ReferenceName) + } + if !reference.FullComparator || reference.SemanticValidation != "exact_public_observation" { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s reference %s is not an exact full comparator", record.Dataset, record.Name, options.ReferenceName) + } + if reference.RowCount != record.RowCount || !slices.Equal(reference.ObservedRows, record.ObservedRows) { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s reference observation differs from production", record.Dataset, record.Name) + } + if reference.Stats.WarmupIterations < 20 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d reference requires at least 20 warmups, got %d", record.Dataset, record.Name, record.Environment.Round, reference.Stats.WarmupIterations) + } + expectedProductionOrder, expectedReferenceOrder := referenceClosureMeasurementOrder(true, record.Environment.Round) + if record.RawPGXWaterfall.MeasurementOrder != expectedProductionOrder || reference.MeasurementOrder != expectedReferenceOrder { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s round %d lacks carryover-balanced production/reference order: got %d/%d, expected %d/%d", record.Dataset, record.Name, record.Environment.Round, record.RawPGXWaterfall.MeasurementOrder, reference.MeasurementOrder, expectedProductionOrder, expectedReferenceOrder) + } + + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + } + if seenRounds[key] == nil { + seenRounds[key] = map[int]struct{}{} + } + if _, duplicate := seenRounds[key][record.Environment.Round]; duplicate { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s has duplicate round %d", record.Dataset, record.Name, record.Environment.Round) + } + seenRounds[key][record.Environment.Round] = struct{}{} + + if series[key] == nil { + series[key] = &closureSeries{ + production: roundSamples{}, + reference: roundSamples{}, + architecture: reference.Architecture, + qualificationSplit: record.Shape.QualificationSplit, + workloadSHA256: record.WorkloadSHA256, + querySHA256: querySHA256, + } + } else if series[key].architecture != reference.Architecture || + series[key].qualificationSplit != record.Shape.QualificationSplit || + series[key].workloadSHA256 != record.WorkloadSHA256 || series[key].querySHA256 != querySHA256 { + return ReferenceClosureReport{}, fmt.Errorf("%s/%s reference, workload, query, or split identity changed across rounds", record.Dataset, record.Name) + } + + for _, sample := range record.RawPGXWaterfall.Samples { + if sample.Total > 0 { + series[key].production[record.Environment.Round] = append(series[key].production[record.Environment.Round], sample.Total) + } + } + + for _, sample := range reference.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[key].reference[record.Environment.Round] = append(series[key].reference[record.Environment.Round], sample.Duration) + } + } + } + + if len(series) == 0 { + return ReferenceClosureReport{}, fmt.Errorf("artifact has no successful PostgreSQL production/reference records") + } + + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + report := ReferenceClosureReport{ + Version: referenceClosureReportVersion, + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + Candidate: productionCandidate, + SourceCommit: sourceIdentity.SourceCommit, + DirtyDiffSHA256: sourceIdentity.DirtyDiffSHA256, + BinarySHA256: sourceIdentity.BinarySHA256, + CorpusSHA256: sourceIdentity.CorpusSHA256, + ReferenceName: options.ReferenceName, + Passed: true, + } + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for idx, key := range keys { + candidate, baseline := matchedRounds(series[key].production, series[key].reference) + entry := ReferenceClosureCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: series[key].qualificationSplit, + WorkloadSHA256: series[key].workloadSHA256, + QuerySHA256: series[key].querySHA256, + ReferenceName: options.ReferenceName, + ReferenceArchitecture: series[key].architecture, + Rounds: len(candidate), + ProductionSamples: sampleCount(candidate), + ReferenceSamples: sampleCount(baseline), + RatioUpperLimit: options.RatioUpperLimit, + AbsoluteFloor: options.AbsoluteResolution, + Passed: true, + ProductionRuntimeReceiptChains: caseRuntimeReceiptChains(records, key), + } + if entry.Rounds < 10 || entry.Rounds > 20 { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("requires 10-20 matched rounds, got %d", entry.Rounds)) + } + for _, round := range sortedRounds(candidate) { + if len(candidate[round]) < 50 || len(baseline[round]) < 50 { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("round %d requires at least 50 samples per side, got %d/%d", round, len(candidate[round]), len(baseline[round]))) + } + } + if entry.Rounds > 0 { + seed := options.Seed + int64(idx)*7919 + entry.ProductionAAResolution = withinSessionAAResolution(candidate, seed+2, gateOptions) + entry.ReferenceAAResolution = withinSessionAAResolution(baseline, seed+3, gateOptions) + entry.AbsoluteResolution = max(options.AbsoluteResolution, entry.ProductionAAResolution, entry.ReferenceAAResolution) + entry.MedianRatio = bootstrapRoundMedianRatio(baseline, candidate, seed, gateOptions) + entry.MedianChange = negateDurationInterval(bootstrapRoundMedianSaving(baseline, candidate, seed+1, gateOptions)) + entry.AbsoluteGapUpper = max(absDuration(entry.MedianChange.Lower), absDuration(entry.MedianChange.Upper)) + if entry.MedianRatio.Upper > options.RatioUpperLimit && entry.AbsoluteGapUpper > entry.AbsoluteResolution { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("ratio upper %.4f exceeds %.4f and absolute gap upper %s exceeds effective resolution %s", entry.MedianRatio.Upper, options.RatioUpperLimit, entry.AbsoluteGapUpper, entry.AbsoluteResolution)) + } + } + if !entry.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, entry) + } + return report, nil +} + +// promotionCandidateForReferenceClosure returns the authorization identity +// represented by a production record. Runtime orientation policies retain the +// policy identity even though the lowering's static Selected field names its +// incumbent arm; other candidates use the concrete applied executor. +func promotionCandidateForReferenceClosure(record CaseResult) string { + if record.Optimization != nil { + for _, outcome := range record.Optimization.TargetOutcomes { + if isOrientationProbePolicy(outcome.EmittedPolicy) { + return outcome.EmittedPolicy + } + } + } + return appliedPostgresArchitecture(record) +} + +// withinSessionAAResolution returns the larger within-session A/A noise estimate for a case. +func withinSessionAAResolution(samples roundSamples, seed int64, options PerfGateOptions) time.Duration { + armA, armB := splitInterleavedDiagnosticSeries(samples) + armA, armB = matchedRounds(armA, armB) + if len(armA) == 0 { + return 0 + } + interval := bootstrapRoundMedianSaving(armA, armB, seed, options) + return max(absDuration(interval.Lower), absDuration(interval.Upper)) +} + +// splitInterleavedDiagnosticSeries estimates within-session resolution for the +// descriptive reference-closure report only. Promotion-grade host A/A evidence +// is built exclusively from explicit arms by collectExplicitAASeries. +func splitInterleavedDiagnosticSeries(samples roundSamples) (roundSamples, roundSamples) { + armA, armB := roundSamples{}, roundSamples{} + for round, values := range samples { + for idx, value := range values { + if idx%2 == 0 { + armA[round] = append(armA[round], value) + } else { + armB[round] = append(armB[round], value) + } + } + } + return armA, armB +} + +// absDuration returns the magnitude of a signed duration. +func absDuration(value time.Duration) time.Duration { + return time.Duration(math.Abs(float64(value))) +} + +// createReferenceClosureReport loads benchmark records, builds a closure report, and writes it as JSON. +func createReferenceClosureReport(artifactPath, outputPath string, options ReferenceClosureOptions) (bool, error) { + records, err := readJSONLFile(artifactPath) + if err != nil { + return false, err + } + report, err := buildReferenceClosureReport(records, options) + if err != nil { + return false, err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return false, err + } + return report.Passed, writeReferenceClosureReport(outputPath, report) +} + +// writeReferenceClosureReport writes a reference-closure report as indented JSON. +func writeReferenceClosureReport(path string, report ReferenceClosureReport) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/reference_closure_report_test.go b/cmd/graphbench/reference_closure_report_test.go new file mode 100644 index 00000000..f2628771 --- /dev/null +++ b/cmd/graphbench/reference_closure_report_test.go @@ -0,0 +1,235 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestBuildReferenceClosureReportPassesRatioOrResolution verifies that a small absolute gap within measurement resolution passes even when production is five percent slower. +func TestBuildReferenceClosureReportPassesRatioOrResolution(t *testing.T) { + records := referenceClosureRecords(10, 50, time.Millisecond, 1050*time.Microsecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 7, + Confidence: 0.975, + BootstrapCount: 250, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.Len(t, report.Cases, 1) + entry := report.Cases[0] + require.Equal(t, 10, entry.Rounds) + require.Equal(t, 500, entry.ProductionSamples) + require.Equal(t, 500, entry.ReferenceSamples) + require.InDelta(t, 1.05, entry.MedianRatio.Estimate, 0.0001) + require.LessOrEqual(t, entry.AbsoluteGapUpper, 100*time.Microsecond) + require.Equal(t, 100*time.Microsecond, entry.AbsoluteFloor) + require.Equal(t, 100*time.Microsecond, entry.AbsoluteResolution) +} + +// TestBuildReferenceClosureReportUsesCaseAAResolution verifies that observed production-side A/A noise raises the per-case absolute resolution above the default floor. +func TestBuildReferenceClosureReportUsesCaseAAResolution(t *testing.T) { + records := referenceClosureRecords(10, 50, 2*time.Millisecond, 1500*time.Microsecond) + for idx := range records { + for sampleIdx := range records[idx].RawPGXWaterfall.Samples { + if sampleIdx%2 == 1 { + records[idx].RawPGXWaterfall.Samples[sampleIdx].Total = 2700 * time.Microsecond + } + } + } + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + }) + + require.NoError(t, err) + require.True(t, report.Passed) + require.Greater(t, report.Cases[0].ProductionAAResolution, 100*time.Microsecond) + require.Equal(t, report.Cases[0].ProductionAAResolution, report.Cases[0].AbsoluteResolution) +} + +// TestBuildReferenceClosureReportFailsMaterialGap verifies that a confidence interval exceeding both ratio and absolute-resolution allowances fails closure. +func TestBuildReferenceClosureReportFailsMaterialGap(t *testing.T) { + records := referenceClosureRecords(10, 50, time.Millisecond, 1500*time.Microsecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + }) + + require.NoError(t, err) + require.False(t, report.Passed) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "ratio upper") +} + +// TestBuildReferenceClosureReportEnforcesProtocolAndExactComparator verifies minimum rounds/samples, exact public observations, and carryover-balanced measurement order. +func TestBuildReferenceClosureReportEnforcesProtocolAndExactComparator(t *testing.T) { + records := referenceClosureRecords(9, 49, time.Millisecond, time.Millisecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + }) + require.NoError(t, err) + require.False(t, report.Passed) + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "10-20 matched rounds") + require.ErrorContains(t, reasonsError(report.Cases[0].Reasons), "at least 50 samples") + + records = referenceClosureRecords(10, 50, time.Millisecond, time.Millisecond) + records[0].PostgresReferences[0].ObservedRows = []string{"[2]"} + _, err = buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + }) + require.ErrorContains(t, err, "observation differs") + + records = referenceClosureRecords(10, 50, time.Millisecond, time.Millisecond) + records[1].PostgresReferences[0].MeasurementOrder = 2 + _, err = buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, + Confidence: 0.975, + }) + require.ErrorContains(t, err, "lacks carryover-balanced") +} + +func TestBuildReferenceClosureReportBindsCandidateSourceWorkloadAndQuery(t *testing.T) { + records := referenceClosureRecords(10, 50, time.Millisecond, time.Millisecond) + report, err := buildReferenceClosureReport(records, ReferenceClosureOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + }) + require.NoError(t, err) + require.Equal(t, referenceClosureReportVersion, report.Version) + require.Equal(t, "candidate", report.Candidate) + require.Equal(t, "deadbeef", report.SourceCommit) + require.Equal(t, cleanWorkingTreeSHA256(), report.DirtyDiffSHA256) + require.Equal(t, strings.Repeat("a", 64), report.BinarySHA256) + require.Equal(t, strings.Repeat("a", 64), report.CorpusSHA256) + require.Equal(t, defaultBootstrapCount, report.BootstrapCount) + require.Len(t, report.Cases, 1) + require.Equal(t, "training", report.Cases[0].QualificationSplit) + require.Equal(t, strings.Repeat("a", 64), report.Cases[0].WorkloadSHA256) + require.True(t, lowercaseSHA256(report.Cases[0].QuerySHA256)) + require.Len(t, report.Cases[0].ProductionRuntimeReceiptChains, 500) +} + +func TestBuildReferenceClosureReportRejectsIdentityDrift(t *testing.T) { + tests := map[string]struct { + mutate func([]CaseResult) + reason string + }{ + "source": { + mutate: func(records []CaseResult) { records[1].Environment.SourceCommit = "other" }, + reason: "source, binary, or corpus identity changed", + }, + "candidate": { + mutate: func(records []CaseResult) { records[1].Optimization.TargetOutcomes[0].Applied = "other" }, + reason: "production executor identity changed", + }, + "workload": { + mutate: func(records []CaseResult) { records[1].WorkloadSHA256 = strings.Repeat("b", 64) }, + reason: "workload, query, or split identity changed", + }, + "query": { + mutate: func(records []CaseResult) { records[1].Cypher += " limit 1" }, + reason: "workload, query, or split identity changed", + }, + "sql fingerprint": { + mutate: func(records []CaseResult) { records[1].SQLFingerprint = strings.Repeat("b", 64) }, + reason: "lacks exact workload and translated-SQL identity", + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + records := referenceClosureRecords(10, 50, time.Millisecond, time.Millisecond) + test.mutate(records) + _, err := buildReferenceClosureReport(records, ReferenceClosureOptions{Seed: 1, Confidence: defaultConfidenceLevel}) + require.ErrorContains(t, err, test.reason) + }) + } +} + +// referenceClosureRecords returns carryover-balanced production/reference rounds with exact observations and uniform warm timings. +func referenceClosureRecords(rounds, samples int, referenceDuration, productionDuration time.Duration) []CaseResult { + records := make([]CaseResult, 0, rounds) + digest := strings.Repeat("a", 64) + cypherQuery := "match p = shortestPath((a)-[:Edge*1..4]->(b)) return length(p)" + sqlQuery := "select 1" + for round := 1; round <= rounds; round++ { + productionOrder, referenceOrder := referenceClosureMeasurementOrder(true, round) + record := CaseResult{ + Dataset: "fixture", + Name: "distance", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[1]"}, + WorkloadSHA256: digest, + Cypher: cypherQuery, + SQL: sqlQuery, + SQLFingerprint: sqlFingerprint(sqlQuery), + Shape: WorkloadShape{QualificationSplit: "training"}, + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: digest, + SourceCommit: "deadbeef", + DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: digest, + Round: round, + WarmupIterations: 20, + }, + RawPGXWaterfall: &PostgresBoundaryWaterfall{ + WarmupIterations: 20, + MeasurementOrder: productionOrder, + SQLFingerprint: sqlFingerprint(sqlQuery), + }, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", Applied: "candidate", + }}}, + PostgresReferences: []PostgresReferenceResult{{ + Name: "s3_unidirectional_trail_cte", + Architecture: "SP-S3-U-D", + FullComparator: true, + SemanticValidation: "exact_public_observation", + MeasurementOrder: referenceOrder, + RowCount: 1, + ObservedRows: []string{"[1]"}, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }}, + } + for iteration := 1; iteration <= samples; iteration++ { + record.RawPGXWaterfall.Samples = append(record.RawPGXWaterfall.Samples, BoundarySample{ + Iteration: iteration, + Total: productionDuration, + Rows: 1, + }) + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: referenceDuration, + }) + invocation := fmt.Sprintf("closure-%d-%d", round, iteration) + fallback := false + record.Stats.Samples = append(record.Stats.Samples, LatencySample{ + Round: round, Iteration: iteration, Classification: "warm", Duration: productionDuration, + RuntimeInvocationID: invocation, RuntimeIdentity: "candidate", RuntimeBranch: "selected", FallbackExecuted: &fallback, + RuntimeReceiptEvents: []RuntimeReceiptEvent{{InvocationID: invocation, Ordinal: 1, RuntimeIdentity: "candidate", RuntimeBranch: "selected"}}, + }) + } + records = append(records, record) + } + return records +} diff --git a/cmd/graphbench/reference_pair_report.go b/cmd/graphbench/reference_pair_report.go new file mode 100644 index 00000000..2b0a7bf0 --- /dev/null +++ b/cmd/graphbench/reference_pair_report.go @@ -0,0 +1,356 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" +) + +// referencePairReportVersion identifies the serialized schema revision for reference pair report. +const referencePairReportVersion = 2 + +const ( + // referencePairProtocolConfirmation requires 20 warmups, 10 to 20 rounds, and 50 samples per arm and round. + referencePairProtocolConfirmation = "confirmation" + + // referencePairProtocolDiscovery permits exploratory comparison with five warmups, five rounds, and ten samples per arm and round. + referencePairProtocolDiscovery = "discovery" +) + +// ReferencePairOptions selects two reference arms and the statistical protocol used for their paired comparison. +type ReferencePairOptions struct { + // Seed controls deterministic random sampling. + Seed int64 + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 + // BootstrapCount sets the number of bootstrap resamples. + BootstrapCount int + // BaselineName identifies the reference arm treated as the comparison baseline. + BaselineName string + // CandidateName identifies the reference arm evaluated against the baseline. + CandidateName string + // Protocol identifies the measurement protocol. + Protocol string +} + +// ReferencePairCase reports identity, sample, ratio, and absolute-change evidence for one reference-arm pair. +type ReferencePairCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Rounds records the number of independent measurement rounds. + Rounds int `json:"rounds"` + // BaselineArchitecture records the executor architecture declared by the baseline arm. + BaselineArchitecture string `json:"baseline_architecture"` + // CandidateArchitecture records the executor architecture declared by the candidate arm. + CandidateArchitecture string `json:"candidate_architecture"` + // BaselineBoundary records the portion of baseline execution included in its latency samples. + BaselineBoundary string `json:"baseline_boundary"` + // CandidateBoundary records the portion of candidate execution included in its latency samples. + CandidateBoundary string `json:"candidate_boundary"` + // BaselineSemanticValidation identifies the observation contract enforced for the baseline arm. + BaselineSemanticValidation string `json:"baseline_semantic_validation"` + // CandidateSemanticValidation identifies the observation contract enforced for the candidate arm. + CandidateSemanticValidation string `json:"candidate_semantic_validation"` + // BaselineSamples records warm timing samples available from the baseline arm. + BaselineSamples int `json:"baseline_samples"` + // CandidateSamples records warm timing samples available from the candidate arm. + CandidateSamples int `json:"candidate_samples"` + // MedianRatio reports the candidate-to-baseline median latency ratio and confidence bounds. + MedianRatio RatioInterval `json:"median_ratio"` + // P95Ratio reports the candidate-to-baseline P95 latency ratio and confidence bounds. + P95Ratio RatioInterval `json:"p95_ratio"` + // MedianChange reports the absolute median latency difference and confidence bounds. + MedianChange DurationInterval `json:"median_change"` + // BaselineAAResolution records the baseline arm's A/A-derived absolute noise floor. + BaselineAAResolution time.Duration `json:"baseline_aa_resolution"` + // CandidateAAResolution records the candidate arm's A/A-derived absolute noise floor. + CandidateAAResolution time.Duration `json:"candidate_aa_resolution"` +} + +// ReferencePairReport contains the input identity, protocol thresholds, and results of paired reference analysis. +type ReferencePairReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Seed controls deterministic random sampling. + Seed int64 `json:"seed"` + // Confidence sets the confidence level used for statistical intervals. + Confidence float64 `json:"confidence_level"` + // ArtifactSHA256 identifies the exact input artifact summarized by the report. + ArtifactSHA256 string `json:"artifact_sha256"` + // BaselineName identifies the reference arm treated as the comparison baseline. + BaselineName string `json:"baseline_name"` + // CandidateName identifies the reference arm evaluated against the baseline. + CandidateName string `json:"candidate_name"` + // Protocol identifies the measurement protocol. + Protocol string `json:"protocol"` + // MinimumWarmups records the minimum untimed iterations required for each compared arm. + MinimumWarmups int `json:"minimum_warmups"` + // MinimumRounds records the minimum independent rounds required for comparison. + MinimumRounds int `json:"minimum_rounds"` + // MaximumRounds records the maximum rounds accepted by the selected protocol. + MaximumRounds int `json:"maximum_rounds"` + // MinimumSamples records the minimum warm samples required from each arm and round. + MinimumSamples int `json:"minimum_samples_per_round"` + // Cases contains paired statistical evidence for each workload present in the selected reference arms. + Cases []ReferencePairCase `json:"cases"` +} + +// buildReferencePairReport validates two reference arms and computes paired ratio and duration intervals by case. +func buildReferencePairReport(records []CaseResult, options ReferencePairOptions) (ReferencePairReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return ReferencePairReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 || options.BaselineName == "" || options.CandidateName == "" || options.BaselineName == options.CandidateName { + return ReferencePairReport{}, fmt.Errorf("valid distinct baseline and candidate reference arms are required") + } + protocol := options.Protocol + if protocol == "" { + protocol = referencePairProtocolConfirmation + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples := 20, 10, 20, 50 + if protocol == referencePairProtocolDiscovery { + minimumWarmups, minimumRounds, maximumRounds, minimumSamples = 5, 5, 20, 10 + } else if protocol != referencePairProtocolConfirmation { + return ReferencePairReport{}, fmt.Errorf("unsupported reference-pair protocol %q", protocol) + } + // pairSeries groups the two reference arms and the identities that must remain stable across rounds. + type pairSeries struct { + // baseline groups duration samples from the designated baseline arm by round. + baseline roundSamples + // candidate groups duration samples from the designated candidate arm by round. + candidate roundSamples + // baselineArchitecture identifies the execution architecture reported by the baseline arm. + baselineArchitecture string + // candidateArchitecture identifies the execution architecture reported by the candidate arm. + candidateArchitecture string + // baselineBoundary identifies the measurement boundary reported by the baseline arm. + baselineBoundary string + // candidateBoundary identifies the measurement boundary reported by the candidate arm. + candidateBoundary string + // baselineValidation retains the baseline observation contract that must remain stable across rounds. + baselineValidation string + // candidateValidation retains the candidate observation contract that must remain stable across rounds. + candidateValidation string + // baselineImplementation identifies the baseline reference implementation. + baselineImplementation string + // candidateImplementation identifies the candidate reference implementation. + candidateImplementation string + // baselineSQLFingerprint identifies the normalized SQL executed by the baseline arm. + baselineSQLFingerprint string + // candidateSQLFingerprint identifies the normalized SQL executed by the candidate arm. + candidateSQLFingerprint string + // binaryIdentity binds all paired rounds to the same executable and source state. + binaryIdentity string + // baselineFirst records by round whether the baseline arm executed before the candidate. + baselineFirst map[int]bool + } + series := map[performanceKey]*pairSeries{} + seen := map[performanceKey]map[int]struct{}{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + continue + } + if record.Status != StatusOK || record.Environment == nil || record.Environment.WarmupIterations < minimumWarmups { + return ReferencePairReport{}, fmt.Errorf("%s/%s lacks a successful %d-warmup PostgreSQL record", record.Dataset, record.Name, minimumWarmups) + } + baseline := findReference(record.PostgresReferences, options.BaselineName) + candidate := findReference(record.PostgresReferences, options.CandidateName) + if baseline == nil || candidate == nil { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d lacks reference pair %s/%s", record.Dataset, record.Name, record.Environment.Round, options.BaselineName, options.CandidateName) + } + fullComparators := baseline.FullComparator && candidate.FullComparator && baseline.SemanticValidation == "exact_public_observation" && candidate.SemanticValidation == "exact_public_observation" + hydrationComparators := !baseline.FullComparator && !candidate.FullComparator && baseline.SemanticValidation == "precomputed_exact_path_inputs" && candidate.SemanticValidation == "precomputed_exact_path_inputs" + orderedComparators := !baseline.FullComparator && !candidate.FullComparator && baseline.ObservationShape == "ordered_ids" && candidate.ObservationShape == "ordered_ids" && baseline.SemanticValidation == "exact_ordered_ids" && candidate.SemanticValidation == "exact_ordered_ids" + if !fullComparators && !hydrationComparators && !orderedComparators { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference pair does not share an exact comparable boundary", record.Dataset, record.Name) + } + if (fullComparators || hydrationComparators) && (baseline.RowCount != record.RowCount || candidate.RowCount != record.RowCount || !slices.Equal(baseline.ObservedRows, record.ObservedRows) || !slices.Equal(candidate.ObservedRows, record.ObservedRows)) { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair observation differs from production", record.Dataset, record.Name) + } + if orderedComparators && (baseline.RowCount != candidate.RowCount || !slices.Equal(baseline.ObservedRows, candidate.ObservedRows)) { + return ReferencePairReport{}, fmt.Errorf("%s/%s ordered-ID reference-pair observations differ", record.Dataset, record.Name) + } + if baseline.ImplementationID == "" || candidate.ImplementationID == "" || baseline.SQLFingerprint == "" || candidate.SQLFingerprint == "" || record.Environment.BinarySHA256 == "" { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d lacks complete reference-pair implementation identity", record.Dataset, record.Name, record.Environment.Round) + } + if baseline.Stats.WarmupIterations < minimumWarmups || candidate.Stats.WarmupIterations < minimumWarmups || baseline.MeasurementOrder <= 0 || candidate.MeasurementOrder <= 0 || baseline.MeasurementOrder == candidate.MeasurementOrder { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d lacks warm, ordered reference-pair measurements", record.Dataset, record.Name, record.Environment.Round) + } + binaryIdentity := fmt.Sprintf("%s\x00%s\x00%s\x00%s\x00%s", record.Environment.BinarySHA256, record.Environment.DirtyDiffSHA256, record.Environment.SourceCommit, record.Environment.GOOS, record.Environment.GOARCH) + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + } + if seen[key] == nil { + seen[key] = map[int]struct{}{} + } + if _, duplicate := seen[key][record.Environment.Round]; duplicate { + return ReferencePairReport{}, fmt.Errorf("%s/%s has duplicate round %d", record.Dataset, record.Name, record.Environment.Round) + } + seen[key][record.Environment.Round] = struct{}{} + + if series[key] == nil { + series[key] = &pairSeries{ + baseline: roundSamples{}, + candidate: roundSamples{}, + baselineArchitecture: baseline.Architecture, + candidateArchitecture: candidate.Architecture, + baselineBoundary: baseline.Boundary, + candidateBoundary: candidate.Boundary, + baselineValidation: baseline.SemanticValidation, + candidateValidation: candidate.SemanticValidation, + baselineImplementation: baseline.ImplementationID, + candidateImplementation: candidate.ImplementationID, + baselineSQLFingerprint: baseline.SQLFingerprint, + candidateSQLFingerprint: candidate.SQLFingerprint, + binaryIdentity: binaryIdentity, + baselineFirst: map[int]bool{}, + } + } else if series[key].baselineArchitecture != baseline.Architecture || series[key].candidateArchitecture != candidate.Architecture || + series[key].baselineBoundary != baseline.Boundary || series[key].candidateBoundary != candidate.Boundary || + series[key].baselineValidation != baseline.SemanticValidation || series[key].candidateValidation != candidate.SemanticValidation || + series[key].baselineImplementation != baseline.ImplementationID || series[key].candidateImplementation != candidate.ImplementationID || + series[key].baselineSQLFingerprint != baseline.SQLFingerprint || series[key].candidateSQLFingerprint != candidate.SQLFingerprint || + series[key].binaryIdentity != binaryIdentity { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair identity changed across rounds", record.Dataset, record.Name) + } + + series[key].baselineFirst[record.Environment.Round] = baseline.MeasurementOrder < candidate.MeasurementOrder + for _, sample := range baseline.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[key].baseline[record.Environment.Round] = append(series[key].baseline[record.Environment.Round], sample.Duration) + } + } + for _, sample := range candidate.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[key].candidate[record.Environment.Round] = append(series[key].candidate[record.Environment.Round], sample.Duration) + } + } + } + if len(series) == 0 { + return ReferencePairReport{}, fmt.Errorf("artifact has no PostgreSQL reference-pair records") + } + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + report := ReferencePairReport{ + Version: referencePairReportVersion, + Seed: options.Seed, + Confidence: options.Confidence, + BaselineName: options.BaselineName, + CandidateName: options.CandidateName, + Protocol: protocol, + MinimumWarmups: minimumWarmups, + MinimumRounds: minimumRounds, + MaximumRounds: maximumRounds, + MinimumSamples: minimumSamples, + } + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for idx, key := range keys { + baseline, candidate := matchedRounds(series[key].baseline, series[key].candidate) + if len(baseline) < minimumRounds || len(baseline) > maximumRounds { + return ReferencePairReport{}, fmt.Errorf("%s/%s requires %d-%d matched rounds, got %d", key.dataset, key.name, minimumRounds, maximumRounds, len(baseline)) + } + rounds := sortedRounds(baseline) + baselineFirstCount := 0 + for roundIdx, round := range rounds { + baselineFirst := series[key].baselineFirst[round] + if baselineFirst { + baselineFirstCount++ + } + if roundIdx > 0 && series[key].baselineFirst[rounds[roundIdx-1]] == baselineFirst { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair arm order does not alternate across rounds", key.dataset, key.name) + } + } + candidateFirstCount := len(rounds) - baselineFirstCount + if baselineFirstCount-candidateFirstCount > 1 || candidateFirstCount-baselineFirstCount > 1 { + return ReferencePairReport{}, fmt.Errorf("%s/%s reference-pair arm order is not balanced", key.dataset, key.name) + } + for _, round := range rounds { + if len(baseline[round]) < minimumSamples || len(candidate[round]) < minimumSamples { + return ReferencePairReport{}, fmt.Errorf("%s/%s round %d requires %d samples per arm", key.dataset, key.name, round, minimumSamples) + } + } + seed := options.Seed + int64(idx)*7919 + report.Cases = append(report.Cases, ReferencePairCase{ + Dataset: key.dataset, + Name: key.name, + Rounds: len(baseline), + BaselineArchitecture: series[key].baselineArchitecture, + CandidateArchitecture: series[key].candidateArchitecture, + BaselineBoundary: series[key].baselineBoundary, + CandidateBoundary: series[key].candidateBoundary, + BaselineSemanticValidation: series[key].baselineValidation, + CandidateSemanticValidation: series[key].candidateValidation, + BaselineSamples: sampleCount(baseline), + CandidateSamples: sampleCount(candidate), + MedianRatio: bootstrapRoundMedianRatio(baseline, candidate, seed, gateOptions), + P95Ratio: bootstrapStratifiedP95Ratio(baseline, candidate, seed+4, gateOptions), + MedianChange: negateDurationInterval(bootstrapRoundMedianSaving(baseline, candidate, seed+1, gateOptions)), + BaselineAAResolution: withinSessionAAResolution(baseline, seed+2, gateOptions), + CandidateAAResolution: withinSessionAAResolution(candidate, seed+3, gateOptions), + }) + } + return report, nil +} + +// findReference returns the named PostgreSQL reference result or nil when it is absent. +func findReference(references []PostgresReferenceResult, name string) *PostgresReferenceResult { + for idx := range references { + if references[idx].Name == name { + return &references[idx] + } + } + return nil +} + +// createReferencePairReport loads benchmark records, builds a reference-pair report, and writes it as JSON. +func createReferencePairReport(artifactPath, outputPath string, options ReferencePairOptions) error { + records, err := readJSONLFile(artifactPath) + if err != nil { + return err + } + report, err := buildReferencePairReport(records, options) + if err != nil { + return err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return err + } + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + encoded = append(encoded, '\n') + if outputPath == "" { + _, err = os.Stdout.Write(encoded) + return err + } + if err := ensureOutputDir(outputPath); err != nil { + return err + } + return os.WriteFile(outputPath, encoded, 0o644) +} diff --git a/cmd/graphbench/reference_pair_report_test.go b/cmd/graphbench/reference_pair_report_test.go new file mode 100644 index 00000000..c6934ec6 --- /dev/null +++ b/cmd/graphbench/reference_pair_report_test.go @@ -0,0 +1,416 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildReferencePairReportComparesExactMatchedArms verifies median/P95 ratios and absolute change across ten carryover-balanced full-comparator rounds. +func TestBuildReferencePairReportComparesExactMatchedArms(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + baselineOrder, candidateOrder := 2, 3 + if round%2 == 0 { + baselineOrder, candidateOrder = 3, 2 + } + record := CaseResult{ + Dataset: "fixture", + Name: "distance", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[2]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 20, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "s3", + Architecture: "SP-S3-U-D", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"[2]"}, + MeasurementOrder: baselineOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + { + Name: "s1", + Architecture: "SP-S1", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"[2]"}, + MeasurementOrder: candidateOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 50; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: time.Millisecond, + }) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: 2 * time.Millisecond, + }) + } + records = append(records, record) + } + + report, err := buildReferencePairReport(records, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + BaselineName: "s3", + CandidateName: "s1", + }) + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.Equal(t, 10, report.Cases[0].Rounds) + require.InDelta(t, 2, report.Cases[0].MedianRatio.Estimate, 0.0001) + require.InDelta(t, 2, report.Cases[0].P95Ratio.Estimate, 0.0001) + require.Equal(t, time.Millisecond, report.Cases[0].MedianChange.Estimate) +} + +// TestBuildReferencePairReportComparesValidatedHydrationBoundaries verifies that two prevalidated hydration implementations remain comparable despite different input-boundary descriptions. +func TestBuildReferencePairReportComparesValidatedHydrationBoundaries(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + baselineOrder, candidateOrder := 2, 3 + if round%2 == 0 { + baselineOrder, candidateOrder = 3, 2 + } + record := CaseResult{ + Dataset: "fixture", + Name: "path", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[path]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 20, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "m0", + Architecture: "MAT-M0", + Boundary: "edge IDs", + SemanticValidation: "precomputed_exact_path_inputs", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: baselineOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + { + Name: "m1", + Architecture: "MAT-M1", + Boundary: "node and edge IDs", + SemanticValidation: "precomputed_exact_path_inputs", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: candidateOrder, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 50; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: time.Millisecond, + }) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: 2 * time.Millisecond, + }) + } + records = append(records, record) + } + + report, err := buildReferencePairReport(records, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + BaselineName: "m0", + CandidateName: "m1", + }) + require.NoError(t, err) + require.Len(t, report.Cases, 1) + require.Equal(t, "precomputed_exact_path_inputs", report.Cases[0].BaselineSemanticValidation) + require.Equal(t, "edge IDs", report.Cases[0].BaselineBoundary) + require.InDelta(t, 2, report.Cases[0].MedianRatio.Estimate, 0.0001) + require.InDelta(t, 2, report.Cases[0].P95Ratio.Estimate, 0.0001) +} + +// TestBuildReferencePairReportRejectsMixedExactBoundaries verifies that a full public-result comparator cannot be timed against a precomputed hydration-only boundary. +func TestBuildReferencePairReportRejectsMixedExactBoundaries(t *testing.T) { + record := CaseResult{ + Dataset: "fixture", + Name: "path", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[path]"}, + Environment: &RunEnvironment{ + Round: 1, + WarmupIterations: 20, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "full", + FullComparator: true, + SemanticValidation: "exact_public_observation", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: 2, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + { + Name: "hydration", + SemanticValidation: "precomputed_exact_path_inputs", + RowCount: 1, + ObservedRows: []string{"[path]"}, + MeasurementOrder: 3, + Stats: DurationStats{ + WarmupIterations: 20, + }, + }, + }, + } + + _, err := buildReferencePairReport([]CaseResult{record}, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BaselineName: "full", + CandidateName: "hydration", + }) + require.ErrorContains(t, err, "does not share an exact comparable boundary") +} + +// TestBuildReferencePairReportSupportsLabeledOrderedIDDiscovery verifies the reduced discovery protocol thresholds and ratio calculation for exact ordered-ID observations. +func TestBuildReferencePairReportSupportsLabeledOrderedIDDiscovery(t *testing.T) { + records := make([]CaseResult, 0, 5) + for round := 1; round <= 5; round++ { + baselineOrder, candidateOrder := 2, 3 + if round%2 == 0 { + baselineOrder, candidateOrder = 3, 2 + } + record := CaseResult{ + Dataset: "fixture", + Name: "ordered", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[public]"}, + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 5, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "search_ordered_ids", + Architecture: "EXPANSION-STEPWISE-FORWARD", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[[1,2],3,[4]]"}, + MeasurementOrder: baselineOrder, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + { + Name: "suffix_seeded_reverse_ordered_ids", + Architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[[1,2],3,[4]]"}, + MeasurementOrder: candidateOrder, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 10; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: 2 * time.Millisecond, + }) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{ + Round: round, + Iteration: iteration, + Classification: "warm", + Duration: time.Millisecond, + }) + } + records = append(records, record) + } + + report, err := buildReferencePairReport(records, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BootstrapCount: 100, + BaselineName: "search_ordered_ids", + CandidateName: "suffix_seeded_reverse_ordered_ids", + Protocol: referencePairProtocolDiscovery, + }) + require.NoError(t, err) + require.Equal(t, referencePairProtocolDiscovery, report.Protocol) + require.Equal(t, 5, report.MinimumWarmups) + require.Equal(t, 5, report.MinimumRounds) + require.Equal(t, 10, report.MinimumSamples) + require.Len(t, report.Cases, 1) + require.InDelta(t, 0.5, report.Cases[0].MedianRatio.Estimate, 0.0001) +} + +// TestBuildReferencePairReportRejectsChangedImplementationIdentity verifies that an arm's implementation fingerprint must remain constant across all measurement rounds. +func TestBuildReferencePairReportRejectsChangedImplementationIdentity(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + records = append(records, referencePairProtocolRecord(round, round%2 == 1)) + } + records[4].PostgresReferences[0].ImplementationID = "changed" + _, err := buildReferencePairReport(records, ReferencePairOptions{ + Confidence: 0.975, + BaselineName: "baseline", + CandidateName: "candidate", + }) + require.ErrorContains(t, err, "identity changed") +} + +// TestBuildReferencePairReportRejectsUnbalancedArmOrder verifies that repeatedly measuring the same arm first violates the carryover-balancing protocol. +func TestBuildReferencePairReportRejectsUnbalancedArmOrder(t *testing.T) { + records := make([]CaseResult, 0, 10) + for round := 1; round <= 10; round++ { + records = append(records, referencePairProtocolRecord(round, true)) + } + _, err := buildReferencePairReport(records, ReferencePairOptions{ + Confidence: 0.975, + BaselineName: "baseline", + CandidateName: "candidate", + }) + require.ErrorContains(t, err, "does not alternate") +} + +// referencePairProtocolRecord returns one exact comparator round with selectable arm order and uniform warm timing samples. +func referencePairProtocolRecord(round int, baselineFirst bool) CaseResult { + baselineOrder, candidateOrder := 2, 3 + if !baselineFirst { + baselineOrder, candidateOrder = candidateOrder, baselineOrder + } + record := CaseResult{ + Dataset: "fixture", + Name: "protocol", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"[1]"}, + Environment: &RunEnvironment{Round: round, WarmupIterations: 20}, + PostgresReferences: []PostgresReferenceResult{ + {Name: "baseline", Architecture: "A", FullComparator: true, SemanticValidation: "exact_public_observation", RowCount: 1, ObservedRows: []string{"[1]"}, MeasurementOrder: baselineOrder, Stats: DurationStats{WarmupIterations: 20}}, + {Name: "candidate", Architecture: "B", FullComparator: true, SemanticValidation: "exact_public_observation", RowCount: 1, ObservedRows: []string{"[1]"}, MeasurementOrder: candidateOrder, Stats: DurationStats{WarmupIterations: 20}}, + }, + } + stampReferencePairIdentity(&record) + for iteration := 1; iteration <= 50; iteration++ { + record.PostgresReferences[0].Stats.Samples = append(record.PostgresReferences[0].Stats.Samples, LatencySample{Round: round, Iteration: iteration, Classification: "warm", Duration: time.Millisecond}) + record.PostgresReferences[1].Stats.Samples = append(record.PostgresReferences[1].Stats.Samples, LatencySample{Round: round, Iteration: iteration, Classification: "warm", Duration: 2 * time.Millisecond}) + } + return record +} + +// stampReferencePairIdentity assigns a stable runtime, implementation, and SQL identity to both reference arms. +func stampReferencePairIdentity(record *CaseResult) { + record.Environment.BinarySHA256 = "binary" + record.Environment.DirtyDiffSHA256 = "dirty" + record.Environment.SourceCommit = "commit" + record.Environment.GOOS = "linux" + record.Environment.GOARCH = "amd64" + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + reference.ImplementationID = reference.Name + "-implementation" + reference.SQLFingerprint = reference.Name + "-sql" + } +} + +// TestBuildReferencePairReportRejectsMismatchedOrderedIDObservations verifies that discovery timing cannot compare arms whose ordered-ID result sequences differ. +func TestBuildReferencePairReportRejectsMismatchedOrderedIDObservations(t *testing.T) { + record := CaseResult{ + Dataset: "fixture", + Name: "ordered", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + Round: 1, + WarmupIterations: 5, + }, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "search_ordered_ids", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[a]"}, + MeasurementOrder: 2, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + { + Name: "suffix_seeded_reverse_ordered_ids", + ObservationShape: "ordered_ids", + SemanticValidation: "exact_ordered_ids", + RowCount: 1, + ObservedRows: []string{"[b]"}, + MeasurementOrder: 3, + Stats: DurationStats{ + WarmupIterations: 5, + }, + }, + }, + } + + _, err := buildReferencePairReport([]CaseResult{record}, ReferencePairOptions{ + Seed: 1, + Confidence: 0.975, + BaselineName: "search_ordered_ids", + CandidateName: "suffix_seeded_reverse_ordered_ids", + Protocol: referencePairProtocolDiscovery, + }) + require.ErrorContains(t, err, "ordered-ID reference-pair observations differ") +} diff --git a/cmd/graphbench/reference_tournament_report.go b/cmd/graphbench/reference_tournament_report.go new file mode 100644 index 00000000..6bb2ee4b --- /dev/null +++ b/cmd/graphbench/reference_tournament_report.go @@ -0,0 +1,473 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "time" +) + +// referenceTournamentReportVersion reserves the stable protocol value used to recognize reference tournament report version across artifacts and executions. +const referenceTournamentReportVersion = 1 + +// ReferenceTournamentOptions defines a predeclared three- or five-arm Williams tournament. +// The first arm is always the incumbent. +type ReferenceTournamentOptions struct { + // Seed makes randomized statistical procedures reproducible. + Seed int64 + // BootstrapCount records the number of bootstrap count. + BootstrapCount int + // Confidence sets the requested statistical confidence level. + Confidence float64 + // MaterialityRatio supplies the materiality ratio input to the ReferenceTournamentOptions contract. + MaterialityRatio float64 + // MaterialityAbsolute supplies the materiality absolute input to the ReferenceTournamentOptions contract. + MaterialityAbsolute time.Duration + // P95RatioLimit supplies the p95 ratio limit input to the ReferenceTournamentOptions contract. + P95RatioLimit float64 + // Arms supplies the arms input to the ReferenceTournamentOptions contract. + Arms []string + // Protocol identifies the protocol. + Protocol string +} + +// ReferenceTournamentPair groups state that must remain consistent while processing reference tournament pair. +type ReferenceTournamentPair struct { + // Arm supplies the arm input to the ReferenceTournamentPair contract. + Arm string `json:"arm"` + // MedianRatio supplies the median ratio input to the ReferenceTournamentPair contract. + MedianRatio RatioInterval `json:"median_ratio_to_incumbent"` + // MedianSaving supplies the median saving input to the ReferenceTournamentPair contract. + MedianSaving DurationInterval `json:"median_saving_vs_incumbent"` + // P95Ratio supplies the p95 ratio input to the ReferenceTournamentPair contract. + P95Ratio RatioInterval `json:"p95_ratio_to_incumbent"` + // Material indicates whether material applies. + Material bool `json:"material"` + // P95Contained indicates whether p95 contained applies. + P95Contained bool `json:"p95_contained"` + // QualifiedWinner indicates whether qualified winner applies. + QualifiedWinner bool `json:"qualified_winner"` +} + +// ReferenceTournamentCase records the evidence and decision for one reference tournament workload. +type ReferenceTournamentCase struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset"` + // Name identifies the name. + Name string `json:"name"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // Winner supplies the winner input to the ReferenceTournamentCase contract. + Winner string `json:"winner,omitempty"` + // Rounds records the number of rounds. + Rounds int `json:"rounds"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` + // Pairs supplies the pairs input to the ReferenceTournamentCase contract. + Pairs []ReferenceTournamentPair `json:"pairs"` +} + +// ReferenceTournamentReport records the evidence and outcome produced by reference tournament. +type ReferenceTournamentReport struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // ArtifactSHA256 binds the referenced artifact content by SHA-256 digest. + ArtifactSHA256 string `json:"artifact_sha256,omitempty"` + // Protocol identifies the protocol. + Protocol string `json:"protocol"` + // Incumbent supplies the incumbent input to the ReferenceTournamentReport contract. + Incumbent string `json:"incumbent"` + // Winner supplies the winner input to the ReferenceTournamentReport contract. + Winner string `json:"winner,omitempty"` + // Arms supplies the arms input to the ReferenceTournamentReport contract. + Arms []string `json:"arms"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // MaterialityRatio supplies the materiality ratio input to the ReferenceTournamentReport contract. + MaterialityRatio float64 `json:"materiality_ratio"` + // MaterialityAbsolute supplies the materiality absolute input to the ReferenceTournamentReport contract. + MaterialityAbsolute time.Duration `json:"materiality_absolute_lower_limit"` + // P95RatioLimit supplies the p95 ratio limit input to the ReferenceTournamentReport contract. + P95RatioLimit float64 `json:"p95_ratio_upper_limit"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // PromotionEligible indicates whether promotion eligible applies. + PromotionEligible bool `json:"promotion_eligible"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []ReferenceTournamentCase `json:"cases"` +} + +// tournamentArmSeries accumulates matched observations used to evaluate tournament arm. +type tournamentArmSeries struct { + // identity retains the identity while tournamentArmSeries is assembled or evaluated. + identity string + // samples retains the samples while tournamentArmSeries is assembled or evaluated. + samples roundSamples +} + +// tournamentCaseSeries accumulates matched observations used to evaluate tournament case. +type tournamentCaseSeries struct { + // split retains the split while tournamentCaseSeries is assembled or evaluated. + split string + // arms retains the arms while tournamentCaseSeries is assembled or evaluated. + arms map[string]*tournamentArmSeries + // rounds retains the rounds while tournamentCaseSeries is assembled or evaluated. + rounds map[int]struct{} +} + +// buildReferenceTournamentReport builds reference tournament report. +func buildReferenceTournamentReport(records []CaseResult, options ReferenceTournamentOptions) (ReferenceTournamentReport, error) { + if err := normalizeReferenceTournamentOptions(&options); err != nil { + return ReferenceTournamentReport{}, err + } + minimumWarmups, minimumRounds, maximumRounds, minimumSamples, err := referenceTournamentRequirements(options.Protocol) + if err != nil { + return ReferenceTournamentReport{}, err + } + + series := map[performanceKey]*tournamentCaseSeries{} + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL || !recordContainsAnyReference(record, options.Arms) { + continue + } + if err := addReferenceTournamentRecord(series, record, options.Arms, minimumWarmups); err != nil { + return ReferenceTournamentReport{}, err + } + } + if len(series) == 0 { + return ReferenceTournamentReport{}, fmt.Errorf("artifact has no PostgreSQL reference tournament records") + } + + report := ReferenceTournamentReport{ + Version: referenceTournamentReportVersion, + Protocol: options.Protocol, + Arms: append([]string(nil), options.Arms...), + Incumbent: options.Arms[0], + Confidence: options.Confidence, + MaterialityRatio: options.MaterialityRatio, + MaterialityAbsolute: options.MaterialityAbsolute, + P95RatioLimit: options.P95RatioLimit, + Passed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + keys := sortedTournamentPerformanceKeys(series) + gate := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + winners := map[string]struct{}{} + for caseIndex, key := range keys { + entry := evaluateReferenceTournamentCase(key, series[key], options, gate, caseIndex, minimumRounds, maximumRounds, minimumSamples) + if entry.Passed { + winners[entry.Winner] = struct{}{} + } else { + report.Passed = false + } + switch entry.QualificationSplit { + case "training": + report.TrainingPassed = report.TrainingPassed && entry.Passed + case "holdout": + report.HoldoutPassed = report.HoldoutPassed && entry.Passed + } + report.Cases = append(report.Cases, entry) + } + + report.TrainingPassed = report.TrainingPassed && tournamentHasSplit(report.Cases, "training") + report.HoldoutPassed = report.HoldoutPassed && tournamentHasSplit(report.Cases, "holdout") + if len(winners) == 1 { + for winner := range winners { + report.Winner = winner + } + } else { + report.Passed = false + } + report.Passed = report.Passed && report.TrainingPassed && report.HoldoutPassed && report.Winner != "" + report.PromotionEligible = options.Protocol == referencePairProtocolConfirmation && report.Passed + return report, nil +} + +// normalizeReferenceTournamentOptions normalizes reference tournament options. +func normalizeReferenceTournamentOptions(options *ReferenceTournamentOptions) error { + if len(options.Arms) != 3 && len(options.Arms) != 5 { + return fmt.Errorf("reference tournament requires exactly 3 or 5 arms") + } + seen := map[string]struct{}{} + for _, arm := range options.Arms { + if arm == "" { + return fmt.Errorf("reference tournament arm must not be empty") + } + if _, duplicate := seen[arm]; duplicate { + return fmt.Errorf("reference tournament arms must be distinct") + } + seen[arm] = struct{}{} + } + if options.Confidence <= 0 || options.Confidence >= 1 { + return fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.MaterialityRatio == 0 { + options.MaterialityRatio = .95 + } + if options.MaterialityRatio <= 0 || options.MaterialityRatio >= 1 { + return fmt.Errorf("materiality ratio must be between 0 and 1") + } + if options.MaterialityAbsolute == 0 { + options.MaterialityAbsolute = 100 * time.Microsecond + } + if options.MaterialityAbsolute < 0 { + return fmt.Errorf("materiality absolute must not be negative") + } + if options.P95RatioLimit == 0 { + options.P95RatioLimit = 1.05 + } + if options.P95RatioLimit <= 0 { + return fmt.Errorf("p95 ratio limit must be positive") + } + if options.Protocol == "" { + options.Protocol = referencePairProtocolConfirmation + } + return nil +} + +// referenceTournamentRequirements supports benchmark evidence processing for reference tournament requirements. +func referenceTournamentRequirements(protocol string) (int, int, int, int, error) { + switch protocol { + case referencePairProtocolDiscovery: + return 5, 5, 20, 10, nil + case referencePairProtocolConfirmation: + return 20, 10, 20, 50, nil + default: + return 0, 0, 0, 0, fmt.Errorf("unsupported reference tournament protocol %q", protocol) + } +} + +// recordContainsAnyReference supports benchmark evidence processing for record contains any reference. +func recordContainsAnyReference(record CaseResult, arms []string) bool { + for _, reference := range record.PostgresReferences { + if slices.Contains(arms, reference.Name) { + return true + } + } + return false +} + +// addReferenceTournamentRecord supports benchmark evidence processing for add reference tournament record. +func addReferenceTournamentRecord(series map[performanceKey]*tournamentCaseSeries, record CaseResult, arms []string, minimumWarmups int) error { + if record.Status != StatusOK || record.Environment == nil || record.Environment.WarmupIterations < minimumWarmups { + return fmt.Errorf("%s/%s lacks a successful %d-warmup PostgreSQL record", record.Dataset, record.Name, minimumWarmups) + } + if record.Shape.QualificationSplit != "training" && record.Shape.QualificationSplit != "holdout" { + return fmt.Errorf("%s/%s requires a training or holdout qualification split", record.Dataset, record.Name) + } + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + } + current := series[key] + if current == nil { + current = &tournamentCaseSeries{ + split: record.Shape.QualificationSplit, + arms: map[string]*tournamentArmSeries{}, + rounds: map[int]struct{}{}, + } + series[key] = current + } else if current.split != record.Shape.QualificationSplit { + return fmt.Errorf("%s/%s changes qualification split across rounds", record.Dataset, record.Name) + } + if record.Environment.Round < 1 { + return fmt.Errorf("%s/%s has invalid tournament round %d", record.Dataset, record.Name, record.Environment.Round) + } + if _, duplicate := current.rounds[record.Environment.Round]; duplicate { + return fmt.Errorf("%s/%s has duplicate tournament round %d", record.Dataset, record.Name, record.Environment.Round) + } + current.rounds[record.Environment.Round] = struct{}{} + if err := validateTournamentRoundOrder(record.Environment.Round, arms, record.PostgresReferences); err != nil { + return fmt.Errorf("%s/%s: %w", record.Dataset, record.Name, err) + } + for _, name := range arms { + if err := addReferenceTournamentArm(current, record, name, minimumWarmups); err != nil { + return err + } + } + return nil +} + +// addReferenceTournamentArm supports benchmark evidence processing for add reference tournament arm. +func addReferenceTournamentArm(current *tournamentCaseSeries, record CaseResult, name string, minimumWarmups int) error { + reference := findReference(record.PostgresReferences, name) + if reference == nil { + return fmt.Errorf("%s/%s lacks tournament arm %s", record.Dataset, record.Name, name) + } + if !reference.FullComparator || reference.SemanticValidation != "exact_public_observation" || reference.RowCount != record.RowCount || !slices.Equal(reference.ObservedRows, record.ObservedRows) { + return fmt.Errorf("%s/%s arm %s is not an exact public comparator", record.Dataset, record.Name, name) + } + if reference.Stats.WarmupIterations < minimumWarmups || reference.ImplementationID == "" || reference.SQLFingerprint == "" { + return fmt.Errorf("%s/%s arm %s lacks warmups or identity", record.Dataset, record.Name, name) + } + identity := reference.Architecture + "\x00" + reference.ImplementationID + "\x00" + reference.SQLFingerprint + "\x00" + reference.Boundary + arm := current.arms[name] + if arm == nil { + arm = &tournamentArmSeries{ + identity: identity, + samples: roundSamples{}, + } + current.arms[name] = arm + } else if arm.identity != identity { + return fmt.Errorf("%s/%s arm %s identity changed", record.Dataset, record.Name, name) + } + for _, sample := range reference.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + arm.samples[record.Environment.Round] = append(arm.samples[record.Environment.Round], sample.Duration) + } + } + return nil +} + +// sortedTournamentPerformanceKeys returns the lookup keys used for sorted tournament performance. +func sortedTournamentPerformanceKeys(series map[performanceKey]*tournamentCaseSeries) []performanceKey { + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + return keys[i].dataset < keys[j].dataset || keys[i].dataset == keys[j].dataset && keys[i].name < keys[j].name + }) + return keys +} + +// evaluateReferenceTournamentCase supports benchmark evidence processing for evaluate reference tournament case. +func evaluateReferenceTournamentCase(key performanceKey, current *tournamentCaseSeries, options ReferenceTournamentOptions, gate PerfGateOptions, caseIndex, minimumRounds, maximumRounds, minimumSamples int) ReferenceTournamentCase { + entry := ReferenceTournamentCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: current.split, + Rounds: len(current.rounds), + Passed: true, + } + if entry.Rounds < minimumRounds || entry.Rounds > maximumRounds { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("requires %d-%d Williams rounds, got %d", minimumRounds, maximumRounds, entry.Rounds)) + } + for _, name := range options.Arms { + for _, round := range sortedRoundSet(current.rounds) { + if len(current.arms[name].samples[round]) < minimumSamples { + entry.Passed = false + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s round %d requires %d samples", name, round, minimumSamples)) + } + } + } + + incumbent := current.arms[options.Arms[0]].samples + bestMedian := time.Duration(1<<63 - 1) + for armIndex, name := range options.Arms[1:] { + baseline, candidate := matchedRounds(incumbent, current.arms[name].samples) + seed := options.Seed + int64(caseIndex*31+armIndex)*7919 + pair := ReferenceTournamentPair{ + Arm: name, + MedianRatio: bootstrapRoundMedianRatio(baseline, candidate, seed, gate), + MedianSaving: bootstrapRoundMedianSaving(baseline, candidate, seed+1, gate), + P95Ratio: bootstrapStratifiedP95Ratio(baseline, candidate, seed+2, gate), + } + pair.Material = pair.MedianRatio.Upper <= options.MaterialityRatio || pair.MedianSaving.Lower >= options.MaterialityAbsolute + pair.P95Contained = pair.P95Ratio.Upper <= options.P95RatioLimit + pair.QualifiedWinner = pair.Material && pair.P95Contained + if pair.QualifiedWinner { + median := time.Duration(durationQuantile(flattenSamples(candidate, sortedRounds(candidate)), .5)) + if median < bestMedian { + bestMedian, entry.Winner = median, name + } + } + entry.Pairs = append(entry.Pairs, pair) + } + if entry.Winner == "" { + entry.Passed = false + entry.Reasons = append(entry.Reasons, "no candidate materially beats the incumbent with p95 containment") + } + return entry +} + +// tournamentHasSplit supports benchmark evidence processing for tournament has split. +func tournamentHasSplit(cases []ReferenceTournamentCase, split string) bool { + for _, entry := range cases { + if entry.QualificationSplit == split { + return true + } + } + return false +} + +// validateTournamentRoundOrder validates tournament round order. +func validateTournamentRoundOrder(round int, arms []string, references []PostgresReferenceResult) error { + base := make([]postgresReferenceSpec, len(arms)) + for idx, arm := range arms { + base[idx] = postgresReferenceSpec{name: arm} + } + expected := referenceSpecsForRound(base, round) + orders := map[string]int{} + for _, reference := range references { + if slices.Contains(arms, reference.Name) { + orders[reference.Name] = reference.MeasurementOrder + } + } + for idx, spec := range expected { + // Production is measurement position one when more than one reference + // arm is selected; the tournament occupies the contiguous suffix. + if orders[spec.name] != idx+2 { + return fmt.Errorf("round %d does not match the declared %d-arm Williams order", round, len(arms)) + } + } + return nil +} + +// createReferenceTournamentReport creates reference tournament report. +func createReferenceTournamentReport(artifactPath, outputPath string, options ReferenceTournamentOptions) (bool, error) { + records, err := readJSONLFile(artifactPath) + if err != nil { + return false, err + } + report, err := buildReferenceTournamentReport(records, options) + if err != nil { + return false, err + } + report.ArtifactSHA256, err = fileSHA256(artifactPath) + if err != nil { + return false, err + } + var output *os.File + if outputPath == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(outputPath); err != nil { + return false, err + } + output, err = os.Create(outputPath) + if err != nil { + return false, err + } + defer output.Close() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + if err := encoder.Encode(report); err != nil { + return false, err + } + return report.PromotionEligible, nil +} diff --git a/cmd/graphbench/reference_tournament_report_test.go b/cmd/graphbench/reference_tournament_report_test.go new file mode 100644 index 00000000..bd58cc21 --- /dev/null +++ b/cmd/graphbench/reference_tournament_report_test.go @@ -0,0 +1,136 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestBuildReferenceTournamentReportQualifiesStableHoldoutWinner verifies build reference tournament report qualifies stable holdout winner behavior. +func TestBuildReferenceTournamentReportQualifiesStableHoldoutWinner(t *testing.T) { + arms := []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"} + var records []CaseResult + for round := 1; round <= 12; round++ { + for _, split := range []string{"training", "holdout"} { + records = append(records, referenceTournamentRecord(arms, round, split, map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, + arms[1]: 7 * time.Millisecond, + arms[2]: 5 * time.Millisecond, + })) + } + } + + report, err := buildReferenceTournamentReport(records, ReferenceTournamentOptions{ + Seed: 1, + BootstrapCount: 100, + Confidence: .975, + Arms: arms, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.True(t, report.Passed) + require.True(t, report.PromotionEligible) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.Equal(t, arms[2], report.Winner) + require.Len(t, report.Cases, 2) + for _, entry := range report.Cases { + require.True(t, entry.Passed) + require.Equal(t, arms[2], entry.Winner) + } +} + +// TestBuildReferenceTournamentReportRejectsOrderAndWinnerDrift verifies build reference tournament report rejects order and winner drift behavior. +func TestBuildReferenceTournamentReportRejectsOrderAndWinnerDrift(t *testing.T) { + arms := []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"} + badOrder := referenceTournamentRecord(arms, 1, "training", map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, arms[1]: 7 * time.Millisecond, arms[2]: 5 * time.Millisecond, + }) + badOrder.PostgresReferences[0].MeasurementOrder = 99 + _, err := buildReferenceTournamentReport([]CaseResult{badOrder}, ReferenceTournamentOptions{ + Confidence: .975, + Arms: arms, + Protocol: referencePairProtocolDiscovery, + }) + require.ErrorContains(t, err, "Williams order") + + var records []CaseResult + for round := 1; round <= 10; round++ { + records = append(records, + referenceTournamentRecord(arms, round, "training", map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, arms[1]: 5 * time.Millisecond, arms[2]: 7 * time.Millisecond, + }), + referenceTournamentRecord(arms, round, "holdout", map[string]time.Duration{ + arms[0]: 10 * time.Millisecond, arms[1]: 7 * time.Millisecond, arms[2]: 5 * time.Millisecond, + }), + ) + } + report, err := buildReferenceTournamentReport(records, ReferenceTournamentOptions{ + Seed: 1, + BootstrapCount: 100, + Confidence: .975, + Arms: arms, + Protocol: referencePairProtocolConfirmation, + }) + require.NoError(t, err) + require.False(t, report.Passed) + require.False(t, report.PromotionEligible) + require.Empty(t, report.Winner) +} + +// referenceTournamentRecord prepares or inspects test evidence for reference tournament record. +func referenceTournamentRecord(arms []string, round int, split string, durations map[string]time.Duration) CaseResult { + record := CaseResult{ + Environment: &RunEnvironment{ + Round: round, + WarmupIterations: 20, + }, + Dataset: "tournament", + Name: "case-" + split, + Shape: WorkloadShape{QualificationSplit: split}, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RowCount: 1, + ObservedRows: []string{"row"}, + } + base := make([]postgresReferenceSpec, len(arms)) + for idx, arm := range arms { + base[idx].name = arm + } + orders := map[string]int{} + for idx, spec := range referenceSpecsForRound(base, round) { + orders[spec.name] = idx + 2 + } + for _, arm := range arms { + samples := make([]LatencySample, 50) + for idx := range samples { + samples[idx] = LatencySample{ + Classification: "warm", + Duration: durations[arm] + time.Duration(idx), + } + } + record.PostgresReferences = append(record.PostgresReferences, PostgresReferenceResult{ + Name: arm, + Architecture: arm, + ImplementationID: arm + "-v1", + SQLFingerprint: arm + "-sql-v1", + Boundary: "relationships", + FullComparator: true, + SemanticValidation: "exact_public_observation", + MeasurementOrder: orders[arm], + RowCount: 1, + ObservedRows: []string{"row"}, + Stats: DurationStats{ + WarmupIterations: 20, + Samples: samples, + }, + }) + } + return record +} diff --git a/cmd/graphbench/references.go b/cmd/graphbench/references.go new file mode 100644 index 00000000..815716dc --- /dev/null +++ b/cmd/graphbench/references.go @@ -0,0 +1,2036 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" +) + +// postgresReferenceSchemaVersion identifies the serialized schema revision for PostgreSQL reference schema. +const postgresReferenceSchemaVersion = 1 + +// postgresReferenceArms lists the independently implemented PostgreSQL comparison arms. +var postgresReferenceArms = []string{ + "round_trip", + "endpoint_validation", + "fixed_suffix_rows", + "minimum_graph_access", + "search_ordered_ids", + "stepwise_forward_aa_ordered_ids", + "root_reuse_ordered_ids", + "late_hydration_ordered_ids", + "factored_suffix_forward_ordered_ids", + "suffix_seeded_reverse_ordered_ids", + "backward_viability_forward_ordered_ids", + "hydration_only", + "ordered_path_ids_hydration_only", + "complete_reference", + "root_reuse_complete", + "late_hydration_complete", + "factored_suffix_forward_complete", + "suffix_seeded_reverse_complete", + "backward_viability_forward_complete", + "m0_directed_hydration_only", + "m1_ordered_ids_hydration_only", + "s3_unidirectional_trail_cte", + "s3_unidirectional_cte_m0_directed", + "s3_unidirectional_cte_m1_ordered_ids", + "s3_bidirectional_trail_cte", + "s1_array_bfs_distance", + "s4_canonical_source_distance", + "s4_canonical_source_witness_m0", + "sp_b1_strict_alternating_distance", + "sp_b1_strict_alternating_witness_m0", + "sp_b2_smaller_frontier_distance", + "sp_b2_smaller_frontier_witness_m0", + "asp_a1_stored_helper_m0", + "asp_i1_inline_predecessor_dag_m0", + "asp_b1_bidirectional_dag_strict_m0", + "asp_b2_bidirectional_dag_smaller_frontier_m0", + "expand_into_pair_join", + "expand_into_lower_degree_scan", + "expand_into_pair_cache", +} + +// validPostgresReferenceArm reports whether a reference-arm selector is declared. +func validPostgresReferenceArm(name string) bool { + return slices.Contains(postgresReferenceArms, name) +} + +// postgresReferenceSpec defines one independent PostgreSQL reference implementation and its observation contract. +type postgresReferenceSpec struct { + // name is the canonical selector and serialized identity for the reference arm. + name string + // legacyName retains the compatibility alias accepted for a reference arm. + legacyName string + // architecture retains the executor architecture that must remain stable across rounds. + architecture string + // implementationID provides a versioned identity for the reference algorithm and materialization strategy. + implementationID string + // stateShape describes recursive state retained by the reference implementation. + stateShape string + // observationShape describes the normalized values returned by the reference boundary. + observationShape string + // semanticValidation describes the exact observation contract enforced for the reference. + semanticValidation string + // boundary identifies the timed boundary exposed by the reference arm. + boundary string + // fullComparator reports whether the reference produces the complete public observation. + fullComparator bool + // aaAliasOf identifies the reference arm reused as an explicit A/A alias. + aaAliasOf string + // timingBoundary describes which portion of reference execution contributes to latency samples. + timingBoundary string + // sql contains the executable SQL for an independent reference arm. + sql string + // parameters supplies resolved parameters to the reference SQL query. + parameters map[string]any + // validationSQL contains SQL used to validate affected entity counts after a write. + validationSQL string + // validationParams supplies parameters used to validate precomputed reference inputs. + validationParams map[string]any +} + +// measureReferences executes references and records its timing observations. +func (s *postgresSQLRunner) measureReferences(ctx context.Context, testCase ScaleCase, params map[string]any, idMap opengraph.IDMap, publicObservation []string, warmupIterations, iterations int) ([]PostgresReferenceResult, error) { + readOptions := s.readTransactionOptions() + specs, err := s.referenceSpecs(ctx, testCase, params) + if err != nil { + return nil, err + } + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + if err := validateReferenceSpecs(specs); err != nil { + return nil, fmt.Errorf("validate PostgreSQL reference identities: %w", err) + } + if len(s.referenceArms) > 0 { + specs, err = selectReferenceSpecs(specs, s.referenceArms) + if err != nil { + return nil, fmt.Errorf("%w for %s/%s", err, testCase.Dataset, testCase.Name) + } + } + specs = referenceSpecsForRound(specs, s.round) + results := make([]PostgresReferenceResult, 0, len(specs)) + for _, spec := range specs { + rowCount, stats, err := measureRawPostgres(ctx, s.db, spec.sql, spec.parameters, warmupIterations, iterations, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s: %w", spec.name, err) + } + var observedRows []string + if spec.fullComparator || spec.validationSQL != "" { + var observedCount int64 + err := s.db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + observedCount, observedRows, err = observeRawRows(tx, spec.sql, spec.parameters, idMap, resultContainsNodeIDs(testCase.Expected), resultContainsPaths(testCase.Expected)) + return err + }, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s exact observation: %w", spec.name, err) + } + if observedCount != rowCount { + return nil, fmt.Errorf("%s exact observation row count changed from %d to %d", spec.name, rowCount, observedCount) + } + if spec.validationSQL != "" { + var ( + validationCount int64 + validationRows []string + ) + + err := s.db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + validationCount, validationRows, err = observeRawRows(tx, spec.validationSQL, spec.validationParams, idMap, resultContainsNodeIDs(testCase.Expected), resultContainsPaths(testCase.Expected)) + return err + }, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s validation reference observation: %w", spec.name, err) + } + if validationCount != observedCount || !slices.Equal(validationRows, observedRows) { + return nil, fmt.Errorf("%s materialized observation differs from validation reference: candidate=%v reference=%v", spec.name, observedRows, validationRows) + } + } + if testCase.Expected.RowCount != nil && rowCount != *testCase.Expected.RowCount { + return nil, fmt.Errorf("%s returned %d rows, expected %d", spec.name, rowCount, *testCase.Expected.RowCount) + } + if spec.semanticValidation != "exact_ordered_ids" { + if err := validateExpectedObservations(testCase.Expected, observedRows); err != nil { + return nil, fmt.Errorf("%s semantic validation: %w", spec.name, err) + } + if publicObservation != nil && !slices.Equal(publicObservation, observedRows) && !validAlternativeShortestPathObservation(testCase, publicObservation, observedRows) { + return nil, fmt.Errorf("%s exact public observation differs: public=%v reference=%v", spec.name, publicObservation, observedRows) + } + } + } + for idx := range stats.Samples { + stats.Samples[idx].Backend = ModePostgresSQL + stats.Samples[idx].Dataset = testCase.Dataset + stats.Samples[idx].Case = testCase.Name + "/reference/" + spec.name + stats.Samples[idx].ConnectionID = s.backendPID + } + plan, planJSON, metrics, err := explainRawPostgres(ctx, s.db, spec.sql, spec.parameters, readOptions...) + if err != nil { + return nil, fmt.Errorf("%s explain: %w", spec.name, err) + } + results = append(results, PostgresReferenceResult{ + SchemaVersion: postgresReferenceSchemaVersion, + Name: spec.name, + LegacyName: spec.legacyName, + Architecture: spec.architecture, + ImplementationID: spec.implementationID, + StateShape: spec.stateShape, + ObservationShape: spec.observationShape, + SemanticValidation: spec.semanticValidation, + Boundary: spec.boundary, + TimingBoundary: spec.timingBoundary, + FullComparator: spec.fullComparator, + AAAliasOf: spec.aaAliasOf, + SQL: spec.sql, + SQLFingerprint: normalizedSQLFingerprint(spec.sql), + RowCount: rowCount, + ObservedRows: observedRows, + Stats: stats, + PostgresPlan: plan, + PostgresPlanJSON: planJSON, + PostgresMetrics: &metrics, + traversalTelemetryParameters: copyReferenceParams(spec.parameters), + }) + } + return results, nil +} + +// selectReferenceSpecs restricts reference arms to explicit selectors and rejects missing requested arms. +func selectReferenceSpecs(specs []postgresReferenceSpec, names []string) ([]postgresReferenceSpec, error) { + selected := make([]postgresReferenceSpec, 0, len(names)) + for _, name := range names { + idx := referenceSpecIndexOrMissing(specs, name) + if idx < 0 { + return nil, fmt.Errorf("requested PostgreSQL reference arm %q is unavailable", name) + } + selected = append(selected, specs[idx]) + } + return selected, nil +} + +// explainRawPostgres runs raw PostgreSQL EXPLAIN and returns normalized plan text, JSON, and metrics. +func explainRawPostgres(ctx context.Context, db graph.Database, sqlQuery string, params map[string]any, transactionOptions ...graph.TransactionOption) ([]string, json.RawMessage, PostgresPlanMetrics, error) { + var ( + plan []string + planJSON json.RawMessage + ) + + err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, TIMING OFF) "+sqlQuery, params) + defer result.Close() + for result.Next() { + if values := result.Values(); len(values) > 0 { + plan = append(plan, fmt.Sprint(values[0])) + } + } + if err := result.Error(); err != nil { + return err + } + jsonResult := tx.Raw("EXPLAIN (ANALYZE, BUFFERS, WAL, SETTINGS, TIMING ON, FORMAT JSON) "+sqlQuery, params) + defer jsonResult.Close() + if jsonResult.Next() && len(jsonResult.Values()) > 0 { + var err error + planJSON, err = encodePostgresPlanJSON(jsonResult.Values()[0]) + if err != nil { + return err + } + } + return jsonResult.Error() + }, transactionOptions...) + if err != nil { + return nil, nil, PostgresPlanMetrics{}, err + } + metrics, err := parsePostgresPlanJSONMetrics(planJSON) + if err != nil { + return nil, nil, PostgresPlanMetrics{}, err + } + return plan, planJSON, metrics, nil +} + +// normalizedReferenceSpec fills legacy reference metadata defaults used for stable identity comparisons. +func normalizedReferenceSpec(spec postgresReferenceSpec) postgresReferenceSpec { + if spec.architecture == "" { + spec.architecture = "component_probe" + } + if spec.implementationID == "" { + spec.implementationID = spec.name + "_v1" + } + if spec.stateShape == "" { + spec.stateShape = "implementation_defined" + } + if spec.observationShape == "" { + spec.observationShape = "component_observation" + } + if spec.timingBoundary == "" { + spec.timingBoundary = "raw_pgx" + } + if spec.semanticValidation == "" { + spec.semanticValidation = "row_count_stability" + if spec.fullComparator { + spec.semanticValidation = "exact_public_observation" + } + } + return spec +} + +// normalizedSQLFingerprint hashes SQL after collapsing insignificant whitespace. +func normalizedSQLFingerprint(sql string) string { + return sqlFingerprint(strings.Join(strings.Fields(sql), " ")) +} + +// validateReferenceSpecs rejects duplicate, incomplete, or semantically inconsistent reference specifications. +func validateReferenceSpecs(specs []postgresReferenceSpec) error { + byName := make(map[string]postgresReferenceSpec, len(specs)) + byImplementation := make(map[string]postgresReferenceSpec, len(specs)) + byFingerprint := make(map[string]postgresReferenceSpec, len(specs)) + for _, spec := range specs { + if spec.name == "" || spec.architecture == "" || spec.implementationID == "" || spec.stateShape == "" || spec.observationShape == "" || spec.timingBoundary == "" || spec.semanticValidation == "" { + return fmt.Errorf("reference %q has an incomplete architecture identity", spec.name) + } + if _, found := byName[spec.name]; found { + return fmt.Errorf("duplicate reference name %q", spec.name) + } + fingerprint := normalizedSQLFingerprint(spec.sql) + if previous, found := byImplementation[spec.implementationID]; found && (previous.stateShape != spec.stateShape || previous.observationShape != spec.observationShape || normalizedSQLFingerprint(previous.sql) != fingerprint) { + return fmt.Errorf("implementation %q changes state, observation, or SQL identity between %q and %q", spec.implementationID, previous.name, spec.name) + } + if previous, found := byFingerprint[fingerprint]; found { + previousCanonical := previous.name + if previous.aaAliasOf != "" { + previousCanonical = previous.aaAliasOf + } + specCanonical := spec.name + if spec.aaAliasOf != "" { + specCanonical = spec.aaAliasOf + } + if specCanonical != previousCanonical { + return fmt.Errorf("references %q and %q have identical normalized SQL without a declared A/A alias", previous.name, spec.name) + } + canonical, alias := byName[previousCanonical], spec + if canonical.name == "" { + canonical = previous + } + if parameterShape(canonical.parameters) != parameterShape(alias.parameters) || canonical.observationShape != alias.observationShape || canonical.timingBoundary != alias.timingBoundary || canonical.fullComparator != alias.fullComparator || canonical.semanticValidation != alias.semanticValidation { + return fmt.Errorf("A/A alias %q does not match canonical arm %q at an identical comparison boundary", alias.name, canonical.name) + } + } + byName[spec.name] = spec + byImplementation[spec.implementationID] = spec + byFingerprint[fingerprint] = spec + } + for _, spec := range specs { + if spec.aaAliasOf == "" { + continue + } + canonical, found := byName[spec.aaAliasOf] + if !found { + return fmt.Errorf("A/A alias %q names missing canonical arm %q", spec.name, spec.aaAliasOf) + } + if normalizedSQLFingerprint(spec.sql) != normalizedSQLFingerprint(canonical.sql) { + return fmt.Errorf("A/A alias %q SQL differs from canonical arm %q", spec.name, canonical.name) + } + } + return nil +} + +// parameterShape returns a type-only description of query parameters for reference identity checks. +func parameterShape(parameters map[string]any) string { + names := make([]string, 0, len(parameters)) + for name := range parameters { + names = append(names, name) + } + sort.Strings(names) + var shape strings.Builder + for _, name := range names { + shape.WriteString(name) + shape.WriteByte('=') + if parameters[name] == nil { + shape.WriteString("") + } else { + shape.WriteString(reflect.TypeOf(parameters[name]).String()) + } + shape.WriteByte(';') + } + return shape.String() +} + +// validAlternativeShortestPathObservation reports whether two observations are both valid shortest-path witnesses. +func validAlternativeShortestPathObservation(testCase ScaleCase, publicRows, referenceRows []string) bool { + if testCase.Expected.ResultKind != "path_set" || strings.Contains(strings.ToLower(testCase.Cypher), "allshortestpaths") { + return false + } + provablyOutbound, err := shortestReferenceIsProvablyOutbound(testCase.Cypher) + if err != nil || !provablyOutbound { + return false + } + + publicPath, publicOK := singleStablePathObservation(publicRows) + referencePath, referenceOK := singleStablePathObservation(referenceRows) + if !publicOK || !referenceOK || !validOutboundStablePath(publicPath, testCase.Shape.EdgeKinds) || !validOutboundStablePath(referencePath, testCase.Shape.EdgeKinds) { + return false + } + if len(publicPath.Relationships) != len(referencePath.Relationships) { + return false + } + + publicStart, publicEnd := publicPath.Nodes[0].Identity, publicPath.Nodes[len(publicPath.Nodes)-1].Identity + referenceStart, referenceEnd := referencePath.Nodes[0].Identity, referencePath.Nodes[len(referencePath.Nodes)-1].Identity + return publicStart == referenceStart && publicEnd == referenceEnd +} + +// singleStablePathObservation returns the sole normalized path when the result contains exactly one valid path. +func singleStablePathObservation(rows []string) (stablePathObservation, bool) { + if len(rows) != 1 { + return stablePathObservation{}, false + } + + var columns []json.RawMessage + if err := json.Unmarshal([]byte(rows[0]), &columns); err != nil || len(columns) != 1 { + return stablePathObservation{}, false + } + + var path stablePathObservation + if err := json.Unmarshal(columns[0], &path); err != nil { + return stablePathObservation{}, false + } + return path, true +} + +// validOutboundStablePath reports whether a stable path follows every relationship in outbound order. +func validOutboundStablePath(path stablePathObservation, allowedKinds []string) bool { + if len(path.Nodes) == 0 || len(path.Nodes) != len(path.Relationships)+1 { + return false + } + for _, node := range path.Nodes { + if strings.HasPrefix(node.Identity, "unmapped-node:") { + return false + } + } + for idx, relationship := range path.Relationships { + if relationship.Start != path.Nodes[idx].Identity || relationship.End != path.Nodes[idx+1].Identity { + return false + } + if len(allowedKinds) != 0 && !slices.Contains(allowedKinds, relationship.Kind) { + return false + } + } + return true +} + +// referenceSpecsForRound returns reference specifications in the predeclared balanced order for a round. +func referenceSpecsForRound(specs []postgresReferenceSpec, round int) []postgresReferenceSpec { + if len(specs) == 3 && round > 0 { + // Odd-sized treatment sets need a doubled Williams design. Across these + // six rows every arm occupies every position twice, and every directed + // first-order carryover pair occurs twice. + schedule := [6][3]int{ + {0, 1, 2}, + {1, 2, 0}, + {2, 0, 1}, + {2, 1, 0}, + {0, 2, 1}, + {1, 0, 2}, + } + row := schedule[(round-1)%len(schedule)] + ordered := make([]postgresReferenceSpec, len(specs)) + for idx, slot := range row { + ordered[idx] = specs[slot] + } + return ordered + } + if len(specs) == 5 && round > 0 { + // Ten-sequence Williams/carryover-balanced schedule predeclared by the + // fixed-suffix expansion tournament. Slots are the caller-selected arms, so B1/B2/B3 can + // share this schedule without hard-coding architecture names here. + schedule := [10][5]int{ + {0, 1, 4, 2, 3}, {1, 2, 0, 3, 4}, {2, 3, 1, 4, 0}, {3, 4, 2, 0, 1}, {4, 0, 3, 1, 2}, + {3, 2, 4, 1, 0}, {4, 3, 0, 2, 1}, {0, 4, 1, 3, 2}, {1, 0, 2, 4, 3}, {2, 1, 3, 0, 4}, + } + row := schedule[(round-1)%len(schedule)] + ordered := make([]postgresReferenceSpec, len(specs)) + for idx, slot := range row { + ordered[idx] = specs[slot] + } + return ordered + } + ordered := append([]postgresReferenceSpec(nil), specs...) + if round > 0 && round%2 == 0 { + slices.Reverse(ordered) + } + return ordered +} + +// referenceSpecs constructs the independent PostgreSQL reference implementations for a scale case. +func (s *postgresSQLRunner) referenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + if testCase.Category == "expand_into_one_hop" { + return s.expandIntoReferenceSpecs(ctx, testCase, params) + } + if testCase.Category == "generated_fixed_suffix_expansion" { + return s.fixedSuffixExpansionReferenceSpecs(ctx, testCase, params) + } + if testCase.Category == "generated_shortest_path" || testCase.Category == "generated_shortest_path_v2" { + // Singleton and all-shortest architectures are kept as distinct arms; + // allShortestPaths uses its relationship-distinct predecessor DAG only. + if strings.Contains(strings.ToLower(testCase.Cypher), "allshortestpaths") { + return s.allShortestReferenceSpecs(ctx, testCase, params) + } + return s.shortestReferenceSpecs(ctx, testCase, params) + } + switch testCase.Name { + case "shortest_distance_bound_pair", "one_shortest_path_bound_pair": + return s.shortestReferenceSpecs(ctx, testCase, params) + case "fixed_suffix_expansion_endpoint_ids", "fixed_suffix_expansion_path_observed": + return s.fixedSuffixExpansionReferenceSpecs(ctx, testCase, params) + default: + return nil, nil + } +} + +// allShortestDAGSearch returns the predecessor-DAG SQL search for all shortest paths in one direction. +func allShortestDAGSearch(direction graph.Direction) string { + distanceJoin, distanceNext := "e.start_id = distance.node_id", "e.end_id" + predecessorJoin := "e.start_id = prior.node_id and e.end_id = paths.node_id" + if direction == graph.DirectionInbound { + distanceJoin, distanceNext = "e.end_id = distance.node_id", "e.start_id" + predecessorJoin = "e.end_id = prior.node_id and e.start_id = paths.node_id" + } + return `with recursive validated(start_id, end_id) as materialized ( + select start_node.id, end_node.id + from node start_node, node end_node + where start_node.graph_id = @graph_id and start_node.id = @start_id + and end_node.graph_id = @graph_id and end_node.id = @end_id +), distance(node_id, depth) as ( + select validated.start_id, 0 from validated + union + select ` + distanceNext + `, distance.depth + 1 + from distance + join edge e on e.graph_id = @graph_id and ` + distanceJoin + ` + where distance.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), target as materialized ( + select depth from distance + where node_id = @end_id and depth >= @min_depth + order by depth limit 1 +), predecessor(node_id, depth, predecessor_id, edge_id) as materialized ( + select paths.node_id, paths.depth, prior.node_id, e.id + from distance paths + join target on paths.depth > 0 and paths.depth <= target.depth + join distance prior on prior.depth = paths.depth - 1 + join edge e on e.graph_id = @graph_id and ` + predecessorJoin + ` + where (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), paths(node_id, depth, edge_ids) as ( + select @end_id::int8, target.depth, array[]::int8[] from target + union all + select predecessor.predecessor_id, paths.depth - 1, array[predecessor.edge_id]::int8[] || paths.edge_ids + from paths join predecessor on predecessor.node_id = paths.node_id and predecessor.depth = paths.depth +), shortest(depth, edge_ids) as materialized ( + select target.depth, paths.edge_ids + from paths join target on true where paths.node_id = @start_id and paths.depth = 0 +)` +} + +// allShortestA1ReferenceSQL supports benchmark evidence processing for all shortest a1 reference sql. +func allShortestA1ReferenceSQL(direction graph.Direction) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + search := `with shortest as materialized ( + select depth, path as edge_ids + from all_shortest_paths_dag( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + ` + ) +)` + return shortestM0FullSQL(search, direction) +} + +// allShortestBidirectionalReferenceSQL exposes a forced two-sided +// predecessor-DAG kernel at the same complete M0 path boundary as ASP-A1. +func allShortestBidirectionalReferenceSQL(functionName string, direction graph.Direction) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + search := `with shortest as materialized ( + select depth, path as edge_ids + from ` + functionName + `( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + `, @state_limit, @frontier_limit, + @predecessor_limit, @enumeration_limit, @output_bytes_limit + ) +)` + return shortestM0FullSQL(search, direction) +} + +// allShortestReferenceSpecs builds the predecessor-DAG reference for an all-shortest-path workload. +func (s *postgresSQLRunner) allShortestReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + probeParams["min_depth"] = int32(1) + if testCase.Shape.MinDepth != nil { + probeParams["min_depth"] = int32(*testCase.Shape.MinDepth) + } + probeParams["max_depth"] = int32(15) + if testCase.Shape.MaxDepth != nil { + probeParams["max_depth"] = int32(*testCase.Shape.MaxDepth) + } + edgeKinds := make(graph.Kinds, 0, len(testCase.Shape.EdgeKinds)) + for _, name := range testCase.Shape.EdgeKinds { + edgeKinds = append(edgeKinds, graph.StringKind(name)) + } + var edgeKindIDs []int16 + if len(edgeKinds) > 0 { + if s.pgDriver == nil { + return nil, fmt.Errorf("map all-shortest reference edge kinds: PostgreSQL driver is unavailable") + } + var err error + edgeKindIDs, err = s.pgDriver.KindMapper().MapKinds(ctx, edgeKinds) + if err != nil { + return nil, fmt.Errorf("map all-shortest reference edge kinds: %w", err) + } + } + probeParams["edge_kind_ids"] = edgeKindIDs + direction, err := shortestReferenceDirection(testCase.Cypher) + if err != nil || direction == graph.DirectionBoth { + return nil, err + } + rootParameter, terminalParameter, err := shortestReferenceEndpointParameters(testCase.Cypher) + if err != nil { + return nil, err + } + probeParams["start_id"] = probeParams[rootParameter] + probeParams["end_id"] = probeParams[terminalParameter] + specs := []postgresReferenceSpec{{ + name: "asp_a1_stored_helper_m0", + architecture: "ASP-A1-DAG", + implementationID: "all_shortest_paths_dag_stored_helper_m0_v1", + stateShape: "minimum-depth helper workspace with relationship-distinct predecessors", + observationShape: "complete all-shortest path multiset", + semanticValidation: "exact_public_observation", + boundary: "complete path composites", + fullComparator: true, + sql: allShortestA1ReferenceSQL(direction), + parameters: probeParams, + }} + + // I1 is valid only inside the same distinct-endpoint, min-one bounded + // contract enforced by the production emitter. A1 remains available as the + // exact control outside that envelope. + startID, startOK := probeParams["start_id"].(int64) + endID, endOK := probeParams["end_id"].(int64) + maximumDepth, maximumOK := probeParams["max_depth"].(int32) + if probeParams["min_depth"] != int32(1) || !maximumOK || maximumDepth < 1 || maximumDepth > 64 || !startOK || !endOK || startID == endID { + return specs, nil + } + search := allShortestDAGSearch(direction) + specs = append(specs, postgresReferenceSpec{ + name: "asp_i1_inline_predecessor_dag_m0", + architecture: "ASP-I1-U-DAG+MAT-M0", + implementationID: "inline_shortest_depth_predecessor_dag_m0_v1", + stateShape: "node/depth discovery plus every relationship-distinct shortest-depth predecessor edge", + observationShape: "complete all-shortest path multiset", + semanticValidation: "exact_public_observation", + boundary: "complete path composites", + fullComparator: true, + sql: shortestM0FullSQL(search, direction), + parameters: probeParams, + }) + + // B1/B2 are intentionally tool/reference-only. Keep automatic production + // selection on ASP-A1 until independent confirmation passes, and do not + // expose candidate arms outside their distinct-endpoint minimum-one envelope. + candidateParams := copyReferenceParams(probeParams) + candidateParams["state_limit"] = int64(100_000) + candidateParams["frontier_limit"] = int64(100_000) + candidateParams["predecessor_limit"] = int64(100_000) + candidateParams["enumeration_limit"] = int64(100_000) + candidateParams["output_bytes_limit"] = int64(64 * 1024 * 1024) + for _, candidate := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // architecture retains the architecture while anonymous record is assembled or evaluated. + architecture string + // implementationID identifies the implementation id. + implementationID string + // functionName identifies the function name. + functionName string + }{ + { + name: "asp_b1_bidirectional_dag_strict_m0", + architecture: "ASP-B1-DAG-ALT-NODE", + implementationID: "typed_two_sided_predecessor_dag_strict_alternating_v1", + functionName: "all_shortest_paths_b1_strict_alternating", + }, + { + name: "asp_b2_bidirectional_dag_smaller_frontier_m0", + architecture: "ASP-B2-DAG-MIN-LEVEL", + implementationID: "typed_two_sided_predecessor_dag_smaller_current_level_v1", + functionName: "all_shortest_paths_b2_smaller_current_level", + }, + } { + specs = append(specs, postgresReferenceSpec{ + name: candidate.name, + architecture: candidate.architecture, + implementationID: candidate.implementationID, + stateShape: "two-sided minimum-node-depth discovery plus every relationship-distinct equal-depth predecessor/successor at one canonical cut", + observationShape: "complete all-shortest path multiset", + semanticValidation: "exact_public_observation", + boundary: "complete path composites", + fullComparator: true, + sql: allShortestBidirectionalReferenceSQL(candidate.functionName, direction), + parameters: copyReferenceParams(candidateParams), + }) + } + return specs, nil +} + +// shortestReferenceSpecs builds eligible shortest-path reference implementations and measurement boundaries. +func (s *postgresSQLRunner) shortestReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + probeParams["min_depth"] = int32(1) + if testCase.Shape.MinDepth != nil { + probeParams["min_depth"] = int32(*testCase.Shape.MinDepth) + } + probeParams["max_depth"] = int32(15) + if testCase.Shape.MaxDepth != nil { + probeParams["max_depth"] = int32(*testCase.Shape.MaxDepth) + } + edgeKinds := make(graph.Kinds, 0, len(testCase.Shape.EdgeKinds)) + for _, name := range testCase.Shape.EdgeKinds { + edgeKinds = append(edgeKinds, graph.StringKind(name)) + } + edgeKindIDs, err := s.pgDriver.KindMapper().MapKinds(ctx, edgeKinds) + if err != nil { + return nil, fmt.Errorf("map shortest reference edge kinds: %w", err) + } + probeParams["edge_kind_ids"] = edgeKindIDs + direction, err := shortestReferenceDirection(testCase.Cypher) + if err != nil { + return nil, fmt.Errorf("classify shortest reference direction: %w", err) + } + if direction == graph.DirectionBoth { + return nil, nil + } + rootParameter, terminalParameter, err := shortestReferenceEndpointParameters(testCase.Cypher) + if err != nil { + return nil, fmt.Errorf("resolve shortest reference endpoint parameters: %w", err) + } + searchParams := copyReferenceParams(probeParams) + searchParams["start_id"] = probeParams[rootParameter] + searchParams["end_id"] = probeParams[terminalParameter] + search := shortestReferenceSearchForDirection(direction) + values, err := readReferenceRow(ctx, s.db, search+` select depth, node_ids, edge_ids from shortest`, searchParams, s.readTransactionOptions()...) + if err != nil { + return nil, fmt.Errorf("precompute shortest hydration IDs: %w", err) + } + if len(values) != 0 && len(values) != 3 { + return nil, fmt.Errorf("precompute shortest hydration IDs returned %d columns, expected 3", len(values)) + } + var nodeIDs, edgeIDs []int64 + if len(values) == 3 { + nodeIDs, err = referenceInt64Slice(values[1]) + if err != nil { + return nil, fmt.Errorf("decode shortest hydration node IDs: %w", err) + } + edgeIDs, err = referenceInt64Slice(values[2]) + if err != nil { + return nil, fmt.Errorf("decode shortest hydration edge IDs: %w", err) + } + } + return buildShortestReferenceSpecs(testCase, searchParams, nodeIDs, edgeIDs, direction), nil +} + +// shortestReferenceEndpointParameters maps public start and end parameters to physical search endpoints for the parsed direction. +func shortestReferenceEndpointParameters(query string) (string, string, error) { + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + return "", "", err + } + if parsed == nil || parsed.SingleQuery == nil || parsed.SingleQuery.SinglePartQuery == nil || parsed.SingleQuery.MultiPartQuery != nil { + return "", "", fmt.Errorf("expected a single-part shortest query") + } + for _, readingClause := range parsed.SingleQuery.SinglePartQuery.ReadingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + bindings := map[string]string{} + if readingClause.Match.Where != nil { + for _, expression := range readingClause.Match.Where.Expressions { + collectIdentityParameterBindings(expression, bindings) + } + } + for _, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) || len(patternPart.PatternElements) < 3 { + continue + } + root, rootOK := patternPart.PatternElements[0].AsNodePattern() + terminal, terminalOK := patternPart.PatternElements[len(patternPart.PatternElements)-1].AsNodePattern() + if !rootOK || !terminalOK || root.Variable == nil || terminal.Variable == nil { + return "", "", fmt.Errorf("shortest reference endpoints must have variables") + } + rootParameter, rootBound := bindings[root.Variable.Symbol] + terminalParameter, terminalBound := bindings[terminal.Variable.Symbol] + if !rootBound || !terminalBound { + return "", "", fmt.Errorf("shortest reference endpoints must have parameter ID equalities") + } + return rootParameter, terminalParameter, nil + } + } + return "", "", fmt.Errorf("shortest pattern not found") +} + +// collectIdentityParameterBindings extracts equality-bound ID parameters for the two variables in a shortest-path pattern. +func collectIdentityParameterBindings(expression cypher.Expression, bindings map[string]string) { + switch typed := expression.(type) { + case *cypher.Conjunction: + for _, child := range typed.Expressions { + collectIdentityParameterBindings(child, bindings) + } + case *cypher.Parenthetical: + collectIdentityParameterBindings(typed.Expression, bindings) + case *cypher.Comparison: + if typed == nil || len(typed.Partials) != 1 || typed.Partials[0].Operator != cypher.OperatorEquals { + return + } + if symbol, ok := identityReferenceSymbol(typed.Left); ok { + if parameter, ok := typed.Partials[0].Right.(*cypher.Parameter); ok { + bindings[symbol] = parameter.Symbol + } + } + if symbol, ok := identityReferenceSymbol(typed.Partials[0].Right); ok { + if parameter, ok := typed.Left.(*cypher.Parameter); ok { + bindings[symbol] = parameter.Symbol + } + } + } +} + +// identityReferenceSymbol returns the variable whose ID is projected directly by a reference query. +func identityReferenceSymbol(expression cypher.Expression) (string, bool) { + function, ok := expression.(*cypher.FunctionInvocation) + if !ok || function == nil || !strings.EqualFold(function.Name, cypher.IdentityFunction) || len(function.Arguments) != 1 { + return "", false + } + variable, ok := function.Arguments[0].(*cypher.Variable) + if !ok || variable == nil || variable.Symbol == "" { + return "", false + } + return variable.Symbol, true +} + +// shortestReferenceIsProvablyOutbound reports whether a supported shortest-path query has outbound direction. +func shortestReferenceIsProvablyOutbound(query string) (bool, error) { + direction, err := shortestReferenceDirection(query) + if err != nil { + return false, err + } + + return direction == graph.DirectionOutbound, nil +} + +// shortestReferenceDirection parses a shortest-path query and returns its single relationship direction. +func shortestReferenceDirection(query string) (graph.Direction, error) { + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + return 0, err + } + if parsed == nil || parsed.SingleQuery == nil || parsed.SingleQuery.SinglePartQuery == nil || parsed.SingleQuery.MultiPartQuery != nil { + return graph.DirectionBoth, nil + } + + var ( + shortestParts int + relationships int + direction graph.Direction + ) + for _, readingClause := range parsed.SingleQuery.SinglePartQuery.ReadingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) { + continue + } + shortestParts++ + for _, patternElement := range patternPart.PatternElements { + if relationship, isRelationship := patternElement.AsRelationshipPattern(); isRelationship { + relationships++ + direction = relationship.Direction + } + } + } + } + + if shortestParts != 1 || relationships != 1 { + return graph.DirectionBoth, nil + } + return direction, nil +} + +// shortestReferenceSearch returns the compact recursive shortest-path search SQL for a projection mode. +func shortestReferenceSearch() string { + return shortestReferenceSearchForDirection(graph.DirectionOutbound) +} + +// shortestReferenceSearchForDirection returns direction-specific shortest-path search SQL and endpoint columns. +func shortestReferenceSearchForDirection(direction graph.Direction) string { + edgeJoin, nextNode := "e.start_id = search.node_id", "e.end_id" + if direction == graph.DirectionInbound { + edgeJoin, nextNode = "e.end_id = search.node_id", "e.start_id" + } + return `with recursive search(node_id, depth, node_ids, edge_ids) as ( + select @start_id::int8, 0, array[@start_id::int8]::int8[], array[]::int8[] + union all + select ` + nextNode + `, search.depth + 1, search.node_ids || ` + nextNode + `, search.edge_ids || e.id + from search + join edge e on e.graph_id = @graph_id and ` + edgeJoin + ` + where search.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(search.edge_ids) +), shortest as materialized ( + select depth, node_ids, edge_ids from search + where node_id = @end_id and depth >= @min_depth + order by depth, edge_ids limit 1 +)` +} + +// shortestEdgeReferenceSearch returns the edge-only shortest-path search SQL for a direction. +func shortestEdgeReferenceSearch(direction graph.Direction) string { + edgeJoin, nextNode := "e.start_id = search.node_id", "e.end_id" + if direction == graph.DirectionInbound { + edgeJoin, nextNode = "e.end_id = search.node_id", "e.start_id" + } + return `with recursive search(node_id, depth, edge_ids) as ( + select @start_id::int8, 0, array[]::int8[] + union all + select ` + nextNode + `, search.depth + 1, search.edge_ids || e.id + from search + join edge e on e.graph_id = @graph_id and ` + edgeJoin + ` + where search.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(search.edge_ids) +), shortest as materialized ( + select depth, edge_ids from search + where node_id = @end_id and depth >= @min_depth + order by depth, edge_ids limit 1 +)` +} + +// shortestDistanceReferenceSearch returns the minimal-state shortest-distance search SQL for a direction. +func shortestDistanceReferenceSearch() string { + return shortestDistanceReferenceSearchForDirection(graph.DirectionOutbound) +} + +// shortestDistanceReferenceSearchForDirection returns direction-specific shortest-distance SQL and endpoint columns. +func shortestDistanceReferenceSearchForDirection(direction graph.Direction) string { + edgeJoin, nextNode := "e.start_id = search.node_id", "e.end_id" + if direction == graph.DirectionInbound { + edgeJoin, nextNode = "e.end_id = search.node_id", "e.start_id" + } + return `with recursive search(node_id, depth) as ( + select @start_id::int8, 0 + union + select ` + nextNode + `, search.depth + 1 + from search + join edge e on e.graph_id = @graph_id and ` + edgeJoin + ` + where search.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), shortest as materialized ( + select depth from search + where node_id = @end_id and depth >= @min_depth + order by depth limit 1 +)` +} + +// shortestCanonicalWitnessSearch returns SQL that reconstructs one deterministic witness from compact predecessor state. +func shortestCanonicalWitnessSearch(reverseForPublicPath bool) string { + edgeIDs := "witness.edge_ids" + if reverseForPublicPath { + edgeIDs = `(select coalesce(array_agg(reversed.edge_id order by reversed.ordinal desc), array[]::int8[]) + from unnest(witness.edge_ids) with ordinality reversed(edge_id, ordinal))` + } + return `with recursive distance(node_id, depth) as ( + select @search_start_id::int8, 0 + union + select e.end_id, distance.depth + 1 + from distance + join edge e on e.graph_id = @graph_id and e.start_id = distance.node_id + where distance.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) +), target as materialized ( + select depth from distance + where node_id = @search_end_id and depth >= @min_depth + order by depth limit 1 +), witness(node_id, depth, edge_ids) as ( + select @search_end_id::int8, target.depth, array[]::int8[] from target + union all + select predecessor.node_id, witness.depth - 1, array[predecessor.edge_id]::int8[] || witness.edge_ids + from witness + join lateral ( + select prior.node_id, e.id as edge_id + from distance prior + join edge e on e.graph_id = @graph_id and e.start_id = prior.node_id and e.end_id = witness.node_id + where prior.depth = witness.depth - 1 + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + order by e.id, prior.node_id limit 1 + ) predecessor on witness.depth > 0 +), shortest as materialized ( + select target.depth, ` + edgeIDs + ` as edge_ids + from witness join target on true where witness.depth = 0 +)` +} + +// shortestBidirectionalCompactReferenceSQL exposes one forced compact kernel at +// the same distance or M0 hydration boundary as its production control. +func shortestBidirectionalCompactReferenceSQL(functionName string, direction graph.Direction, pathObserved bool) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + search := `with shortest as materialized ( + select depth, path as edge_ids + from ` + functionName + `( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + `, @state_limit, @frontier_limit, @predecessor_limit + ) +)` + if !pathObserved { + return search + ` select depth from shortest` + } + return search + shortestM0MaterializationSelect(direction) +} + +// buildShortestReferenceSpecs assembles exact shortest-path comparators supported by the workload shape. +func buildShortestReferenceSpecs(testCase ScaleCase, probeParams map[string]any, nodeIDs, edgeIDs []int64, direction graph.Direction) []postgresReferenceSpec { + searchNE := shortestReferenceSearchForDirection(direction) + searchE := shortestEdgeReferenceSearch(direction) + fullSQL := shortestDistanceReferenceSearchForDirection(direction) + ` select depth from shortest` + boundary := "distance scalar" + pathObserved := testCase.Name == "one_shortest_path_bound_pair" || testCase.Expected.ResultKind == "path_set" + compactBidirectionalParams := copyReferenceParams(probeParams) + compactBidirectionalParams["state_limit"] = int64(100_000) + compactBidirectionalParams["frontier_limit"] = int64(100_000) + compactBidirectionalParams["predecessor_limit"] = int64(100_000) + if pathObserved { + fullSQL = searchNE + ` +select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + shortest.edge_ids, + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from shortest join node root on root.graph_id = @graph_id and root.id = @start_id` + boundary = "complete path composite" + } + hydrationParams := copyReferenceParams(probeParams) + hydrationParams["node_ids"] = nodeIDs + hydrationParams["edge_ids"] = edgeIDs + hydrationSQL := `select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + @edge_ids::int8[], + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from node root where root.graph_id = @graph_id and root.id = @start_id` + specs := []postgresReferenceSpec{ + { + name: "round_trip", + boundary: "prepared protocol and transaction", + sql: `select 1`, + parameters: nil, + }, + { + name: "endpoint_validation", + boundary: "validated endpoint IDs", + sql: `select id from node where graph_id = @graph_id and id = any(array[@start_id::int8, @end_id::int8]) order by id`, + parameters: probeParams, + }, + { + name: "minimum_graph_access", + boundary: "root adjacency edge IDs", + sql: `select e.id from edge e where e.graph_id = @graph_id and e.start_id = @start_id and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) order by e.id`, + parameters: probeParams, + }, + { + name: "search_ordered_ids", + architecture: "SP-S3-U-NE", + observationShape: "ordered_ids", + stateShape: "ordered node and edge ID arrays", + boundary: "depth plus ordered node/edge IDs", + sql: searchNE + ` select depth, node_ids, edge_ids from shortest`, + parameters: probeParams, + }, + } + if edgeIDs != nil { + specs = append(specs, postgresReferenceSpec{ + name: "hydration_only", + boundary: "complete path composite from precomputed ordered edge IDs", + sql: hydrationSQL, + parameters: hydrationParams, + }) + if pathObserved && direction != graph.DirectionBoth { + specs = append(specs, + postgresReferenceSpec{ + name: "m0_directed_hydration_only", + architecture: "MAT-M0", + implementationID: "directed_set_hydration_" + strings.ToLower(direction.String()) + "_v1", + stateShape: "precomputed ordered edge IDs; node order derived from directed edge endpoints", + observationShape: "complete path composite", + semanticValidation: "precomputed_exact_path_inputs", + boundary: "directed complete path composite from precomputed ordered edge IDs", + sql: shortestM0HydrationSQL(direction), + parameters: hydrationParams, + validationSQL: hydrationSQL, + validationParams: hydrationParams, + }, + postgresReferenceSpec{ + name: "m1_ordered_ids_hydration_only", + architecture: "MAT-M1", + implementationID: "ordered_ids_set_hydration_v1", + stateShape: "precomputed ordered node and edge IDs", + observationShape: "complete path composite", + semanticValidation: "precomputed_exact_path_inputs", + boundary: "complete path composite from precomputed ordered node and edge IDs", + sql: shortestM1HydrationSQL(), + parameters: hydrationParams, + validationSQL: hydrationSQL, + validationParams: hydrationParams, + }, + ) + } + } + specs = append(specs, postgresReferenceSpec{ + name: "s3_unidirectional_trail_cte", + legacyName: "complete_reference_s1_array_cte", + architecture: shortestArchitectureForCase(testCase), + implementationID: "inline_recursive_cte_unidirectional_v3", + stateShape: shortestS3UStateShape(testCase), + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: fullSQL, + parameters: probeParams, + }) + if !pathObserved && direction == graph.DirectionInbound { + canonicalParams := copyReferenceParams(probeParams) + canonicalParams["start_id"], canonicalParams["end_id"] = probeParams["end_id"], probeParams["start_id"] + specs = append(specs, postgresReferenceSpec{ + name: "s4_canonical_source_distance", + architecture: "SP-I1-C-D", + implementationID: "canonical_relationship_source_distance_v1", + stateShape: "relationship-source-oriented node and depth set state", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestDistanceReferenceSearchForDirection(graph.DirectionOutbound) + ` select depth from shortest`, + parameters: canonicalParams, + }) + } + if shortestS1DistanceEligible(testCase, probeParams, direction, pathObserved) { + s1Params := copyReferenceParams(probeParams) + s1Params["state_limit"] = int32(100_000) + specs = append(specs, postgresReferenceSpec{ + name: "s1_array_bfs_distance", + architecture: "SP-S1", + implementationID: "typed_plpgsql_array_bfs_distance_v1", + stateShape: "array-resident frontier and visited node IDs with explicit state ceiling; no path or predecessor state", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestS1DistanceSQL(fullSQL, direction), + parameters: s1Params, + }) + } + if pathObserved && direction != graph.DirectionBoth { + specs = append(specs, + postgresReferenceSpec{ + name: "s3_unidirectional_cte_m0_directed", + architecture: "SP-S3-U-E+MAT-M0", + implementationID: "s3_u_edge_search_directed_set_materializer_" + strings.ToLower(direction.String()) + "_v1", + stateShape: "edge-only recursive trail; materializer derives node order from directed edge endpoints", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestM0FullSQL(searchE, direction), + parameters: probeParams, + }, + postgresReferenceSpec{ + name: "s3_unidirectional_cte_m1_ordered_ids", + architecture: "SP-S3-U-NE+MAT-M1", + implementationID: "s3_u_node_edge_search_ordered_ids_set_materializer_v1", + stateShape: "ordered node-and-edge recursive trails; materializer hydrates both streams by ordinal", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestM1FullSQL(searchNE), + parameters: probeParams, + }, + ) + + witnessParams := copyReferenceParams(probeParams) + witnessParams["search_start_id"], witnessParams["search_end_id"] = probeParams["start_id"], probeParams["end_id"] + reverseForPublicPath := false + if direction == graph.DirectionInbound { + witnessParams["search_start_id"], witnessParams["search_end_id"] = probeParams["end_id"], probeParams["start_id"] + reverseForPublicPath = true + } + witnessSearch := shortestCanonicalWitnessSearch(reverseForPublicPath) + specs = append(specs, postgresReferenceSpec{ + name: "s4_canonical_source_witness_m0", + architecture: "SP-I1-C-WE+MAT-M0", + implementationID: "canonical_source_compact_witness_m0_v1", + stateShape: "node/depth discovery plus one deterministic predecessor per witness depth; no recursive full trails", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestM0FullSQL(witnessSearch, direction), + parameters: witnessParams, + }) + } + if direction != graph.DirectionBoth { + if pathObserved { + specs = append(specs, + postgresReferenceSpec{ + name: "sp_b1_strict_alternating_witness_m0", + architecture: "SP-B1-C-ALT-NODE-WE+MAT-M0", + implementationID: "typed_bidirectional_strict_alternating_node_witness_m0_v1", + stateShape: "ID-only per-side FIFO, minimum-depth seen state, and one deterministic predecessor per accepted node", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b1_strict_alternating", direction, true), + parameters: compactBidirectionalParams, + }, + postgresReferenceSpec{ + name: "sp_b2_smaller_frontier_witness_m0", + architecture: "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + implementationID: "typed_bidirectional_smaller_current_level_witness_m0_v1", + stateShape: "ID-only per-side complete levels, minimum-depth seen state, and one deterministic predecessor per accepted node", + observationShape: "public_observation", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b2_smaller_current_level", direction, true), + parameters: compactBidirectionalParams, + }, + ) + } else { + specs = append(specs, + postgresReferenceSpec{ + name: "sp_b1_strict_alternating_distance", + architecture: "SP-B1-C-ALT-NODE-D", + implementationID: "typed_bidirectional_strict_alternating_node_distance_v1", + stateShape: "ID-only per-side FIFO and minimum-depth seen state; witness predecessor retained outside the observation boundary", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b1_strict_alternating", direction, false), + parameters: compactBidirectionalParams, + }, + postgresReferenceSpec{ + name: "sp_b2_smaller_frontier_distance", + architecture: "SP-B2-C-MIN-LEVEL-D", + implementationID: "typed_bidirectional_smaller_current_level_distance_v1", + stateShape: "ID-only per-side complete levels and minimum-depth seen state; witness predecessor retained outside the observation boundary", + observationShape: "distance scalar", + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalCompactReferenceSQL("shortest_path_b2_smaller_current_level", direction, false), + parameters: compactBidirectionalParams, + }, + ) + } + } + specs = append(specs, postgresReferenceSpec{ + name: "s3_bidirectional_trail_cte", + legacyName: "candidate_s2_bidirectional_cte", + architecture: "SP-S3-B", + implementationID: "inline_recursive_cte_bidirectional_trails_v2", + stateShape: "paired per-row relationship trail arrays", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: shortestBidirectionalReferenceSQL(testCase, direction), + parameters: probeParams, + }) + return specs +} + +// shortestS1DistanceEligible reports whether a case can use the bounded single-direction distance prototype. +func shortestS1DistanceEligible(testCase ScaleCase, parameters map[string]any, direction graph.Direction, pathObserved bool) bool { + if pathObserved || direction == graph.DirectionBoth { + return false + } + minDepth := 1 + if testCase.Shape.MinDepth != nil { + minDepth = *testCase.Shape.MinDepth + } + return minDepth <= 1 && !reflect.DeepEqual(parameters["start_id"], parameters["end_id"]) +} + +// shortestS1DistanceSQL wraps a shortest-path query with the bounded S1 distance prototype. +func shortestS1DistanceSQL(fallbackSQL string, direction graph.Direction) string { + inbound := "false" + if direction == graph.DirectionInbound { + inbound = "true" + } + return `with s1 as materialized ( + select * from graphbench_s1_distance_bfs( + @graph_id, @start_id, @end_id, @min_depth, @max_depth, + @edge_kind_ids, ` + inbound + `, @state_limit + ) +) +select depth from s1 where matched +union all +select fallback.depth from (` + fallbackSQL + `) fallback +where (select overflow from s1) +limit 1` +} + +// shortestArchitectureForCase chooses the witness-producing or distance-only S3 reference architecture from the case's observable result contract. +func shortestArchitectureForCase(testCase ScaleCase) string { + if testCase.Expected.ResultKind == "path_set" || testCase.Name == "one_shortest_path_bound_pair" { + return "SP-S3-U-NE" + } + return "SP-S3-U-D" +} + +// shortestM0HydrationSQL returns SQL that hydrates paths from ordered relationship IDs. +func shortestM0HydrationSQL(direction graph.Direction) string { + return `with shortest(edge_ids) as (select @edge_ids::int8[])` + shortestM0MaterializationSelect(direction) +} + +// shortestM0FullSQL combines edge-only search with M0 path hydration. +func shortestM0FullSQL(search string, direction graph.Direction) string { + return search + shortestM0MaterializationSelect(direction) +} + +// shortestM0MaterializationSelect is intentionally outbound-only. The S3-U +// reference search emits an ordered, graph-scoped outbound edge stream, so M0 +// can derive each next node directly from edge.end_id without recursively +// rediscovering connectivity. +func shortestM0MaterializationSelect(direction graph.Direction) string { + nextNode := "edge.end_id" + if direction == graph.DirectionInbound { + nextNode = "edge.start_id" + } + return ` +select row( + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] || + coalesce(hydrated.nodes, array[]::nodeComposite[]), + coalesce(hydrated.edges, array[]::edgeComposite[]) +)::pathComposite +from shortest +join node root on root.graph_id = @graph_id and root.id = @start_id +cross join lateral ( + select + array_agg((terminal.id, terminal.kind_ids, terminal.properties)::nodeComposite order by path_edge.ordinality)::nodeComposite[] as nodes, + array_agg((edge.id, edge.start_id, edge.end_id, edge.kind_id, edge.properties)::edgeComposite order by path_edge.ordinality)::edgeComposite[] as edges, + count(*) as hydrated_count + from unnest(shortest.edge_ids) with ordinality as path_edge(id, ordinality) + join edge on edge.graph_id = @graph_id and edge.id = path_edge.id + join node terminal on terminal.graph_id = @graph_id and terminal.id = ` + nextNode + ` +) hydrated +where hydrated.hydrated_count = cardinality(shortest.edge_ids)` +} + +// shortestM1HydrationSQL returns SQL that hydrates paths from ordered node and relationship IDs. +func shortestM1HydrationSQL() string { + return `with shortest(node_ids, edge_ids) as (select @node_ids::int8[], @edge_ids::int8[])` + shortestM1MaterializationSelect() +} + +// shortestM1FullSQL combines node-and-edge search with M1 path hydration. +func shortestM1FullSQL(search string) string { + return search + shortestM1MaterializationSelect() +} + +// shortestM1MaterializationSelect hydrates the ordered node and edge streams +// independently and restores public path order with ordinality. M0 and M1 use +// the same S3-U search in full-comparator measurements so their delta isolates +// materialization rather than search state generation. +func shortestM1MaterializationSelect() string { + return ` +select row( + coalesce(hydrated_nodes.nodes, array[]::nodeComposite[]), + coalesce(hydrated_edges.edges, array[]::edgeComposite[]) +)::pathComposite +from shortest +cross join lateral ( + select + array_agg((node.id, node.kind_ids, node.properties)::nodeComposite order by path_node.ordinality)::nodeComposite[] as nodes, + count(*) as hydrated_count + from unnest(shortest.node_ids) with ordinality as path_node(id, ordinality) + join node on node.graph_id = @graph_id and node.id = path_node.id +) hydrated_nodes +cross join lateral ( + select + array_agg((edge.id, edge.start_id, edge.end_id, edge.kind_id, edge.properties)::edgeComposite order by path_edge.ordinality)::edgeComposite[] as edges, + count(*) as hydrated_count + from unnest(shortest.edge_ids) with ordinality as path_edge(id, ordinality) + join edge on edge.graph_id = @graph_id and edge.id = path_edge.id +) hydrated_edges +where cardinality(shortest.node_ids) = cardinality(shortest.edge_ids) + 1 + and hydrated_nodes.hydrated_count = cardinality(shortest.node_ids) + and hydrated_edges.hydrated_count = cardinality(shortest.edge_ids)` +} + +// shortestS3UStateShape describes recursive state retained by the selected unidirectional search projection. +func shortestS3UStateShape(testCase ScaleCase) string { + if testCase.Expected.ResultKind == "path_set" || testCase.Name == "one_shortest_path_bound_pair" { + return "per-row node and relationship trail arrays" + } + return "distance frontier node and depth only; no path or predecessor state" +} + +// shortestBidirectionalReferenceSQL returns the bidirectional shortest-path reference query for the requested result shape. +func shortestBidirectionalReferenceSQL(testCase ScaleCase, direction graph.Direction) string { + forwardJoin, forwardNext := "e.start_id = forward.node_id", "e.end_id" + backwardJoin, backwardNext := "e.end_id = backward.node_id", "e.start_id" + if direction == graph.DirectionInbound { + forwardJoin, forwardNext = "e.end_id = forward.node_id", "e.start_id" + backwardJoin, backwardNext = "e.start_id = backward.node_id", "e.end_id" + } + search := `with recursive +forward(node_id, depth, edge_ids) as ( + select @start_id::int8, 0, array[]::int8[] + union all + select ` + forwardNext + `, forward.depth + 1, forward.edge_ids || e.id + from forward join edge e on e.graph_id = @graph_id and ` + forwardJoin + ` + where forward.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(forward.edge_ids) +), backward(node_id, depth, edge_ids) as ( + select @end_id::int8, 0, array[]::int8[] + union all + select ` + backwardNext + `, backward.depth + 1, e.id || backward.edge_ids + from backward join edge e on e.graph_id = @graph_id and ` + backwardJoin + ` + where backward.depth < @max_depth + and (cardinality(@edge_kind_ids::int2[]) = 0 or e.kind_id = any(@edge_kind_ids::int2[])) + and e.id != all(backward.edge_ids) +), shortest as materialized ( + select forward.depth + backward.depth as depth, forward.edge_ids || backward.edge_ids as edge_ids + from forward join backward using (node_id) + where forward.depth + backward.depth between @min_depth and @max_depth + and not exists (select 1 from unnest(forward.edge_ids) edge_id where edge_id = any(backward.edge_ids)) + order by depth, edge_ids limit 1 +)` + if testCase.Expected.ResultKind != "path_set" { + return search + ` select depth from shortest` + } + return search + ` +select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + shortest.edge_ids, + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from shortest join node root on root.graph_id = @graph_id and root.id = @start_id` +} + +// fixedSuffixExpansionReferenceSpecs builds exact reference implementations for fixed-suffix expansion cases. +func (s *postgresSQLRunner) fixedSuffixExpansionReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + kindNames := []string{"ExpansionRoot", "SuffixHead", "SuffixMiddle", "SuffixTerminal", "Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"} + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + for _, name := range kindNames { + kindID, err := s.pgDriver.KindMapper().MapKind(ctx, graph.StringKind(name)) + if err != nil { + return nil, fmt.Errorf("map reference kind %s: %w", name, err) + } + probeParams[name+"_kind"] = kindID + } + probeParams["min_depth"] = int32(0) + if testCase.Shape.MinDepth != nil { + probeParams["min_depth"] = int32(*testCase.Shape.MinDepth) + } + probeParams["max_depth"] = int32(15) + if testCase.Shape.MaxDepth != nil { + probeParams["max_depth"] = int32(*testCase.Shape.MaxDepth) + } + specs := buildFixedSuffixExpansionReferenceSpecs(testCase, probeParams) + if !referenceHydrationRequested(s.referenceArms) { + return specs, nil + } + searchIdx := referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids") + values, err := readReferenceRow(ctx, s.db, specs[searchIdx].sql, specs[searchIdx].parameters, s.readTransactionOptions()...) + if err != nil { + return nil, fmt.Errorf("precompute fixed-suffix expansion hydration IDs: %w", err) + } + if len(values) == 0 { + completeIdx := referenceSpecIndex(specs, "complete_reference") + emptyHydration := postgresReferenceSpec{ + name: "hydration_only", + architecture: "hydration", + implementationID: "typed_empty_v1", + stateShape: "empty ordered ID input", + observationShape: "typed empty path result", + semanticValidation: "not_applicable_empty_input", + boundary: "typed empty path result", + sql: `select null::pathComposite where false`, + parameters: probeParams, + } + orderedEmptyHydration := emptyHydration + orderedEmptyHydration.name = "ordered_path_ids_hydration_only" + specs = slices.Insert(specs, completeIdx, emptyHydration, orderedEmptyHydration) + return specs, nil + } + if len(values) != 3 { + return nil, fmt.Errorf("precompute fixed-suffix expansion hydration IDs returned %d columns, expected 3", len(values)) + } + nodeIDs, err := referenceInt64Slice(values[0]) + if err != nil || len(nodeIDs) == 0 { + return nil, fmt.Errorf("decode fixed-suffix expansion hydration node IDs: %w", err) + } + edgeIDs, err := referenceInt64Slice(values[2]) + if err != nil { + return nil, fmt.Errorf("decode fixed-suffix expansion hydration edge IDs: %w", err) + } + hydrationParams := copyReferenceParams(probeParams) + hydrationParams["root_id"] = nodeIDs[0] + hydrationParams["node_ids"] = nodeIDs + hydrationParams["edge_ids"] = edgeIDs + hydration := postgresReferenceSpec{ + name: "hydration_only", + boundary: "one complete path composite from precomputed ordered edge IDs", + sql: `select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + @edge_ids::int8[], + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from node root where root.graph_id = @graph_id and root.id = @root_id`, + parameters: hydrationParams, + } + orderedHydration := postgresReferenceSpec{ + name: "ordered_path_ids_hydration_only", + architecture: "hydration", + implementationID: "inline_ordered_path_ids_v1", + stateShape: "precomputed ordered node and edge ID arrays", + observationShape: "complete path composite", + boundary: "one complete path composite from precomputed ordered node and edge IDs", + sql: `select row( + coalesce(( + select array_agg((n.id, n.kind_ids, n.properties)::nodeComposite order by path_node.ordinality) + from unnest(@node_ids::int8[]) with ordinality as path_node(id, ordinality) + join node n on n.graph_id = @graph_id and n.id = path_node.id + ), array[]::nodeComposite[]), + coalesce(( + select array_agg((e.id, e.start_id, e.end_id, e.kind_id, e.properties)::edgeComposite order by path_edge.ordinality) + from unnest(@edge_ids::int8[]) with ordinality as path_edge(id, ordinality) + join edge e on e.graph_id = @graph_id and e.id = path_edge.id + ), array[]::edgeComposite[]) +)::pathComposite`, + parameters: hydrationParams, + } + completeIdx := referenceSpecIndex(specs, "complete_reference") + specs = slices.Insert(specs, completeIdx, hydration, orderedHydration) + return specs, nil +} + +// referenceHydrationRequested reports whether the selected arm requires precomputed hydration inputs. +func referenceHydrationRequested(referenceArms []string) bool { + return len(referenceArms) == 0 || slices.Contains(referenceArms, "hydration_only") || slices.Contains(referenceArms, "ordered_path_ids_hydration_only") +} + +// buildFixedSuffixExpansionReferenceSpecs assembles fixed-suffix search and hydration references for one case. +func buildFixedSuffixExpansionReferenceSpecs(testCase ScaleCase, probeParams map[string]any) []postgresReferenceSpec { + roots := `roots(root_id) as materialized ( + select n.id from node n + where n.graph_id = @graph_id + and @ExpansionRoot_kind::int2 = any(n.kind_ids) + and n.properties ->> 'root_key' = @root_key +)` + suffix := `suffix_rows(boundary_id, head_id, terminal_id, suffix_edge_ids, suffix_node_ids) as materialized ( + select boundary.id, suffix_head.id, suffix_terminal.id, + array[enter_suffix.id, continue_suffix.id, complete_suffix.id]::int8[], + array[boundary.id, suffix_head.id, suffix_middle.id, suffix_terminal.id]::int8[] + from (select 1 from roots limit 1) root_presence + cross join edge enter_suffix + join node boundary on boundary.graph_id = @graph_id and boundary.id = enter_suffix.start_id + join node suffix_head on suffix_head.graph_id = @graph_id and suffix_head.id = enter_suffix.end_id and @SuffixHead_kind::int2 = any(suffix_head.kind_ids) + join edge continue_suffix on continue_suffix.graph_id = @graph_id and continue_suffix.start_id = suffix_head.id and continue_suffix.kind_id = @ContinueSuffix_kind + join node suffix_middle on suffix_middle.graph_id = @graph_id and suffix_middle.id = continue_suffix.end_id and @SuffixMiddle_kind::int2 = any(suffix_middle.kind_ids) + join edge complete_suffix on complete_suffix.graph_id = @graph_id and complete_suffix.start_id = suffix_middle.id and complete_suffix.kind_id = @CompleteSuffix_kind + join node suffix_terminal on suffix_terminal.graph_id = @graph_id and suffix_terminal.id = complete_suffix.end_id and @SuffixTerminal_kind::int2 = any(suffix_terminal.kind_ids) + where enter_suffix.graph_id = @graph_id and enter_suffix.kind_id = @EnterSuffix_kind + and continue_suffix.id <> enter_suffix.id + and complete_suffix.id <> enter_suffix.id and complete_suffix.id <> continue_suffix.id +)` + forwardExpansion := `expansion_paths(root_id, node_id, node_ids, edge_ids, depth) as ( + select root_id, root_id, array[root_id]::int8[], array[]::int8[], 0 from roots + union all + select expansion_paths.root_id, e.end_id, expansion_paths.node_ids || e.end_id, expansion_paths.edge_ids || e.id, expansion_paths.depth + 1 + from expansion_paths join edge e + on e.graph_id = @graph_id and e.start_id = expansion_paths.node_id and e.kind_id = @Expand_kind + join node next_node on next_node.graph_id = @graph_id and next_node.id = e.end_id + where expansion_paths.depth < @max_depth and e.id != all(expansion_paths.edge_ids) +)` + scalarForwardExpansion := strings.Replace(forwardExpansion, "\n join node next_node on next_node.graph_id = @graph_id and next_node.id = e.end_id", "", 1) + allExpansionNodesExist := `not exists ( + select 1 from unnest(expansion_paths.node_ids) as expansion_node_id(id) + left join node expansion_node on expansion_node.graph_id = @graph_id and expansion_node.id = expansion_node_id.id + where expansion_node.id is null + )` + legacyForward := `with recursive ` + roots + `, ` + forwardExpansion + `, paths as materialized ( + select expansion_paths.node_ids || array[suffix_head.id, suffix_middle.id, suffix_terminal.id]::int8[] as node_ids, + suffix_head.id as head_id, suffix_terminal.id as terminal_id, + expansion_paths.edge_ids || enter_suffix.id || continue_suffix.id || complete_suffix.id as edge_ids + from expansion_paths + join edge enter_suffix on enter_suffix.graph_id = @graph_id and enter_suffix.start_id = expansion_paths.node_id and enter_suffix.kind_id = @EnterSuffix_kind and enter_suffix.id != all(expansion_paths.edge_ids) + join node suffix_head on suffix_head.graph_id = @graph_id and suffix_head.id = enter_suffix.end_id and @SuffixHead_kind::int2 = any(suffix_head.kind_ids) + join edge continue_suffix on continue_suffix.graph_id = @graph_id and continue_suffix.start_id = suffix_head.id and continue_suffix.kind_id = @ContinueSuffix_kind + and continue_suffix.id != enter_suffix.id and continue_suffix.id != all(expansion_paths.edge_ids) + join node suffix_middle on suffix_middle.graph_id = @graph_id and suffix_middle.id = continue_suffix.end_id and @SuffixMiddle_kind::int2 = any(suffix_middle.kind_ids) + join edge complete_suffix on complete_suffix.graph_id = @graph_id and complete_suffix.start_id = suffix_middle.id and complete_suffix.kind_id = @CompleteSuffix_kind + and complete_suffix.id != enter_suffix.id and complete_suffix.id != continue_suffix.id and complete_suffix.id != all(expansion_paths.edge_ids) + join node suffix_terminal on suffix_terminal.graph_id = @graph_id and suffix_terminal.id = complete_suffix.end_id and @SuffixTerminal_kind::int2 = any(suffix_terminal.kind_ids) + where expansion_paths.depth >= @min_depth +)` + lateHydratedForward := `with recursive ` + roots + `, ` + scalarForwardExpansion + `, paths as materialized ( + select expansion_paths.node_ids || array[suffix_head.id, suffix_middle.id, suffix_terminal.id]::int8[] as node_ids, + suffix_head.id as head_id, suffix_terminal.id as terminal_id, + expansion_paths.edge_ids || enter_suffix.id || continue_suffix.id || complete_suffix.id as edge_ids + from expansion_paths + join edge enter_suffix on enter_suffix.graph_id = @graph_id and enter_suffix.start_id = expansion_paths.node_id and enter_suffix.kind_id = @EnterSuffix_kind and enter_suffix.id != all(expansion_paths.edge_ids) + join node suffix_head on suffix_head.graph_id = @graph_id and suffix_head.id = enter_suffix.end_id and @SuffixHead_kind::int2 = any(suffix_head.kind_ids) + join edge continue_suffix on continue_suffix.graph_id = @graph_id and continue_suffix.start_id = suffix_head.id and continue_suffix.kind_id = @ContinueSuffix_kind + and continue_suffix.id != enter_suffix.id and continue_suffix.id != all(expansion_paths.edge_ids) + join node suffix_middle on suffix_middle.graph_id = @graph_id and suffix_middle.id = continue_suffix.end_id and @SuffixMiddle_kind::int2 = any(suffix_middle.kind_ids) + join edge complete_suffix on complete_suffix.graph_id = @graph_id and complete_suffix.start_id = suffix_middle.id and complete_suffix.kind_id = @CompleteSuffix_kind + and complete_suffix.id != enter_suffix.id and complete_suffix.id != continue_suffix.id and complete_suffix.id != all(expansion_paths.edge_ids) + join node suffix_terminal on suffix_terminal.graph_id = @graph_id and suffix_terminal.id = complete_suffix.end_id and @SuffixTerminal_kind::int2 = any(suffix_terminal.kind_ids) + where expansion_paths.depth >= @min_depth and ` + allExpansionNodesExist + ` +)` + factoredForward := `with recursive ` + roots + `, ` + suffix + `, ` + scalarForwardExpansion + `, paths as materialized ( + select expansion_paths.node_ids || suffix_rows.suffix_node_ids[2:4] as node_ids, + suffix_rows.head_id, suffix_rows.terminal_id, + expansion_paths.edge_ids || suffix_rows.suffix_edge_ids as edge_ids + from expansion_paths join suffix_rows on suffix_rows.boundary_id = expansion_paths.node_id + where expansion_paths.depth >= @min_depth + and not exists (select 1 from unnest(expansion_paths.edge_ids) as expansion_edge(id) where expansion_edge.id = any(suffix_rows.suffix_edge_ids)) + and ` + allExpansionNodesExist + ` +)` + reverse := `with recursive ` + roots + `, ` + suffix + `, boundary_ids(boundary_id) as materialized ( + select distinct boundary_id from suffix_rows +), reverse_trails(boundary_id, node_id, node_ids, edge_ids, depth) as ( + select boundary_id, boundary_id, array[boundary_id]::int8[], array[]::int8[], 0 from boundary_ids + union all + select reverse_trails.boundary_id, e.start_id, array_prepend(e.start_id, reverse_trails.node_ids), + array_prepend(e.id, reverse_trails.edge_ids), reverse_trails.depth + 1 + from reverse_trails join edge e + on e.graph_id = @graph_id and e.end_id = reverse_trails.node_id and e.kind_id = @Expand_kind + where reverse_trails.depth < @max_depth and e.id != all(reverse_trails.edge_ids) +), paths as materialized ( + select reverse_trails.node_ids || suffix_rows.suffix_node_ids[2:4] as node_ids, + suffix_rows.head_id, suffix_rows.terminal_id, + reverse_trails.edge_ids || suffix_rows.suffix_edge_ids as edge_ids + from reverse_trails + join roots on roots.root_id = reverse_trails.node_id + join suffix_rows on suffix_rows.boundary_id = reverse_trails.boundary_id + where reverse_trails.depth >= @min_depth + and not exists (select 1 from unnest(reverse_trails.edge_ids) as expansion_edge(id) where expansion_edge.id = any(suffix_rows.suffix_edge_ids)) + and not exists ( + select 1 from unnest(reverse_trails.node_ids) as expansion_node_id(id) + left join node expansion_node on expansion_node.graph_id = @graph_id and expansion_node.id = expansion_node_id.id + where expansion_node.id is null + ) +)` + viability := `with recursive ` + roots + `, ` + suffix + `, boundary_ids(boundary_id) as materialized ( + select distinct boundary_id from suffix_rows +), viable(node_id, reverse_distance) as ( + select boundary_id, 0 from boundary_ids + union + select e.start_id, viable.reverse_distance + 1 + from viable join edge e + on e.graph_id = @graph_id and e.end_id = viable.node_id and e.kind_id = @Expand_kind + where viable.reverse_distance < @max_depth +), expansion_paths(root_id, node_id, node_ids, edge_ids, depth) as ( + select root_id, root_id, array[root_id]::int8[], array[]::int8[], 0 from roots + where exists (select 1 from viable where viable.node_id = roots.root_id and viable.reverse_distance <= @max_depth) + union all + select expansion_paths.root_id, e.end_id, expansion_paths.node_ids || e.end_id, expansion_paths.edge_ids || e.id, expansion_paths.depth + 1 + from expansion_paths join edge e + on e.graph_id = @graph_id and e.start_id = expansion_paths.node_id and e.kind_id = @Expand_kind + where expansion_paths.depth < @max_depth and e.id != all(expansion_paths.edge_ids) + and exists (select 1 from viable where viable.node_id = e.end_id and viable.reverse_distance <= @max_depth - expansion_paths.depth - 1) +), paths as materialized ( + select expansion_paths.node_ids || suffix_rows.suffix_node_ids[2:4] as node_ids, + suffix_rows.head_id, suffix_rows.terminal_id, + expansion_paths.edge_ids || suffix_rows.suffix_edge_ids as edge_ids + from expansion_paths join suffix_rows on suffix_rows.boundary_id = expansion_paths.node_id + where expansion_paths.depth >= @min_depth + and not exists (select 1 from unnest(expansion_paths.edge_ids) as expansion_edge(id) where expansion_edge.id = any(suffix_rows.suffix_edge_ids)) + and ` + allExpansionNodesExist + ` +)` + fullSQL := legacyForward + ` select head_id, terminal_id from paths` + boundary := "endpoint ID pairs" + pathObserved := testCase.Observes.Paths || testCase.Expected.ResultKind == "path_set" + complete := func(search string) string { + if !pathObserved { + return search + ` select head_id, terminal_id from paths` + } + return search + ` +select ordered_edge_ids_to_path( + @graph_id, + (root.id, root.kind_ids, root.properties)::nodeComposite, + paths.edge_ids, + array[(root.id, root.kind_ids, root.properties)::nodeComposite]::nodeComposite[] +)::pathComposite +from paths join node root on root.graph_id = @graph_id and root.id = paths.node_ids[1]` + } + if pathObserved { + fullSQL = complete(legacyForward) + boundary = "complete path composite" + } + orderedLegacy := legacyForward + ` select node_ids, head_id, edge_ids from paths` + orderedReference := func(spec postgresReferenceSpec) postgresReferenceSpec { + spec.semanticValidation = "exact_ordered_ids" + spec.validationSQL = orderedLegacy + spec.validationParams = probeParams + return spec + } + return []postgresReferenceSpec{ + { + name: "round_trip", + architecture: "protocol", + stateShape: "none", + boundary: "prepared protocol and transaction", + sql: `select 1`, + }, + { + name: "endpoint_validation", + architecture: "root_validation", + stateShape: "root ID bag", + boundary: "validated root ID", + sql: `select n.id from node n where n.graph_id = @graph_id and @ExpansionRoot_kind::int2 = any(n.kind_ids) and n.properties ->> 'root_key' = @root_key`, + parameters: probeParams, + }, + { + name: "fixed_suffix_rows", + architecture: "factored_suffix", + stateShape: "boundary and ordered suffix IDs", + boundary: "exact suffix rows and distinct boundary IDs", + sql: `with ` + roots + `, ` + suffix + ` select boundary_id, head_id, terminal_id, suffix_edge_ids from suffix_rows`, + parameters: probeParams, + }, + { + name: "minimum_graph_access", + architecture: "root_adjacency", + stateShape: "edge IDs", + boundary: "root adjacency edge IDs", + sql: `with ` + roots + ` select e.id from roots join edge e on e.graph_id = @graph_id and e.start_id = roots.root_id and e.kind_id = @Expand_kind order by e.id`, + parameters: probeParams, + }, + orderedReference(postgresReferenceSpec{ + name: "search_ordered_ids", + architecture: "EXPANSION-STEPWISE-FORWARD-SQL", + observationShape: "ordered_ids", + stateShape: "root/boundary IDs and ordered relationship trail", + boundary: "ordered node/edge IDs without hydration", + sql: orderedLegacy, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "stepwise_forward_aa_ordered_ids", + architecture: "EXPANSION-STEPWISE-FORWARD-AA", + aaAliasOf: "search_ordered_ids", + observationShape: "ordered_ids", + stateShape: "root/boundary IDs and ordered relationship trail", + boundary: "ordered node/edge IDs", + sql: orderedLegacy, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "root_reuse_ordered_ids", + architecture: "EXPANSION-STEPWISE-FORWARD-AA", + aaAliasOf: "search_ordered_ids", + observationShape: "ordered_ids", + stateShape: "root/boundary IDs and ordered relationship trail", + boundary: "ordered node/edge IDs", + sql: orderedLegacy, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "late_hydration_ordered_ids", + architecture: "EXPANSION-LATE-HYDRATED-FORWARD", + observationShape: "ordered_ids", + stateShape: "scalar expansion state and ordered relationship trail", + boundary: "ordered node/edge IDs", + sql: lateHydratedForward + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "factored_suffix_forward_ordered_ids", + architecture: "EXPANSION-FACTORED-SUFFIX-FORWARD", + observationShape: "ordered_ids", + stateShape: "scalar forward trails joined to exact suffix bag", + boundary: "ordered node/edge IDs", + sql: factoredForward + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "suffix_seeded_reverse_ordered_ids", + architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + observationShape: "ordered_ids", + stateShape: "scalar reverse trails with prepended relationship IDs", + boundary: "ordered node/edge IDs", + sql: reverse + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + orderedReference(postgresReferenceSpec{ + name: "backward_viability_forward_ordered_ids", + architecture: "EXPANSION-BACKWARD-VIABILITY-FORWARD", + observationShape: "ordered_ids", + stateShape: "depth-aware viability filter plus exact forward trails", + boundary: "ordered node/edge IDs", + sql: viability + ` select node_ids, head_id, edge_ids from paths`, + parameters: probeParams, + }), + { + name: "complete_reference", + architecture: "EXPANSION-STEPWISE-FORWARD-SQL", + stateShape: "forward relationship trails", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: fullSQL, + parameters: probeParams, + }, + { + name: "root_reuse_complete", + architecture: "EXPANSION-STEPWISE-FORWARD-AA", + aaAliasOf: "complete_reference", + stateShape: "forward relationship trails", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(legacyForward), + parameters: probeParams, + }, + { + name: "late_hydration_complete", + architecture: "EXPANSION-LATE-HYDRATED-FORWARD", + stateShape: "scalar expansion state with final-only hydration", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(lateHydratedForward), + parameters: probeParams, + }, + { + name: "factored_suffix_forward_complete", + architecture: "EXPANSION-FACTORED-SUFFIX-FORWARD", + stateShape: "exact forward trails joined to suffix bag", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(factoredForward), + parameters: probeParams, + }, + { + name: "suffix_seeded_reverse_complete", + architecture: "EXPANSION-SUFFIX-SEEDED-REVERSE", + stateShape: "exact reverse trails joined back to suffix bag", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(reverse), + parameters: probeParams, + }, + { + name: "backward_viability_forward_complete", + architecture: "EXPANSION-BACKWARD-VIABILITY-FORWARD", + stateShape: "permissive viability plus exact forward trails", + observationShape: observationShapeForCase(testCase), + semanticValidation: "exact_public_observation", + boundary: boundary, + fullComparator: true, + sql: complete(viability), + parameters: probeParams, + }, + } +} + +// observationShapeForCase selects full public path observations when the case exposes paths and endpoint IDs otherwise. +func observationShapeForCase(testCase ScaleCase) string { + if testCase.Observes.Paths || testCase.Expected.ResultKind == "path_set" { + return "public_observation" + } + return "endpoint_ids" +} + +// referenceSpecIndex returns a reference arm's index and panics when the arm is absent. +func referenceSpecIndex(specs []postgresReferenceSpec, name string) int { + for idx, spec := range specs { + if spec.name == name { + return idx + } + } + panic("missing PostgreSQL reference spec " + name) +} + +// referenceSpecIndexOrMissing returns a reference arm's index or -1 when absent. +func referenceSpecIndexOrMissing(specs []postgresReferenceSpec, name string) int { + for idx, spec := range specs { + if spec.name == name { + return idx + } + } + return -1 +} + +// readReferenceRow reads reference row and propagates I/O or decoding failures. +func readReferenceRow(ctx context.Context, db graph.Database, sqlQuery string, params map[string]any, transactionOptions ...graph.TransactionOption) ([]any, error) { + var values []any + err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw(sqlQuery, params) + defer result.Close() + if !result.Next() { + if err := result.Error(); err != nil { + return err + } + return nil + } + values = append(values, result.Values()...) + return result.Error() + }, transactionOptions...) + if err != nil { + return nil, err + } + + return values, nil +} + +// referenceInt64Slice normalizes supported driver array representations to []int64. +func referenceInt64Slice(value any) ([]int64, error) { + switch typed := value.(type) { + case []int64: + result := make([]int64, len(typed)) + copy(result, typed) + return result, nil + case []int32: + result := make([]int64, len(typed)) + for idx, item := range typed { + result[idx] = int64(item) + } + return result, nil + case []any: + result := make([]int64, len(typed)) + for idx, item := range typed { + switch integer := item.(type) { + case int64: + result[idx] = integer + case int32: + result[idx] = int64(integer) + default: + return nil, fmt.Errorf("array item %d has type %T", idx, item) + } + } + return result, nil + default: + return nil, fmt.Errorf("expected integer array, got %T", value) + } +} + +// copyReferenceParams duplicates reference params without aliasing mutable state. +func copyReferenceParams(params map[string]any) map[string]any { + copy := make(map[string]any, len(params)+10) + for name, value := range params { + copy[name] = value + } + return copy +} + +// measureRawPostgres executes raw PostgreSQL and records its timing observations. +func measureRawPostgres(ctx context.Context, db graph.Database, sqlQuery string, params map[string]any, warmupIterations, iterations int, transactionOptions ...graph.TransactionOption) (int64, DurationStats, error) { + run := func() (int64, error) { + var count int64 + err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw(sqlQuery, params) + defer result.Close() + for result.Next() { + count++ + _ = result.Values() + } + return result.Error() + }, transactionOptions...) + if err != nil { + return 0, err + } + + return count, nil + } + coldStart := time.Now() + rowCount, err := run() + if err != nil { + return 0, DurationStats{}, err + } + coldDuration := time.Since(coldStart) + for range warmupIterations { + nextCount, err := run() + if err != nil { + return 0, DurationStats{}, err + } + if nextCount != rowCount { + return 0, DurationStats{}, fmt.Errorf("reference row count changed from %d to %d", rowCount, nextCount) + } + } + durations := make([]time.Duration, iterations) + for idx := range iterations { + start := time.Now() + nextCount, err := run() + if err != nil { + return 0, DurationStats{}, err + } + if nextCount != rowCount { + return 0, DurationStats{}, fmt.Errorf("reference row count changed from %d to %d", rowCount, nextCount) + } + durations[idx] = time.Since(start) + } + stats, err := computeDurationStats(durations) + if err != nil { + return 0, DurationStats{}, err + } + stats.WarmupIterations = warmupIterations + stats.Samples = append([]LatencySample{{ + Iteration: 0, + Classification: "cold", + Duration: coldDuration, + }}, stats.Samples...) + return rowCount, stats, nil +} diff --git a/cmd/graphbench/references_expand_into.go b/cmd/graphbench/references_expand_into.go new file mode 100644 index 00000000..d598be2b --- /dev/null +++ b/cmd/graphbench/references_expand_into.go @@ -0,0 +1,183 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + + "github.com/specterops/dawgs/graph" +) + +// expandIntoInputPairs reserves the stable protocol value used to recognize expand into input pairs across artifacts and executions. +const expandIntoInputPairs = `input_pairs(pair_ordinal, start_id, end_id) as materialized ( + select pair_ordinal, start_id, @end_id::int8 + from unnest(@start_ids::int8[]) with ordinality input(start_id, pair_ordinal) +)` + +// expandIntoReferenceSpecs builds three exact one-hop bound-pair arms sharing the same public relationship boundary. +func (s *postgresSQLRunner) expandIntoReferenceSpecs(ctx context.Context, testCase ScaleCase, params map[string]any) ([]postgresReferenceSpec, error) { + probeParams := copyReferenceParams(params) + probeParams["graph_id"] = s.graphID + edgeKinds := make(graph.Kinds, 0, len(testCase.Shape.EdgeKinds)) + for _, name := range testCase.Shape.EdgeKinds { + edgeKinds = append(edgeKinds, graph.StringKind(name)) + } + var edgeKindIDs []int16 + if len(edgeKinds) > 0 { + if s.pgDriver == nil { + return nil, fmt.Errorf("map ExpandInto reference edge kinds: PostgreSQL driver is unavailable") + } + var err error + edgeKindIDs, err = s.pgDriver.KindMapper().MapKinds(ctx, edgeKinds) + if err != nil { + return nil, fmt.Errorf("map ExpandInto reference edge kinds: %w", err) + } + } + probeParams["edge_kind_ids"] = edgeKindIDs + return buildExpandIntoReferenceSpecs(probeParams, testCase.Shape.Direction), nil +} + +// buildExpandIntoReferenceSpecs constructs the exact SQL arms after graph/kind parameters are resolved. +func buildExpandIntoReferenceSpecs(probeParams map[string]any, direction string) []postgresReferenceSpec { + pairJoinPredicate := expandIntoPairPredicate(direction, "matched", "input_pairs") + startDegreePredicate := expandIntoEndpointPredicate(direction, true, "start_adj", "input_pairs") + endDegreePredicate := expandIntoEndpointPredicate(direction, false, "end_adj", "input_pairs") + startScanPredicate := expandIntoEndpointPredicate(direction, true, "outbound", "input_pairs") + endScanPredicate := expandIntoEndpointPredicate(direction, false, "inbound", "input_pairs") + startPairPredicate := expandIntoPairPredicate(direction, "outbound", "input_pairs") + endPairPredicate := expandIntoPairPredicate(direction, "inbound", "input_pairs") + cachePairPredicate := expandIntoPairPredicate(direction, "matched", "distinct_pairs") + + pairJoin := `with ` + expandIntoInputPairs + ` +select (matched.id, matched.start_id, matched.end_id, matched.kind_id, matched.properties)::edgeComposite +from input_pairs +join edge matched on matched.graph_id = @graph_id + and ` + pairJoinPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or matched.kind_id = any(@edge_kind_ids::int2[]))` + + lowerDegree := `with ` + expandIntoInputPairs + ` +select (matched.id, matched.start_id, matched.end_id, matched.kind_id, matched.properties)::edgeComposite +from input_pairs +join lateral ( + with degrees as materialized ( + select + (select count(*) from edge start_adj + where start_adj.graph_id = @graph_id and ` + startDegreePredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or start_adj.kind_id = any(@edge_kind_ids::int2[]))) as start_degree, + (select count(*) from edge end_adj + where end_adj.graph_id = @graph_id and ` + endDegreePredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or end_adj.kind_id = any(@edge_kind_ids::int2[]))) as end_degree + ) + select candidate.* + from degrees + join lateral ( + select outbound.* from edge outbound + where degrees.start_degree <= degrees.end_degree + and outbound.graph_id = @graph_id and ` + startScanPredicate + ` + and ` + startPairPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or outbound.kind_id = any(@edge_kind_ids::int2[])) + union all + select inbound.* from edge inbound + where degrees.end_degree < degrees.start_degree + and inbound.graph_id = @graph_id and ` + endScanPredicate + ` + and ` + endPairPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or inbound.kind_id = any(@edge_kind_ids::int2[])) + ) candidate on true +) matched on true` + + pairCache := `with ` + expandIntoInputPairs + `, +distinct_pairs(start_id, end_id) as materialized ( + select distinct start_id, end_id from input_pairs +), pair_matches(start_id, end_id, id, edge_start_id, edge_end_id, kind_id, properties) as materialized ( + select distinct_pairs.start_id, distinct_pairs.end_id, + matched.id, matched.start_id, matched.end_id, matched.kind_id, matched.properties + from distinct_pairs + join edge matched on matched.graph_id = @graph_id + and ` + cachePairPredicate + ` + and (cardinality(@edge_kind_ids::int2[]) = 0 or matched.kind_id = any(@edge_kind_ids::int2[])) +) +select (pair_matches.id, pair_matches.edge_start_id, pair_matches.edge_end_id, pair_matches.kind_id, pair_matches.properties)::edgeComposite +from input_pairs +join pair_matches on pair_matches.start_id = input_pairs.start_id and pair_matches.end_id = input_pairs.end_id` + + return []postgresReferenceSpec{ + { + name: "expand_into_pair_join", + architecture: "EXPAND-INTO-PAIR-JOIN", + implementationID: "expand_into_parameterized_pair_join_v2", + stateShape: "outer pair rows joined directly to matching relationships", + observationShape: "complete relationship composites", + semanticValidation: "exact_public_observation", + boundary: "complete matching relationships", + fullComparator: true, + sql: pairJoin, + parameters: probeParams, + }, + { + name: "expand_into_lower_degree_scan", + architecture: "EXPAND-INTO-LOWER-DEGREE", + implementationID: "expand_into_typed_lower_degree_scan_v2", + stateShape: "per-pair typed directional degrees plus one disjoint adjacency scan", + observationShape: "complete relationship composites", + semanticValidation: "exact_public_observation", + boundary: "complete matching relationships", + fullComparator: true, + sql: lowerDegree, + parameters: probeParams, + }, + { + name: "expand_into_pair_cache", + architecture: "EXPAND-INTO-PAIR-CACHE", + implementationID: "expand_into_distinct_pair_match_cache_v2", + stateShape: "statement-local distinct pair keys and every matching relationship row", + observationShape: "complete relationship composites with duplicate outer-row multiplicity reapplied", + semanticValidation: "exact_public_observation", + boundary: "complete matching relationships", + fullComparator: true, + sql: pairCache, + parameters: probeParams, + }, + } +} + +// expandIntoPairPredicate returns the complete physical edge predicate for one +// logical bound pair. The directionless form uses one OR predicate rather than +// UNION ALL so a self-loop is emitted once, matching Cypher relationship +// multiplicity. +func expandIntoPairPredicate(direction, edgeAlias, pairAlias string) string { + outbound := fmt.Sprintf("%s.start_id = %s.start_id and %s.end_id = %s.end_id", edgeAlias, pairAlias, edgeAlias, pairAlias) + inbound := fmt.Sprintf("%s.end_id = %s.start_id and %s.start_id = %s.end_id", edgeAlias, pairAlias, edgeAlias, pairAlias) + switch direction { + case "inbound": + return inbound + case "directionless": + return "((" + outbound + ") or (" + inbound + "))" + default: + return outbound + } +} + +// expandIntoEndpointPredicate returns the physical adjacency predicate for the +// logical start or end endpoint used by the lower-degree reference arm. +func expandIntoEndpointPredicate(direction string, logicalStart bool, edgeAlias, pairAlias string) string { + pairColumn := "end_id" + if logicalStart { + pairColumn = "start_id" + } + physicalColumn := pairColumn + if direction == "inbound" { + if physicalColumn == "start_id" { + physicalColumn = "end_id" + } else { + physicalColumn = "start_id" + } + } + if direction == "directionless" { + return fmt.Sprintf("(%s.start_id = %s.%s or %s.end_id = %s.%s)", edgeAlias, pairAlias, pairColumn, edgeAlias, pairAlias, pairColumn) + } + return fmt.Sprintf("%s.%s = %s.%s", edgeAlias, physicalColumn, pairAlias, pairColumn) +} diff --git a/cmd/graphbench/references_expand_into_test.go b/cmd/graphbench/references_expand_into_test.go new file mode 100644 index 00000000..e4ab94a8 --- /dev/null +++ b/cmd/graphbench/references_expand_into_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// TestExpandIntoReferencesShareExactRelationshipBoundary verifies all three study arms preserve rows, cross-kind matches, and duplicate input multiplicity. +func TestExpandIntoReferencesShareExactRelationshipBoundary(t *testing.T) { + params := map[string]any{ + "graph_id": int32(7), "start_ids": []int64{1, 2, 1}, "end_id": int64(3), "edge_kind_ids": []int16{4, 5}, + } + specs := buildExpandIntoReferenceSpecs(params, "outbound") + require.Len(t, specs, 3) + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + require.NoError(t, validateReferenceSpecs(specs)) + + pairJoin := specs[referenceSpecIndex(specs, "expand_into_pair_join")] + require.True(t, pairJoin.fullComparator) + require.Contains(t, pairJoin.sql, "unnest(@start_ids::int8[]) with ordinality") + require.Contains(t, pairJoin.sql, "matched.start_id = input_pairs.start_id and matched.end_id = input_pairs.end_id") + require.Contains(t, pairJoin.sql, "matched.kind_id = any(@edge_kind_ids::int2[])") + + lowerDegree := specs[referenceSpecIndex(specs, "expand_into_lower_degree_scan")] + require.Contains(t, lowerDegree.sql, "degrees as materialized") + require.Contains(t, lowerDegree.sql, "degrees.start_degree <= degrees.end_degree") + require.Contains(t, lowerDegree.sql, "degrees.end_degree < degrees.start_degree") + require.Contains(t, lowerDegree.sql, "union all") + + cache := specs[referenceSpecIndex(specs, "expand_into_pair_cache")] + require.Contains(t, cache.sql, "select distinct start_id, end_id from input_pairs") + require.Contains(t, cache.sql, "pair_matches") + require.Contains(t, cache.observationShape, "duplicate outer-row multiplicity") + for _, spec := range specs { + require.Equal(t, "exact_public_observation", spec.semanticValidation) + require.Equal(t, params, spec.parameters) + } +} + +// TestExpandIntoReferencesPreserveInboundAndDirectionlessPairs verifies every +// study arm uses the same physical pair semantics and does not double-count a +// directionless self-loop. +func TestExpandIntoReferencesPreserveInboundAndDirectionlessPairs(t *testing.T) { + inbound := buildExpandIntoReferenceSpecs(map[string]any{}, "inbound") + require.Contains(t, inbound[0].sql, "matched.end_id = input_pairs.start_id") + require.Contains(t, inbound[1].sql, "outbound.end_id = input_pairs.start_id") + require.Contains(t, inbound[1].sql, "inbound.start_id = input_pairs.end_id") + require.Contains(t, inbound[2].sql, "matched.end_id = distinct_pairs.start_id") + + directionless := buildExpandIntoReferenceSpecs(map[string]any{}, "directionless") + for _, spec := range directionless { + require.Contains(t, spec.sql, " or (") + require.NotContains(t, spec.sql, "union all\n select matched") + } + require.Contains(t, directionless[1].sql, "degrees.start_degree <= degrees.end_degree") + require.Contains(t, directionless[1].sql, "degrees.end_degree < degrees.start_degree") +} + +// TestExpandIntoScaleCasesParse verifies the shared three-way study corpus remains valid Cypher input. +func TestExpandIntoScaleCasesParse(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + found := 0 + stateClasses := map[string]bool{} + for _, testCase := range corpus.Cases { + if testCase.Category != "expand_into_one_hop" { + continue + } + found++ + stateClasses[testCase.Shape.ExpectedStateClass] = true + _, err := frontend.ParseCypher(frontend.NewContext(), testCase.Cypher) + require.NoError(t, err, testCase.Name) + } + require.Equal(t, 11, found) + require.True(t, stateClasses["source_lower_degree"]) + require.True(t, stateClasses["target_lower_degree"]) +} + +// TestExpandIntoReferenceArmsAreDeclared verifies command-line selection accepts every three-way study arm. +func TestExpandIntoReferenceArmsAreDeclared(t *testing.T) { + for _, name := range []string{"expand_into_pair_join", "expand_into_lower_degree_scan", "expand_into_pair_cache"} { + require.True(t, validPostgresReferenceArm(name), name) + } +} diff --git a/cmd/graphbench/references_test.go b/cmd/graphbench/references_test.go new file mode 100644 index 00000000..8485138b --- /dev/null +++ b/cmd/graphbench/references_test.go @@ -0,0 +1,882 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// outboundShortestPathQuery is the canonical bound-endpoint path query shared by reference-arm tests. +const outboundShortestPathQuery = "MATCH p = shortestPath((s)-[*0..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + +// TestSupplementalPostgresReadHelpersPropagateTransactionOptions verifies that +// reference timing, precomputation, and plan capture all retain the caller's +// stable-snapshot transaction contract. +func TestSupplementalPostgresReadHelpersPropagateTransactionOptions(t *testing.T) { + database := &referenceTransactionOptionTestDatabase{expectedDriverConfig: "stable-snapshot"} + transactionOption := func(config *graph.TransactionConfig) { + config.DriverConfig = "stable-snapshot" + } + + rowCount, _, err := measureRawPostgres(context.Background(), database, "select value", nil, 0, 1, transactionOption) + require.NoError(t, err) + require.Equal(t, int64(1), rowCount) + + values, err := readReferenceRow(context.Background(), database, "select value", nil, transactionOption) + require.NoError(t, err) + require.Equal(t, []any{int64(1)}, values) + + plan, planJSON, _, err := explainRawPostgres(context.Background(), database, "select value", nil, transactionOption) + require.NoError(t, err) + require.NotEmpty(t, plan) + require.NotEmpty(t, planJSON) + + require.Equal(t, []bool{true, true, true, true}, database.transactionOptionsApplied) +} + +// referenceTransactionOptionTestDatabase records transaction configuration and +// supplies the narrow raw-query surface used by supplemental reference helpers. +type referenceTransactionOptionTestDatabase struct { + // Database supplies the database input to the referenceTransactionOptionTestDatabase contract. + graph.Database + // expectedDriverConfig retains the expected driver config while referenceTransactionOptionTestDatabase is assembled or evaluated. + expectedDriverConfig any + // transactionOptionsApplied retains the transaction options applied while referenceTransactionOptionTestDatabase is assembled or evaluated. + transactionOptionsApplied []bool +} + +// ReadTransaction applies the supplied options before executing a synthetic raw transaction. +func (s *referenceTransactionOptionTestDatabase) ReadTransaction(_ context.Context, delegate graph.TransactionDelegate, options ...graph.TransactionOption) error { + config := &graph.TransactionConfig{} + for _, option := range options { + option(config) + } + s.transactionOptionsApplied = append(s.transactionOptionsApplied, config.DriverConfig == s.expectedDriverConfig) + return delegate(&referenceTransactionOptionTestTransaction{}) +} + +// referenceTransactionOptionTestTransaction returns one scalar row or one valid plan document. +type referenceTransactionOptionTestTransaction struct { + // Transaction supplies the transaction input to the referenceTransactionOptionTestTransaction contract. + graph.Transaction +} + +// Raw returns the minimal row shape expected by the helper under test. +func (s *referenceTransactionOptionTestTransaction) Raw(statement string, _ map[string]any) graph.Result { + if strings.Contains(statement, "FORMAT JSON") { + return &referenceTransactionOptionTestResult{rows: [][]any{{`[{"Plan":{"Node Type":"Result","Actual Rows":1,"Actual Loops":1}}]`}}} + } + if strings.HasPrefix(statement, "EXPLAIN ") { + return &referenceTransactionOptionTestResult{rows: [][]any{{"Result"}}} + } + return &referenceTransactionOptionTestResult{rows: [][]any{{int64(1)}}} +} + +// referenceTransactionOptionTestResult iterates a fixed set of raw rows. +type referenceTransactionOptionTestResult struct { + // Result supplies the result input to the referenceTransactionOptionTestResult contract. + graph.Result + // rows retains the rows while referenceTransactionOptionTestResult is assembled or evaluated. + rows [][]any + // index retains the index while referenceTransactionOptionTestResult is assembled or evaluated. + index int +} + +// Next advances to the next fixed row. +func (s *referenceTransactionOptionTestResult) Next() bool { + if s.index >= len(s.rows) { + return false + } + s.index++ + return true +} + +// Values returns the current fixed row. +func (s *referenceTransactionOptionTestResult) Values() []any { + return s.rows[s.index-1] +} + +// Error reports a successful fixed result. +func (s *referenceTransactionOptionTestResult) Error() error { + return nil +} + +// Close satisfies graph.Result. +func (s *referenceTransactionOptionTestResult) Close() {} + +// TestShortestReferenceSpecsAreGraphScopedAndSeparateRawFromFullOutput verifies the complete arm inventory, graph partition predicates, precomputed hydration inputs, and full-comparator metadata. +func TestShortestReferenceSpecsAreGraphScopedAndSeparateRawFromFullOutput(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(1), "end_id": int64(2), "max_depth": int32(15)} + specs := buildShortestReferenceSpecs(ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: outboundShortestPathQuery, + }, params, []int64{1, 2, 3}, []int64{10, 11}, graph.DirectionOutbound) + + require.Len(t, specs, 14) + require.Equal(t, "round_trip", specs[0].name) + require.Equal(t, int32(42), specs[1].parameters["graph_id"]) + require.Equal(t, "minimum_graph_access", specs[2].name) + require.Contains(t, specs[3].sql, "e.graph_id = @graph_id") + require.Contains(t, specs[3].boundary, "ordered node/edge IDs") + require.Equal(t, []int64{10, 11}, specs[4].parameters["edge_ids"]) + require.Equal(t, []int64{1, 2, 3}, specs[6].parameters["node_ids"]) + + s3u := specs[referenceSpecIndex(specs, "s3_unidirectional_trail_cte")] + require.True(t, s3u.fullComparator) + require.Equal(t, "complete_reference_s1_array_cte", s3u.legacyName) + require.Equal(t, "SP-S3-U-NE", s3u.architecture) + require.Contains(t, s3u.sql, "ordered_edge_ids_to_path") + + s3b := specs[referenceSpecIndex(specs, "s3_bidirectional_trail_cte")] + require.Equal(t, "candidate_s2_bidirectional_cte", s3b.legacyName) + require.Equal(t, "SP-S3-B", s3b.architecture) + require.True(t, s3b.fullComparator) + require.Contains(t, s3b.sql, "forward join backward") + require.Contains(t, s3b.sql, "e.graph_id = @graph_id") + require.Contains(t, s3b.sql, "edge_id = any(backward.edge_ids)") +} + +// TestShortestDistanceReferenceCarriesNoTrailOrPredecessorState verifies that distance-only recursion stores just the frontier node and depth, avoiding node and edge trail arrays. +func TestShortestDistanceReferenceCarriesNoTrailOrPredecessorState(t *testing.T) { + specs := buildShortestReferenceSpecs(ScaleCase{ + Name: "shortest_distance_bound_pair", + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + }, map[string]any{}, nil, nil, graph.DirectionOutbound) + reference := specs[referenceSpecIndex(specs, "s3_unidirectional_trail_cte")] + + require.Equal(t, "distance frontier node and depth only; no path or predecessor state", reference.stateShape) + require.Contains(t, reference.sql, "search(node_id, depth)") + require.NotContains(t, reference.sql, "node_ids") + require.NotContains(t, reference.sql, "edge_ids") +} + +// TestCompactBidirectionalReferencesExposeMatchedDistanceAndWitnessBoundaries +// verifies the four frozen arms share caps while preserving observation shape. +func TestCompactBidirectionalReferencesExposeMatchedDistanceAndWitnessBoundaries(t *testing.T) { + params := map[string]any{ + "graph_id": int32(42), "start_id": int64(1), "end_id": int64(3), + "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{1}, + } + distance := buildShortestReferenceSpecs(ScaleCase{ + Expected: ExpectedResult{ResultKind: "scalar"}, + }, params, nil, nil, graph.DirectionOutbound) + for _, name := range []string{"sp_b1_strict_alternating_distance", "sp_b2_smaller_frontier_distance"} { + spec := distance[referenceSpecIndex(distance, name)] + require.True(t, spec.fullComparator) + require.Equal(t, "distance scalar", spec.observationShape) + require.Equal(t, int64(100_000), spec.parameters["state_limit"]) + require.Equal(t, int64(100_000), spec.parameters["frontier_limit"]) + require.Equal(t, int64(100_000), spec.parameters["predecessor_limit"]) + require.Contains(t, spec.sql, "select depth, path as edge_ids") + require.NotContains(t, spec.sql, "ordered_edge_ids_to_path") + } + + witness := buildShortestReferenceSpecs(ScaleCase{ + Name: "one_shortest_path_bound_pair", + Expected: ExpectedResult{ResultKind: "path_set"}, + }, params, nil, nil, graph.DirectionInbound) + for _, name := range []string{"sp_b1_strict_alternating_witness_m0", "sp_b2_smaller_frontier_witness_m0"} { + spec := witness[referenceSpecIndex(witness, name)] + require.True(t, spec.fullComparator) + require.Equal(t, "public_observation", spec.observationShape) + require.Contains(t, spec.sql, "@edge_kind_ids, true") + require.Contains(t, spec.sql, "terminal.id = edge.start_id") + } +} + +// TestCanonicalSourceDistanceReferenceSwapsInboundEndpointsAndPhysicalDirection verifies that the inbound-only canonical arm searches from the logical terminal using reversed physical adjacency. +func TestCanonicalSourceDistanceReferenceSwapsInboundEndpointsAndPhysicalDirection(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(10), "end_id": int64(20), "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{1}} + testCase := ScaleCase{ + Name: "hidden_fanin", + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + } + inbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionInbound) + canonical := inbound[referenceSpecIndex(inbound, "s4_canonical_source_distance")] + require.Equal(t, "SP-I1-C-D", canonical.architecture) + require.Equal(t, int64(20), canonical.parameters["start_id"]) + require.Equal(t, int64(10), canonical.parameters["end_id"]) + require.Contains(t, canonical.sql, "e.start_id = search.node_id") + require.Contains(t, canonical.sql, "select e.end_id") + require.NotContains(t, canonical.sql, "edge_ids") + require.True(t, canonical.fullComparator) + + outbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionOutbound) + require.Equal(t, -1, referenceSpecIndexOrMissing(outbound, "s4_canonical_source_distance")) +} + +// TestShortestS1DistancePrototypeIsDistinctBoundedAndFallsBack verifies S1 metadata, its state guard and SQL fallback, and propagation of inbound traversal direction. +func TestShortestS1DistancePrototypeIsDistinctBoundedAndFallsBack(t *testing.T) { + minDepth, maxDepth := 1, 8 + params := map[string]any{ + "graph_id": int32(1), "start_id": int64(10), "end_id": int64(20), + "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{2}, + } + testCase := ScaleCase{ + Name: "distance", + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + specs := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionOutbound) + s1 := specs[referenceSpecIndex(specs, "s1_array_bfs_distance")] + + require.Equal(t, "SP-S1", s1.architecture) + require.Equal(t, "typed_plpgsql_array_bfs_distance_v1", s1.implementationID) + require.True(t, s1.fullComparator) + require.Equal(t, int32(100_000), s1.parameters["state_limit"]) + require.Contains(t, s1.sql, "graphbench_s1_distance_bfs") + require.Contains(t, s1.sql, "where (select overflow from s1)") + require.Contains(t, s1.sql, shortestDistanceReferenceSearchForDirection(graph.DirectionOutbound)) + + inbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionInbound) + require.Contains(t, inbound[referenceSpecIndex(inbound, "s1_array_bfs_distance")].sql, "@edge_kind_ids, true, @state_limit") +} + +// TestShortestS1DistancePrototypeRejectsUnsupportedShapes verifies that S1 is omitted for minimum depth above one, path results, and identical bound endpoints. +func TestShortestS1DistancePrototypeRejectsUnsupportedShapes(t *testing.T) { + minDepth, maxDepth := 2, 8 + params := map[string]any{"start_id": int64(10), "end_id": int64(20)} + distance := ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "scalar", + }, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + require.Equal(t, -1, referenceSpecIndexOrMissing(buildShortestReferenceSpecs(distance, params, nil, nil, graph.DirectionOutbound), "s1_array_bfs_distance")) + + minDepth = 1 + path := ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + require.Equal(t, -1, referenceSpecIndexOrMissing(buildShortestReferenceSpecs(path, params, nil, nil, graph.DirectionOutbound), "s1_array_bfs_distance")) + + params["end_id"] = int64(10) + require.Equal(t, -1, referenceSpecIndexOrMissing(buildShortestReferenceSpecs(distance, params, nil, nil, graph.DirectionOutbound), "s1_array_bfs_distance")) +} + +// TestShortestPathReferencesCompareM0AndM1WithMinimalSearchState verifies exact M0/M1 comparator arms while preserving their edge-only versus node-and-edge hydration boundaries. +func TestShortestPathReferencesCompareM0AndM1WithMinimalSearchState(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(1), "end_id": int64(3), "max_depth": int32(4)} + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: outboundShortestPathQuery, + }, + params, + []int64{1, 2, 3}, + []int64{10, 11}, + graph.DirectionOutbound, + ) + + m0 := specs[referenceSpecIndex(specs, "s3_unidirectional_cte_m0_directed")] + m1 := specs[referenceSpecIndex(specs, "s3_unidirectional_cte_m1_ordered_ids")] + require.Equal(t, "SP-S3-U-E+MAT-M0", m0.architecture) + require.Equal(t, "SP-S3-U-NE+MAT-M1", m1.architecture) + require.True(t, m0.fullComparator) + require.True(t, m1.fullComparator) + require.Equal(t, "exact_public_observation", m0.semanticValidation) + require.Equal(t, "exact_public_observation", m1.semanticValidation) + require.Contains(t, m0.sql, shortestEdgeReferenceSearch(graph.DirectionOutbound)) + require.Contains(t, m1.sql, shortestReferenceSearch()) + require.NotContains(t, m0.sql, "node_ids") + require.NotContains(t, m0.sql, "ordered_edge_ids_to_path") + require.NotContains(t, m1.sql, "ordered_edge_ids_to_path") + require.Contains(t, m0.sql, "terminal.id = edge.end_id") + require.Contains(t, m1.sql, "unnest(shortest.node_ids) with ordinality") + require.Contains(t, m0.sql, "edge.graph_id = @graph_id") + require.Contains(t, m1.sql, "node.graph_id = @graph_id") +} + +// TestCanonicalWitnessReferenceUsesCompactDiscoveryAndRestoresInboundPathOrder verifies distance-only discovery, separate witness reconstruction, swapped inbound endpoints, and restoration of public path order. +func TestCanonicalWitnessReferenceUsesCompactDiscoveryAndRestoresInboundPathOrder(t *testing.T) { + params := map[string]any{"graph_id": int32(42), "start_id": int64(10), "end_id": int64(20), "min_depth": int32(1), "max_depth": int32(8), "edge_kind_ids": []int16{1}} + testCase := ScaleCase{ + Name: "path", + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + } + inbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionInbound) + witness := inbound[referenceSpecIndex(inbound, "s4_canonical_source_witness_m0")] + require.Equal(t, "SP-I1-C-WE+MAT-M0", witness.architecture) + require.Equal(t, int64(20), witness.parameters["search_start_id"]) + require.Equal(t, int64(10), witness.parameters["search_end_id"]) + require.Contains(t, witness.sql, "distance(node_id, depth)") + require.Contains(t, witness.sql, "witness(node_id, depth, edge_ids)") + require.Contains(t, witness.sql, "e.start_id = distance.node_id") + require.Contains(t, witness.sql, "order by reversed.ordinal desc") + require.Contains(t, witness.sql, "terminal.id = edge.start_id") + require.NotContains(t, witness.sql, "distance(node_id, depth, edge_ids)") + require.True(t, witness.fullComparator) + + outbound := buildShortestReferenceSpecs(testCase, params, nil, nil, graph.DirectionOutbound) + outboundWitness := outbound[referenceSpecIndex(outbound, "s4_canonical_source_witness_m0")] + require.Equal(t, int64(10), outboundWitness.parameters["search_start_id"]) + require.NotContains(t, outboundWitness.sql, "reversed.ordinal") +} + +// TestAllShortestDAGReferenceRetainsEveryShortestDepthPredecessor verifies that all-shortest search records every depth-minimal predecessor and reconstructs paths in both physical directions without LIMIT-based tie loss. +func TestAllShortestDAGReferenceRetainsEveryShortestDepthPredecessor(t *testing.T) { + outbound := allShortestDAGSearch(graph.DirectionOutbound) + require.Contains(t, outbound, "distance(node_id, depth)") + require.Contains(t, outbound, "predecessor(node_id, depth, predecessor_id, edge_id)") + require.Contains(t, outbound, "paths(node_id, depth, edge_ids)") + require.Contains(t, outbound, "e.start_id = prior.node_id and e.end_id = paths.node_id") + require.Contains(t, outbound, "paths.depth <= target.depth") + require.NotContains(t, outbound, "limit 1\n ) predecessor") + + inbound := allShortestDAGSearch(graph.DirectionInbound) + require.Contains(t, inbound, "e.end_id = distance.node_id") + require.Contains(t, inbound, "e.end_id = prior.node_id and e.start_id = paths.node_id") +} + +// TestShortestReferenceIdentitiesAndInboundMinimalState verifies normalized arm identities and inbound M0 SQL that recurses over edge trails without carrying node arrays. +func TestShortestReferenceIdentitiesAndInboundMinimalState(t *testing.T) { + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: "MATCH p = shortestPath((s)<-[*1..4]-(e)) RETURN p", + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, + map[string]any{"graph_id": int32(42), "start_id": int64(1), "end_id": int64(3), "max_depth": int32(4)}, + []int64{1, 2, 3}, []int64{10, 11}, graph.DirectionInbound, + ) + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + require.NoError(t, validateReferenceSpecs(specs)) + m0 := specs[referenceSpecIndex(specs, "s3_unidirectional_cte_m0_directed")] + require.Contains(t, m0.sql, "e.end_id = search.node_id") + require.Contains(t, m0.sql, "terminal.id = edge.start_id") + require.NotContains(t, m0.sql, "node_ids") +} + +// TestShortestPathMaterializerOnlyReferencesExcludeSearch verifies that M0/M1 hydration-only arms consume precomputed exact inputs and neither their timing nor validation SQL performs recursive search. +func TestShortestPathMaterializerOnlyReferencesExcludeSearch(t *testing.T) { + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "one_shortest_path_bound_pair", + Cypher: outboundShortestPathQuery, + }, + map[string]any{}, + []int64{1, 2}, + []int64{10}, + graph.DirectionOutbound, + ) + + m0 := specs[referenceSpecIndex(specs, "m0_directed_hydration_only")] + m1 := specs[referenceSpecIndex(specs, "m1_ordered_ids_hydration_only")] + require.False(t, m0.fullComparator) + require.False(t, m1.fullComparator) + require.NotContains(t, m0.sql, "with recursive") + require.NotContains(t, m1.sql, "with recursive") + require.Equal(t, "precomputed_exact_path_inputs", m0.semanticValidation) + require.Equal(t, "precomputed_exact_path_inputs", m1.semanticValidation) + require.NotEmpty(t, m0.validationSQL) + require.NotEmpty(t, m1.validationSQL) + require.NotContains(t, m0.validationSQL, "with recursive") + require.NotContains(t, m1.validationSQL, "with recursive") +} + +// TestShortestReferencesPreserveZeroLengthPathInputs verifies non-nil empty edge arrays, singleton node arrays, minimum-depth predicates, and bidirectional acceptance of zero-edge paths. +func TestShortestReferencesPreserveZeroLengthPathInputs(t *testing.T) { + zeroEdges, err := referenceInt64Slice([]int64{}) + require.NoError(t, err) + require.NotNil(t, zeroEdges) + + params := map[string]any{ + "graph_id": int32(42), + "start_id": int64(1), + "end_id": int64(1), + "min_depth": int32(0), + "max_depth": int32(4), + "edge_kind_ids": []int16{}, + } + specs := buildShortestReferenceSpecs( + ScaleCase{ + Name: "zero_shortest_path", + Cypher: outboundShortestPathQuery, + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, + params, + []int64{1}, + zeroEdges, + graph.DirectionOutbound, + ) + + require.Contains(t, shortestReferenceSearch(), "depth >= @min_depth") + require.Contains(t, shortestDistanceReferenceSearch(), "depth >= @min_depth") + require.Equal(t, zeroEdges, specs[referenceSpecIndex(specs, "m0_directed_hydration_only")].parameters["edge_ids"]) + require.Equal(t, []int64{1}, specs[referenceSpecIndex(specs, "m1_ordered_ids_hydration_only")].parameters["node_ids"]) + require.Contains(t, specs[referenceSpecIndex(specs, "s3_bidirectional_trail_cte")].sql, "between @min_depth and @max_depth") +} + +// TestShortestMaterializersRequireProvablyOutboundPattern verifies direction parsing and withholds ordered outbound hydration arms only for directionless patterns. +func TestShortestMaterializersRequireProvablyOutboundPattern(t *testing.T) { + for _, testCase := range []struct { + // name identifies the direction case in subtest diagnostics. + name string + + // query is the pattern whose relationship direction is classified. + query string + + // outbound is true when parsing must select physical outbound traversal. + outbound bool + + // supported is true when directional reference materializers must be available. + supported bool + }{ + { + name: "outbound", + query: "MATCH p = shortestPath((s)-[*1..4]->(e)) RETURN p", + outbound: true, + supported: true, + }, + { + name: "inbound", + query: "MATCH p = shortestPath((s)<-[*1..4]-(e)) RETURN p", + supported: true, + }, + { + name: "directionless", + query: "MATCH p = shortestPath((s)-[*1..4]-(e)) RETURN p", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + direction, err := shortestReferenceDirection(testCase.query) + require.NoError(t, err) + require.Equal(t, testCase.outbound, direction == graph.DirectionOutbound) + + specs := buildShortestReferenceSpecs( + ScaleCase{ + Cypher: testCase.query, + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, + map[string]any{}, + []int64{1, 2}, + []int64{10}, + direction, + ) + if testCase.supported { + require.NotEqual(t, -1, referenceSpecIndexOrMissing(specs, "s3_unidirectional_cte_m0_directed")) + } else { + require.Equal(t, -1, referenceSpecIndexOrMissing(specs, "s3_unidirectional_cte_m0_directed")) + require.Equal(t, -1, referenceSpecIndexOrMissing(specs, "m1_ordered_ids_hydration_only")) + } + }) + } +} + +// TestShortestReferenceEndpointParametersFollowPatternRootOrder verifies that endpoint bindings follow left-to-right pattern roles rather than arrow direction or variable spelling. +func TestShortestReferenceEndpointParametersFollowPatternRootOrder(t *testing.T) { + for _, testCase := range []struct { + // name identifies the endpoint-order case in subtest diagnostics. + name string + + // query contains the bound variables whose pattern positions are resolved. + query string + + // root is the parameter attached to the left pattern endpoint. + root string + + // terminal is the parameter attached to the right pattern endpoint. + terminal string + }{ + { + name: "outbound", + query: `MATCH p = shortestPath((s)-[:Traverse*1..8]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)`, + root: "start_id", + terminal: "end_id", + }, + { + name: "inbound same symbols", + query: `MATCH p = shortestPath((s)<-[:Traverse*1..8]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)`, + root: "start_id", + terminal: "end_id", + }, + { + name: "inbound reversed symbols", + query: `MATCH p = shortestPath((e)<-[:Traverse*1..8]-(s)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN length(p)`, + root: "end_id", + terminal: "start_id", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + root, terminal, err := shortestReferenceEndpointParameters(testCase.query) + require.NoError(t, err) + require.Equal(t, testCase.root, root) + require.Equal(t, testCase.terminal, terminal) + }) + } +} + +// TestAlternativeOneShortestPathTieIsSemanticallyValid verifies acceptance of an equal-length valid tie and rejection of longer, wrong-kind, or unmapped alternatives. +func TestAlternativeOneShortestPathTieIsSemanticallyValid(t *testing.T) { + testCase := ScaleCase{ + Cypher: outboundShortestPathQuery, + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + Shape: WorkloadShape{ + EdgeKinds: []string{"Edge"}, + }, + } + public := []string{`[{"nodes":[{"identity":"start"},{"identity":"left"},{"identity":"end"}],"relationships":[{"start":"start","end":"left","kind":"Edge"},{"start":"left","end":"end","kind":"Edge"}]}]`} + alternative := []string{`[{"nodes":[{"identity":"start"},{"identity":"right"},{"identity":"end"}],"relationships":[{"start":"start","end":"right","kind":"Edge"},{"start":"right","end":"end","kind":"Edge"}]}]`} + longer := []string{`[{"nodes":[{"identity":"start"},{"identity":"right"},{"identity":"other"},{"identity":"end"}],"relationships":[{"start":"start","end":"right","kind":"Edge"},{"start":"right","end":"other","kind":"Edge"},{"start":"other","end":"end","kind":"Edge"}]}]`} + wrongKind := []string{`[{"nodes":[{"identity":"start"},{"identity":"right"},{"identity":"end"}],"relationships":[{"start":"start","end":"right","kind":"Wrong"},{"start":"right","end":"end","kind":"Wrong"}]}]`} + unmapped := []string{`[{"nodes":[{"identity":"start"},{"identity":"unmapped-node:42"},{"identity":"end"}],"relationships":[{"start":"start","end":"unmapped-node:42","kind":"Edge"},{"start":"unmapped-node:42","end":"end","kind":"Edge"}]}]`} + + require.True(t, validAlternativeShortestPathObservation(testCase, public, alternative)) + require.False(t, validAlternativeShortestPathObservation(testCase, public, longer)) + require.False(t, validAlternativeShortestPathObservation(testCase, public, wrongKind)) + require.False(t, validAlternativeShortestPathObservation(testCase, public, unmapped)) +} + +// TestReferenceSpecsAlternateOrderByRound verifies fallback odd/even forward-reverse execution ordering without mutating the declared arm sequence. +func TestReferenceSpecsAlternateOrderByRound(t *testing.T) { + specs := []postgresReferenceSpec{{name: "first"}, {name: "second"}} + require.Equal(t, []postgresReferenceSpec{{name: "first"}, {name: "second"}}, referenceSpecsForRound(specs, 1)) + require.Equal(t, []postgresReferenceSpec{{name: "second"}, {name: "first"}}, referenceSpecsForRound(specs, 2)) + require.Equal(t, "first", specs[0].name) +} + +// TestThreeArmReferenceSpecsUseCarryoverBalancedSchedule verifies the doubled +// Williams design balances both execution position and directed carryover. +func TestThreeArmReferenceSpecsUseCarryoverBalancedSchedule(t *testing.T) { + specs := []postgresReferenceSpec{{name: "A"}, {name: "B"}, {name: "C"}} + expected := [][]string{ + {"A", "B", "C"}, + {"B", "C", "A"}, + {"C", "A", "B"}, + {"C", "B", "A"}, + {"A", "C", "B"}, + {"B", "A", "C"}, + } + positions := map[string][3]int{} + carryover := map[[2]string]int{} + for round, want := range expected { + got := referenceSpecNames(referenceSpecsForRound(specs, round+1)) + require.Equal(t, want, got) + for position, arm := range got { + counts := positions[arm] + counts[position]++ + positions[arm] = counts + if position > 0 { + carryover[[2]string{got[position-1], arm}]++ + } + } + } + require.Equal(t, expected[0], referenceSpecNames(referenceSpecsForRound(specs, 7))) + for _, arm := range []string{"A", "B", "C"} { + require.Equal(t, [3]int{2, 2, 2}, positions[arm]) + } + for _, pair := range [][2]string{{"A", "B"}, {"A", "C"}, {"B", "A"}, {"B", "C"}, {"C", "A"}, {"C", "B"}} { + require.Equal(t, 2, carryover[pair], pair) + } + require.Equal(t, "A", specs[0].name) +} + +// TestFiveArmReferenceSpecsUsePredeclaredBalancedSchedule verifies selected rows of the ten-round five-arm schedule and its periodic repetition. +func TestFiveArmReferenceSpecsUsePredeclaredBalancedSchedule(t *testing.T) { + specs := []postgresReferenceSpec{{name: "T1"}, {name: "T2"}, {name: "T3"}, {name: "T4"}, {name: "T5"}} + require.Equal(t, []string{"T1", "T2", "T5", "T3", "T4"}, referenceSpecNames(referenceSpecsForRound(specs, 1))) + require.Equal(t, []string{"T4", "T3", "T5", "T2", "T1"}, referenceSpecNames(referenceSpecsForRound(specs, 6))) + require.Equal(t, []string{"T1", "T2", "T5", "T3", "T4"}, referenceSpecNames(referenceSpecsForRound(specs, 11))) +} + +// referenceSpecNames returns reference names in their declared execution order. +func referenceSpecNames(specs []postgresReferenceSpec) []string { + names := make([]string, len(specs)) + for idx, spec := range specs { + names[idx] = spec.name + } + return names +} + +// TestAllShortestPathCaseUsesDistinctFullMultisetDAGReferences verifies that +// stored A1, inline I1, and both exact two-sided candidates retain distinct +// treatment identities. +func TestAllShortestPathCaseUsesDistinctFullMultisetDAGReferences(t *testing.T) { + runner := &postgresSQLRunner{} + specs, err := runner.referenceSpecs(context.Background(), ScaleCase{ + Category: "generated_shortest_path", + Cypher: "MATCH p = allShortestPaths((s)-[:Traverse*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + }, map[string]any{"start_id": int64(1), "end_id": int64(2)}) + + require.NoError(t, err) + require.Len(t, specs, 4) + require.Equal(t, []string{ + "asp_a1_stored_helper_m0", + "asp_i1_inline_predecessor_dag_m0", + "asp_b1_bidirectional_dag_strict_m0", + "asp_b2_bidirectional_dag_smaller_frontier_m0", + }, referenceSpecNames(specs)) + require.Equal(t, []string{"ASP-A1-DAG", "ASP-I1-U-DAG+MAT-M0", "ASP-B1-DAG-ALT-NODE", "ASP-B2-DAG-MIN-LEVEL"}, []string{ + specs[0].architecture, specs[1].architecture, specs[2].architecture, specs[3].architecture, + }) + for _, spec := range specs { + require.True(t, validPostgresReferenceArm(spec.name), spec.name) + require.True(t, spec.fullComparator) + require.Equal(t, "complete all-shortest path multiset", spec.observationShape) + require.Equal(t, "exact_public_observation", spec.semanticValidation) + require.Contains(t, spec.sql, "pathComposite") + } + for _, spec := range specs[2:] { + require.Equal(t, int64(100_000), spec.parameters["state_limit"]) + require.Equal(t, int64(100_000), spec.parameters["frontier_limit"]) + require.Equal(t, int64(100_000), spec.parameters["predecessor_limit"]) + require.Equal(t, int64(100_000), spec.parameters["enumeration_limit"]) + require.Equal(t, int64(64*1024*1024), spec.parameters["output_bytes_limit"]) + require.Contains(t, spec.sql, "@enumeration_limit, @output_bytes_limit") + } + require.Contains(t, specs[0].sql, "all_shortest_paths_dag") + require.Contains(t, specs[1].sql, "with recursive validated") + require.Contains(t, specs[2].sql, "all_shortest_paths_b1_strict_alternating") + require.Contains(t, specs[3].sql, "all_shortest_paths_b2_smaller_current_level") +} + +// TestAllShortestBidirectionalReferencesStayInsideNarrowEnvelope verifies +// min-zero, over-depth, and equal endpoints retain only the exact A1 control. +func TestAllShortestBidirectionalReferencesStayInsideNarrowEnvelope(t *testing.T) { + runner := &postgresSQLRunner{} + minimumZero, maximumFour, maximumSixtyFive := 0, 4, 65 + for _, test := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // shape retains the shape while anonymous record is assembled or evaluated. + shape WorkloadShape + // params retains the params while anonymous record is assembled or evaluated. + params map[string]any + }{ + { + name: "zero minimum", + shape: WorkloadShape{ + MinDepth: &minimumZero, + MaxDepth: &maximumFour, + }, + params: map[string]any{"start_id": int64(1), "end_id": int64(2)}, + }, + { + name: "maximum sixty five", + shape: WorkloadShape{MaxDepth: &maximumSixtyFive}, + params: map[string]any{"start_id": int64(1), "end_id": int64(2)}, + }, + { + name: "equal endpoints", + shape: WorkloadShape{MaxDepth: &maximumFour}, + params: map[string]any{"start_id": int64(1), "end_id": int64(1)}, + }, + } { + t.Run(test.name, func(t *testing.T) { + specs, err := runner.referenceSpecs(context.Background(), ScaleCase{ + Category: "generated_shortest_path", + Cypher: "MATCH p = allShortestPaths((s)-[:Traverse*0..65]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p", + Shape: test.shape, + }, test.params) + require.NoError(t, err) + require.Len(t, specs, 1) + require.Equal(t, "ASP-A1-DAG", specs[0].architecture) + }) + } +} + +// TestFixedSuffixExpansionReferenceSpecsAvoidAmbiguousArrayContainmentOperators verifies all seventeen arms use explicit membership predicates and retain each strategy's defining recursive SQL shape. +func TestFixedSuffixExpansionReferenceSpecsAvoidAmbiguousArrayContainmentOperators(t *testing.T) { + specs := buildFixedSuffixExpansionReferenceSpecs(ScaleCase{ + Name: "fixed_suffix_expansion_endpoint_ids", + }, map[string]any{"graph_id": int32(42)}) + + require.Len(t, specs, 17) + for _, spec := range specs { + require.NotContains(t, spec.sql, " @> ") + } + require.Contains(t, specs[1].sql, "= any(n.kind_ids)") + require.Contains(t, specs[referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids")].sql, "array_prepend(e.id, reverse_trails.edge_ids)") + require.Contains(t, specs[referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids")].sql, "union all") + require.Contains(t, specs[referenceSpecIndex(specs, "backward_viability_forward_ordered_ids")].sql, "viable(node_id, reverse_distance)") + require.Contains(t, specs[referenceSpecIndex(specs, "factored_suffix_forward_ordered_ids")].sql, "suffix_rows") +} + +// TestFixedSuffixHydrationPrecomputeIsSelectionAware verifies that default and explicit hydration-only selections request precomputed path inputs. +func TestFixedSuffixHydrationPrecomputeIsSelectionAware(t *testing.T) { + require.True(t, referenceHydrationRequested(nil)) + require.True(t, referenceHydrationRequested([]string{"hydration_only"})) + require.True(t, referenceHydrationRequested([]string{"ordered_path_ids_hydration_only"})) + require.False(t, referenceHydrationRequested([]string{"suffix_seeded_reverse_ordered_ids"})) +} + +// TestGeneratedFixedSuffixExpansionReferencesUseDeclaredDepthAndObservation verifies propagation of maximum depth and selection of ID-row versus fully hydrated path output SQL. +func TestGeneratedFixedSuffixExpansionReferencesUseDeclaredDepthAndObservation(t *testing.T) { + minDepth, maxDepth := 0, 16 + runner := &postgresSQLRunner{} + testCase := ScaleCase{ + Name: "generated_fixed_suffix_expansion_endpoint_d16_f1000", + Category: "generated_fixed_suffix_expansion", + Expected: ExpectedResult{ResultKind: "id_rows"}, + Shape: WorkloadShape{ + MinDepth: &minDepth, + MaxDepth: &maxDepth, + }, + } + // Reference routing occurs before kind mapping; the generated category is + // asserted separately from the SQL builder so this remains a unit test. + require.NotNil(t, runner) + specs := buildFixedSuffixExpansionReferenceSpecs(testCase, map[string]any{"min_depth": int32(0), "max_depth": int32(16)}) + require.Contains(t, specs[referenceSpecIndex(specs, "complete_reference")].sql, "select head_id, terminal_id") + require.NotContains(t, specs[referenceSpecIndex(specs, "complete_reference")].sql, "ordered_edge_ids_to_path") + require.Equal(t, int32(16), specs[referenceSpecIndex(specs, "suffix_seeded_reverse_ordered_ids")].parameters["max_depth"]) + + testCase.Observes.Paths = true + testCase.Expected.ResultKind = "path_set" + pathSpecs := buildFixedSuffixExpansionReferenceSpecs(testCase, map[string]any{"min_depth": int32(0), "max_depth": int32(16)}) + require.Contains(t, pathSpecs[referenceSpecIndex(pathSpecs, "suffix_seeded_reverse_complete")].sql, "ordered_edge_ids_to_path") +} + +// TestParseConfigValidatesPostgresReferenceArmSelector verifies ordered arm selection, implicit reference enablement, and rejection of unknown or duplicate arm names. +func TestParseConfigValidatesPostgresReferenceArmSelector(t *testing.T) { + cfg, err := parseConfig([]string{"-postgres-reference-arms", "suffix_seeded_reverse_ordered_ids,factored_suffix_forward_complete"}, func(string) string { return "" }) + require.NoError(t, err) + require.True(t, cfg.PostgresReferences) + require.Equal(t, []string{"suffix_seeded_reverse_ordered_ids", "factored_suffix_forward_complete"}, cfg.PostgresReferenceArms) + + _, err = parseConfig([]string{"-postgres-reference-arms", "does_not_exist"}, func(string) string { return "" }) + require.ErrorContains(t, err, "unknown PostgreSQL reference arm") + _, err = parseConfig([]string{"-postgres-reference-arms", "round_trip,round_trip"}, func(string) string { return "" }) + require.ErrorContains(t, err, "duplicate PostgreSQL reference arm") +} + +// TestRequestedReferenceArmCannotDisappearFromCase verifies that an explicitly requested arm must be available for the particular workload shape. +func TestRequestedReferenceArmCannotDisappearFromCase(t *testing.T) { + _, err := selectReferenceSpecs([]postgresReferenceSpec{{name: "available"}}, []string{"missing"}) + require.ErrorContains(t, err, `requested PostgreSQL reference arm "missing" is unavailable`) +} + +// TestReferenceIdentityRejectsUndeclaredDuplicateSQL verifies that normalized duplicate SQL requires an explicit A/A alias linking the second arm to the first. +func TestReferenceIdentityRejectsUndeclaredDuplicateSQL(t *testing.T) { + specs := []postgresReferenceSpec{ + normalizedReferenceSpec(postgresReferenceSpec{ + name: "one", + architecture: "SP-S1", + stateShape: "state", + observationShape: "ordered_ids", + sql: "select 1", + }), + normalizedReferenceSpec(postgresReferenceSpec{ + name: "two", + architecture: "SP-S2", + stateShape: "state", + observationShape: "ordered_ids", + sql: " select 1 ", + }), + } + require.ErrorContains(t, validateReferenceSpecs(specs), "without a declared A/A alias") + + specs[1].aaAliasOf = "one" + require.NoError(t, validateReferenceSpecs(specs)) +} + +// TestReferenceIdentityRejectsImplementationShapeDrift verifies that a shared implementation ID cannot describe different state shapes or SQL bodies. +func TestReferenceIdentityRejectsImplementationShapeDrift(t *testing.T) { + specs := []postgresReferenceSpec{ + normalizedReferenceSpec(postgresReferenceSpec{ + name: "one", + architecture: "SP-S1", + implementationID: "same", + stateShape: "edge IDs", + observationShape: "ordered_ids", + sql: "select 1", + }), + normalizedReferenceSpec(postgresReferenceSpec{ + name: "two", + architecture: "SP-S1", + implementationID: "same", + stateShape: "node and edge IDs", + observationShape: "ordered_ids", + sql: "select 2", + }), + } + require.ErrorContains(t, validateReferenceSpecs(specs), "changes state, observation, or SQL identity") +} + +// TestFixedSuffixExpansionRootReuseIsExplicitAAAlias verifies that root-reuse arms declare their byte-equivalent ordered-ID and complete-reference counterparts. +func TestFixedSuffixExpansionRootReuseIsExplicitAAAlias(t *testing.T) { + specs := buildFixedSuffixExpansionReferenceSpecs(ScaleCase{ + Name: "fixed_suffix_expansion_endpoint_ids", + }, map[string]any{"graph_id": int32(42)}) + for idx := range specs { + specs[idx] = normalizedReferenceSpec(specs[idx]) + } + require.NoError(t, validateReferenceSpecs(specs)) + require.Equal(t, "search_ordered_ids", specs[referenceSpecIndex(specs, "root_reuse_ordered_ids")].aaAliasOf) + require.Equal(t, "complete_reference", specs[referenceSpecIndex(specs, "root_reuse_complete")].aaAliasOf) +} + +// TestFixedSuffixExpansionOrderedIDReferencesValidateAgainstCanonicalObservation verifies that every ordered-ID strategy uses the canonical search SQL and parameters for semantic validation. +func TestFixedSuffixExpansionOrderedIDReferencesValidateAgainstCanonicalObservation(t *testing.T) { + specs := buildFixedSuffixExpansionReferenceSpecs(ScaleCase{ + Name: "fixed_suffix_expansion_endpoint_ids", + }, map[string]any{"graph_id": int32(42)}) + canonical := specs[referenceSpecIndex(specs, "search_ordered_ids")] + + for _, name := range []string{ + "search_ordered_ids", + "factored_suffix_forward_ordered_ids", + "suffix_seeded_reverse_ordered_ids", + "backward_viability_forward_ordered_ids", + } { + spec := specs[referenceSpecIndex(specs, name)] + require.Equal(t, "exact_ordered_ids", spec.semanticValidation) + require.Equal(t, canonical.sql, spec.validationSQL) + require.Equal(t, canonical.parameters, spec.validationParams) + } +} + +// TestReferenceInt64SliceAcceptsDriverArrayRepresentations verifies normalization of int64, int32, and mixed driver arrays while rejecting nonnumeric elements with their index. +func TestReferenceInt64SliceAcceptsDriverArrayRepresentations(t *testing.T) { + require.Equal(t, []int64{1, 2}, mustReferenceInt64Slice(t, []int64{1, 2})) + require.Equal(t, []int64{3, 4}, mustReferenceInt64Slice(t, []int32{3, 4})) + require.Equal(t, []int64{5, 6}, mustReferenceInt64Slice(t, []any{int64(5), int32(6)})) + _, err := referenceInt64Slice([]any{"not-an-id"}) + require.ErrorContains(t, err, "array item 0") +} + +// mustReferenceInt64Slice converts a reference value to integers and fails the test on invalid input. +func mustReferenceInt64Slice(t *testing.T, value any) []int64 { + t.Helper() + result, err := referenceInt64Slice(value) + require.NoError(t, err) + return result +} diff --git a/cmd/graphbench/resource_gate.go b/cmd/graphbench/resource_gate.go new file mode 100644 index 00000000..23ac3140 --- /dev/null +++ b/cmd/graphbench/resource_gate.go @@ -0,0 +1,1050 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "sort" + "strings" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// resourceGateVersion identifies the serialized schema revision for resource gate. +const resourceGateVersion = 5 + +// ResourceGateReport reports whether production and reference plan resources remain within their allowed envelopes. +type ResourceGateReport struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // ArtifactSHA256 binds this report to the exact input JSONL artifact. + ArtifactSHA256 string `json:"artifact_sha256"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Cases contains resource-envelope decisions for each evaluated production or reference executor. + Cases []ResourceGateCase `json:"cases"` +} + +// ResourceGateCase attributes resource-gate failures to one production or reference executor architecture. +type ResourceGateCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Round identifies the measured record that produced this decision. + Round int `json:"round,omitempty"` + // Block identifies the paired measurement block for this record. + Block int `json:"block,omitempty"` + // RunUUID binds the resource decision to one run series. + RunUUID string `json:"run_uuid,omitempty"` + // Arm identifies the measured executor arm. + Arm string `json:"arm,omitempty"` + // ArmOrder supplies the arm order input to the ResourceGateCase contract. + ArmOrder int `json:"arm_order,omitempty"` + // Reference identifies the reference arm evaluated by the resource gate. + Reference string `json:"reference,omitempty"` + // Tier identifies the resource envelope applied to the case. + Tier string `json:"tier"` + // QualificationSplit identifies training, frozen holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // Architecture identifies the executor architecture. + Architecture string `json:"architecture,omitempty"` + // FallbackArchitecture identifies the executor architecture used after fallback. + FallbackArchitecture string `json:"fallback_architecture,omitempty"` + // Passed reports whether every required gate condition succeeded. + Passed bool `json:"passed"` + // Reasons lists explanations for the reported disposition. + Reasons []string `json:"reasons,omitempty"` + // NumericLimits records declared telemetry ceilings applied by this gate. + NumericLimits map[string]int64 `json:"numeric_limits,omitempty"` + // NumericObserved records invocation-local high-water marks compared with the limits. + NumericObserved map[string]int64 `json:"numeric_observed,omitempty"` + // RuntimeReceiptChains preserves complete measured branch chains alongside + // the resource decision. + RuntimeReceiptChains [][]RuntimeReceiptEvent `json:"runtime_receipt_chains,omitempty"` +} + +// createResourceGateReport evaluates production and reference plan metrics against resource ceilings and writes the report. +func createResourceGateReport(artifact, output string) (bool, error) { + records, err := readJSONLFile(artifact) + if err != nil { + return false, err + } + artifactSHA256, err := fileSHA256(artifact) + if err != nil { + return false, err + } + report := ResourceGateReport{ + Version: resourceGateVersion, + ArtifactSHA256: artifactSHA256, + Passed: true, + } + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL { + continue + } + gateCase := evaluateProductionResourceGateCase(record) + if !gateCase.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, gateCase) + for _, reference := range record.PostgresReferences { + if !reference.FullComparator || reference.Architecture == "" { + continue + } + referenceCase := ResourceGateCase{ + Dataset: record.Dataset, + Name: record.Name, + Round: gateCase.Round, + Block: gateCase.Block, + RunUUID: gateCase.RunUUID, + Arm: gateCase.Arm, + ArmOrder: gateCase.ArmOrder, + Reference: reference.Name, + Tier: gateCase.Tier, + QualificationSplit: gateCase.QualificationSplit, + Architecture: reference.Architecture, + Passed: true, + } + if reference.PostgresMetrics == nil { + referenceCase.Reasons = append(referenceCase.Reasons, "structured PostgreSQL reference plan metrics are missing") + } else if compactBidirectionalWorkspaceArchitecture(reference.Architecture) { + appendWorkspaceResourceReasons(&referenceCase, reference.PostgresMetrics) + } else if reference.Architecture != "SP-S0" { + appendPortableResourceReasons(&referenceCase, reference.PostgresMetrics) + } + appendTelemetryResourceReasons(&referenceCase, reference.TraversalTelemetry, telemetryRequiredForArchitecture(reference.Architecture)) + appendWorkspaceCeilingReasons(&referenceCase, record.Environment, reference.TraversalTelemetry, compactBidirectionalWorkspaceArchitecture(reference.Architecture), compactBidirectionalWorkspaceArchitecture(reference.Architecture)) + referenceCase.Passed = len(referenceCase.Reasons) == 0 + if !referenceCase.Passed { + report.Passed = false + } + report.Cases = append(report.Cases, referenceCase) + } + } + if len(report.Cases) == 0 { + return false, fmt.Errorf("resource artifact contains no PostgreSQL cases") + } + sort.Slice(report.Cases, func(i, j int) bool { + if report.Cases[i].Dataset != report.Cases[j].Dataset { + return report.Cases[i].Dataset < report.Cases[j].Dataset + } + if report.Cases[i].Name != report.Cases[j].Name { + return report.Cases[i].Name < report.Cases[j].Name + } + if report.Cases[i].Round != report.Cases[j].Round { + return report.Cases[i].Round < report.Cases[j].Round + } + return report.Cases[i].Reference < report.Cases[j].Reference + }) + + var raw []byte + if raw, err = json.MarshalIndent(report, "", " "); err != nil { + return false, err + } + if output == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + } else { + err = os.WriteFile(output, append(raw, '\n'), 0o644) + } + if err != nil { + return false, err + } + + return report.Passed, nil +} + +// evaluateProductionResourceGateCase derives the complete production decision +// from one artifact record. Qualification reuses this exact evaluator so a +// serialized report cannot suppress spill, WAL, attribution, fallback, or cap +// failures while retaining the candidate artifact digest. +func evaluateProductionResourceGateCase(record CaseResult) ResourceGateCase { + gateCase := ResourceGateCase{ + Dataset: record.Dataset, + Name: record.Name, + Tier: record.Shape.FixtureTier, + QualificationSplit: record.Shape.QualificationSplit, + Passed: true, + RuntimeReceiptChains: runtimeReceiptChains(record.Stats.Samples), + } + if record.Environment != nil { + gateCase.Round = record.Environment.Round + gateCase.Block = record.Environment.Block + gateCase.RunUUID = record.Environment.RunUUID + gateCase.Arm = record.Environment.Arm + gateCase.ArmOrder = record.Environment.ArmOrder + } + if gateCase.Tier == "" { + gateCase.Tier = "legacy" + } + if gateCase.QualificationSplit == "" { + gateCase.QualificationSplit = "legacy" + } + gateCase.Architecture = appliedPostgresArchitecture(record) + portableCandidate := gateCase.Architecture != "" && gateCase.Architecture != "SP-S0" + workspaceCandidate := compactWorkspaceArchitecture(gateCase.Architecture) + if gateCase.Architecture == "SP-S0-DIRECT" { + if loops, found, err := postgresPlanFunctionLoops(record.PostgresPlanJSON, "bidirectional_sp_harness"); err != nil { + gateCase.Reasons = append(gateCase.Reasons, "direct preflight fallback attribution failed: "+err.Error()) + } else if !found { + gateCase.Reasons = append(gateCase.Reasons, "direct preflight fallback plan node is missing") + } else if loops > 0 { + portableCandidate = false + gateCase.FallbackArchitecture = "SP-S0" + } + } + if record.Status != StatusOK { + gateCase.Reasons = append(gateCase.Reasons, "record status is "+record.Status) + } + if record.PostgresMetrics == nil { + gateCase.Reasons = append(gateCase.Reasons, "structured PostgreSQL plan metrics are missing") + } else if workspaceCandidate { + appendWorkspaceResourceReasons(&gateCase, record.PostgresMetrics) + } else if portableCandidate { + appendPortableResourceReasons(&gateCase, record.PostgresMetrics) + } + if contract, guarded := guardedInlineResourceContractForArchitecture(gateCase.Architecture); guarded { + appendGuardedInlineResourceBindingReasons(&gateCase, record, contract) + } + telemetryRequired := telemetryRequiredForRecord(record, gateCase.Architecture) + appendTelemetryResourceReasons(&gateCase, record.TraversalTelemetry, telemetryRequired) + appendFallbackExpectationReasons(&gateCase, record) + appendWorkspaceCeilingReasons(&gateCase, record.Environment, record.TraversalTelemetry, workspaceCandidate, compactBidirectionalWorkspaceArchitecture(gateCase.Architecture)) + gateCase.Passed = len(gateCase.Reasons) == 0 + return gateCase +} + +// compactWorkspaceArchitecture reports whether an executor deliberately uses +// bounded session-local typed workspace rather than portable recursive state. +func compactWorkspaceArchitecture(architecture string) bool { + switch architecture { + case "ASP-A1-DAG", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL", + "SP-S4-C-D", + "SP-S4-C-WE+MAT-M0", + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0": + return true + default: + return false + } +} + +// compactBidirectionalWorkspaceArchitecture identifies reference arms whose +// measured boundary deliberately includes the reusable spb_* workspace. +func compactBidirectionalWorkspaceArchitecture(architecture string) bool { + switch architecture { + case "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + "ASP-B1-DAG-ALT-NODE", + "ASP-B2-DAG-MIN-LEVEL": + return true + default: + return false + } +} + +// telemetryRequiredForArchitecture identifies candidates whose qualification +// depends on executor-visible work rather than outer EXPLAIN counters. This +// architecture-only check also applies to explicit reference arms, so guarded +// inline I1 production requirements deliberately belong to the record-aware +// check below instead. +func telemetryRequiredForArchitecture(architecture string) bool { + return strings.HasPrefix(architecture, "SP-B1-") || + strings.HasPrefix(architecture, "SP-B2-") || + strings.HasPrefix(architecture, "ASP-B1-") || + strings.HasPrefix(architecture, "ASP-B2-") || + isOrientationProbePolicy(architecture) || + isSuffixReverseGuardPolicy(architecture) +} + +// telemetryRequiredForRecord supports benchmark evidence processing for telemetry required for record. +func telemetryRequiredForRecord(record CaseResult, architecture string) bool { + if _, guarded := guardedInlineResourceContractForArchitecture(architecture); guarded { + return true + } + if telemetryRequiredForArchitecture(architecture) { + return true + } + if record.Optimization != nil { + for _, outcome := range record.Optimization.TargetOutcomes { + if isOrientationProbePolicy(outcome.EmittedPolicy) || isSuffixReverseGuardPolicy(outcome.EmittedPolicy) || guardedInlineResourcePolicy(outcome.EmittedPolicy) { + return true + } + } + } + return record.TraversalTelemetry != nil && + (isOrientationProbePolicy(record.TraversalTelemetry.Summary.EmittedIdentity) || + isOrientationProbePolicy(record.TraversalTelemetry.Summary.SelectorVersion) || + isSuffixReverseGuardPolicy(record.TraversalTelemetry.Summary.EmittedIdentity) || + isSuffixReverseGuardPolicy(record.TraversalTelemetry.Summary.SelectorVersion) || + guardedInlineResourcePolicy(record.TraversalTelemetry.Summary.EmittedIdentity)) +} + +// guardedInlineResourceContract groups state that must remain consistent while processing guarded inline resource contract. +type guardedInlineResourceContract struct { + // architecture retains the architecture while guardedInlineResourceContract is assembled or evaluated. + architecture string + // family retains the family while guardedInlineResourceContract is assembled or evaluated. + family string + // telemetryFamily retains the telemetry family while guardedInlineResourceContract is assembled or evaluated. + telemetryFamily TraversalTelemetryFamily + // policy retains the policy while guardedInlineResourceContract is assembled or evaluated. + policy string + // namespace retains the namespace while guardedInlineResourceContract is assembled or evaluated. + namespace string + // label retains the label while guardedInlineResourceContract is assembled or evaluated. + label string +} + +// guardedInlineResourceContractForArchitecture supports benchmark evidence processing for guarded inline resource contract for architecture. +func guardedInlineResourceContractForArchitecture(architecture string) (guardedInlineResourceContract, bool) { + switch architecture { + case string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness): + return guardedInlineResourceContract{ + architecture: architecture, + family: "SP", + telemetryFamily: TraversalTelemetryFamilySP, + policy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + namespace: "inline_shortest_path", + label: "inline canonical SP", + }, true + case string(optimize.ShortestPathExecutorASPI1DAG): + return guardedInlineResourceContract{ + architecture: architecture, + family: "ASP", + telemetryFamily: TraversalTelemetryFamilyASP, + policy: optimize.ShortestPathPolicyASPI1GuardedV1, + namespace: "inline_asp", + label: "inline ASP", + }, true + case string(optimize.ShortestPathExecutorI2GuardedDistance): + return guardedInlineResourceContract{architecture: architecture, family: "SP", telemetryFamily: TraversalTelemetryFamilySP, policy: optimize.ShortestPathPolicyI2DistanceGuardedV1, namespace: "inline_shortest_distance", label: "inline SP distance"}, true + case string(optimize.ShortestPathExecutorI2GuardedDistanceV2), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP): + return guardedInlineResourceContract{architecture: architecture, family: "SP", telemetryFamily: TraversalTelemetryFamilySP, policy: optimize.ShortestPathPolicyI2DistanceGuardedV2, namespace: "inline_shortest_distance", label: "inline SP distance V2"}, true + default: + return guardedInlineResourceContract{}, false + } +} + +// guardedInlineResourcePolicy supports benchmark evidence processing for guarded inline resource policy. +func guardedInlineResourcePolicy(policy string) bool { + return policy == optimize.ShortestPathPolicyI1CanonicalGuardedV1 || policy == optimize.ShortestPathPolicyASPI1GuardedV1 || policy == optimize.ShortestPathPolicyI2DistanceGuardedV1 || policy == optimize.ShortestPathPolicyI2DistanceGuardedV2 +} + +// appendGuardedInlineResourceBindingReasons prevents an unguarded comparator +// with the same executor architecture from satisfying production resource +// evidence. Production I1 must bind the translated outcome and telemetry to +// its exact policy and to the observation-specific typed counter namespace. +func appendGuardedInlineResourceBindingReasons(gateCase *ResourceGateCase, record CaseResult, contract guardedInlineResourceContract) { + emittedPolicy := "" + outcomeFound := false + if record.Optimization != nil { + for _, outcome := range record.Optimization.TargetOutcomes { + applied := outcome.Applied + if applied == "" { + applied = outcome.Selected + } + if outcome.Family == contract.family && applied == contract.architecture { + emittedPolicy = outcome.EmittedPolicy + outcomeFound = true + break + } + } + } + if !outcomeFound || emittedPolicy != contract.policy { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "%s production architecture requires emitted policy %q; found %q", + contract.label, contract.policy, emittedPolicy, + )) + } + + telemetry := record.TraversalTelemetry + if telemetry == nil { + return + } + if telemetry.Summary.EmittedIdentity != contract.policy { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "%s production telemetry requires emitted identity %q; found %q", + contract.label, contract.policy, telemetry.Summary.EmittedIdentity, + )) + } + if telemetry.Diagnostic == nil { + return + } + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, contract.telemetryFamily) { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "%s production telemetry requires declared counter family %q", + contract.label, contract.telemetryFamily, + )) + } + if observationRequiresHydration(telemetry.Summary.ObservationMode) && + !slices.Contains(telemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + gateCase.Reasons = append(gateCase.Reasons, contract.label+" production telemetry requires declared hydration counters for its observation mode") + } + + inlineASP := telemetry.Diagnostic.Counters.InlineASP + inlineShortestPath := telemetry.Diagnostic.Counters.InlineShortestPath + inlineShortestDistance := telemetry.Diagnostic.Counters.InlineShortestDistance + switch contract.namespace { + case "inline_shortest_path": + if inlineShortestPath == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline canonical SP production telemetry requires inline_shortest_path counters") + } + if inlineASP != nil { + gateCase.Reasons = append(gateCase.Reasons, "inline canonical SP production telemetry must not use inline_asp counters") + } + case "inline_asp": + if inlineASP == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline ASP production telemetry requires inline_asp counters") + } + if inlineShortestPath != nil { + gateCase.Reasons = append(gateCase.Reasons, "inline ASP production telemetry must not use inline_shortest_path counters") + } + case "inline_shortest_distance": + if inlineShortestDistance == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance production telemetry requires inline_shortest_distance counters") + } else if inlineShortestDistance.OutputRows != nil && *inlineShortestDistance.OutputRows != record.RowCount { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance typed output does not match the exact public observation") + } + if inlineASP != nil || inlineShortestPath != nil { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance production telemetry must not use predecessor counter namespaces") + } + if telemetry.Diagnostic.PlanReplay != nil { + if outputRows, found := telemetry.Diagnostic.PlanReplay.Counters["sp_i2_output_rows"]; found && outputRows != record.RowCount { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance plan output does not match the exact public observation") + } + } + } +} + +// appendFallbackExpectationReasons appends fallback expectation reasons. +func appendFallbackExpectationReasons(gateCase *ResourceGateCase, record CaseResult) { + expectation := record.Shape.FallbackExpectation + if expectation == "" { + if telemetryRequiredForRecord(record, appliedPostgresArchitecture(record)) { + gateCase.Reasons = append(gateCase.Reasons, "candidate resource qualification requires a typed fallback expectation") + } + return + } + if record.TraversalTelemetry == nil { + gateCase.Reasons = append(gateCase.Reasons, "fallback expectation lacks runtime telemetry") + return + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.FallbackExecuted == nil { + gateCase.Reasons = append(gateCase.Reasons, "fallback runtime outcome is unavailable") + return + } + switch expectation { + case "required": + if !*summary.FallbackExecuted { + gateCase.Reasons = append(gateCase.Reasons, "declared overflow-fallback expectation did not execute its exact fallback") + } + case "forbidden": + if *summary.FallbackExecuted { + gateCase.Reasons = append(gateCase.Reasons, "normal/envelope candidate unexpectedly executed fallback") + } + case "allowed": + default: + gateCase.Reasons = append(gateCase.Reasons, "unknown fallback expectation "+expectation) + } +} + +// appendWorkspaceCeilingReasons appends workspace ceiling reasons. +func appendWorkspaceCeilingReasons(gateCase *ResourceGateCase, environment *RunEnvironment, telemetry *TraversalExecutionTelemetry, workspaceArchitecture, ceilingsRequired bool) { + if !workspaceArchitecture { + return + } + if environment == nil || environment.SessionMemoryCeilingBytes <= 0 || environment.PoolMemoryCeilingBytes <= 0 { + if ceilingsRequired { + gateCase.Reasons = append(gateCase.Reasons, "workspace candidate requires positive declared session and pool memory ceilings") + } + return + } + if telemetry == nil || telemetry.Diagnostic == nil || telemetry.Diagnostic.Counters.Workspace == nil { + gateCase.Reasons = append(gateCase.Reasons, "declared workspace memory ceilings lack measured session and pool high-water evidence") + return + } + if environment.PoolSize <= 0 { + gateCase.Reasons = append(gateCase.Reasons, "workspace candidate requires a declared positive pool size") + return + } + workspace := telemetry.Diagnostic.Counters.Workspace + if workspace.SessionPeakBytes == nil { + gateCase.Reasons = append(gateCase.Reasons, "declared session memory ceiling lacks a measured session high-water value") + } else if *workspace.SessionPeakBytes > environment.SessionMemoryCeilingBytes { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("session workspace peak %d exceeds declared ceiling %d", *workspace.SessionPeakBytes, environment.SessionMemoryCeilingBytes)) + } + if workspace.PoolPeakBytes == nil { + gateCase.Reasons = append(gateCase.Reasons, "declared pool memory ceiling lacks a measured pool high-water value") + } else if *workspace.PoolPeakBytes > environment.PoolMemoryCeilingBytes { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("pool workspace peak %d exceeds declared ceiling %d", *workspace.PoolPeakBytes, environment.PoolMemoryCeilingBytes)) + } + if environment.PoolSize > 1 && telemetry != nil && telemetry.Diagnostic != nil && + telemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] == "single_connection_diagnostic_pool.session_peak_bytes" { + gateCase.Reasons = append(gateCase.Reasons, "pool workspace ceiling lacks an aggregate multi-session high-water measurement") + } +} + +// appendTelemetryResourceReasons validates identity attribution and numeric +// cap evidence from a distinct untimed diagnostic invocation. +func appendTelemetryResourceReasons(gateCase *ResourceGateCase, telemetry *TraversalExecutionTelemetry, required bool) { + if telemetry == nil { + if required { + gateCase.Reasons = append(gateCase.Reasons, "required traversal execution telemetry is missing") + } + return + } + if err := ValidateTraversalExecutionTelemetry(telemetry); err != nil { + gateCase.Reasons = append(gateCase.Reasons, err.Error()) + return + } + + summary := telemetry.Summary + if summary.RuntimeOutcomeAvailable != nil && !*summary.RuntimeOutcomeAvailable { + if required { + gateCase.Reasons = append(gateCase.Reasons, "candidate qualification requires an observed runtime traversal outcome") + } + return + } + if !slices.Contains(summary.PlannedIdentities, summary.RuntimeIdentity) { + gateCase.Reasons = append(gateCase.Reasons, "runtime traversal identity is not a planned candidate") + } + if summary.AppliedIdentity != summary.RuntimeIdentity { + gateCase.Reasons = append(gateCase.Reasons, "applied traversal identity does not match runtime identity") + } + if summary.FallbackExecuted != nil && *summary.FallbackExecuted && summary.RuntimeIdentity != summary.FallbackIdentity { + gateCase.Reasons = append(gateCase.Reasons, "fallback traversal identity does not match runtime identity") + } + if required && telemetry.Level != TraversalTelemetryLevelDiagnostic { + gateCase.Reasons = append(gateCase.Reasons, "candidate qualification requires an untimed diagnostic replay") + return + } + if telemetry.Diagnostic == nil { + return + } + counterStatus := telemetry.Diagnostic.CounterStatus + if counterStatus == "" { + counterStatus = TraversalTelemetryCounterStatusComplete + } + if required && counterStatus != TraversalTelemetryCounterStatusComplete { + gateCase.Reasons = append(gateCase.Reasons, "candidate qualification requires complete executor counters; diagnostic status is "+string(counterStatus)) + return + } + if required && (isOrientationProbePolicy(summary.EmittedIdentity) || isOrientationProbePolicy(summary.SelectorVersion)) { + requiredFamilies := []TraversalTelemetryFamily{TraversalTelemetryFamilyOrientation, TraversalTelemetryFamilyOrdinary} + if observationRequiresHydration(summary.ObservationMode) { + requiredFamilies = append(requiredFamilies, TraversalTelemetryFamilyHydration) + } + for _, family := range requiredFamilies { + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, family) { + gateCase.Reasons = append(gateCase.Reasons, "orientation qualification is missing required counter family "+string(family)) + } + } + appendOrientationAttributionReasons(gateCase, telemetry.Diagnostic) + } + if required && (isSuffixReverseGuardPolicy(summary.EmittedIdentity) || isSuffixReverseGuardPolicy(summary.SelectorVersion)) { + requiredFamilies := []TraversalTelemetryFamily{TraversalTelemetryFamilySuffixGuard, TraversalTelemetryFamilyOrdinary} + if observationRequiresHydration(summary.ObservationMode) { + requiredFamilies = append(requiredFamilies, TraversalTelemetryFamilyHydration) + } + for _, family := range requiredFamilies { + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, family) { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard qualification is missing required counter family "+string(family)) + } + } + appendSuffixGuardAttributionReasons(gateCase, telemetry.Diagnostic) + } + if required && summary.EmittedIdentity == optimize.ShortestPathPolicyASPI1GuardedV1 { + appendInlinePredecessorAttributionReasons(gateCase, telemetry.Diagnostic, "inline ASP") + } + if required && summary.EmittedIdentity == optimize.ShortestPathPolicyI1CanonicalGuardedV1 { + appendInlinePredecessorAttributionReasons(gateCase, telemetry.Diagnostic, "inline canonical SP") + } + if required && (summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV1 || + summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV2) { + appendInlineDistanceAttributionReasons(gateCase, telemetry) + } + + observed := traversalNumericObservations(telemetry.Diagnostic.Counters) + gateCase.NumericLimits = make(map[string]int64, len(summary.Caps)) + gateCase.NumericObserved = map[string]int64{} + for name, limit := range summary.Caps { + gateCase.NumericLimits[name] = limit + if limit < 0 { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("traversal cap %s is negative", name)) + continue + } + value, found := observed[name] + if !found { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("required numeric traversal counter %s is missing", name)) + continue + } + gateCase.NumericObserved[name] = value + allowed := limit + if traversalCapUsesSentinel(name) { + allowed++ + } + if value < 0 { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("traversal counter %s=%d is negative", name, value)) + } else if value > allowed { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("traversal counter %s=%d exceeds ceiling %d", name, value, allowed)) + } + } +} + +// appendInlineDistanceAttributionReasons binds the SP-I2 runtime receipt to +// exact named-plan counters. Qualification accepts one marker and one executor +// loop only, proves the inactive arm remained uninitialized, and requires the +// typed counters to agree with their plan-derived sources. +func appendInlineDistanceAttributionReasons(gateCase *ResourceGateCase, telemetry *TraversalExecutionTelemetry) { + if telemetry == nil || telemetry.Diagnostic == nil || telemetry.Diagnostic.PlanReplay == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance qualification requires exact plan branch evidence") + return + } + inline := telemetry.Diagnostic.Counters.InlineShortestDistance + if inline == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance counters are missing") + return + } + + plan := telemetry.Diagnostic.PlanReplay.Counters + required := []string{ + "sp_i2_distance_rows", "sp_i2_target_rows", "sp_i2_output_rows", + "sp_i2_candidate_marker_rows", "sp_i2_fallback_marker_rows", + "sp_i2_candidate_branch_rows", "sp_i2_fallback_branch_rows", + "sp_i2_candidate_executor_loops", "sp_i2_fallback_executor_loops", + } + if telemetry.Summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV2 { + required = append(required, "sp_i2_admission_rows", "sp_i2_admission_loops") + if spI2DirectDevelopmentIdentity(telemetry.Summary.RequestedIdentity) { + required = append(required, "sp_i2_direct_rows", "sp_i2_direct_loops") + } + } + for _, name := range required { + if _, found := plan[name]; !found { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance execution is missing exact plan counter "+name) + return + } + } + + candidateMarker := plan["sp_i2_candidate_marker_rows"] + fallbackMarker := plan["sp_i2_fallback_marker_rows"] + candidateRows := plan["sp_i2_candidate_branch_rows"] + fallbackRows := plan["sp_i2_fallback_branch_rows"] + outputRows := plan["sp_i2_output_rows"] + candidateLoops := plan["sp_i2_candidate_executor_loops"] + fallbackLoops := plan["sp_i2_fallback_executor_loops"] + stateRows := plan["sp_i2_distance_rows"] + if (candidateMarker != 0 && candidateMarker != 1) || (fallbackMarker != 0 && fallbackMarker != 1) || candidateMarker+fallbackMarker != 1 { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance execution must attribute exactly one candidate or fallback marker") + } + if candidateRows < 0 || candidateRows > 1 || fallbackRows < 0 || fallbackRows > 1 || outputRows != candidateRows+fallbackRows { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance output does not equal its complementary branch rows") + } + directRows := plan["sp_i2_direct_rows"] + if directRows < 0 || directRows > 1 { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance direct floor must return at most one row") + } + if plan["sp_i2_target_rows"]+directRows != candidateRows { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance candidate branch does not agree with its target receipt") + } + + typed := map[string]*int64{ + "sp_i2_distance_rows": inline.StateRows, + "sp_i2_output_rows": inline.OutputRows, + "sp_i2_candidate_marker_rows": inline.CandidateMarkerRows, + "sp_i2_fallback_marker_rows": inline.FallbackMarkerRows, + "sp_i2_candidate_branch_rows": inline.CandidateBranchRows, + "sp_i2_fallback_branch_rows": inline.FallbackBranchRows, + "sp_i2_candidate_executor_loops": inline.CandidateExecutorLoops, + "sp_i2_fallback_executor_loops": inline.FallbackExecutorLoops, + } + if telemetry.Summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV2 { + typed["sp_i2_admission_rows"] = inline.AdmissionProbeRows + typed["sp_i2_admission_loops"] = inline.AdmissionProbeLoops + typed["sp_i2_target_rows"] = inline.TargetRows + if inline.FrontierGuardDominated == nil || inline.CapRelationship == "" || inline.ObservedOverflowReason == "" { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance V2 admission telemetry is incomplete") + } + if spI2DirectDevelopmentIdentity(telemetry.Summary.RequestedIdentity) { + typed["sp_i2_direct_rows"] = inline.DirectProbeRows + typed["sp_i2_direct_loops"] = inline.DirectProbeLoops + } + } + for name, value := range typed { + if value == nil || *value != plan[name] { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance typed counter does not match plan counter "+name) + } + } + if inline.FrontierRows == nil || inline.StateRows == nil || *inline.FrontierRows != *inline.StateRows { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance conservative frontier bound does not match bounded state rows") + } + + summary := telemetry.Summary + stateLimit, stateLimitFound := summary.Caps["state_rows"] + frontierLimit, frontierLimitFound := summary.Caps["frontier_rows"] + validCaps := stateLimitFound && frontierLimitFound && stateLimit > 0 && frontierLimit > 0 + if !validCaps { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance attribution requires positive state and frontier caps") + } + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.FallbackExecuted == nil || summary.Overflow == nil { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance marker selection lacks a complete runtime receipt") + return + } + if candidateMarker == 1 { + expectedBranch := "inline_canonical_distance" + if directRows == 1 { + expectedBranch = "inline_direct_distance" + } else if outputRows == 0 { + expectedBranch = "inline_canonical_distance_no_path" + } + if candidateLoops != 1 || fallbackLoops != 0 || fallbackRows != 0 { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance candidate selection did not suppress the fallback executor and output arm") + } + if directRows == 1 && (stateRows != 0 || plan["sp_i2_admission_rows"] != 0 || plan["sp_i2_target_rows"] != 0) { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance direct floor did not suppress recursive admission and target work") + } + // FrontierRows is deliberately a conservative alias for the complete + // bounded state relation, not an independently observable peak level. + // Requiring that relation to remain within both reported caps is + // conservative and admits the exact boundary without guessing which + // aggregate gate would otherwise have fired. + if validCaps && (stateRows > stateLimit || stateRows > frontierLimit) { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance candidate selection exceeds its state or conservative frontier cap") + } + expectedIdentity := string(optimize.ShortestPathExecutorI2GuardedDistance) + if summary.EmittedIdentity == optimize.ShortestPathPolicyI2DistanceGuardedV2 { + expectedIdentity = summary.RequestedIdentity + } + if summary.RuntimeIdentity != expectedIdentity || summary.RuntimeBranch != expectedBranch || *summary.FallbackExecuted || *summary.Overflow { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance candidate marker contradicts the runtime receipt") + } + } + if fallbackMarker == 1 { + if fallbackLoops != 1 || candidateLoops != 0 || candidateRows != 0 { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance fallback selection did not suppress the candidate executor and output arm") + } + // The current diagnostic contract exposes one bounded-state count and a + // conservative frontier alias. It cannot identify which aggregate gate + // fired, but an exact cap+1 row for at least one reported bound is required + // to corroborate the overflow branch. Qualified production caps are equal, + // so every production overflow has this observable sentinel. + if validCaps && stateRows != stateLimit+1 && stateRows != frontierLimit+1 { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance fallback selection lacks an exact state or conservative frontier cap+1 sentinel") + } + if summary.RuntimeIdentity != string(optimize.ShortestPathExecutorS4CanonicalDistance) || summary.RuntimeBranch != "exact_s4_distance_fallback" || !*summary.FallbackExecuted || !*summary.Overflow { + gateCase.Reasons = append(gateCase.Reasons, "inline SP distance fallback marker contradicts the runtime receipt") + } + } +} + +// appendSuffixGuardAttributionReasons proves one and only one output arm ran, +// and rejects plans that accidentally retain orientation-v2's topology work. +func appendSuffixGuardAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic) { + if diagnostic == nil || diagnostic.PlanReplay == nil { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard qualification requires exact plan branch and sentinel evidence") + return + } + counters := diagnostic.PlanReplay.Counters + candidate, candidatePresent := counters["suffix_guard_candidate_marker_rows"] + fallback, fallbackPresent := counters["suffix_guard_fallback_marker_rows"] + if !candidatePresent || !fallbackPresent || (candidate != 0 && candidate != 1) || (fallback != 0 && fallback != 1) || candidate+fallback != 1 { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard execution must attribute exactly one candidate or fallback marker") + } + candidateRows, candidateRowsPresent := counters["suffix_guard_candidate_branch_rows"] + fallbackRows, fallbackRowsPresent := counters["suffix_guard_fallback_branch_rows"] + outputRows, outputRowsPresent := counters["suffix_guard_output_rows"] + if !candidateRowsPresent || !fallbackRowsPresent || !outputRowsPresent || outputRows != candidateRows+fallbackRows { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard execution is missing exact complementary output-branch evidence") + } + candidateLoops, candidateLoopsPresent := counters["suffix_guard_candidate_executor_loops"] + fallbackLoops, fallbackLoopsPresent := counters["suffix_guard_fallback_executor_loops"] + if !candidateLoopsPresent || !fallbackLoopsPresent { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard execution is missing candidate or fallback executor-loop evidence") + } + if candidate == 1 && (candidateLoops != 1 || fallbackLoops != 0 || fallbackRows != 0) { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard candidate selection did not suppress the fallback executor and output arm") + } + if fallback == 1 && (fallbackLoops != 1 || candidateLoops != 0 || candidateRows != 0) { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard fallback selection did not suppress the candidate executor and output arm") + } + for name := range counters { + if strings.HasPrefix(name, "orientation_") && + (strings.Contains(name, "degree") || strings.Contains(name, "score") || strings.Contains(name, "decision")) { + gateCase.Reasons = append(gateCase.Reasons, "suffix-reverse guard plan unexpectedly contains orientation topology work "+name) + } + } +} + +// appendInlineASPAttributionReasons appends inline asp attribution reasons. +func appendInlineASPAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic) { + appendInlinePredecessorAttributionReasons(gateCase, diagnostic, "inline ASP") +} + +// appendInlinePredecessorAttributionReasons appends inline predecessor attribution reasons. +func appendInlinePredecessorAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic, label string) { + if diagnostic == nil || diagnostic.PlanReplay == nil { + gateCase.Reasons = append(gateCase.Reasons, label+" qualification requires exact plan branch evidence") + return + } + counters := diagnostic.PlanReplay.Counters + candidate, candidatePresent := counters["asp_i1_candidate_marker_rows"] + fallback, fallbackPresent := counters["asp_i1_fallback_marker_rows"] + if !candidatePresent || !fallbackPresent || candidate+fallback != 1 { + gateCase.Reasons = append(gateCase.Reasons, label+" execution must attribute exactly one candidate or fallback marker") + } + candidateBranchRows, candidateBranchPresent := counters["asp_i1_candidate_branch_rows"] + fallbackBranchRows, fallbackBranchPresent := counters["asp_i1_fallback_branch_rows"] + if !candidateBranchPresent || !fallbackBranchPresent { + gateCase.Reasons = append(gateCase.Reasons, label+" execution is missing exact candidate or fallback output-branch row evidence") + } + candidateExecutorLoops, candidateExecutorPresent := counters["asp_i1_candidate_executor_loops"] + fallbackExecutorLoops, fallbackExecutorPresent := counters["asp_i1_fallback_executor_loops"] + if !candidateExecutorPresent || !fallbackExecutorPresent { + gateCase.Reasons = append(gateCase.Reasons, label+" execution is missing exact candidate or fallback executor-loop evidence") + } + if candidate == 1 && fallbackBranchRows != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" fallback output arm emitted rows while the candidate was selected") + } + if candidate == 1 && fallbackExecutorLoops != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" fallback executor ran while the candidate was selected") + } + if candidate == 1 && candidateExecutorLoops != 1 { + gateCase.Reasons = append(gateCase.Reasons, label+" candidate marker must bind exactly one selected executor loop") + } + if fallback == 1 && candidateBranchRows != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" candidate output arm emitted rows while fallback was selected") + } + if fallback == 1 && candidateExecutorLoops != 0 { + gateCase.Reasons = append(gateCase.Reasons, label+" candidate executor ran while fallback was selected") + } + if fallback == 1 && fallbackExecutorLoops != 1 { + gateCase.Reasons = append(gateCase.Reasons, label+" fallback marker must bind exactly one selected executor loop") + } +} + +// appendOrientationAttributionReasons appends orientation attribution reasons. +func appendOrientationAttributionReasons(gateCase *ResourceGateCase, diagnostic *TraversalExecutionDiagnostic) { + if diagnostic == nil || diagnostic.PlanReplay == nil { + gateCase.Reasons = append(gateCase.Reasons, "orientation qualification requires exact plan branch and probe evidence") + return + } + counters := diagnostic.PlanReplay.Counters + candidate, candidatePresent := counters["orientation_executed_candidate_rows"] + incumbent, incumbentPresent := counters["orientation_executed_incumbent_rows"] + if !candidatePresent || !incumbentPresent { + gateCase.Reasons = append(gateCase.Reasons, "orientation execution is missing exact selected and unselected arm markers") + } + if candidate+incumbent != 1 { + gateCase.Reasons = append(gateCase.Reasons, "orientation execution must attribute exactly one selected arm and zero unselected-arm work") + } + if candidate == 1 && counters["orientation_incumbent_branch_loops"] != 0 { + gateCase.Reasons = append(gateCase.Reasons, "orientation incumbent arm performed work while the candidate was selected") + } + if incumbent == 1 && counters["orientation_candidate_branch_loops"] != 0 { + gateCase.Reasons = append(gateCase.Reasons, "orientation candidate arm performed work while the incumbent was selected") + } + for _, name := range []string{ + "orientation_root_probe_loops", "orientation_suffix_probe_loops", "orientation_boundary_probe_loops", + "orientation_forward_degree_probe_loops", "orientation_reverse_degree_probe_loops", "orientation_decision_loops", + } { + loops, present := counters[name] + if !present { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("orientation probe %s has no execution-count evidence", name)) + } else if loops > 1 { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf("orientation probe %s executed more than once", name)) + } + } +} + +// traversalCapUsesSentinel reports whether the counter may observe the +// deliberate cap+1 row used to prove overflow before exact fallback. +func traversalCapUsesSentinel(name string) bool { + return strings.Contains(name, "probe") || strings.Contains(name, "suffix") || strings.Contains(name, "state") || + strings.Contains(name, "frontier") || strings.Contains(name, "queue") || + strings.Contains(name, "seen") || strings.Contains(name, "predecessor") || + strings.Contains(name, "output") +} + +// traversalNumericObservations maps typed diagnostic counters to stable gate names. +func traversalNumericObservations(counters TraversalDiagnosticCounters) map[string]int64 { + observed := map[string]int64{} + set := func(name string, value *int64) { + if value != nil { + observed[name] = *value + } + } + if ordinary := counters.Ordinary; ordinary != nil { + set("root_rows", ordinary.Roots) + set("edge_candidates", ordinary.EdgeCandidates) + set("state_rows", ordinary.PeakState) + set("output_paths", ordinary.EmittedTrails) + } + if orientation := counters.Orientation; orientation != nil { + set("forward_seed_rows", orientation.ForwardSeeds) + set("reverse_seed_rows", orientation.ReverseSeeds) + set("probe_rows", orientation.ProbeRows) + if orientation.ForwardDegreeSamples != nil && orientation.ReverseDegreeSamples != nil { + degreePeak := max(*orientation.ForwardDegreeSamples, *orientation.ReverseDegreeSamples) + observed["directional_degree_rows"] = degreePeak + } + set("survival_rows", orientation.ShallowSurvivalRows) + set("branch_loops", orientation.BranchLoops) + } + if guard := counters.SuffixGuard; guard != nil { + set("root_rows", guard.RootPresenceRows) + set("suffix_rows", guard.SuffixRows) + set("reverse_seed_rows", guard.SuffixRows) + set("state_rows", guard.StateRows) + set("output_rows", guard.OutputRows) + } + if component := counters.SuffixComponent; component != nil { + set("suffix_rows", component.SuffixRows) + set("reverse_seed_rows", component.SuffixRows) + set("state_rows", component.ReverseStateRows) + set("output_rows", component.OutputRows) + if component.OrderedNodeHydrationRows != nil && component.OrderedEdgeHydrationRows != nil { + observed["hydration_rows"] = *component.OrderedNodeHydrationRows + *component.OrderedEdgeHydrationRows + } + } + if shortest := counters.ShortestPath; shortest != nil { + set("state_rows", shortest.SeenPeak) + set("frontier_rows", shortest.FrontierPeak) + set("queue_rows", shortest.QueuePeak) + set("seen_rows", shortest.SeenPeak) + set("predecessor_rows", shortest.PredecessorPeak) + set("meeting_rows", shortest.MeetingCandidates) + set("witness_rows", shortest.WitnessRows) + } + if all := counters.AllShortestPaths; all != nil { + set("state_rows", all.Search.SeenPeak) + set("frontier_rows", all.Search.FrontierPeak) + set("queue_rows", all.Search.QueuePeak) + set("seen_rows", all.Search.SeenPeak) + set("predecessor_rows", all.PredecessorPeak) + set("output_paths", all.OutputPaths) + set("output_rows", all.EnumeratedCandidates) + set("output_edge_cells", all.OutputEdgeCells) + set("output_bytes", all.OutputBytes) + } + if inline := counters.InlineASP; inline != nil { + set("state_rows", inline.DistanceRows) + set("predecessor_rows", inline.PredecessorRows) + set("output_rows", inline.EnumerationRows) + set("output_paths", inline.OutputPaths) + set("output_bytes", inline.OutputBytes) + } + if inline := counters.InlineShortestPath; inline != nil { + set("state_rows", inline.DistanceRows) + set("predecessor_rows", inline.PredecessorRows) + set("output_rows", inline.EnumerationRows) + set("output_paths", inline.OutputPaths) + set("output_bytes", inline.OutputBytes) + } + if inline := counters.InlineShortestDistance; inline != nil { + set("state_rows", inline.StateRows) + set("frontier_rows", inline.FrontierRows) + set("queue_rows", inline.FrontierRows) + set("output_rows", inline.OutputRows) + } + if hydration := counters.Hydration; hydration != nil { + set("hydration_rows", hydration.Rows) + set("hydration_bytes", hydration.Bytes) + } + return observed +} + +// appendWorkspaceResourceReasons adds failures for excessive executor or session workspace usage. +func appendWorkspaceResourceReasons(gateCase *ResourceGateCase, metrics *PostgresPlanMetrics) { + if metrics.Buffers.TempRead != 0 || metrics.Buffers.TempWritten != 0 { + gateCase.Reasons = append(gateCase.Reasons, "compact workspace candidate spilled to executor temporary storage") + } + if metrics.WALRecords != 0 || metrics.WALBytes != 0 { + gateCase.Reasons = append(gateCase.Reasons, "non-mutating compact workspace candidate emitted WAL") + } +} + +// appendPortableResourceReasons adds failures for spill, loops, or cardinality evidence that violates portable limits. +func appendPortableResourceReasons(gateCase *ResourceGateCase, metrics *PostgresPlanMetrics) { + buffers := metrics.Buffers + if buffers.TempRead != 0 || buffers.TempWritten != 0 { + gateCase.Reasons = append(gateCase.Reasons, "portable candidate used temporary buffers") + } + if buffers.LocalHit != 0 || buffers.LocalRead != 0 || buffers.LocalDirtied != 0 || buffers.LocalWritten != 0 { + gateCase.Reasons = append(gateCase.Reasons, "portable candidate used local workspace") + } + if metrics.WALRecords != 0 || metrics.WALBytes != 0 { + gateCase.Reasons = append(gateCase.Reasons, "non-mutating portable candidate emitted WAL") + } +} + +// postgresPlanFunctionLoops sums actual loops for PostgreSQL plan nodes invoking the named function. +func postgresPlanFunctionLoops(raw json.RawMessage, function string) (int64, bool, error) { + if len(raw) == 0 { + return 0, false, nil + } + var document []map[string]any + if err := json.Unmarshal(raw, &document); err != nil { + return 0, false, err + } + if len(document) == 0 { + return 0, false, nil + } + root, ok := document[0]["Plan"].(map[string]any) + if !ok { + return 0, false, nil + } + var loops int64 + found := false + var walk func(map[string]any) + walk = func(node map[string]any) { + alias, _ := node["Alias"].(string) + functionName, _ := node["Function Name"].(string) + if alias == function || functionName == function { + found = true + if actualLoops, ok := node["Actual Loops"].(float64); ok { + loops += int64(actualLoops) + } + } + children, _ := node["Plans"].([]any) + for _, child := range children { + if childNode, ok := child.(map[string]any); ok { + walk(childNode) + } + } + } + walk(root) + return loops, found, nil +} + +// appliedPostgresArchitecture returns the effective PostgreSQL executor architecture, including fallback attribution. +func appliedPostgresArchitecture(record CaseResult) string { + if record.Optimization == nil { + return "" + } + for _, outcome := range record.Optimization.TargetOutcomes { + if outcome.Family == "SP" || outcome.Family == "ASP" || outcome.Family == "fixed_suffix_expansion" || outcome.Family == "fixed_prefix_terminal_expansion" { + if outcome.Applied != "" { + return outcome.Applied + } + return outcome.Selected + } + } + return "" +} diff --git a/cmd/graphbench/resource_gate_test.go b/cmd/graphbench/resource_gate_test.go new file mode 100644 index 00000000..355195fa --- /dev/null +++ b/cmd/graphbench/resource_gate_test.go @@ -0,0 +1,1018 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestResourceGateReportBindsExactInputArtifact verifies that schema v5 reports +// retain the SHA-256 digest of the exact JSONL bytes supplied to the gate. +func TestResourceGateReportBindsExactInputArtifact(t *testing.T) { + tempDir := t.TempDir() + artifact := filepath.Join(tempDir, "records.jsonl") + record := CaseResult{ + Environment: &RunEnvironment{ + Round: 3, + Block: 3, + RunUUID: "resource-run", + Arm: "candidate", + ArmOrder: 2, + }, + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S4-C-D", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + artifactRaw, err := os.ReadFile(artifact) + require.NoError(t, err) + expectedDigest := sha256.Sum256(artifactRaw) + + output := filepath.Join(tempDir, "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.True(t, passed) + + var report ResourceGateReport + reportRaw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(reportRaw, &report)) + require.Equal(t, resourceGateVersion, report.Version) + require.Equal(t, hex.EncodeToString(expectedDigest[:]), report.ArtifactSHA256) + require.True(t, isLowerHexSHA256(report.ArtifactSHA256)) + require.Equal(t, 3, report.Cases[0].Round) + require.Equal(t, 3, report.Cases[0].Block) + require.Equal(t, "resource-run", report.Cases[0].RunUUID) + require.Equal(t, "candidate", report.Cases[0].Arm) + require.Equal(t, 2, report.Cases[0].ArmOrder) +} + +// TestResourceGateAllowsCompactSessionWorkspaceButRejectsExecutorSpill verifies that local workspace writes are permitted for the compact architecture while temporary-buffer spill fails the gate. +func TestResourceGateAllowsCompactSessionWorkspaceButRejectsExecutorSpill(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S4-C-D", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.True(t, passed) + + record.PostgresMetrics.Buffers.TempWritten = 1 + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "spill-report.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateRecognizesCompactBidirectionalWorkspaceArchitectures freezes +// local-workspace attribution for production and full-comparator B1/B2 arms. +func TestResourceGateRecognizesCompactBidirectionalWorkspaceArchitectures(t *testing.T) { + for _, architecture := range []string{ + "SP-B1-C-ALT-NODE-D", + "SP-B1-C-ALT-NODE-WE+MAT-M0", + "SP-B2-C-MIN-LEVEL-D", + "SP-B2-C-MIN-LEVEL-WE+MAT-M0", + } { + require.True(t, compactWorkspaceArchitecture(architecture), architecture) + require.True(t, compactBidirectionalWorkspaceArchitecture(architecture), architecture) + } + require.True(t, compactWorkspaceArchitecture("SP-S4-C-D")) + require.False(t, compactBidirectionalWorkspaceArchitecture("SP-S4-C-D")) +} + +// TestResourceGateRecognizesASPProductionArchitecture verifies that the applied all-shortest-path lowering, rather than a fallback label, identifies the production architecture. +func TestResourceGateRecognizesASPProductionArchitecture(t *testing.T) { + record := CaseResult{ + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "ASP", + Applied: "ASP-A1-DAG", + }}, + }, + } + require.Equal(t, "ASP-A1-DAG", appliedPostgresArchitecture(record)) +} + +// TestResourceGateChecksFullComparatorReferenceResources verifies that temporary-buffer usage in a full comparator becomes its own failing report case with arm attribution. +func TestResourceGateChecksFullComparatorReferenceResources(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + PostgresReferences: []PostgresReferenceResult{{ + Name: "s4", + Architecture: "SP-S4-C-D", + FullComparator: true, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + TempWritten: 1, + }, + }, + }}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 2) + require.Equal(t, "s4", report.Cases[1].Reference) + require.Contains(t, report.Cases[1].Reasons, "portable candidate used temporary buffers") +} + +// TestResourceGateAttributesDirectPreflightIncumbentFallback verifies that a direct-preflight plan executing the recursive harness is attributed to SP-S0 fallback while a skipped harness remains direct. +func TestResourceGateAttributesDirectPreflightIncumbentFallback(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "fallback", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0-DIRECT", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + PostgresPlanJSON: json.RawMessage(`[{"Plan":{"Plans":[{"Alias":"bidirectional_sp_harness","Actual Loops":1}]}}]`), + }, + { + Dataset: "fixture", + Name: "direct", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0-DIRECT", + }}, + }, + PostgresPlanJSON: json.RawMessage(`[{"Plan":{"Plans":[{"Function Name":"bidirectional_sp_harness","Actual Loops":0}]}}]`), + PostgresMetrics: &PostgresPlanMetrics{}, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.True(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Equal(t, "SP-S0", report.Cases[1].FallbackArchitecture) +} + +// TestResourceGateFailsClosedWithoutStructuredMetrics verifies that a successful portable candidate still fails resource gating when structured PostgreSQL metrics are absent. +func TestResourceGateFailsClosedWithoutStructuredMetrics(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "missing-metrics", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S4-C-D", + }}, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Contains(t, report.Cases[0].Reasons, "structured PostgreSQL plan metrics are missing") +} + +// TestResourceGateRejectsDirectPreflightWorkspaceOnDirectHit verifies that a true direct hit cannot claim local workspace writes when the recursive harness executed zero times. +func TestResourceGateRejectsDirectPreflightWorkspaceOnDirectHit(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "direct", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0-DIRECT", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + PostgresPlanJSON: json.RawMessage(`[{"Plan":{"Plans":[{"Alias":"bidirectional_sp_harness","Actual Loops":0}]}}]`), + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateAllowsStressDiagnosticsAndExactFallback verifies that spill is diagnostic on stress fixtures and compact workspace use is allowed for an explicitly selected exact fallback. +func TestResourceGateAllowsStressDiagnosticsAndExactFallback(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + records := []CaseResult{ + { + Dataset: "fixture", + Name: "stress", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "stress", + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + TempWritten: 1, + }, + }, + }, + { + Dataset: "fixture", + Name: "fallback", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + }, + Optimization: &translate.OptimizationSummary{ + TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Selected: "SP-S0", + }}, + }, + PostgresMetrics: &PostgresPlanMetrics{ + Buffers: Buffers{ + LocalWritten: 1, + }, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, records)) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.True(t, passed) +} + +// TestResourceGateEnforcesTelemetryIdentityAndNumericSentinels verifies a +// candidate may observe exactly cap+1, while larger work or contradictory +// runtime attribution fails closed. +func TestResourceGateEnforcesTelemetryIdentityAndNumericSentinels(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = "SP-B1-C-ALT-NODE-D" + telemetry.Summary.PlannedIdentities = []string{"SP-B1-C-ALT-NODE-D", "SP-S4-C-D"} + telemetry.Summary.EmittedIdentity = "sp-bidirectional-tournament-v1" + telemetry.Summary.RuntimeIdentity = "SP-B1-C-ALT-NODE-D" + telemetry.Summary.AppliedIdentity = "SP-B1-C-ALT-NODE-D" + telemetry.Summary.RuntimeOutcomeAvailable = telemetryBool(true) + telemetry.Summary.Provenance["runtime_outcome_available"] = "executor.receipt" + telemetry.Summary.Caps = map[string]int64{"state_rows": 32} + telemetry.Summary.Provenance["caps.state_rows"] = "policy.state_cap" + delete(telemetry.Summary.Provenance, "caps.state") + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(33) + record := CaseResult{ + Dataset: "fixture", + Name: "candidate", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "envelope", + FallbackExpectation: "forbidden", + }, + Environment: &RunEnvironment{ + PoolSize: 1, + SessionMemoryCeilingBytes: 1 << 20, + PoolMemoryCeilingBytes: 1 << 20, + }, + TraversalTelemetry: &telemetry, + PostgresMetrics: &PostgresPlanMetrics{}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-B1-C-ALT-NODE-D", + }}, + }, + } + record.TraversalTelemetry.Diagnostic.Counters.Workspace = &TraversalWorkspaceCounters{ + SessionPeakBytes: telemetryInt64(4096), + PoolPeakBytes: telemetryInt64(4096), + } + record.TraversalTelemetry.Diagnostic.RequiredFamilies = append( + record.TraversalTelemetry.Diagnostic.RequiredFamilies, + TraversalTelemetryFamilyWorkspace, + ) + record.TraversalTelemetry.Diagnostic.Provenance["workspace.session_peak_bytes"] = "test.session" + record.TraversalTelemetry.Diagnostic.Provenance["workspace.pool_peak_bytes"] = "test.pool" + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passingReportPath := filepath.Join(t.TempDir(), "cap-plus-one.json") + passed, err := createResourceGateReport(artifact, passingReportPath) + require.NoError(t, err) + passingReportRaw, err := os.ReadFile(passingReportPath) + require.NoError(t, err) + var passingReport ResourceGateReport + require.NoError(t, json.Unmarshal(passingReportRaw, &passingReport)) + require.True(t, passed, passingReport.Cases) + + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(34) + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "overflow.json")) + require.NoError(t, err) + require.False(t, passed) + + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(-1) + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + negativeReportPath := filepath.Join(t.TempDir(), "negative.json") + passed, err = createResourceGateReport(artifact, negativeReportPath) + require.NoError(t, err) + require.False(t, passed) + negativeReportRaw, err := os.ReadFile(negativeReportPath) + require.NoError(t, err) + var negativeReport ResourceGateReport + require.NoError(t, json.Unmarshal(negativeReportRaw, &negativeReport)) + require.Contains(t, negativeReport.Cases[0].Reasons, "traversal counter state_rows=-1 is negative") + + telemetry.Diagnostic.Counters.Ordinary.PeakState = telemetryInt64(32) + telemetry.Summary.AppliedIdentity = "SP-S4-C-D" + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "identity.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateRequiresDiagnosticTelemetryForBidirectionalCandidates verifies +// opaque function work cannot qualify from outer EXPLAIN evidence alone. +func TestResourceGateRequiresDiagnosticTelemetryForBidirectionalCandidates(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "missing-telemetry", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + PostgresMetrics: &PostgresPlanMetrics{}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-B2-C-MIN-LEVEL-D", + }}, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "missing.json")) + require.NoError(t, err) + require.False(t, passed) + + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = "SP-B2-C-MIN-LEVEL-D" + telemetry.Summary.PlannedIdentities = []string{"SP-B2-C-MIN-LEVEL-D", "SP-S4-C-D"} + telemetry.Summary.EmittedIdentity = "sp-bidirectional-tournament-v1" + telemetry.Summary.RuntimeIdentity = "SP-B2-C-MIN-LEVEL-D" + telemetry.Summary.AppliedIdentity = "SP-B2-C-MIN-LEVEL-D" + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusHiddenUnavailable + telemetry.Diagnostic.IncompleteReasons = []string{"function scan hides invocation counters"} + record.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "incomplete.json") + passed, err = createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Contains(t, report.Cases[0].Reasons, "candidate qualification requires complete executor counters; diagnostic status is hidden_counters_unavailable") +} + +// TestResourceGateRejectsDeclaredMemoryCeilingsWithoutMeasuredWorkspace verifies resource gate rejects declared memory ceilings without measured workspace behavior. +func TestResourceGateRejectsDeclaredMemoryCeilingsWithoutMeasuredWorkspace(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "declared-only", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + PostgresMetrics: &PostgresPlanMetrics{}, + Environment: &RunEnvironment{ + PoolSize: 1, + SessionMemoryCeilingBytes: 1024, + PoolMemoryCeilingBytes: 4096, + }, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S4-C-D", + }}}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.False(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Contains(t, report.Cases[0].Reasons, "declared workspace memory ceilings lack measured session and pool high-water evidence") +} + +func TestResourceGateRequiresExactV2InlineDistanceAttribution(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) + telemetry.Summary.PlannedIdentities = []string{ + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D), + string(optimize.ShortestPathExecutorS4CanonicalDistance), + } + telemetry.Summary.EmittedIdentity = optimize.ShortestPathPolicyI2DistanceGuardedV2 + telemetry.Summary.RuntimeIdentity = string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) + telemetry.Summary.AppliedIdentity = string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) + telemetry.Summary.RuntimeOutcomeAvailable = telemetryBool(true) + telemetry.Summary.Caps = map[string]int64{"state_rows": 100, "frontier_rows": 100} + telemetry.Summary.Provenance["runtime_outcome_available"] = "test.receipt" + telemetry.Summary.Provenance["caps.state_rows"] = "test.cap.state" + telemetry.Summary.Provenance["caps.frontier_rows"] = "test.cap.frontier" + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilySP} + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusComplete + telemetry.Diagnostic.Counters.InlineShortestDistance = &InlineDistanceTraversalCounters{} + telemetry.Diagnostic.PlanReplay = &TraversalPlanReplayEvidence{ + Source: "test-plan", + Counters: map[string]int64{ + "sp_i2_distance_rows": 1, + "sp_i2_target_rows": 1, + "sp_i2_output_rows": 1, + "sp_i2_candidate_marker_rows": 1, + "sp_i2_fallback_marker_rows": 0, + "sp_i2_candidate_branch_rows": 1, + "sp_i2_fallback_branch_rows": 0, + "sp_i2_candidate_executor_loops": 1, + "sp_i2_fallback_executor_loops": 0, + "sp_i2_admission_rows": 1, + "sp_i2_admission_loops": 1, + // The direct counter is deliberately absent. V2 E1D must fail closed. + }, + Provenance: map[string]string{"counters.sp_i2_distance_rows": "test.plan"}, + } + gateCase := ResourceGateCase{} + appendInlineDistanceAttributionReasons(&gateCase, &telemetry) + require.Contains(t, gateCase.Reasons, "inline SP distance execution is missing exact plan counter sp_i2_direct_rows") +} + +// TestResourceGateRequiresCompleteOrientationPolicyAndExactBranchAttribution verifies resource gate requires complete orientation policy and exact branch attribution behavior. +func TestResourceGateRequiresCompleteOrientationPolicyAndExactBranchAttribution(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = "EXPANSION-SUFFIX-SEEDED-REVERSE" + telemetry.Summary.PlannedIdentities = []string{"EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-STEPWISE-FORWARD"} + telemetry.Summary.EmittedIdentity = "orientation-probe-v1" + telemetry.Summary.RuntimeIdentity = "EXPANSION-SUFFIX-SEEDED-REVERSE" + telemetry.Summary.AppliedIdentity = "EXPANSION-SUFFIX-SEEDED-REVERSE" + telemetry.Summary.SelectorVersion = "orientation-probe-v1" + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.CounterStatus = TraversalTelemetryCounterStatusPlanPartial + telemetry.Diagnostic.IncompleteReasons = []string{"plan evidence only"} + telemetry.Diagnostic.PlanReplay = &TraversalPlanReplayEvidence{ + Source: "test", + Counters: map[string]int64{"orientation_executed_candidate_rows": 1}, + Flags: map[string]bool{}, + Provenance: map[string]string{"counters.orientation_executed_candidate_rows": "test.marker"}, + } + record := CaseResult{ + Dataset: "fixture", + Name: "orientation", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + PostgresMetrics: &PostgresPlanMetrics{}, + TraversalTelemetry: &telemetry, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "fixed_suffix_expansion", + Applied: "EXPANSION-SUFFIX-SEEDED-REVERSE", + EmittedPolicy: "orientation-probe-v1", + }}}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "report.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateScopesStressFallbackToDeclaredExpectation verifies resource gate scopes stress fallback to declared expectation behavior. +func TestResourceGateScopesStressFallbackToDeclaredExpectation(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + withoutExpectation := CaseResult{ + Dataset: "fixture", + Name: "stress-no-overflow", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "stress"}, + PostgresMetrics: &PostgresPlanMetrics{}, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Applied: "SP-S0", + }}}, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{withoutExpectation})) + passed, err := createResourceGateReport(artifact, filepath.Join(t.TempDir(), "no-expectation.json")) + require.NoError(t, err) + require.True(t, passed) + + withExpectation := withoutExpectation + withExpectation.Name = "stress-overflow" + withExpectation.Shape.FallbackExpectation = "required" + telemetry := validTraversalTelemetry() + telemetry.Summary.FallbackExecuted = telemetryBool(false) + withExpectation.TraversalTelemetry = &telemetry + require.NoError(t, writeJSONLFile(artifact, []CaseResult{withExpectation})) + passed, err = createResourceGateReport(artifact, filepath.Join(t.TempDir(), "expected.json")) + require.NoError(t, err) + require.False(t, passed) +} + +// TestResourceGateValidatesExactOrientationMarkersAndProbeCounts verifies resource gate validates exact orientation markers and probe counts behavior. +func TestResourceGateValidatesExactOrientationMarkersAndProbeCounts(t *testing.T) { + probeCounters := map[string]int64{ + "orientation_executed_candidate_rows": 1, + "orientation_executed_incumbent_rows": 0, + "orientation_root_probe_loops": 1, + "orientation_suffix_probe_loops": 1, + "orientation_boundary_probe_loops": 1, + "orientation_forward_degree_probe_loops": 1, + "orientation_reverse_degree_probe_loops": 1, + "orientation_decision_loops": 1, + "orientation_candidate_branch_loops": 1, + "orientation_incumbent_branch_loops": 0, + } + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: probeCounters}} + gateCase := &ResourceGateCase{} + appendOrientationAttributionReasons(gateCase, diagnostic) + require.Empty(t, gateCase.Reasons) + + probeCounters["orientation_executed_incumbent_rows"] = 1 + probeCounters["orientation_root_probe_loops"] = 2 + delete(probeCounters, "orientation_suffix_probe_loops") + appendOrientationAttributionReasons(gateCase, diagnostic) + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "exactly one selected arm") + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "executed more than once") + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "no execution-count evidence") + + probeCounters["orientation_executed_incumbent_rows"] = 0 + probeCounters["orientation_root_probe_loops"] = 1 + probeCounters["orientation_suffix_probe_loops"] = 1 + probeCounters["orientation_incumbent_branch_loops"] = 1 + gateCase.Reasons = nil + appendOrientationAttributionReasons(gateCase, diagnostic) + require.Contains(t, gateCase.Reasons, "orientation incumbent arm performed work while the candidate was selected") +} + +// TestResourceGateValidatesSuffixGuardInactiveArmAndRejectsTopologyWork +// verifies the reverse-first guard's resource contract is independent from +// orientation-v2 and proves the unselected executor stayed inactive. +func TestResourceGateValidatesSuffixGuardInactiveArmAndRejectsTopologyWork(t *testing.T) { + counters := map[string]int64{ + "suffix_guard_candidate_marker_rows": 1, "suffix_guard_fallback_marker_rows": 0, + "suffix_guard_candidate_branch_rows": 1, "suffix_guard_fallback_branch_rows": 0, + "suffix_guard_output_rows": 1, + "suffix_guard_candidate_executor_loops": 1, "suffix_guard_fallback_executor_loops": 0, + } + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: counters}} + gateCase := &ResourceGateCase{} + appendSuffixGuardAttributionReasons(gateCase, diagnostic) + require.Empty(t, gateCase.Reasons) + + counters["suffix_guard_fallback_executor_loops"] = 1 + appendSuffixGuardAttributionReasons(gateCase, diagnostic) + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "did not suppress the fallback executor") + + counters["suffix_guard_fallback_executor_loops"] = 0 + counters["orientation_forward_degree_rows"] = 10 + gateCase.Reasons = nil + appendSuffixGuardAttributionReasons(gateCase, diagnostic) + require.Contains(t, strings.Join(gateCase.Reasons, "\n"), "unexpectedly contains orientation topology work") +} + +// TestResourceGateRequiresSingularInlineASPBranchAndInactiveArm verifies resource gate requires singular inline asp branch and inactive arm behavior. +func TestResourceGateRequiresSingularInlineASPBranchAndInactiveArm(t *testing.T) { + gateCase := &ResourceGateCase{} + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: map[string]int64{ + "asp_i1_candidate_marker_rows": 1, + "asp_i1_fallback_marker_rows": 0, + "asp_i1_candidate_branch_rows": 1, + "asp_i1_fallback_branch_rows": 0, + "asp_i1_candidate_executor_loops": 1, + "asp_i1_fallback_executor_loops": 0, + }}} + appendInlineASPAttributionReasons(gateCase, diagnostic) + require.Empty(t, gateCase.Reasons) + + for _, missing := range []string{"asp_i1_candidate_branch_rows", "asp_i1_fallback_branch_rows"} { + value := diagnostic.PlanReplay.Counters[missing] + delete(diagnostic.PlanReplay.Counters, missing) + missingCase := &ResourceGateCase{} + appendInlineASPAttributionReasons(missingCase, diagnostic) + require.Contains(t, missingCase.Reasons, "inline ASP execution is missing exact candidate or fallback output-branch row evidence") + diagnostic.PlanReplay.Counters[missing] = value + } + for _, missing := range []string{"asp_i1_candidate_executor_loops", "asp_i1_fallback_executor_loops"} { + value := diagnostic.PlanReplay.Counters[missing] + delete(diagnostic.PlanReplay.Counters, missing) + missingCase := &ResourceGateCase{} + appendInlineASPAttributionReasons(missingCase, diagnostic) + require.Contains(t, missingCase.Reasons, "inline ASP execution is missing exact candidate or fallback executor-loop evidence") + diagnostic.PlanReplay.Counters[missing] = value + } + + diagnostic.PlanReplay.Counters["asp_i1_fallback_executor_loops"] = 1 + executedInactiveCase := &ResourceGateCase{} + appendInlineASPAttributionReasons(executedInactiveCase, diagnostic) + require.Contains(t, executedInactiveCase.Reasons, "inline ASP fallback executor ran while the candidate was selected") + diagnostic.PlanReplay.Counters["asp_i1_fallback_executor_loops"] = 0 + + diagnostic.PlanReplay.Counters["asp_i1_fallback_marker_rows"] = 1 + diagnostic.PlanReplay.Counters["asp_i1_fallback_branch_rows"] = 1 + appendInlineASPAttributionReasons(gateCase, diagnostic) + require.Contains(t, gateCase.Reasons, "inline ASP execution must attribute exactly one candidate or fallback marker") + require.Contains(t, gateCase.Reasons, "inline ASP fallback output arm emitted rows while the candidate was selected") +} + +// TestResourceGateScopesGuardedI1TelemetryAndInactiveArm verifies resource gate scopes guarded i1 telemetry and inactive arm behavior. +func TestResourceGateScopesGuardedI1TelemetryAndInactiveArm(t *testing.T) { + require.False(t, telemetryRequiredForArchitecture(string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness))) + require.False(t, telemetryRequiredForArchitecture(string(optimize.ShortestPathExecutorASPI1DAG))) + require.True(t, telemetryRequiredForRecord( + guardedI1ResourceRecord(string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness)), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + )) + + gateCase := &ResourceGateCase{} + diagnostic := &TraversalExecutionDiagnostic{PlanReplay: &TraversalPlanReplayEvidence{Counters: map[string]int64{ + "asp_i1_candidate_marker_rows": 1, + "asp_i1_fallback_marker_rows": 0, + "asp_i1_candidate_branch_rows": 1, + "asp_i1_fallback_branch_rows": 0, + "asp_i1_candidate_executor_loops": 1, + "asp_i1_fallback_executor_loops": 0, + }}} + appendInlinePredecessorAttributionReasons(gateCase, diagnostic, "inline canonical SP") + require.Empty(t, gateCase.Reasons) + + diagnostic.PlanReplay.Counters["asp_i1_fallback_marker_rows"] = 1 + diagnostic.PlanReplay.Counters["asp_i1_fallback_branch_rows"] = 1 + appendInlinePredecessorAttributionReasons(gateCase, diagnostic, "inline canonical SP") + require.Contains(t, gateCase.Reasons, "inline canonical SP execution must attribute exactly one candidate or fallback marker") + require.Contains(t, gateCase.Reasons, "inline canonical SP fallback output arm emitted rows while the candidate was selected") +} + +// TestResourceGateDoesNotRequireGuardedTelemetryForExplicitI1References verifies resource gate does not require guarded telemetry for explicit i1 references behavior. +func TestResourceGateDoesNotRequireGuardedTelemetryForExplicitI1References(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "records.jsonl") + record := CaseResult{ + Dataset: "fixture", + Name: "explicit-references", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{FixtureTier: "normal"}, + PostgresMetrics: &PostgresPlanMetrics{}, + PostgresReferences: []PostgresReferenceResult{ + { + Name: "sp-i1-reference", + Architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + FullComparator: true, + PostgresMetrics: &PostgresPlanMetrics{}, + }, + { + Name: "asp-i1-reference", + Architecture: string(optimize.ShortestPathExecutorASPI1DAG), + FullComparator: true, + PostgresMetrics: &PostgresPlanMetrics{}, + }, + }, + } + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.True(t, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 3) + for _, gateCase := range report.Cases { + require.True(t, gateCase.Passed, "%+v", gateCase) + require.NotContains(t, gateCase.Reasons, "required traversal execution telemetry is missing") + } +} + +// TestResourceGateBindsGuardedI1PolicyAndCounterNamespace verifies resource gate binds guarded i1 policy and counter namespace behavior. +func TestResourceGateBindsGuardedI1PolicyAndCounterNamespace(t *testing.T) { + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // architecture retains the architecture while anonymous record is assembled or evaluated. + architecture string + // mutate retains the mutate while anonymous record is assembled or evaluated. + mutate func(*CaseResult) + // passed indicates whether passed applies. + passed bool + // reason retains the reason while anonymous record is assembled or evaluated. + reason string + }{ + { + name: "canonical SP valid", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + passed: true, + }, + { + name: "ASP valid", + architecture: string(optimize.ShortestPathExecutorASPI1DAG), + passed: true, + }, + { + name: "canonical SP missing outcome policy", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + record.Optimization.TargetOutcomes[0].EmittedPolicy = "" + }, + reason: "inline canonical SP production architecture requires emitted policy", + }, + { + name: "canonical SP wrong telemetry policy", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Summary.EmittedIdentity = optimize.ShortestPathPolicyASPI1GuardedV1 + }, + reason: "inline canonical SP production telemetry requires emitted identity", + }, + { + name: "canonical SP wrong counter namespace", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + diagnostic := record.TraversalTelemetry.Diagnostic + diagnostic.Counters.InlineASP = diagnostic.Counters.InlineShortestPath + diagnostic.Counters.InlineShortestPath = nil + diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilyASP, TraversalTelemetryFamilyHydration} + diagnostic.Provenance = guardedI1CounterProvenance("inline_asp") + }, + reason: "inline canonical SP production telemetry requires inline_shortest_path counters", + }, + { + name: "canonical SP missing contract counter family", + architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilyHydration} + }, + reason: `inline canonical SP production telemetry requires declared counter family "shortest_path"`, + }, + { + name: "ASP missing hydration family", + architecture: string(optimize.ShortestPathExecutorASPI1DAG), + mutate: func(record *CaseResult) { + diagnostic := record.TraversalTelemetry.Diagnostic + diagnostic.RequiredFamilies = []TraversalTelemetryFamily{TraversalTelemetryFamilyASP} + diagnostic.Counters.Hydration = nil + }, + reason: "inline ASP production telemetry requires declared hydration counters for its observation mode", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + record := guardedI1ResourceRecord(test.architecture) + if test.mutate != nil { + test.mutate(&record) + } + artifact := filepath.Join(t.TempDir(), "records.jsonl") + require.NoError(t, writeJSONLFile(artifact, []CaseResult{record})) + output := filepath.Join(t.TempDir(), "report.json") + passed, err := createResourceGateReport(artifact, output) + require.NoError(t, err) + require.Equal(t, test.passed, passed) + + var report ResourceGateReport + raw, err := os.ReadFile(output) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &report)) + require.Len(t, report.Cases, 1) + if test.reason != "" { + require.Contains(t, strings.Join(report.Cases[0].Reasons, "\n"), test.reason) + } + }) + } +} + +// guardedI1ResourceRecord prepares or inspects test evidence for guarded i1 resource record. +func guardedI1ResourceRecord(architecture string) CaseResult { + contract, _ := guardedInlineResourceContractForArchitecture(architecture) + fallback := string(optimize.ShortestPathExecutorS4CanonicalWitness) + requiredFamily := TraversalTelemetryFamilySP + observationMode := "one_path" + if architecture == string(optimize.ShortestPathExecutorASPI1DAG) { + fallback = string(optimize.ShortestPathExecutorASPA1DAG) + requiredFamily = TraversalTelemetryFamilyASP + observationMode = "all_paths" + } + + inlineCounters := &InlinePredecessorTraversalCounters{ + DistanceRows: telemetryInt64(3), + PredecessorRows: telemetryInt64(2), + EnumerationRows: telemetryInt64(1), + OutputPaths: telemetryInt64(1), + OutputBytes: telemetryInt64(64), + CandidateMarkerRows: telemetryInt64(1), + FallbackMarkerRows: telemetryInt64(0), + CandidateBranchRows: telemetryInt64(1), + FallbackBranchRows: telemetryInt64(0), + CandidateExecutorLoops: telemetryInt64(1), + FallbackExecutorLoops: telemetryInt64(0), + } + diagnosticCounters := TraversalDiagnosticCounters{} + if requiredFamily == TraversalTelemetryFamilySP { + diagnosticCounters.InlineShortestPath = inlineCounters + } else { + diagnosticCounters.InlineASP = inlineCounters + } + diagnosticCounters.Hydration = &TraversalHydrationCounters{ + PathCount: telemetryInt64(1), + NodeLookups: telemetryInt64(2), + EdgeLookups: telemetryInt64(1), + Loops: telemetryInt64(1), + Rows: telemetryInt64(1), + TimeNS: telemetryInt64(100), + Bytes: telemetryInt64(64), + } + planCounters := map[string]int64{ + "asp_i1_distance_rows": 3, + "asp_i1_predecessor_rows": 2, + "asp_i1_enumeration_rows": 1, + "asp_i1_output_rows": 1, + "asp_i1_candidate_marker_rows": 1, + "asp_i1_fallback_marker_rows": 0, + "asp_i1_candidate_branch_rows": 1, + "asp_i1_fallback_branch_rows": 0, + "asp_i1_candidate_executor_loops": 1, + "asp_i1_fallback_executor_loops": 0, + } + planProvenance := map[string]string{} + for name := range planCounters { + planProvenance["counters."+name] = "test.plan." + name + } + + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Summary.RequestedIdentity = architecture + telemetry.Summary.PlannedIdentities = []string{architecture, fallback} + telemetry.Summary.EmittedIdentity = contract.policy + telemetry.Summary.RuntimeIdentity = architecture + telemetry.Summary.AppliedIdentity = architecture + telemetry.Summary.ObservationMode = observationMode + telemetry.Summary.RuntimeOutcomeAvailable = telemetryBool(true) + telemetry.Summary.Caps = map[string]int64{ + "state_rows": 100, "predecessor_rows": 100, "output_rows": 100, "output_bytes": 1024, + } + telemetry.Summary.Provenance["observation_mode"] = "test.observation" + telemetry.Summary.Provenance["runtime_outcome_available"] = "test.receipt" + for capName := range telemetry.Summary.Caps { + telemetry.Summary.Provenance["caps."+capName] = "test.cap." + capName + } + telemetry.Diagnostic = &TraversalExecutionDiagnostic{ + InvocationID: "guarded-i1-resource", + ConnectionID: "backend-1", + TimedSample: telemetryBool(false), + RequiredFamilies: []TraversalTelemetryFamily{requiredFamily, TraversalTelemetryFamilyHydration}, + Counters: diagnosticCounters, + CounterStatus: TraversalTelemetryCounterStatusComplete, + PlanReplay: &TraversalPlanReplayEvidence{ + Source: "test-plan", + Counters: planCounters, + Provenance: planProvenance, + }, + Provenance: guardedI1CounterProvenance(contract.namespace), + } + + return CaseResult{ + Dataset: "fixture", + Name: "guarded-i1", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Shape: WorkloadShape{ + FixtureTier: "normal", + FallbackExpectation: "forbidden", + }, + PostgresMetrics: &PostgresPlanMetrics{}, + TraversalTelemetry: &telemetry, + Optimization: &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: contract.family, + Candidate: architecture, + Selected: architecture, + Applied: architecture, + EmittedPolicy: contract.policy, + }}}, + } +} + +// inlineI1CounterProvenance prepares or inspects test evidence for inline i1 counter provenance. +func inlineI1CounterProvenance(namespace string) map[string]string { + provenance := map[string]string{} + for _, name := range []string{ + "distance_rows", "predecessor_rows", "enumeration_rows", "output_paths", "output_bytes", + "candidate_marker_rows", "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", + "candidate_executor_loops", "fallback_executor_loops", + } { + provenance[namespace+"."+name] = "test." + namespace + "." + name + } + return provenance +} + +// guardedI1CounterProvenance prepares or inspects test evidence for guarded i1 counter provenance. +func guardedI1CounterProvenance(namespace string) map[string]string { + provenance := inlineI1CounterProvenance(namespace) + for _, name := range []string{"path_count", "node_lookups", "edge_lookups", "loops", "rows", "time_ns", "bytes"} { + provenance["hydration."+name] = "test.hydration." + name + } + return provenance +} diff --git a/cmd/graphbench/results.go b/cmd/graphbench/results.go index f333b327..a96872e0 100644 --- a/cmd/graphbench/results.go +++ b/cmd/graphbench/results.go @@ -17,92 +17,706 @@ package main import ( + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io" "os" "path/filepath" + "slices" "sort" "time" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/testutil" ) const ( - StatusOK = "ok" - StatusRowMismatch = "row_mismatch" - StatusError = "error" + // StatusOK marks a benchmark case whose execution and expectations succeeded. + StatusOK = "ok" + + // StatusRowMismatch marks a benchmark case whose observed row count differed from its expectation. + StatusRowMismatch = "row_mismatch" + + // StatusError marks a benchmark case that failed during execution. + StatusError = "error" + + // StatusNotImplemented marks a benchmark case unsupported by the selected backend. StatusNotImplemented = "not_implemented" ) +// DurationStats summarizes warmup policy, measured latency samples, quantiles, and sample sufficiency. type DurationStats struct { - Iterations int `json:"iterations"` - Median time.Duration `json:"median"` - P95 time.Duration `json:"p95"` - Max time.Duration `json:"max"` + // Iterations records the number of iterations. + Iterations int `json:"iterations"` + // WarmupIterations records the number of warmup iterations. + WarmupIterations int `json:"warmup_iterations"` + // Median supplies the median input to the DurationStats contract. + Median time.Duration `json:"median"` + // P95 supplies the p95 input to the DurationStats contract. + P95 time.Duration `json:"p95"` + // P99 supplies the p99 input to the DurationStats contract. + P99 time.Duration `json:"p99"` + // P99Gated reports whether the sample count is sufficient to enforce the P99 noise threshold. + P99Gated bool `json:"p99_gated"` + // Max supplies the max input to the DurationStats contract. + Max time.Duration `json:"max"` + // Samples contains the individual measurements. + Samples []LatencySample `json:"samples,omitempty"` + // ReceiptStabilization records the single excluded receipt-bearing + // invocation immediately preceding timed iteration one. + ReceiptStabilization *RuntimeStabilizationReceipt `json:"receipt_stabilization,omitempty"` +} + +// RuntimeStabilizationReceipt preserves the identity and event chain of the +// excluded pre-timing invocation without inserting its latency into Samples. +type RuntimeStabilizationReceipt struct { + InvocationID string `json:"invocation_id"` + RequestedIdentity string `json:"requested_identity"` + RuntimeIdentity string `json:"runtime_identity"` + RuntimeBranch string `json:"runtime_branch"` + FallbackExecuted *bool `json:"fallback_executed"` + Events []RuntimeReceiptEvent `json:"events"` +} + +// RuntimeReceiptEvent records one ordered executor transition observed during +// a measured traversal invocation. Multiple events preserve nested fallback +// chains such as I1 -> S4 -> S3 without reducing them to the terminal arm. +type RuntimeReceiptEvent struct { + // InvocationID binds this event to the session-local timed invocation that emitted it. + InvocationID string `json:"invocation_id,omitempty"` + // Ordinal supplies the ordinal input to the RuntimeReceiptEvent contract. + Ordinal int `json:"ordinal"` + // RuntimeIdentity identifies the runtime identity. + RuntimeIdentity string `json:"runtime_identity"` + // RuntimeBranch supplies the runtime branch input to the RuntimeReceiptEvent contract. + RuntimeBranch string `json:"runtime_branch"` + // FallbackExecuted indicates whether fallback executed applies. + FallbackExecuted bool `json:"fallback_executed"` +} + +// LatencySample records one labeled duration and its measurement order. +type LatencySample struct { + // Round identifies the measurement round. + Round int `json:"round"` + // Block identifies the measurement block used to control carryover effects. + Block int `json:"block,omitempty"` + // Arm identifies the measurement arm that produced the sample. + Arm string `json:"arm,omitempty"` + // ArmOrder supplies the arm order input to the LatencySample contract. + ArmOrder int `json:"arm_order,omitempty"` + // RunUUID links the sample to its resumable benchmark run series. + RunUUID string `json:"run_uuid,omitempty"` + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // Case identifies the workload whose iteration produced the sample. + Case string `json:"case"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // ConnectionID identifies the connection id. + ConnectionID string `json:"connection_id,omitempty"` + // Classification supplies the classification input to the LatencySample contract. + Classification string `json:"classification"` + // Duration records elapsed time for this observation. + Duration time.Duration `json:"duration"` + // RequestedIdentity identifies the requested identity. + RequestedIdentity string `json:"requested_identity,omitempty"` + // RuntimeIdentity identifies the runtime identity. + RuntimeIdentity string `json:"runtime_identity,omitempty"` + // RuntimeBranch supplies the runtime branch input to the LatencySample contract. + RuntimeBranch string `json:"runtime_branch,omitempty"` + // FallbackExecuted records whether the candidate delegated to its exact incumbent. + FallbackExecuted *bool `json:"fallback_executed,omitempty"` + // RuntimeAttestation identifies the boundary that supplied runtime identity. + RuntimeAttestation string `json:"runtime_attestation,omitempty"` + // RuntimeInvocationID uniquely identifies the session-local timed invocation. + RuntimeInvocationID string `json:"runtime_invocation_id,omitempty"` + // RuntimeReceiptEvents preserves the complete ordered runtime branch chain + // for this exact measured invocation. + RuntimeReceiptEvents []RuntimeReceiptEvent `json:"runtime_receipt_events,omitempty"` +} + +// ConcurrencySample records one concurrent worker iteration and its connection and latency stages. +type ConcurrencySample struct { + // Worker identifies the concurrent worker that produced the sample. + Worker int `json:"worker"` + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // ConnectionID identifies the connection id. + ConnectionID string `json:"connection_id"` + // Classification supplies the classification input to the ConcurrencySample contract. + Classification string `json:"classification"` + // PoolWait records latency spent acquiring a database connection from the pool. + PoolWait time.Duration `json:"pool_wait"` + // Transaction records latency spent beginning and configuring the transaction. + Transaction time.Duration `json:"transaction_setup"` + // ExecuteDrain records latency spent executing and draining all rows. + ExecuteDrain time.Duration `json:"execute_decode_drain"` + // Total supplies the total input to the ConcurrencySample contract. + Total time.Duration `json:"total"` +} + +// ConcurrencyBlock summarizes all samples and connection usage for one concurrency level. +type ConcurrencyBlock struct { + // Concurrency supplies the concurrency input to the ConcurrencyBlock contract. + Concurrency int `json:"concurrency"` + // PoolSize sets the database connection-pool size. + PoolSize int `json:"pool_size"` + // Operations records successful query operations completed by a concurrency block. + Operations int `json:"operations"` + // Wall records end-to-end wall time for a concurrency block. + Wall time.Duration `json:"wall"` + // QPS reports completed query iterations per second. + QPS float64 `json:"qps"` + // Samples contains the individual measurements. + Samples []ConcurrencySample `json:"samples"` +} + +// PostgresReferenceResult records one independent PostgreSQL reference arm's identity, plan, observations, and timings. +type PostgresReferenceResult struct { + // SchemaVersion identifies the PostgreSQL reference-result schema revision. + SchemaVersion int `json:"schema_version"` + // Name identifies the independently measured reference arm. + Name string `json:"name"` + // LegacyName retains a compatibility alias for the reference arm. + LegacyName string `json:"legacy_name,omitempty"` + // Architecture identifies the executor architecture. + Architecture string `json:"architecture"` + // ImplementationID provides a versioned identity for the measured reference algorithm and materializer. + ImplementationID string `json:"implementation_id"` + // StateShape describes recursive state retained by the reference implementation. + StateShape string `json:"state_shape"` + // ObservationShape describes normalized values returned by the reference boundary. + ObservationShape string `json:"observation_shape"` + // SemanticValidation identifies the exact observation contract enforced for the reference. + SemanticValidation string `json:"semantic_validation"` + // Boundary identifies the measured execution boundary. + Boundary string `json:"boundary"` + // TimingBoundary describes which reference stages contribute to latency samples. + TimingBoundary string `json:"timing_boundary"` + // FullComparator indicates that the reference returns the complete public observation. + FullComparator bool `json:"full_comparator"` + // MeasurementOrder supplies the measurement order input to the PostgresReferenceResult contract. + MeasurementOrder int `json:"measurement_order,omitempty"` + // AAAliasOf identifies the reference arm reused for an explicit A/A comparison. + AAAliasOf string `json:"aa_alias_of,omitempty"` + // SQL contains the rendered SQL statement. + SQL string `json:"sql"` + // SQLFingerprint identifies normalized SQL without retaining the statement text. + SQLFingerprint string `json:"sql_fingerprint"` + // RowCount records the number of row count. + RowCount int64 `json:"row_count"` + // ObservedRows contains stable serialized observations used for correctness comparison. + ObservedRows []string `json:"observed_rows,omitempty"` + // Stats contains latency statistics for the enclosing result or reference. + Stats DurationStats `json:"stats"` + // PostgresPlan contains normalized PostgreSQL text-plan lines. + PostgresPlan []string `json:"postgres_plan,omitempty"` + // PostgresPlanJSON contains structured PostgreSQL EXPLAIN evidence. + PostgresPlanJSON json.RawMessage `json:"postgres_plan_json,omitempty"` + // PostgresMetrics contains normalized PostgreSQL plan resource metrics. + PostgresMetrics *PostgresPlanMetrics `json:"postgres_metrics,omitempty"` + // TraversalTelemetry contains lightweight execution identity and optional untimed diagnostic counters. + TraversalTelemetry *TraversalExecutionTelemetry `json:"traversal_execution_telemetry,omitempty"` + // traversalTelemetryParameters retains invocation parameters only until all + // timed samples finish and the optional replay is attached. + traversalTelemetryParameters map[string]any +} + +// CompileSample breaks one Cypher compilation into parse, translate, and render stages. +type CompileSample struct { + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // Parse records Cypher parse latency. + Parse time.Duration `json:"parse"` + // Optimize records query optimization latency. + Optimize time.Duration `json:"optimize"` + // TranslateIncludingOptimize records combined translation and optimization latency. + TranslateIncludingOptimize time.Duration `json:"translate_including_optimize"` + // Render records SQL rendering latency after translation. + Render time.Duration `json:"render"` + // Total supplies the total input to the CompileSample contract. + Total time.Duration `json:"total"` + // Allocations records allocation count while measuring the client-side stage. + Allocations uint64 `json:"allocations"` + // AllocatedBytes records bytes allocated while measuring the client-side stage. + AllocatedBytes uint64 `json:"allocated_bytes"` +} + +// ClientWaterfall summarizes compile and raw-request samples at the client boundary. +type ClientWaterfall struct { + // IntervalsOverlap warns that nested compilation stages cannot be summed as exclusive costs. + IntervalsOverlap bool `json:"intervals_overlap"` + // Notes contains human-readable caveats attached to the artifact or case. + Notes string `json:"notes"` + // Samples contains the individual measurements. + Samples []CompileSample `json:"samples"` +} + +// BoundarySample breaks one raw PostgreSQL request into client-side latency stages. +type BoundarySample struct { + // Iteration identifies the measured iteration within its worker or round. + Iteration int `json:"iteration"` + // PoolWait records latency spent acquiring a PostgreSQL connection from the pool. + PoolWait time.Duration `json:"pool_wait"` + // Transaction records latency spent beginning and configuring the transaction. + Transaction time.Duration `json:"transaction_setup"` + // BindPrepare records PostgreSQL bind and statement-prepare latency. + BindPrepare time.Duration `json:"bind_prepare"` + // FirstRow records latency until the first result row becomes available. + FirstRow time.Duration `json:"first_row"` + // AllRowsDecode records client time to decode the complete result set. + AllRowsDecode time.Duration `json:"all_rows_decode"` + // DrainClose records latency spent draining remaining rows and closing the iterator. + DrainClose time.Duration `json:"drain_close"` + // Total supplies the total input to the BoundarySample contract. + Total time.Duration `json:"total"` + // Rows records the number of rows. + Rows int64 `json:"rows"` + // Allocations records allocation count while measuring the client-side stage. + Allocations uint64 `json:"allocations"` + // AllocatedBytes records bytes allocated while measuring the client-side stage. + AllocatedBytes uint64 `json:"allocated_bytes"` + // ConnectionID identifies the PostgreSQL backend that ran a closure sample. + // It is set only for prepared-state closure evidence. + ConnectionID string `json:"connection_id,omitempty"` + // WorkspaceBytes records PostgreSQL temporary workspace bytes sampled before + // the raw query transaction rolls back. It is set only by closure capture. + WorkspaceBytes *int64 `json:"workspace_bytes,omitempty"` + // ObservationSHA256 binds the normalized public rows decoded by this exact + // raw-PGX execution. It is set only by closure capture. + ObservationSHA256 string `json:"observation_sha256,omitempty"` +} + +// PostgresBoundaryWaterfall summarizes PostgreSQL planning, execution, and client overhead samples. +type PostgresBoundaryWaterfall struct { + // Boundary identifies the measured execution boundary. + Boundary string `json:"boundary"` + // SQLFingerprint identifies normalized SQL without retaining the statement text. + SQLFingerprint string `json:"sql_fingerprint"` + // WarmupIterations records the number of warmup iterations. + WarmupIterations int `json:"warmup_iterations"` + // MeasurementOrder supplies the measurement order input to the PostgresBoundaryWaterfall contract. + MeasurementOrder int `json:"measurement_order,omitempty"` + // Samples contains the individual measurements. + Samples []BoundarySample `json:"samples"` +} + +// PostgresBoundaryWorkspaceHighWater records direct PostgreSQL temporary +// workspace high-water marks observed while a closure boundary is active. +// Pool measurements are exact for the closure's required size-one pool. +type PostgresBoundaryWorkspaceHighWater struct { + // PerQueryPeakBytes is the maximum workspace observed within one raw-PGX query transaction. + PerQueryPeakBytes int64 `json:"per_query_peak_bytes"` + // FreshSessionPeakBytes is the maximum workspace observed on the dedicated fresh connection. + FreshSessionPeakBytes int64 `json:"fresh_session_peak_bytes"` + // SessionPeakBytes is the maximum workspace observed on the pooled physical session. + SessionPeakBytes int64 `json:"session_peak_bytes"` + // PoolPeakBytes is the maximum workspace across all pooled sessions. + PoolPeakBytes int64 `json:"pool_peak_bytes"` +} + +// PostgresBoundaryClosure records the non-production timing and workspace +// strata needed to distinguish fresh-session preparation from reusable pooled +// execution. It is diagnostic-only and never contains routing decisions. +type PostgresBoundaryClosure struct { + // Boundary identifies the exact client/database boundary being measured. + Boundary string `json:"boundary"` + // SQLFingerprint binds each stratum to the exact translated SQL. + SQLFingerprint string `json:"sql_fingerprint"` + // FreshSessionPreparedMiss is one first execution on a newly opened session. + // It is simultaneously the fresh-session and prepared-statement miss stratum. + FreshSessionPreparedMiss BoundarySample `json:"fresh_session_prepared_miss"` + // SameSessionPreparedHits records repeated executions on that same fresh session. + SameSessionPreparedHits []BoundarySample `json:"same_session_prepared_hits"` + // PoolPreparedMiss is the first execution on the size-one pooled session. + PoolPreparedMiss BoundarySample `json:"pool_prepared_miss"` + // PoolReacquiredPreparedHits records executions after releasing and reacquiring + // the closure's size-one pooled session. + PoolReacquiredPreparedHits []BoundarySample `json:"pool_reacquired_prepared_hits"` + // Workspace contains per-query, session, and pool temporary-workspace high-water marks. + Workspace PostgresBoundaryWorkspaceHighWater `json:"workspace"` } +// PostgresPlanMetrics aggregates structural, cardinality, timing, and buffer evidence from a PostgreSQL plan. type PostgresPlanMetrics struct { - PlanningMS *float64 `json:"planning_ms,omitempty"` + // PlanningMS records PostgreSQL planning time in milliseconds. + PlanningMS *float64 `json:"planning_ms,omitempty"` + // ExecutionMS records PostgreSQL execution time in milliseconds. ExecutionMS *float64 `json:"execution_ms,omitempty"` - Buffers Buffers `json:"buffers,omitempty"` + // Buffers contains shared, local, and temporary buffer activity attributed to the plan. + Buffers Buffers `json:"buffers,omitempty"` + // TempFiles records temporary files created by the backend session. + TempFiles int64 `json:"temp_files,omitempty"` + // TempBytes records temporary bytes written by the backend session. + TempBytes int64 `json:"temp_bytes,omitempty"` + // WALRecords records write-ahead-log records attributed to the plan node. + WALRecords int64 `json:"wal_records,omitempty"` + // WALBytes records write-ahead-log bytes attributed to the plan node. + WALBytes int64 `json:"wal_bytes,omitempty"` + // RootRows records rows emitted by root selection in the PostgreSQL plan. + RootRows int64 `json:"root_rows,omitempty"` + // RecursiveRows records rows emitted by recursive traversal state. + RecursiveRows int64 `json:"recursive_rows,omitempty"` + // RecursiveLoops records loops performed by recursive plan nodes. + RecursiveLoops int64 `json:"recursive_loops,omitempty"` + // FrontierRows records rows retained in the active traversal frontier. + FrontierRows int64 `json:"frontier_rows,omitempty"` + // WitnessRows records rows retained for shortest-path witness reconstruction. + WitnessRows int64 `json:"witness_rows,omitempty"` + // MeetingRows records bidirectional search rows where frontiers meet. + MeetingRows int64 `json:"meeting_rows,omitempty"` + // HydrationRows records rows processed while hydrating paths from ID trails. + HydrationRows int64 `json:"hydration_rows,omitempty"` + // ForwardEdgeProbes records relationship probes performed by forward search. + ForwardEdgeProbes int64 `json:"forward_edge_probes,omitempty"` + // ReverseEdgeProbes records relationship probes performed by reverse search. + ReverseEdgeProbes int64 `json:"reverse_edge_probes,omitempty"` + // RootLookupLoops records repeated plan loops used to locate traversal roots. + RootLookupLoops int64 `json:"root_lookup_loops,omitempty"` + // BoundaryLookupLoops records loops used to resolve traversal boundaries. + BoundaryLookupLoops int64 `json:"boundary_lookup_loops,omitempty"` + // HydrationLoops records loops performed while hydrating search results. + HydrationLoops int64 `json:"hydration_loops,omitempty"` + // EndpointProbeRows records rows examined by endpoint preflight probing. + EndpointProbeRows int64 `json:"endpoint_probe_rows,omitempty"` + // ReverseStateProbeRows records reverse-search state rows examined by probing. + ReverseStateProbeRows int64 `json:"reverse_state_probe_rows,omitempty"` + // EndpointGuardOverflow reports whether endpoint-seeded search exceeded its configured guard. + EndpointGuardOverflow bool `json:"endpoint_guard_overflow,omitempty"` + // StateGuardOverflow reports whether recursive state exceeded its configured guard. + StateGuardOverflow bool `json:"state_guard_overflow,omitempty"` + // ExpansionFallbackExecuted reports whether guarded expansion switched to its exact fallback executor. + ExpansionFallbackExecuted bool `json:"expansion_fallback_executed,omitempty"` + // PlanNodes lists normalized PostgreSQL plan-node metrics in traversal order. + PlanNodes []PostgresPlanNodeMetric `json:"plan_nodes,omitempty"` + // Provenance maps derived metric names to the plan evidence used to compute them. + Provenance map[string]string `json:"provenance,omitempty"` +} + +// PostgresPlanNodeMetric captures one PostgreSQL plan node's identity, counters, and buffers. +type PostgresPlanNodeMetric struct { + // PlanNodeID identifies this node within the normalized pre-order plan tree. + PlanNodeID int64 `json:"plan_node_id,omitempty"` + // ParentPlanNodeID identifies the direct parent node; the root has no parent. + ParentPlanNodeID int64 `json:"parent_plan_node_id,omitempty"` + // NodeType identifies the PostgreSQL plan node type. + NodeType string `json:"node_type"` + // ParentRelationship identifies the relationship by which this plan node is attached to its parent. + ParentRelationship string `json:"parent_relationship,omitempty"` + // CTEName names the recursive common-table expression referenced by the plan node. + CTEName string `json:"cte_name,omitempty"` + // RelationName identifies the PostgreSQL relation scanned by the plan node. + RelationName string `json:"relation_name,omitempty"` + // Alias contains the display alias assigned to the plan node. + Alias string `json:"alias,omitempty"` + // IndexName names the PostgreSQL index scanned by the plan node. + IndexName string `json:"index_name,omitempty"` + // FunctionName identifies a SQL function invoked by a Function Scan without exposing its internal work. + FunctionName string `json:"function_name,omitempty"` + // SubplanName names an initplan, subplan, or CTE body used for stable branch attribution. + SubplanName string `json:"subplan_name,omitempty"` + // PlanRows records the number of plan rows. + PlanRows int64 `json:"plan_rows,omitempty"` + // PlanWidth supplies the plan width input to the PostgresPlanNodeMetric contract. + PlanWidth int64 `json:"plan_width,omitempty"` + // ActualRows records rows actually emitted by the plan node. + ActualRows int64 `json:"actual_rows,omitempty"` + // ActualLoops records how many times the PostgreSQL plan node executed. + ActualLoops int64 `json:"actual_loops,omitempty"` + // RowsRemovedByFilter records rows PostgreSQL reports as rejected by this node's filter. + RowsRemovedByFilter int64 `json:"rows_removed_by_filter,omitempty"` + // ActualTotalMS records total observed time for the PostgreSQL plan node. + ActualTotalMS float64 `json:"actual_total_ms,omitempty"` + // Buffers contains shared, local, and temporary buffer activity attributed to the plan. + Buffers Buffers `json:"buffers,omitempty"` + // Provenance identifies the plan evidence from which this node metric was measured. + Provenance string `json:"provenance"` } +// Buffers contains PostgreSQL buffer activity split by storage class and operation. type Buffers struct { - SharedHit int64 `json:"shared_hit,omitempty"` - SharedRead int64 `json:"shared_read,omitempty"` + // SharedHit records shared PostgreSQL buffer cache hits. + SharedHit int64 `json:"shared_hit,omitempty"` + // SharedRead records shared PostgreSQL buffers read by the plan. + SharedRead int64 `json:"shared_read,omitempty"` + // SharedDirtied records shared PostgreSQL buffers dirtied by the plan. SharedDirtied int64 `json:"shared_dirtied,omitempty"` - TempRead int64 `json:"temp_read,omitempty"` - TempWritten int64 `json:"temp_written,omitempty"` + // SharedWritten records shared PostgreSQL buffers written by the plan. + SharedWritten int64 `json:"shared_written,omitempty"` + // LocalHit records local PostgreSQL buffer cache hits. + LocalHit int64 `json:"local_hit,omitempty"` + // LocalRead records local PostgreSQL buffers read by the plan. + LocalRead int64 `json:"local_read,omitempty"` + // LocalDirtied records local PostgreSQL buffers dirtied by the plan. + LocalDirtied int64 `json:"local_dirtied,omitempty"` + // LocalWritten records local PostgreSQL buffers written by the plan. + LocalWritten int64 `json:"local_written,omitempty"` + // TempRead records temporary PostgreSQL buffers read by the plan. + TempRead int64 `json:"temp_read,omitempty"` + // TempWritten records temporary PostgreSQL buffers written by the plan. + TempWritten int64 `json:"temp_written,omitempty"` } +// CaseResult records one workload execution with provenance, observations, plan evidence, and latency samples. type CaseResult struct { - Source string `json:"source"` - Dataset string `json:"dataset"` - Name string `json:"name"` - Category string `json:"category"` - ExecutionMode ExecutionMode `json:"execution_mode"` - Status string `json:"status"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - NodeParams map[string]string `json:"node_params,omitempty"` - ExpectedRowCount *int64 `json:"expected_row_count,omitempty"` - RowCount int64 `json:"row_count,omitempty"` - Stats DurationStats `json:"stats,omitempty"` - SQL string `json:"sql,omitempty"` - PostgresPlan []string `json:"postgres_plan,omitempty"` - PostgresMetrics *PostgresPlanMetrics `json:"postgres_metrics,omitempty"` - Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` - Neo4jOperators []string `json:"neo4j_operators,omitempty"` - Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` - Baseline *BaselineComparison `json:"baseline,omitempty"` - FallbackReason string `json:"fallback_reason,omitempty"` - Error string `json:"error,omitempty"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Environment captures the environment in which the measurement ran. + Environment *RunEnvironment `json:"environment,omitempty"` + // PostgresEnvironment captures PostgreSQL settings required for comparability. + PostgresEnvironment *PostgresEnvironment `json:"postgres_environment,omitempty"` + // Fixture captures the fixture identity and cardinality contract. + Fixture *FixtureMetadata `json:"fixture,omitempty"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // WorkloadSHA256 binds the result to the case declaration and execution mode. + WorkloadSHA256 string `json:"workload_sha256"` + // Category groups cases by workload category. + Category string `json:"category"` + // Shape describes the workload shape used for selection and comparison. + Shape WorkloadShape `json:"shape"` + // ExecutionMode identifies the backend execution mode that produced the case result. + ExecutionMode ExecutionMode `json:"execution_mode"` + // Status supplies the status input to the CaseResult contract. + Status string `json:"status"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params map[string]any `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // ExpectedRowCount sets the required result-row count when known. + ExpectedRowCount *int64 `json:"expected_row_count,omitempty"` + // ObservedRows contains stable serialized observations used for correctness comparison. + ObservedRows []string `json:"observed_rows,omitempty"` + // RowCount records the number of row count. + RowCount int64 `json:"row_count,omitempty"` + // MatchedCount records entities selected by the measured mutation. + MatchedCount *int64 `json:"matched_count,omitempty"` + // AffectedCount records entities actually changed by the measured mutation. + AffectedCount *int64 `json:"affected_count,omitempty"` + // PostState contains the observed results of post-write validation queries. + PostState []StateQueryResult `json:"post_state,omitempty"` + // Stats contains latency statistics for the enclosing result or reference. + Stats DurationStats `json:"stats,omitempty"` + // Concurrency contains opt-in worker-count measurements for this case. + Concurrency []ConcurrencyBlock `json:"concurrency,omitempty"` + // PostgresReferences contains independent PostgreSQL reference results for the case. + PostgresReferences []PostgresReferenceResult `json:"postgres_references,omitempty"` + // ClientWaterfall contains Cypher compilation and client-boundary timing samples. + ClientWaterfall *ClientWaterfall `json:"client_waterfall,omitempty"` + // RawPGXWaterfall contains raw PGX boundary timings used for PostgreSQL cost attribution. + RawPGXWaterfall *PostgresBoundaryWaterfall `json:"raw_pgx_waterfall,omitempty"` + // RawPGXRoundTrip records legacy aggregate raw-PGX round-trip latency. + RawPGXRoundTrip *PostgresBoundaryWaterfall `json:"raw_pgx_round_trip,omitempty"` + // PostgresBoundaryClosure records diagnostic-only prepared-state and + // temporary-workspace closure evidence for fixed-suffix routing. + PostgresBoundaryClosure *PostgresBoundaryClosure `json:"postgres_boundary_closure,omitempty"` + // SQL contains the rendered SQL statement. + SQL string `json:"sql,omitempty"` + // SQLFingerprint identifies normalized SQL without retaining the statement text. + SQLFingerprint string `json:"sql_fingerprint,omitempty"` + // PostgresPlan contains normalized PostgreSQL text-plan lines. + PostgresPlan []string `json:"postgres_plan,omitempty"` + // PostgresPlanJSON contains structured PostgreSQL EXPLAIN evidence. + PostgresPlanJSON json.RawMessage `json:"postgres_plan_json,omitempty"` + // PostgresMetrics contains normalized PostgreSQL plan resource metrics. + PostgresMetrics *PostgresPlanMetrics `json:"postgres_metrics,omitempty"` + // TraversalTelemetry contains lightweight execution identity and optional untimed diagnostic counters. + TraversalTelemetry *TraversalExecutionTelemetry `json:"traversal_execution_telemetry,omitempty"` + // Neo4jPlan contains the normalized Neo4j operator tree. + Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` + // Neo4jOperators lists normalized Neo4j operators found in the captured plan. + Neo4jOperators []string `json:"neo4j_operators,omitempty"` + // Optimization captures translation optimization and lowering decisions. + Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` + // ParseCache reports parse-cache hit and miss statistics for the case. + ParseCache *pg.ParseCacheStats `json:"parse_cache,omitempty"` + // Baseline contains the latency comparison with a matching baseline record. + Baseline *BaselineComparison `json:"baseline,omitempty"` + // FallbackReason explains why execution used a fallback architecture. + FallbackReason string `json:"fallback_reason,omitempty"` + // ExistingGraph selects read-only execution against a pre-existing graph. + ExistingGraph *ExistingGraphRun `json:"existing_graph,omitempty"` + // Error supplies the error input to the CaseResult contract. + Error string `json:"error,omitempty"` + // StableObservation reports whether ObservedRows contains a backend-independent normalized result. + StableObservation bool `json:"observation_captured,omitempty"` +} + +// StateQueryResult records a post-write validation query's row count and optional scalar value. +type StateQueryResult struct { + // Name labels the post-write state assertion that produced this result. + Name string `json:"name"` + // RowCount records the number of row count. + RowCount int64 `json:"row_count"` + // ScalarInt contains the observed scalar value when the state query expects one. + ScalarInt *int64 `json:"scalar_int,omitempty"` } +// BaselineComparison compares current median latency with a previously recorded baseline. type BaselineComparison struct { + // BaselineMedian supplies the baseline median input to the BaselineComparison contract. BaselineMedian time.Duration `json:"baseline_median"` - CurrentMedian time.Duration `json:"current_median"` - Change time.Duration `json:"change"` - Ratio float64 `json:"ratio"` + // CurrentMedian supplies the current median input to the BaselineComparison contract. + CurrentMedian time.Duration `json:"current_median"` + // Change records current latency relative to the selected baseline. + Change time.Duration `json:"change"` + // Ratio reports the candidate-to-baseline latency ratio. + Ratio float64 `json:"ratio"` } +// validateBackendObservations checks row counts and stable observations across successful backend results. +func validateBackendObservations(records []CaseResult) error { + // observationKey identifies one dataset, case, backend, and round during observation validation. + type observationKey struct { + // dataset names the fixture shared by observations compared across backends. + dataset string + // name identifies the workload case compared across backends. + name string + } + + postgres := map[observationKey][]string{} + for _, record := range records { + if record.ExecutionMode == ModePostgresSQL && record.Status == StatusOK && record.StableObservation && record.ObservedRows != nil { + postgres[observationKey{ + dataset: record.Dataset, + name: record.Name, + }] = record.ObservedRows + } + } + + for _, record := range records { + if record.ExecutionMode != ModeNeo4j || record.Status != StatusOK || !record.StableObservation || record.ObservedRows == nil { + continue + } + key := observationKey{ + dataset: record.Dataset, + name: record.Name, + } + if expected, found := postgres[key]; found && !slices.Equal(expected, record.ObservedRows) { + return fmt.Errorf("backend observations differ for %s/%s: postgres=%v neo4j=%v", record.Dataset, record.Name, expected, record.ObservedRows) + } + } + + return nil +} + +// newCaseResult initializes workload identity, expectations, observation policy, and successful status for one case. func newCaseResult(testCase ScaleCase, mode ExecutionMode, params map[string]any) CaseResult { return CaseResult{ Source: testCase.Source, Dataset: testCase.Dataset, Name: testCase.Name, + WorkloadSHA256: scaleCaseWorkloadIdentity(testCase, mode), Category: testCase.Category, + Shape: testCase.Shape, ExecutionMode: mode, Status: StatusOK, Cypher: testCase.Cypher, Params: params, NodeParams: testCase.NodeParams, + NodeListParams: testCase.NodeListParams, ExpectedRowCount: testCase.Expected.RowCount, + StableObservation: testCase.Expected.ResultKind == "id_rows" || + testCase.Expected.ResultKind == "scalar" || + (testCase.Expected.ResultKind == "path_set" && (len(testCase.Expected.PathRows) > 0 || + testCase.Expected.RowCount != nil && *testCase.Expected.RowCount == 0)), } } +// scaleCaseWorkloadIdentity hashes the logical workload fields that must match across artifacts. +func scaleCaseWorkloadIdentity(testCase ScaleCase, mode ExecutionMode) string { + payload := struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Backend identifies the execution backend. + Backend ExecutionMode `json:"backend"` + // Case contains the complete workload declaration included in the identity digest. + Case ScaleCase `json:"case"` + }{ + Version: 1, + Source: testCase.Source, + Backend: mode, + Case: testCase, + } + raw, err := json.Marshal(payload) + if err != nil { + return "" + } + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// attachFixtureMetadata adds fixture metadata to the owning artifact. +func attachFixtureMetadata(record *CaseResult, fixture FixtureMetadata) { + if record == nil { + return + } + record.Fixture = &fixture + payload := struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // LogicalWorkloadSHA256 identifies query semantics independently of runtime measurements. + LogicalWorkloadSHA256 string `json:"logical_workload_sha256"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Checksum binds fixture identity to its canonical logical contents. + Checksum string `json:"checksum"` + // NodeCount records logical fixture nodes declared or loaded. + NodeCount int `json:"node_count"` + // EdgeCount records logical fixture relationships declared or loaded. + EdgeCount int `json:"edge_count"` + // PhysicalNodeCount records physical node rows present in the backend fixture. + PhysicalNodeCount int64 `json:"physical_node_count,omitempty"` + // PhysicalEdgeCount records physical relationship rows present in the backend fixture. + PhysicalEdgeCount int64 `json:"physical_edge_count,omitempty"` + // Configuration captures the generator parameters that define the fixture shape. + Configuration string `json:"configuration,omitempty"` + // Shortest contains expectations derived from a generated shortest-path fixture. + Shortest *ShortestFixtureExpectations `json:"shortest,omitempty"` + // FixedSuffixExpansion contains expectations derived from a fixed-suffix expansion fixture. + FixedSuffixExpansion *FixedSuffixExpansionFixtureExpectations `json:"fixed_suffix_expansion,omitempty"` + // EndpointSeededExpansion contains expectations derived from an endpoint-seeded expansion fixture. + EndpointSeededExpansion *EndpointSeededExpansionFixtureExpectations `json:"endpoint_seeded_expansion,omitempty"` + }{ + Version: 1, + LogicalWorkloadSHA256: record.WorkloadSHA256, + Dataset: fixture.Dataset, + Checksum: fixture.Checksum, + NodeCount: fixture.NodeCount, + EdgeCount: fixture.EdgeCount, + PhysicalNodeCount: fixture.PhysicalNodeCount, + PhysicalEdgeCount: fixture.PhysicalEdgeCount, + Configuration: fixture.Configuration, + Shortest: fixture.Shortest, + FixedSuffixExpansion: fixture.FixedSuffixExpansion, + EndpointSeededExpansion: fixture.EndpointSeededExpansion, + } + raw, err := json.Marshal(payload) + if err != nil { + record.WorkloadSHA256 = "" + return + } + digest := sha256.Sum256(raw) + record.WorkloadSHA256 = hex.EncodeToString(digest[:]) +} + +// computeDurationStats validates measured durations and derives median, tail, maximum, and labeled sample data. func computeDurationStats(durations []time.Duration) (DurationStats, error) { if len(durations) == 0 { return DurationStats{}, fmt.Errorf("duration stats require at least one duration") @@ -115,14 +729,91 @@ func computeDurationStats(durations []time.Duration) (DurationStats, error) { n := len(sortedDurations) p95Index := (95*n+99)/100 - 1 + p99Index := (99*n+99)/100 - 1 return DurationStats{ Iterations: n, Median: sortedDurations[n/2], P95: sortedDurations[p95Index], + P99: sortedDurations[p99Index], + P99Gated: n >= 10_000, Max: sortedDurations[n-1], + Samples: func() []LatencySample { + samples := make([]LatencySample, len(durations)) + for idx, duration := range durations { + samples[idx] = LatencySample{ + Round: 1, + Iteration: idx + 1, + Classification: "warm", + Duration: duration, + } + } + return samples + }(), }, nil } +// labelLatencySamples attaches backend, dataset, and case identity to every latency sample in stats. +func labelLatencySamples(stats *DurationStats, mode ExecutionMode, testCase ScaleCase) { + for idx := range stats.Samples { + stats.Samples[idx].Backend = mode + stats.Samples[idx].Case = testCase.Name + stats.Samples[idx].Dataset = testCase.Dataset + } +} + +// setSampleRound assigns a measurement round to every latency sample in stats. +func setSampleRound(stats *DurationStats, round int) { + for idx := range stats.Samples { + stats.Samples[idx].Round = round + } +} + +// setSampleRunMetadata copies run, arm, block, and round identity onto every latency sample in stats. +func setSampleRunMetadata(stats *DurationStats, environment RunEnvironment) { + for idx := range stats.Samples { + stats.Samples[idx].Round = environment.Round + stats.Samples[idx].Block = environment.Block + stats.Samples[idx].Arm = environment.Arm + stats.Samples[idx].ArmOrder = environment.ArmOrder + stats.Samples[idx].RunUUID = environment.RunUUID + } +} + +// setSampleTraversalRuntimeMetadata binds every timed sample to the singular +// invocation-local replay outcome obtained for the same case, parameters, SQL, +// and physical session. This supports diagnostics but deliberately does not +// claim per-timed-invocation attribution; promotion gates require the stronger +// "timed_invocation" attestation. +func setSampleTraversalRuntimeMetadata(stats *DurationStats, telemetry *TraversalExecutionTelemetry) { + if stats == nil || telemetry == nil { + return + } + for idx := range stats.Samples { + if stats.Samples[idx].RuntimeAttestation == "timed_invocation" { + continue + } + stats.Samples[idx].RequestedIdentity = telemetry.Summary.RequestedIdentity + stats.Samples[idx].RuntimeIdentity = telemetry.Summary.RuntimeIdentity + stats.Samples[idx].RuntimeBranch = telemetry.Summary.RuntimeBranch + stats.Samples[idx].FallbackExecuted = telemetry.Summary.FallbackExecuted + stats.Samples[idx].RuntimeAttestation = "same_case_invocation_local_replay" + } +} + +// setCaseRunMetadata assigns case run metadata across the supplied records. +func setCaseRunMetadata(record *CaseResult, metadata testutil.BaselineMetadata, environment RunEnvironment) { + if record == nil { + return + } + record.Metadata = metadata + record.Environment = &environment + setSampleRunMetadata(&record.Stats, environment) + for idx := range record.PostgresReferences { + setSampleRunMetadata(&record.PostgresReferences[idx].Stats, environment) + } +} + +// applyRowExpectation marks a successful result as mismatched when its row count violates the declared expectation. func applyRowExpectation(result *CaseResult) { if result.ExpectedRowCount != nil && result.RowCount != *result.ExpectedRowCount { result.Status = StatusRowMismatch @@ -130,6 +821,7 @@ func applyRowExpectation(result *CaseResult) { } } +// writeJSONLFile writes records to standard output or replaces the requested JSON Lines artifact. func writeJSONLFile(path string, records []CaseResult) (err error) { if path == "" { return writeJSONL(os.Stdout, records) @@ -152,6 +844,95 @@ func writeJSONLFile(path string, records []CaseResult) (err error) { return writeJSONL(output, records) } +// appendJSONLFile validates compatibility with existing records before appending new JSON Lines entries. +func appendJSONLFile(path string, records []CaseResult) (err error) { + if path == "" { + return errors.New("append JSONL path must not be empty") + } + if err := ensureOutputDir(path); err != nil { + return err + } + + if existing, readErr := readJSONLFile(path); readErr == nil { + if err := validateJSONLAppend(existing, records); err != nil { + return err + } + } else if !errors.Is(readErr, os.ErrNotExist) { + return readErr + } + + output, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0o600) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + return writeJSONL(output, records) +} + +// validateJSONLAppend ensures appended records share run identity and do not duplicate case rounds. +func validateJSONLAppend(existing, appended []CaseResult) error { + if len(existing) == 0 || len(appended) == 0 { + return nil + } + + left, right := existing[0].Environment, appended[0].Environment + if left == nil || right == nil { + return errors.New("append JSONL requires run environment metadata") + } + if left.RunUUID != right.RunUUID || left.Arm != right.Arm || left.BinarySHA256 != right.BinarySHA256 || left.DirtyDiffSHA256 != right.DirtyDiffSHA256 { + return fmt.Errorf("append JSONL run identity mismatch: existing run=%q arm=%q binary=%q diff=%q, appended run=%q arm=%q binary=%q diff=%q", + left.RunUUID, left.Arm, left.BinarySHA256, left.DirtyDiffSHA256, + right.RunUUID, right.Arm, right.BinarySHA256, right.DirtyDiffSHA256) + } + + // recordKey identifies one run, dataset, case, mode, and round during append validation. + type recordKey struct { + // dataset names the fixture component of the append-deduplication key. + dataset string + // name identifies the workload case within its dataset. + name string + // mode separates records for different execution backends within the same round. + mode ExecutionMode + // round identifies the measurement round used to balance execution order. + round int + } + seen := make(map[recordKey]struct{}, len(existing)) + for _, record := range existing { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + seen[recordKey{ + dataset: record.Dataset, + name: record.Name, + mode: record.ExecutionMode, + round: round, + }] = struct{}{} + } + for _, record := range appended { + round := 0 + if record.Environment != nil { + round = record.Environment.Round + } + key := recordKey{ + dataset: record.Dataset, + name: record.Name, + mode: record.ExecutionMode, + round: round, + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("append JSONL duplicate record for %s/%s/%s round %d", key.dataset, key.name, key.mode, key.round) + } + seen[key] = struct{}{} + } + return nil +} + +// writeJSONL encodes each case result as one JSON Lines record in input order. func writeJSONL(w io.Writer, records []CaseResult) error { encoder := json.NewEncoder(w) for _, record := range records { @@ -163,6 +944,7 @@ func writeJSONL(w io.Writer, records []CaseResult) error { return nil } +// readJSONLFile reads JSON Lines file and propagates I/O or decoding failures. func readJSONLFile(path string) ([]CaseResult, error) { input, err := os.Open(path) if err != nil { @@ -184,6 +966,7 @@ func readJSONLFile(path string) ([]CaseResult, error) { return nil, err } + normalizeHistoricalReferences(&record) records = append(records, record) } @@ -191,6 +974,42 @@ func readJSONLFile(path string) ([]CaseResult, error) { return records, nil } +// normalizeHistoricalReferences canonicalizes historical references for stable comparison. +func normalizeHistoricalReferences(record *CaseResult) { + for idx := range record.PostgresReferences { + reference := &record.PostgresReferences[idx] + if reference.SchemaVersion != 0 { + continue + } + reference.SchemaVersion = 1 + switch reference.Name { + case "complete_reference_s1_array_cte": + reference.LegacyName = reference.Name + reference.Name = "s3_unidirectional_trail_cte" + reference.Architecture = "SP-S3-U-NE" + reference.ImplementationID = "inline_recursive_cte_unidirectional_v1" + case "candidate_s2_bidirectional_cte": + reference.LegacyName = reference.Name + reference.Name = "s3_bidirectional_trail_cte" + reference.Architecture = "SP-S3-B" + reference.ImplementationID = "inline_recursive_cte_bidirectional_trails_v1" + } + if reference.StateShape == "" { + reference.StateShape = "legacy_unspecified" + } + if reference.ObservationShape == "" { + reference.ObservationShape = reference.Boundary + } + if reference.SemanticValidation == "" { + reference.SemanticValidation = "legacy_row_count_only" + if !reference.FullComparator { + reference.SemanticValidation = "row_count_stability" + } + } + } +} + +// ensureOutputDir creates the parent directory needed for an output file. func ensureOutputDir(path string) error { dir := filepath.Dir(path) if dir == "." || dir == "" { @@ -200,6 +1019,7 @@ func ensureOutputDir(path string) error { return os.MkdirAll(dir, 0o755) } +// applyBaseline attaches median latency deltas and ratios from matching baseline records. func applyBaseline(path string, records []CaseResult) error { baseline, err := readJSONLFile(path) if err != nil { @@ -229,6 +1049,7 @@ func applyBaseline(path string, records []CaseResult) error { return nil } +// resultKey joins result identity fields into the append-validation key. func resultKey(dataset, name string, mode ExecutionMode) string { return dataset + "\x00" + name + "\x00" + string(mode) } diff --git a/cmd/graphbench/results_test.go b/cmd/graphbench/results_test.go index 0ee87344..60134938 100644 --- a/cmd/graphbench/results_test.go +++ b/cmd/graphbench/results_test.go @@ -17,18 +17,51 @@ package main import ( + "path/filepath" "testing" "time" "github.com/stretchr/testify/require" ) +// TestAppendJSONLFileValidatesRunIdentityAndDuplicateRounds verifies append-only accumulation across rounds while rejecting duplicate keys and changes to arm or run UUID. +func TestAppendJSONLFileValidatesRunIdentityAndDuplicateRounds(t *testing.T) { + path := filepath.Join(t.TempDir(), "rounds.jsonl") + record := func(round int, arm, runUUID, binary string) CaseResult { + return CaseResult{ + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + Round: round, + Arm: arm, + RunUUID: runUUID, + BinarySHA256: binary, + DirtyDiffSHA256: "diff", + }, + } + } + + require.NoError(t, appendJSONLFile(path, []CaseResult{record(1, "candidate", "run-1", "binary")})) + require.NoError(t, appendJSONLFile(path, []CaseResult{record(2, "candidate", "run-1", "binary")})) + records, err := readJSONLFile(path) + require.NoError(t, err) + require.Len(t, records, 2) + + require.ErrorContains(t, appendJSONLFile(path, []CaseResult{record(2, "candidate", "run-1", "binary")}), "duplicate record") + require.ErrorContains(t, appendJSONLFile(path, []CaseResult{record(3, "incumbent", "run-1", "binary")}), "run identity mismatch") + require.ErrorContains(t, appendJSONLFile(path, []CaseResult{record(3, "candidate", "run-2", "binary")}), "run identity mismatch") +} + +// TestComputeDurationStatsRejectsEmptyDurations verifies that aggregate statistics cannot be fabricated without at least one timing observation. func TestComputeDurationStatsRejectsEmptyDurations(t *testing.T) { _, err := computeDurationStats(nil) require.ErrorContains(t, err, "at least one duration") } +// TestComputeDurationStatsCopiesAndSortsDurations verifies aggregate values, preservation of input/sample order, default warm labels, backend metadata, and round relabeling. func TestComputeDurationStatsCopiesAndSortsDurations(t *testing.T) { durations := []time.Duration{ 30 * time.Millisecond, @@ -42,12 +75,47 @@ func TestComputeDurationStatsCopiesAndSortsDurations(t *testing.T) { require.Equal(t, 3, stats.Iterations) require.Equal(t, 20*time.Millisecond, stats.Median) require.Equal(t, 30*time.Millisecond, stats.P95) + require.Equal(t, 30*time.Millisecond, stats.P99) + require.False(t, stats.P99Gated) require.Equal(t, 30*time.Millisecond, stats.Max) require.Equal(t, 30*time.Millisecond, durations[0]) require.Equal(t, 10*time.Millisecond, durations[1]) require.Equal(t, 20*time.Millisecond, durations[2]) + require.Equal(t, []LatencySample{ + { + Round: 1, + Iteration: 1, + Classification: "warm", + Duration: 30 * time.Millisecond, + }, + { + Round: 1, + Iteration: 2, + Classification: "warm", + Duration: 10 * time.Millisecond, + }, + { + Round: 1, + Iteration: 3, + Classification: "warm", + Duration: 20 * time.Millisecond, + }, + }, stats.Samples) + + labelLatencySamples(&stats, ModePostgresSQL, ScaleCase{ + Name: "case", + Dataset: "fixture", + }) + require.Equal(t, ModePostgresSQL, stats.Samples[0].Backend) + require.Equal(t, "case", stats.Samples[0].Case) + require.Equal(t, "fixture", stats.Samples[0].Dataset) + + setSampleRound(&stats, 7) + require.Equal(t, 7, stats.Samples[0].Round) + require.Equal(t, 7, stats.Samples[2].Round) } +// TestComputeDurationStatsUsesNearestRankP95 verifies that twenty ordered samples select the nineteenth value for P95 while retaining the twentieth as maximum. func TestComputeDurationStatsUsesNearestRankP95(t *testing.T) { durations := make([]time.Duration, 20) for idx := range durations { @@ -60,3 +128,86 @@ func TestComputeDurationStatsUsesNearestRankP95(t *testing.T) { require.Equal(t, 19*time.Millisecond, stats.P95) require.Equal(t, 20*time.Millisecond, stats.Max) } + +// TestCheckStateExpectationChecksRowsAndScalar verifies simultaneous row/scalar acceptance and a scalar-specific diagnostic on mismatch. +func TestCheckStateExpectationChecksRowsAndScalar(t *testing.T) { + rowCount := int64(1) + scalar := int64(3) + + require.NoError(t, checkStateExpectation( + StateQueryResult{ + RowCount: 1, + ScalarInt: &scalar, + }, + ExpectedResult{ + RowCount: &rowCount, + ScalarInt: &scalar, + }, + )) + + wrong := int64(4) + require.ErrorContains(t, checkStateExpectation( + StateQueryResult{ + RowCount: 1, + ScalarInt: &scalar, + }, + ExpectedResult{ScalarInt: &wrong}, + ), "expected scalar integer 4") +} + +// TestValidateBackendObservationsPreservesDuplicateStableRows verifies multiset semantics: equal duplicate rows match across backends, but dropping one duplicate does not. +func TestValidateBackendObservationsPreservesDuplicateStableRows(t *testing.T) { + records := []CaseResult{ + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + StableObservation: true, + ObservedRows: []string{`["a"]`, `["a"]`}, + }, + { + Dataset: "fixture", + Name: "case", + ExecutionMode: ModeNeo4j, + Status: StatusOK, + StableObservation: true, + ObservedRows: []string{`["a"]`, `["a"]`}, + }, + } + require.NoError(t, validateBackendObservations(records)) + + records[1].ObservedRows = []string{`["a"]`} + require.ErrorContains(t, validateBackendObservations(records), "backend observations differ") +} + +// TestNewCaseResultCrossChecksExactPathSets verifies that path observations +// become stable cross-backend evidence only when an exact nonempty path set or +// an exact empty result is declared. +func TestNewCaseResultCrossChecksExactPathSets(t *testing.T) { + record := newCaseResult(ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + }, + }, ModePostgresSQL, nil) + require.False(t, record.StableObservation) + + record = newCaseResult(ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + PathRows: []ExpectedPath{{ + Nodes: []string{"start"}, + }}, + }, + }, ModePostgresSQL, nil) + require.True(t, record.StableObservation) + + zero := int64(0) + record = newCaseResult(ScaleCase{ + Expected: ExpectedResult{ + ResultKind: "path_set", + RowCount: &zero, + }, + }, ModePostgresSQL, nil) + require.True(t, record.StableObservation) +} diff --git a/cmd/graphbench/run_lock.go b/cmd/graphbench/run_lock.go new file mode 100644 index 00000000..d267a48d --- /dev/null +++ b/cmd/graphbench/run_lock.go @@ -0,0 +1,54 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "syscall" +) + +// destructiveRunLock holds the filesystem lock that serializes destructive benchmark runs. +type destructiveRunLock struct { + // file owns the lock file descriptor until the destructive run completes. + file *os.File +} + +// acquireDestructiveRunLock acquires a nonblocking filesystem lock that serializes destructive runs. +func acquireDestructiveRunLock(path string) (*destructiveRunLock, error) { + if path == "" { + return nil, fmt.Errorf("destructive lock path must not be empty") + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, fmt.Errorf("create destructive lock directory: %w", err) + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("open destructive lock: %w", err) + } + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = file.Close() + return nil, fmt.Errorf("another GraphBench process holds destructive lock %s: %w", path, err) + } + if err := file.Truncate(0); err == nil { + _, _ = fmt.Fprintf(file, "pid=%d\n", os.Getpid()) + } + return &destructiveRunLock{file: file}, nil +} + +// Close releases the advisory process lock and closes its file descriptor. +func (s *destructiveRunLock) Close() error { + if s == nil || s.file == nil { + return nil + } + unlockErr := syscall.Flock(int(s.file.Fd()), syscall.LOCK_UN) + closeErr := s.file.Close() + if unlockErr != nil { + return unlockErr + } + return closeErr +} diff --git a/cmd/graphbench/run_lock_test.go b/cmd/graphbench/run_lock_test.go new file mode 100644 index 00000000..d3dc322c --- /dev/null +++ b/cmd/graphbench/run_lock_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDestructiveRunLockRejectsOverlap verifies that a held lock prevents a second destructive GraphBench process from using the same lock path. +func TestDestructiveRunLockRejectsOverlap(t *testing.T) { + path := filepath.Join(t.TempDir(), "graphbench.lock") + first, err := acquireDestructiveRunLock(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, first.Close()) }) + + _, err = acquireDestructiveRunLock(path) + require.ErrorContains(t, err, "another GraphBench process") +} diff --git a/cmd/graphbench/scale_corpus_contract_test.go b/cmd/graphbench/scale_corpus_contract_test.go new file mode 100644 index 00000000..cbd5678a --- /dev/null +++ b/cmd/graphbench/scale_corpus_contract_test.go @@ -0,0 +1,842 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// scaleCorpusRequiredIDs lists representative corpus cases required by the regression contract. +var scaleCorpusRequiredIDs = []string{ + "REC-01", "REC-02", "REC-04", "REC-06", "REC-08", + "TRUST-01", "TRUST-02", + "PRUNE-01", "PRUNE-02", "PRUNE-03", "PRUNE-04", + "HOP-01", "HOP-02", "HOP-03", "HOP-04", "HOP-05", "HOP-07", "HOP-09", + "SCAN-01", "SCAN-02", "SCAN-03", "SCAN-04", "SCAN-05", "SCAN-07", "SCAN-08", + "LOOKUP-02", "LOOKUP-04", "LOOKUP-05", "LOOKUP-09", "LOOKUP-11", "LOOKUP-13", "LOOKUP-15", "LOOKUP-16", +} + +// TestGeneratedScaleCasesParseAndExecuteRealBackends verifies that each generated family has parseable Cypher and an explicit support decision for PostgreSQL and Neo4j. +func TestGeneratedScaleCasesParseAndExecuteRealBackends(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + covered := map[string]int{} + for _, testCase := range corpus.Cases { + if !strings.HasPrefix(testCase.Dataset, "generated_shortest_paths_") && !strings.HasPrefix(testCase.Dataset, "generated_fixed_suffix_expansion_") && !strings.HasPrefix(testCase.Dataset, "generated_endpoint_seeded_expansion_") { + continue + } + _, err := frontend.ParseCypher(frontend.NewContext(), testCase.Cypher) + require.NoError(t, err, testCase.Name) + _, postgresUnsupported := testCase.UnsupportedReason(ModePostgresSQL) + _, neo4jUnsupported := testCase.UnsupportedReason(ModeNeo4j) + require.True(t, testCase.Supports(ModePostgresSQL) || postgresUnsupported, testCase.Name) + require.True(t, testCase.Supports(ModeNeo4j) || neo4jUnsupported, testCase.Name) + if strings.HasPrefix(testCase.Dataset, "generated_shortest_paths_") { + covered["shortest"]++ + } else if strings.HasPrefix(testCase.Dataset, "generated_fixed_suffix_expansion_") { + covered["fixed_suffix_expansion"]++ + } else { + covered["endpoint_seeded_expansion"]++ + } + } + require.Positive(t, covered["shortest"]) + require.Positive(t, covered["fixed_suffix_expansion"]) + require.Positive(t, covered["endpoint_seeded_expansion"]) +} + +// TestGeneratedFixedSuffixV3OrientationCorpusFreezesTrainingAndHoldoutMatrices +// verifies exact cohort sizes, independent training dimensions, fresh holdout +// depths, canonical cohort tags, and graph-derived result cardinalities. +func TestGeneratedFixedSuffixV3OrientationCorpusFreezesTrainingAndHoldoutMatrices(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + // fraction identifies one reachable-endpoint and fanout combination. + type fraction struct { + // reachable retains the reachable while fraction is assembled or evaluated. + reachable int + + // fanout retains the fanout while fraction is assembled or evaluated. + fanout int + } + trainingDepths := map[int]bool{} + trainingFanouts := map[int]bool{} + trainingFractions := map[fraction]bool{} + trainingDisconnected := map[int]bool{} + trainingFanIn := map[int]bool{} + trainingMultiplicity := map[int]bool{} + trainingRoots := map[int]bool{} + trainingObservations := map[string]bool{} + trainingBoundaryControls := map[[2]bool]bool{} + trainingZeroDepth := map[bool]bool{} + trainingPayloads := map[int]bool{} + holdoutDepths := map[int]bool{} + declaredCohort := map[performanceKey]string{} + trainingCount, holdoutCount := 0, 0 + + for _, testCase := range corpus.Cases { + if !strings.HasPrefix(testCase.Dataset, "generated_fixed_suffix_expansion_v3_") || + !slices.Contains(testCase.Tags, "orientation-v2-training") && + !slices.Contains(testCase.Tags, "orientation-v2-holdout") { + continue + } + + config, ok := parseFixedSuffixExpansionV3DatasetName(testCase.Dataset) + require.True(t, ok, testCase.Name) + require.NotNil(t, testCase.Expected.RowCount, testCase.Name) + require.NotNil(t, testCase.Shape.MaxDepth, testCase.Name) + require.Equal(t, config.ExpansionDepth, *testCase.Shape.MaxDepth, testCase.Name) + if testCase.Expected.ResultKind == "path_set" { + require.Len(t, testCase.Expected.PathRows, int(*testCase.Expected.RowCount), + testCase.Name+" must predeclare every stable path observation") + require.True(t, newCaseResult(testCase, ModePostgresSQL, testCase.Params).StableObservation, testCase.Name) + } + declaredCohort[performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + }] = testCase.Shape.QualificationSplit + + metadata, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + require.NotNil(t, metadata.FixedSuffixExpansion, testCase.Name) + require.Equal(t, metadata.FixedSuffixExpansion.CompleteOutputTrails, *testCase.Expected.RowCount, testCase.Name) + + trainingTag := slices.Contains(testCase.Tags, "orientation-v2-training") + holdoutTag := slices.Contains(testCase.Tags, "orientation-v2-holdout") + require.NotEqual(t, trainingTag, holdoutTag, testCase.Name) + switch testCase.Shape.QualificationSplit { + case "training": + require.True(t, trainingTag, testCase.Name) + require.False(t, holdoutTag, testCase.Name) + trainingCount++ + trainingDepths[config.ExpansionDepth] = true + trainingFanouts[config.Fanout] = true + trainingFractions[fraction{ + reachable: *config.ExactReachableSuffixSources, + fanout: config.Fanout, + }] = true + trainingDisconnected[config.DisconnectedSuffixSources] = true + trainingFanIn[config.ReverseFanIn] = true + trainingMultiplicity[config.SuffixPathsPerBoundary] = true + trainingRoots[config.RootMatchCount] = true + observation := "endpoint" + if testCase.Observes.Paths { + observation = "path" + } + trainingObservations[observation] = true + trainingBoundaryControls[[2]bool{config.AddProductiveBoundaryCycle, config.AddProductiveBoundarySelfLoop}] = true + trainingZeroDepth[*config.RootHasZeroDepthSuffix] = true + trainingPayloads[config.PropertyPayloadSize] = true + case "holdout": + require.False(t, trainingTag, testCase.Name) + require.True(t, holdoutTag, testCase.Name) + holdoutCount++ + holdoutDepths[config.ExpansionDepth] = true + default: + t.Fatalf("%s has invalid v3 orientation split %q", testCase.Name, testCase.Shape.QualificationSplit) + } + } + + require.Equal(t, 8, trainingCount) + require.Equal(t, 4, holdoutCount) + require.GreaterOrEqual(t, len(trainingDepths), 4) + require.GreaterOrEqual(t, len(trainingFanouts), 4) + require.GreaterOrEqual(t, len(trainingFractions), 4) + require.GreaterOrEqual(t, len(trainingDisconnected), 4) + require.GreaterOrEqual(t, len(trainingFanIn), 3) + require.GreaterOrEqual(t, len(trainingMultiplicity), 3) + require.GreaterOrEqual(t, len(trainingRoots), 4) + require.Equal(t, map[string]bool{"endpoint": true, "path": true}, trainingObservations) + require.Equal(t, map[bool]bool{ + false: true, + true: true, + }, trainingZeroDepth) + require.GreaterOrEqual(t, len(trainingPayloads), 3) + for _, combination := range [][2]bool{{false, false}, {true, false}, {false, true}, {true, true}} { + require.True(t, trainingBoundaryControls[combination], "missing cycle/self-loop combination %v", combination) + } + require.Equal(t, map[int]bool{7: true, 11: true, 13: true, 15: true}, holdoutDepths) + for depth := range holdoutDepths { + require.False(t, trainingDepths[depth], "holdout depth %d is already present in training", depth) + } + require.Len(t, orientationV2CanonicalCases, len(declaredCohort)) + for _, frozen := range orientationV2CanonicalCases { + require.Equal(t, frozen.split, declaredCohort[performanceKey{ + dataset: frozen.dataset, + name: frozen.name, + backend: ModePostgresSQL, + }], frozen.name) + } + canonical, err := canonicalOrientationV2Cohort() + require.NoError(t, err) + _, trainingSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"orientation-v2-training"}}) + require.NoError(t, err) + require.Equal(t, canonical.trainingDeclarationSHA256, trainingSelection.DeclarationSHA256) + _, confirmationSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"orientation-v2-training", "orientation-v2-holdout"}}) + require.NoError(t, err) + require.Equal(t, canonical.declarationSHA256, confirmationSelection.DeclarationSHA256) +} + +// TestSuffixReverseRetryV1OpenControlsStayFresh verifies the P1 development +// roster has dedicated open identities for reverse-fan-in, no-path exhaustion, +// and candidate-buffer byte retry. These controls must never borrow a frozen +// orientation holdout merely because it has a superficially similar shape. +func TestSuffixReverseRetryV1OpenControlsStayFresh(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + want := map[string]struct { + depth int + fanIn int + payload int + rows int64 + }{ + "GFSE-P1-TRAIN-D09-F017-R0-X2-I1024-M1-Q1-high_reverse_fanin_path": {depth: 9, fanIn: 1024, rows: 1}, + "GFSE-P1-TRAIN-D09-F513-R0-X512-no_path_exhaustion": {depth: 9, rows: 0}, + "GFSE-P1-TRAIN-D00-F001-R0-X0-M4-P2100000-output_byte_retry_path": {depth: 0, payload: 2_100_000, rows: 4}, + } + found := map[string]bool{} + for _, testCase := range corpus.Cases { + expectation, selected := want[testCase.Name] + if !selected { + continue + } + found[testCase.Name] = true + require.Contains(t, testCase.Tags, "suffix-reverse-retry-v1-training") + require.Contains(t, testCase.Tags, "p1-open") + require.NotContains(t, testCase.Tags, "holdout") + require.Equal(t, "training", testCase.Shape.QualificationSplit) + require.True(t, testCase.Observes.Paths) + require.True(t, testCase.Shape.PathMaterializationRequired) + require.NotNil(t, testCase.Expected.RowCount) + require.Equal(t, expectation.rows, *testCase.Expected.RowCount) + require.Len(t, testCase.Expected.PathRows, int(expectation.rows)) + + config, ok := parseFixedSuffixExpansionV2DatasetName(testCase.Dataset) + require.True(t, ok, testCase.Name) + require.Equal(t, expectation.depth, config.ExpansionDepth) + require.Equal(t, expectation.fanIn, config.ReverseFanIn) + require.Equal(t, expectation.payload, config.PropertyPayloadSize) + } + require.Equal(t, map[string]bool{ + "GFSE-P1-TRAIN-D09-F017-R0-X2-I1024-M1-Q1-high_reverse_fanin_path": true, + "GFSE-P1-TRAIN-D09-F513-R0-X512-no_path_exhaustion": true, + "GFSE-P1-TRAIN-D00-F001-R0-X0-M4-P2100000-output_byte_retry_path": true, + }, found) +} + +// TestEndpointSeededExpansionCorpusCoversGuardOutcomes verifies corpus representatives for admitted execution plus endpoint-guard and state-guard overflow fallbacks. +func TestEndpointSeededExpansionCorpusCoversGuardOutcomes(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + required := map[string]bool{"guard-admitted": false, "endpoint-guard-overflow": false, "state-guard-overflow": false} + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_endpoint_seeded_expansion" { + continue + } + for tag := range required { + if slices.Contains(testCase.Tags, tag) { + required[tag] = true + } + } + } + for tag, found := range required { + require.True(t, found, "endpoint-seeded corpus is missing %s", tag) + } +} + +// TestGeneratedShortestDistanceCorpusCoversQualificationEnvelope verifies distance cases spanning deep, wide, inbound, disconnected, cyclic, parallel-edge, and self-loop shapes. +func TestGeneratedShortestDistanceCorpusCoversQualificationEnvelope(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredTags := map[string]bool{ + "depth-32": false, "depth-64": false, "fanout-512": false, "fanout-1000": false, + "inbound": false, "disconnected": false, "cycle": false, "parallel-edges": false, "self-loop": false, + } + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_shortest_path" || !slices.Contains(testCase.Tags, "distance") { + continue + } + for tag := range requiredTags { + if slices.Contains(testCase.Tags, tag) { + requiredTags[tag] = true + } + } + } + for tag, covered := range requiredTags { + require.True(t, covered, "shortest distance corpus is missing %s", tag) + } +} + +func TestSPInlineDistanceCorpusCoversProductionNoPath(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + found := false + for _, testCase := range corpus.Cases { + if !slices.Contains(testCase.Tags, "sp-i2") || !slices.Contains(testCase.Tags, "disconnected") { + continue + } + require.Equal(t, "inbound", testCase.Shape.Direction) + require.Contains(t, testCase.Tags, "distance") + require.Equal(t, 1, testCase.Shape.RelationshipKindCount) + require.False(t, testCase.Shape.PathMaterializationRequired) + require.Equal(t, "empty", testCase.Shape.ResultCardinalityClass) + found = true + } + require.True(t, found, "SP-I2 corpus is missing an inbound typed no-path case") +} + +func TestFixedSuffixV2SparsePathHasStableOracle(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + var testCase ScaleCase + for _, candidate := range corpus.Cases { + if candidate.Name == "GFSE-V2-D16-F1000-R1-X1-M1-sparse_path" { + testCase = candidate + break + } + } + require.NotEmpty(t, testCase.Name) + require.Equal(t, "path_set", testCase.Expected.ResultKind) + require.Len(t, testCase.Expected.PathRows, 2) + require.True(t, newCaseResult(testCase, ModePostgresSQL, testCase.Params).StableObservation) +} + +// TestGeneratedShortestPathCorpusCoversMaterializerEnvelope verifies hydrated-path cases spanning deep, wide, inbound, zero-depth, disconnected, cyclic, parallel-edge, and self-loop shapes. +func TestGeneratedShortestPathCorpusCoversMaterializerEnvelope(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredTags := map[string]bool{ + "depth-32": false, "depth-64": false, "fanout-512": false, "fanout-1000": false, + "inbound": false, "zero-depth": false, "disconnected": false, "cycle": false, "parallel-edges": false, "self-loop": false, + } + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_shortest_path" || !slices.Contains(testCase.Tags, "path") { + continue + } + for tag := range requiredTags { + if slices.Contains(testCase.Tags, tag) { + requiredTags[tag] = true + } + } + } + for tag, covered := range requiredTags { + require.True(t, covered, "shortest path corpus is missing %s", tag) + } +} + +// TestGeneratedAllShortestCorpusCoversInlineQualificationEnvelope keeps the +// training corpus broad enough to qualify early-stop behavior independently +// from the frozen depth-8 holdouts. Cap-threshold branch execution is covered +// by the live guarded-statement integration tests because corpus cases do not +// override immutable production caps. +func TestGeneratedAllShortestCorpusCoversInlineQualificationEnvelope(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredTraining := map[string]bool{ + "early-depth-1": false, "early-depth-2": false, "early-depth-3": false, + "max-16": false, "max-64": false, "inbound": false, + "cycle-dead-tail": false, "reconvergence": false, "disconnected": false, + } + hasQualifiedHoldout := false + for _, testCase := range corpus.Cases { + if testCase.Category != "generated_shortest_path_v2" || !slices.Contains(testCase.Tags, "all-shortest") { + continue + } + if testCase.Shape.QualificationSplit == "holdout" && testCase.Shape.RelationshipKindCount == 1 && testCase.Shape.MaxDepth != nil && *testCase.Shape.MaxDepth >= 3 { + hasQualifiedHoldout = true + } + if testCase.Shape.QualificationSplit != "training" { + continue + } + for tag := range requiredTraining { + if slices.Contains(testCase.Tags, tag) { + requiredTraining[tag] = true + } + } + } + for tag, covered := range requiredTraining { + require.True(t, covered, "all-shortest training corpus is missing %s", tag) + } + require.True(t, hasQualifiedHoldout, "all-shortest corpus lacks a typed single-kind holdout at maximum depth 3 or greater") +} + +// scaleCorpusCaseID joins a scale case's dataset and name into its contract identifier. +func scaleCorpusCaseID(name string) string { + if separator := strings.IndexByte(name, '_'); separator >= 0 { + return name[:separator] + } + return name +} + +// scaleCorpusRequiredIDSet returns the required representative scale-case identifiers as a set. +func scaleCorpusRequiredIDSet() map[string]struct{} { + required := make(map[string]struct{}, len(scaleCorpusRequiredIDs)) + for _, id := range scaleCorpusRequiredIDs { + required[id] = struct{}{} + } + return required +} + +// TestScaleCorpusRequiredRepresentativesDeclareCardinality verifies every required query-form tag is present and declares row counts or complete mutation cardinalities. +func TestScaleCorpusRequiredRepresentativesDeclareCardinality(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + required := scaleCorpusRequiredIDSet() + covered := map[string]int{} + for _, testCase := range corpus.Cases { + id := scaleCorpusCaseID(testCase.Name) + if _, isRequired := required[id]; !isRequired { + continue + } + + covered[id]++ + require.Contains(t, testCase.Tags, id, "%s must retain its stable query-form tag", testCase.Name) + if testCase.WriteScenario == nil { + require.NotNil(t, testCase.Expected.RowCount, "%s must declare expected row cardinality", testCase.Name) + } else { + require.NotNil(t, testCase.WriteScenario.ExpectedMatched, "%s must declare expected matched cardinality", testCase.Name) + require.NotNil(t, testCase.WriteScenario.ExpectedAffected, "%s must declare expected affected cardinality", testCase.Name) + } + } + + for _, id := range scaleCorpusRequiredIDs { + require.Positive(t, covered[id], "required scale corpus is missing %s", id) + } +} + +// TestScaleCorpusDistinguishesProjectionClasses verifies that ID-only, shallow, and fully hydrated tags agree with result kind and observation requirements. +func TestScaleCorpusDistinguishesProjectionClasses(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + requiredClasses := map[string]bool{ + "projection-id-only": false, + "projection-shallow-ids-kind": false, + "projection-full-hydration": false, + } + for _, testCase := range corpus.Cases { + for _, tag := range testCase.Tags { + if _, required := requiredClasses[tag]; !required { + continue + } + + requiredClasses[tag] = true + switch tag { + case "projection-id-only": + require.Equal(t, "id_set", testCase.Expected.ResultKind) + require.False(t, testCase.Observes.Nodes) + require.False(t, testCase.Observes.Relationships) + require.False(t, testCase.Observes.Properties) + case "projection-shallow-ids-kind": + require.Equal(t, "shallow_ids_kind", testCase.Expected.ResultKind) + require.False(t, testCase.Observes.Nodes) + require.False(t, testCase.Observes.Relationships) + require.False(t, testCase.Observes.Properties) + case "projection-full-hydration": + require.True(t, testCase.Observes.Nodes || testCase.Observes.Relationships) + require.True(t, testCase.Observes.Properties) + } + } + } + + for projectionClass, found := range requiredClasses { + require.True(t, found, "scale corpus is missing %s", projectionClass) + } +} + +// TestFixedSuffixExpansionIDRowsUseStableFixtureIdentitiesAndPreserveDuplicates verifies four identical logical endpoint pairs remain explicit expected rows rather than being deduplicated or backend-ID based. +func TestFixedSuffixExpansionIDRowsUseStableFixtureIdentitiesAndPreserveDuplicates(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + for _, testCase := range corpus.Cases { + if testCase.Name != "fixed_suffix_expansion_endpoint_ids" { + continue + } + + require.Equal(t, [][]string{ + {"fse-head", "fse-terminal"}, + {"fse-head", "fse-terminal"}, + {"fse-head", "fse-terminal"}, + {"fse-head", "fse-terminal"}, + }, testCase.Expected.IDRows) + return + } + + t.Fatal("fixed_suffix_expansion_endpoint_ids case not found") +} + +// TestGeneratedSPI1InboundV1CorpusFreezesTrainingAndUnopenedHoldoutMatrices +// verifies the preregistered canonical-witness cohort without executing or +// inspecting any holdout timing. The contract binds exact generated topology, +// stable path observations, split tags, query identity, and selection digests. +func TestGeneratedSPI1InboundV1CorpusFreezesTrainingAndUnopenedHoldoutMatrices(t *testing.T) { + const ( + query = "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN p" + querySHA256 = "1024577967901503995d4ec0c76540e96b65f4d25e015ccb6eeffb500a5596f9" + ) + + // expectedCase binds one frozen workload declaration to its expected fixture and result shape. + type expectedCase struct { + // dataset retains the dataset while expectedCase is assembled or evaluated. + dataset string + // config retains the config while expectedCase is assembled or evaluated. + config testutil.ShortestPathScaleV2Config + // fixtureSHA256 binds the referenced fixture content by SHA-256 digest. + fixtureSHA256 string + // split retains the split while expectedCase is assembled or evaluated. + split string + // target retains the target while expectedCase is assembled or evaluated. + target string + // resultDepth retains the result depth while expectedCase is assembled or evaluated. + resultDepth int + // stateClass retains the state class while expectedCase is assembled or evaluated. + stateClass string + // extraTags retains the extra tags while expectedCase is assembled or evaluated. + extraTags []string + } + expected := map[string]expectedCase{ + "GSP-I1-V1-TRAIN-D04-FI016-full": { + dataset: "generated_shortest_paths_v2_d4_o0_r4_fo0_fi16_l2_k0_t0_w0_x4_p0_c0_s0", + config: spI1InboundFixtureConfig(4, 4, 16, 2, 4), + fixtureSHA256: "29b0c923d7e3312ba1f19d09076006692dfc66379a524d160da8d74d9c7c3889", + split: "training", + target: "sp-v2-inbound-end", + resultDepth: 4, + stateClass: "inbound_predecessor_full_depth_fanin_16", + }, + "GSP-I1-V1-TRAIN-D16-FI256-early-d04": { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + config: spI1InboundFixtureConfig(16, 8, 256, 8, 16), + fixtureSHA256: "a297da4f7be1cb8621d173cd763e1fcc902b560e23d8fdfbbc9565d10c308bce", + split: "training", + target: "sp-v2-inbound-linear-04", + resultDepth: 4, + stateClass: "inbound_predecessor_early_target_fanin_256", + extraTags: []string{"early-target", "early-depth-4"}, + }, + "GSP-I1-V1-TRAIN-D16-FI256-full": { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + config: spI1InboundFixtureConfig(16, 8, 256, 8, 16), + fixtureSHA256: "a297da4f7be1cb8621d173cd763e1fcc902b560e23d8fdfbbc9565d10c308bce", + split: "training", + target: "sp-v2-inbound-end", + resultDepth: 16, + stateClass: "inbound_predecessor_full_depth_fanin_256", + }, + "GSP-I1-V1-TRAIN-D16-FI256-disconnected": { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + config: spI1InboundFixtureConfig(16, 8, 256, 8, 16), + fixtureSHA256: "a297da4f7be1cb8621d173cd763e1fcc902b560e23d8fdfbbc9565d10c308bce", + split: "training", + target: "sp-v2-disconnected-end", + resultDepth: -1, + stateClass: "inbound_predecessor_disconnected_fanin_256", + extraTags: []string{"disconnected", "max-miss"}, + }, + "GSP-I1-V1-HOLDOUT-D08-FI031-full": { + dataset: "generated_shortest_paths_v2_d8_o0_r3_fo0_fi31_l3_k0_t0_w0_x7_p0_c0_s0", + config: spI1InboundFixtureConfig(8, 3, 31, 3, 7), + fixtureSHA256: "47acf96f7862e639a8a33bc28f2c9b9e4457320e44c8e88b0b06ab2f25691e63", + split: "holdout", + target: "sp-v2-inbound-end", + resultDepth: 8, + stateClass: "inbound_predecessor_full_depth_fanin_31", + }, + "GSP-I1-V1-HOLDOUT-D32-FI191-full": { + dataset: "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + config: spI1InboundFixtureConfig(32, 11, 191, 21, 13), + fixtureSHA256: "da33b5d223d8513ff4af240613a8f976e12be6b398536bb9b4f8d5a184d9443b", + split: "holdout", + target: "sp-v2-inbound-end", + resultDepth: 32, + stateClass: "inbound_predecessor_full_depth_fanin_191", + }, + "GSP-I1-V1-HOLDOUT-D32-FI191-disconnected": { + dataset: "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + config: spI1InboundFixtureConfig(32, 11, 191, 21, 13), + fixtureSHA256: "da33b5d223d8513ff4af240613a8f976e12be6b398536bb9b4f8d5a184d9443b", + split: "holdout", + target: "sp-v2-disconnected-end", + resultDepth: -1, + stateClass: "inbound_predecessor_disconnected_fanin_191", + extraTags: []string{"disconnected", "max-miss"}, + }, + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + seen := map[string]bool{} + trainingDepths, holdoutDepths := map[int]bool{}, map[int]bool{} + trainingCount, holdoutCount := 0, 0 + for _, testCase := range corpus.Cases { + trainingTag := slices.Contains(testCase.Tags, "sp-i1-inbound-v1-training") + holdoutTag := slices.Contains(testCase.Tags, "sp-i1-inbound-v1-holdout") + if !trainingTag && !holdoutTag { + continue + } + require.NotEqual(t, trainingTag, holdoutTag, testCase.Name) + contract, found := expected[testCase.Name] + require.True(t, found, "unexpected SP-I1 inbound-v1 declaration %s", testCase.Name) + require.False(t, seen[testCase.Name], testCase.Name) + seen[testCase.Name] = true + + require.True(t, strings.HasSuffix(testCase.Source, "/cases/generated_sp_i1_inbound_v1.json"), testCase.Name) + require.Equal(t, contract.dataset, testCase.Dataset, testCase.Name) + require.Equal(t, "generated_shortest_path_v2", testCase.Category, testCase.Name) + require.Equal(t, query, testCase.Cypher, testCase.Name) + require.Equal(t, querySHA256, pg.TraversalPolicyQuerySHA256(testCase.Cypher), testCase.Name) + require.Equal(t, map[string]string{"root_id": "sp-v2-inbound-root", "end_id": contract.target}, testCase.NodeParams, testCase.Name) + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, testCase.CandidateModes, testCase.Name) + require.Empty(t, testCase.UnsupportedModes, testCase.Name) + require.Equal(t, ObservedValues{ + Paths: true, + Nodes: true, + Relationships: true, + Properties: true, + }, testCase.Observes, testCase.Name) + + shape := testCase.Shape + require.Equal(t, contract.split, shape.QualificationSplit, testCase.Name) + require.Equal(t, "forbidden", shape.FallbackExpectation, testCase.Name) + require.Equal(t, "bound_id", shape.RootPredicate, testCase.Name) + require.Equal(t, "bound_id", shape.TerminalPredicate, testCase.Name) + require.Equal(t, []string{"Traverse"}, shape.EdgeKinds, testCase.Name) + require.Equal(t, "inbound", shape.Direction, testCase.Name) + require.Equal(t, 1, shape.RelationshipKindCount, testCase.Name) + require.Equal(t, "normal", shape.FixtureTier, testCase.Name) + require.Equal(t, contract.stateClass, shape.ExpectedStateClass, testCase.Name) + require.NotNil(t, shape.MinDepth, testCase.Name) + require.NotNil(t, shape.MaxDepth, testCase.Name) + require.Equal(t, 1, *shape.MinDepth, testCase.Name) + require.Equal(t, 64, *shape.MaxDepth, testCase.Name) + require.True(t, shape.PathMaterializationRequired, testCase.Name) + + config, ok := parseShortestPathV2DatasetName(testCase.Dataset) + require.True(t, ok, testCase.Name) + require.Equal(t, contract.config, config, testCase.Name) + metadata, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + require.Equal(t, contract.fixtureSHA256, metadata.Checksum, testCase.Name) + require.NotNil(t, metadata.Shortest, testCase.Name) + require.Equal(t, int64(config.Depth), metadata.Shortest.ExpectedMinimumDistance, testCase.Name) + + expectedTags := []string{"generated", "v2", "normal-tier", "path", "inbound", "hidden-fan-in"} + expectedTags = append(expectedTags, contract.extraTags...) + if contract.split == "training" { + trainingCount++ + trainingDepths[config.Depth] = true + expectedTags = append(expectedTags, "sp-i1-inbound-v1-training") + } else { + holdoutCount++ + holdoutDepths[config.Depth] = true + expectedTags = append(expectedTags, "holdout", "sp-i1-inbound-v1-holdout") + } + require.Equal(t, expectedTags, testCase.Tags, testCase.Name) + + require.NotNil(t, testCase.Expected.RowCount, testCase.Name) + require.Equal(t, "path_set", testCase.Expected.ResultKind, testCase.Name) + if contract.resultDepth < 0 { + require.Zero(t, *testCase.Expected.RowCount, testCase.Name) + require.Empty(t, testCase.Expected.PathRows, testCase.Name) + require.Equal(t, "empty", shape.ResultCardinalityClass, testCase.Name) + } else { + require.Equal(t, int64(1), *testCase.Expected.RowCount, testCase.Name) + require.Equal(t, []ExpectedPath{spI1InboundExpectedPath(config.Depth, contract.resultDepth)}, testCase.Expected.PathRows, testCase.Name) + require.Equal(t, "singleton", shape.ResultCardinalityClass, testCase.Name) + } + } + + require.Len(t, seen, 7) + for name := range expected { + require.True(t, seen[name], "missing SP-I1 inbound-v1 declaration %s", name) + } + require.Len(t, spI1CanonicalCases, len(expected)) + canonicalSeen := map[string]bool{} + for _, canonical := range spI1CanonicalCases { + contract, found := expected[canonical.name] + require.True(t, found, "unexpected frozen SP-I1 case %s", canonical.name) + require.False(t, canonicalSeen[canonical.name], canonical.name) + canonicalSeen[canonical.name] = true + require.Equal(t, contract.dataset, canonical.dataset, canonical.name) + require.Equal(t, contract.split, canonical.split, canonical.name) + } + require.Equal(t, seen, canonicalSeen, "corpus and qualification reporter must freeze the same SP-I1 cases") + require.Equal(t, 4, trainingCount) + require.Equal(t, 3, holdoutCount) + require.Equal(t, map[int]bool{4: true, 16: true}, trainingDepths) + require.Equal(t, map[int]bool{8: true, 32: true}, holdoutDepths) + for depth := range holdoutDepths { + require.False(t, trainingDepths[depth], "holdout depth %d is present in training", depth) + } + + training, trainingSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training"}}) + require.NoError(t, err) + require.Len(t, training.Cases, 4) + require.True(t, trainingSelection.DiagnosticOnly) + require.Equal(t, 8, trainingSelection.SelectedDeclarationCount) + require.Equal(t, "1162e6563678dad742d8fe89d250936862b4a73deab247cde4b5ddebdfdd93ce", trainingSelection.DeclarationSHA256) + require.Equal(t, "cc07b55331e15f4e268043d1ed36abf7deec7217771a1b30913db6e738d27f7a", resolvedSelectionSHA256(trainingSelection.Resolved)) + require.Equal(t, "3da3c4b1cea3fa64fbaa1958f7bf8048639241522ccf6e46defd10d2d8c9ccd6", spI1InboundRuntimeCorpusIdentity(training)) + + confirmation, confirmationSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training", "sp-i1-inbound-v1-holdout"}}) + require.NoError(t, err) + require.Len(t, confirmation.Cases, 7) + require.True(t, confirmationSelection.DiagnosticOnly) + require.Equal(t, 14, confirmationSelection.SelectedDeclarationCount) + require.Equal(t, "31f6041f342b3ed8059d4d1396a76f073c3fc877472d06632a8bad16b5a4cbfd", confirmationSelection.DeclarationSHA256) + require.Equal(t, "16a8756a7c32695f0314b3552c80d2a500226c7a44c57847c916a96e775aa0c5", resolvedSelectionSHA256(confirmationSelection.Resolved)) + require.Equal(t, "219ee26cae52d8b81c6c91f9c517692c544ef4cec1aa9b9314fbc4e8f5ad3c5c", spI1InboundRuntimeCorpusIdentity(confirmation)) +} + +func TestGeneratedSPI2DistanceV1CorpusFreezesProtectedCohort(t *testing.T) { + const query = "MATCH p = shortestPath((r)<-[:Traverse*1..64]-(e)) WHERE id(r) = $root_id AND id(e) = $end_id RETURN length(p)" + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + seen := map[string]bool{} + trainingDepths, holdoutDepths := map[int]bool{}, map[int]bool{} + trainingCount, holdoutCount := 0, 0 + for _, testCase := range corpus.Cases { + training := slices.Contains(testCase.Tags, spI2TrainingTag) + holdout := slices.Contains(testCase.Tags, spI2HoldoutTag) + if !training && !holdout { + continue + } + require.NotEqual(t, training, holdout, testCase.Name) + require.False(t, seen[testCase.Name], testCase.Name) + seen[testCase.Name] = true + require.True(t, strings.HasSuffix(testCase.Source, "/cases/generated_sp_i2_distance_v1.json"), testCase.Name) + require.Equal(t, query, testCase.Cypher, testCase.Name) + require.Equal(t, spI2QuerySHA256, pg.TraversalPolicyQuerySHA256(testCase.Cypher), testCase.Name) + require.Equal(t, ObservedValues{}, testCase.Observes, testCase.Name) + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, testCase.CandidateModes, testCase.Name) + require.Equal(t, "forbidden", testCase.Shape.FallbackExpectation, testCase.Name) + require.Equal(t, "inbound", testCase.Shape.Direction, testCase.Name) + require.Equal(t, []string{"Traverse"}, testCase.Shape.EdgeKinds, testCase.Name) + require.Equal(t, 1, testCase.Shape.RelationshipKindCount, testCase.Name) + require.NotNil(t, testCase.Shape.MinDepth, testCase.Name) + require.NotNil(t, testCase.Shape.MaxDepth, testCase.Name) + require.Equal(t, 1, *testCase.Shape.MinDepth, testCase.Name) + require.Equal(t, 64, *testCase.Shape.MaxDepth, testCase.Name) + require.False(t, testCase.Shape.PathMaterializationRequired, testCase.Name) + require.NotNil(t, testCase.Expected.RowCount, testCase.Name) + require.Equal(t, "scalar", testCase.Expected.ResultKind, testCase.Name) + + config, ok := parseShortestPathV2DatasetName(testCase.Dataset) + require.True(t, ok, testCase.Name) + metadata, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + require.True(t, lowercaseSHA256(metadata.Checksum), testCase.Name) + if training { + trainingCount++ + trainingDepths[config.Depth] = true + require.Equal(t, "training", testCase.Shape.QualificationSplit, testCase.Name) + } else { + holdoutCount++ + holdoutDepths[config.Depth] = true + require.Equal(t, "holdout", testCase.Shape.QualificationSplit, testCase.Name) + } + } + require.Len(t, seen, 10) + require.Equal(t, 6, trainingCount) + require.Equal(t, 4, holdoutCount) + require.Equal(t, map[int]bool{3: true, 6: true, 8: true, 16: true}, trainingDepths) + require.Equal(t, map[int]bool{5: true, 13: true, 21: true}, holdoutDepths) + for depth := range holdoutDepths { + require.False(t, trainingDepths[depth], "holdout depth %d is present in training", depth) + } + require.Len(t, spI2CanonicalCases, len(seen)) + + training, trainingSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{spI2TrainingTag}}) + require.NoError(t, err) + require.Len(t, training.Cases, 6) + require.Equal(t, "32053ee421c155d9d2f2c55bb2dbb56aa1df2fb7b227b2639ecdf36d789146f3", trainingSelection.DeclarationSHA256) + require.Equal(t, spI2TrainingResolvedSHA, resolvedSelectionSHA256(trainingSelection.Resolved)) + require.Equal(t, spI2TrainingCorpusSHA256, corpusIdentity(training)) + + confirmation, fullSelection, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{spI2TrainingTag, spI2HoldoutTag}}) + require.NoError(t, err) + require.Len(t, confirmation.Cases, 10) + require.Equal(t, "3b63388e19beaebc1f621e944e53a4377430aa928bada3a8afe18d636138e3e9", fullSelection.DeclarationSHA256) + require.Equal(t, spI2FullResolvedSHA, resolvedSelectionSHA256(fullSelection.Resolved)) + require.Equal(t, spI2FullCorpusSHA256, corpusIdentity(confirmation)) +} + +// spI1InboundFixtureConfig prepares or inspects test evidence for sp i1 inbound fixture config. +func spI1InboundFixtureConfig(depth, rootFanIn, intermediateFanIn, fanInLevel, disconnectedWidth int) testutil.ShortestPathScaleV2Config { + return testutil.ShortestPathScaleV2Config{ + Depth: depth, + ReverseRootFanIn: rootFanIn, + IntermediateReverseFanIn: intermediateFanIn, + FanInLevel: fanInLevel, + DisconnectedWidth: disconnectedWidth, + } +} + +// spI1InboundExpectedPath prepares or inspects test evidence for sp i1 inbound expected path. +func spI1InboundExpectedPath(fixtureDepth, resultDepth int) ExpectedPath { + nodes := []string{"sp-v2-inbound-root"} + for level := 1; level < resultDepth; level++ { + nodes = append(nodes, fmt.Sprintf("sp-v2-inbound-linear-%02d", level)) + } + if resultDepth == fixtureDepth { + nodes = append(nodes, "sp-v2-inbound-end") + } else { + nodes = append(nodes, fmt.Sprintf("sp-v2-inbound-linear-%02d", resultDepth)) + } + kinds := make([]string, resultDepth) + keys := make([]string, resultDepth) + for idx := range resultDepth { + kinds[idx] = "Traverse" + keys[idx] = fmt.Sprintf("inbound-primary-%02d", fixtureDepth-idx) + } + return ExpectedPath{ + Nodes: nodes, + RelationshipKinds: kinds, + RelationshipKeys: keys, + } +} + +// spI1InboundRuntimeCorpusIdentity normalizes the package-test corpus root to +// the repository-root spelling used by GraphBench capture commands. +func spI1InboundRuntimeCorpusIdentity(corpus ScaleCorpus) string { + canonical := ScaleCorpus{Cases: append([]ScaleCase(nil), corpus.Cases...)} + for idx := range canonical.Cases { + if offset := strings.Index(canonical.Cases[idx].Source, "benchmark/testdata/scale/"); offset >= 0 { + canonical.Cases[idx].Source = canonical.Cases[idx].Source[offset:] + } + } + return corpusIdentity(canonical) +} diff --git a/cmd/graphbench/selection.go b/cmd/graphbench/selection.go new file mode 100644 index 00000000..7c843dda --- /dev/null +++ b/cmd/graphbench/selection.go @@ -0,0 +1,252 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "slices" + "sort" +) + +// selectionManifestVersion identifies the serialized schema revision for selection manifest. +const selectionManifestVersion = 2 + +// CorpusSelectors contains exact dataset, category, case, and tag filters supplied by the user. +type CorpusSelectors struct { + // Cases lists exact case names requested by the user. + Cases []string `json:"cases,omitempty"` + // Datasets lists exact dataset selectors supplied by the user. + Datasets []string `json:"datasets,omitempty"` + // Categories lists workload categories used to filter the corpus. + Categories []string `json:"categories,omitempty"` + // Tags lists exact tag selectors supplied by the user. + Tags []string `json:"tags,omitempty"` +} + +// ResolvedCaseSelector identifies a selected case together with its declared category. +type ResolvedCaseSelector struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Category groups cases by workload category. + Category string `json:"category"` +} + +// SelectionManifest records requested filters, resolved workloads, and completeness evidence for one run. +type SelectionManifest struct { + // Version identifies the serialized schema revision. + Version int `json:"version"` + // Requested preserves the exact corpus filters supplied by the user. + Requested CorpusSelectors `json:"requested"` + // Resolved lists exact case selectors retained after corpus filtering. + Resolved []ResolvedCaseSelector `json:"resolved"` + // DiagnosticOnly marks a selection that is informative but ineligible for complete gating. + DiagnosticOnly bool `json:"diagnostic_only"` + // FullDeclarationCount records all case/backend declarations before selection. + FullDeclarationCount int `json:"full_declaration_count"` + // SelectedDeclarationCount records declarations retained by the resolved selection. + SelectedDeclarationCount int `json:"selected_declaration_count"` + // OmittedDeclarationCount records declarations omitted by the resolved selection. + OmittedDeclarationCount int `json:"omitted_declaration_count"` + // ProtectedDeclarationCount records protocol-only declarations omitted before ordinary selector resolution. + ProtectedDeclarationCount int `json:"protected_declaration_count,omitempty"` + // ProtectedDeclarationSHA256 identifies the exact protocol-only declarations omitted from the runnable universe. + ProtectedDeclarationSHA256 string `json:"protected_declaration_sha256,omitempty"` + // DeclarationSHA256 identifies the canonical set of declared workloads. + DeclarationSHA256 string `json:"declaration_sha256"` +} + +// validateSelectionManifestAccounting distinguishes protocol-protected +// omissions from ordinary filtered omissions. An unfiltered artifact remains +// complete only when every omitted declaration is explicitly protected and +// bound by one digest. +func validateSelectionManifestAccounting(manifest SelectionManifest) error { + if manifest.Version != selectionManifestVersion || manifest.FullDeclarationCount < 1 || + manifest.SelectedDeclarationCount < 1 || manifest.OmittedDeclarationCount < 0 || + manifest.FullDeclarationCount != manifest.SelectedDeclarationCount+manifest.OmittedDeclarationCount || + manifest.ProtectedDeclarationCount < 0 || manifest.ProtectedDeclarationCount > manifest.OmittedDeclarationCount { + return fmt.Errorf("selection manifest has inconsistent declaration accounting") + } + if manifest.ProtectedDeclarationCount == 0 { + if manifest.ProtectedDeclarationSHA256 != "" { + return fmt.Errorf("selection manifest has a protected digest without protected declarations") + } + } else if !lowercaseSHA256(manifest.ProtectedDeclarationSHA256) { + return fmt.Errorf("selection manifest lacks a valid protected declaration digest") + } + if !manifest.DiagnosticOnly && manifest.OmittedDeclarationCount != manifest.ProtectedDeclarationCount { + return fmt.Errorf("complete selection manifest contains non-protected omissions") + } + return nil +} + +// selectScaleCorpus filters corpus cases and returns both selected cases and a hashed selection manifest. +func selectScaleCorpus(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + if err := validateCorpusSelectors(corpus, selectors); err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + return selectScaleCorpusValidated(corpus, selectors) +} + +// selectScaleCorpusValidated resolves selectors against a universe whose +// selector names have already been validated. Keeping validation separate lets +// protocol-only workloads remain known selectors while ordinary execution +// deliberately omits them from its runnable universe. +func selectScaleCorpusValidated(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + filtered := len(selectors.Cases)+len(selectors.Datasets)+len(selectors.Categories)+len(selectors.Tags) > 0 + manifest := SelectionManifest{ + Version: selectionManifestVersion, + Requested: selectors, + DiagnosticOnly: filtered, + FullDeclarationCount: len(corpus.DeclaredBackends()), + } + selected := ScaleCorpus{} + for _, testCase := range corpus.Cases { + if matchesSelectors(testCase, selectors) { + selected.Cases = append(selected.Cases, testCase) + manifest.Resolved = append(manifest.Resolved, ResolvedCaseSelector{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Category: testCase.Category, + }) + } + } + if len(selected.Cases) == 0 { + return ScaleCorpus{}, SelectionManifest{}, fmt.Errorf("selectors resolved to an empty corpus") + } + manifest.SelectedDeclarationCount = len(selected.DeclaredBackends()) + manifest.OmittedDeclarationCount = manifest.FullDeclarationCount - manifest.SelectedDeclarationCount + manifest.DeclarationSHA256 = declarationSHA256(selected.DeclaredBackends()) + return selected, manifest, nil +} + +// validateCorpusSelectors rejects duplicate, ambiguous, or unknown exact selectors. +func validateCorpusSelectors(corpus ScaleCorpus, selectors CorpusSelectors) error { + caseMatches := map[string][]ScaleCase{} + datasets := map[string]struct{}{} + categories := map[string]struct{}{} + tags := map[string]struct{}{} + for _, testCase := range corpus.Cases { + caseMatches[testCase.Name] = append(caseMatches[testCase.Name], testCase) + datasets[testCase.Dataset] = struct{}{} + categories[testCase.Category] = struct{}{} + for _, tag := range testCase.Tags { + tags[tag] = struct{}{} + } + } + for _, name := range selectors.Cases { + matches := caseMatches[name] + if len(matches) == 0 { + return fmt.Errorf("unknown case selector %q", name) + } + if len(matches) != 1 { + return fmt.Errorf("ambiguous case selector %q resolves to %d cases", name, len(matches)) + } + } + for _, selector := range []struct { + // kind names the selector dimension for validation errors. + kind string + // values contains the requested selectors to validate in this dimension. + values []string + // known indexes accepted selector values for exact validation. + known map[string]struct{} + }{ + { + kind: "dataset", + values: selectors.Datasets, + known: datasets, + }, + { + kind: "category", + values: selectors.Categories, + known: categories, + }, + { + kind: "tag", + values: selectors.Tags, + known: tags, + }, + } { + for _, value := range selector.values { + if _, found := selector.known[value]; !found { + return fmt.Errorf("unknown %s selector %q", selector.kind, value) + } + } + } + return nil +} + +// matchesSelectors reports whether a scale case matches every nonempty selector dimension. +func matchesSelectors(testCase ScaleCase, selectors CorpusSelectors) bool { + if len(selectors.Cases) > 0 && !slices.Contains(selectors.Cases, testCase.Name) { + return false + } + if len(selectors.Datasets) > 0 && !slices.Contains(selectors.Datasets, testCase.Dataset) { + return false + } + if len(selectors.Categories) > 0 && !slices.Contains(selectors.Categories, testCase.Category) { + return false + } + if len(selectors.Tags) > 0 { + matched := false + for _, tag := range selectors.Tags { + matched = matched || slices.Contains(testCase.Tags, tag) + } + if !matched { + return false + } + } + return true +} + +// selectionIdentity returns the common selection manifest shared by every artifact record. +func selectionIdentity(records []CaseResult) (SelectionManifest, error) { + var selected *SelectionManifest + for _, record := range records { + if record.Environment == nil || record.Environment.Selection == nil { + return SelectionManifest{}, fmt.Errorf("%s/%s has no selection manifest", record.Dataset, record.Name) + } + if selected == nil { + copy := *record.Environment.Selection + selected = © + continue + } + current := record.Environment.Selection + if selected.Version != current.Version || selected.DeclarationSHA256 != current.DeclarationSHA256 || + selected.DiagnosticOnly != current.DiagnosticOnly || selected.FullDeclarationCount != current.FullDeclarationCount || + selected.SelectedDeclarationCount != current.SelectedDeclarationCount || selected.OmittedDeclarationCount != current.OmittedDeclarationCount || + selected.ProtectedDeclarationCount != current.ProtectedDeclarationCount || + selected.ProtectedDeclarationSHA256 != current.ProtectedDeclarationSHA256 || + resolvedSelectionSHA256(selected.Resolved) != resolvedSelectionSHA256(current.Resolved) { + return SelectionManifest{}, fmt.Errorf("artifact contains inconsistent selection manifests") + } + } + + if selected == nil { + return SelectionManifest{}, fmt.Errorf("artifact contains no records") + } + return *selected, nil +} + +// resolvedSelectionSHA256 hashes selected dataset, case, and category tuples in deterministic order. +func resolvedSelectionSHA256(resolved []ResolvedCaseSelector) string { + items := append([]ResolvedCaseSelector(nil), resolved...) + sort.Slice(items, func(i, j int) bool { + if items[i].Dataset != items[j].Dataset { + return items[i].Dataset < items[j].Dataset + } + return items[i].Name < items[j].Name + }) + + digest := sha256.New() + for _, item := range items { + fmt.Fprintf(digest, "%s\x00%s\x00%s\n", item.Dataset, item.Name, item.Category) + } + return hex.EncodeToString(digest.Sum(nil)) +} diff --git a/cmd/graphbench/sp_i1_qualification.go b/cmd/graphbench/sp_i1_qualification.go new file mode 100644 index 00000000..0dd5f0c0 --- /dev/null +++ b/cmd/graphbench/sp_i1_qualification.go @@ -0,0 +1,2233 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const ( + // spI1QualificationVersion reserves the stable protocol value used to recognize sp i1 qualification version across artifacts and executions. + spI1QualificationVersion = 1 + + // spI1FreezeVersion reserves the stable protocol value used to recognize sp i1 freeze version across artifacts and executions. + spI1FreezeVersion = 1 + + // spI1TrainingTag reserves the stable protocol value used to recognize sp i1 training tag across artifacts and executions. + spI1TrainingTag = "sp-i1-inbound-v1-training" + + // spI1HoldoutTag reserves the stable protocol value used to recognize sp i1 holdout tag across artifacts and executions. + spI1HoldoutTag = "sp-i1-inbound-v1-holdout" + + // spI1QuerySHA256 reserves the stable protocol value used to recognize sp i1 query sha256 across artifacts and executions. + spI1QuerySHA256 = "1024577967901503995d4ec0c76540e96b65f4d25e015ccb6eeffb500a5596f9" + + // spI1TrainingCorpusSHA256 reserves the stable protocol value used to recognize sp i1 training corpus sha256 across artifacts and executions. + spI1TrainingCorpusSHA256 = "3da3c4b1cea3fa64fbaa1958f7bf8048639241522ccf6e46defd10d2d8c9ccd6" + + // spI1FullCorpusSHA256 reserves the stable protocol value used to recognize sp i1 full corpus sha256 across artifacts and executions. + spI1FullCorpusSHA256 = "219ee26cae52d8b81c6c91f9c517692c544ef4cec1aa9b9314fbc4e8f5ad3c5c" + + // spI1TrainingResolvedSHA reserves the stable protocol value used to recognize sp i1 training resolved sha across artifacts and executions. + spI1TrainingResolvedSHA = "cc07b55331e15f4e268043d1ed36abf7deec7217771a1b30913db6e738d27f7a" + + // spI1FullResolvedSHA reserves the stable protocol value used to recognize sp i1 full resolved sha across artifacts and executions. + spI1FullResolvedSHA = "16a8756a7c32695f0314b3552c80d2a500226c7a44c57847c916a96e775aa0c5" +) + +// spI1CanonicalCases freezes the training and holdout workloads admitted to SP-I1 qualification. +var spI1CanonicalCases = []struct { + // dataset identifies the generated fixture containing the workload. + dataset string + + // name identifies the workload within the fixture dataset. + name string + + // split assigns the workload to training or unopened holdout evidence. + split string +}{ + { + dataset: "generated_shortest_paths_v2_d4_o0_r4_fo0_fi16_l2_k0_t0_w0_x4_p0_c0_s0", + name: "GSP-I1-V1-TRAIN-D04-FI016-full", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + name: "GSP-I1-V1-TRAIN-D16-FI256-early-d04", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + name: "GSP-I1-V1-TRAIN-D16-FI256-full", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d16_o0_r8_fo0_fi256_l8_k0_t0_w0_x16_p0_c0_s0", + name: "GSP-I1-V1-TRAIN-D16-FI256-disconnected", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d8_o0_r3_fo0_fi31_l3_k0_t0_w0_x7_p0_c0_s0", + name: "GSP-I1-V1-HOLDOUT-D08-FI031-full", + split: "holdout", + }, + { + dataset: "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + name: "GSP-I1-V1-HOLDOUT-D32-FI191-full", + split: "holdout", + }, + { + dataset: "generated_shortest_paths_v2_d32_o0_r11_fo0_fi191_l21_k0_t0_w0_x13_p0_c0_s0", + name: "GSP-I1-V1-HOLDOUT-D32-FI191-disconnected", + split: "holdout", + }, +} + +// spI1CanonicalCohort groups state that must remain consistent while processing sp i1 canonical cohort. +type spI1CanonicalCohort struct { + // keys retains the keys while spI1CanonicalCohort is assembled or evaluated. + keys map[performanceKey]struct{} + // trainingKeys retains the training keys while spI1CanonicalCohort is assembled or evaluated. + trainingKeys map[performanceKey]struct{} + // holdoutKeys retains the holdout keys while spI1CanonicalCohort is assembled or evaluated. + holdoutKeys map[performanceKey]struct{} + // declarationSHA256 binds the referenced declaration content by SHA-256 digest. + declarationSHA256 string + // trainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + trainingDeclarationSHA256 string + // holdoutDeclarationSHA256 binds the referenced holdout declaration content by SHA-256 digest. + holdoutDeclarationSHA256 string + // trainingCorpusSHA256 binds the referenced training corpus content by SHA-256 digest. + trainingCorpusSHA256 string + // fullCorpusSHA256 binds the referenced full corpus content by SHA-256 digest. + fullCorpusSHA256 string + // trainingResolvedSHA256 binds the referenced training resolved content by SHA-256 digest. + trainingResolvedSHA256 string + // fullResolvedSHA256 binds the referenced full resolved content by SHA-256 digest. + fullResolvedSHA256 string +} + +// spI1CanonicalDeclaration groups state that must remain consistent while processing sp i1 canonical declaration. +type spI1CanonicalDeclaration struct { + // testCase retains the test case while spI1CanonicalDeclaration is assembled or evaluated. + testCase ScaleCase + // fixture retains the fixture while spI1CanonicalDeclaration is assembled or evaluated. + fixture FixtureMetadata +} + +// canonicalSPI1Declarations resolves the frozen SP-I1 workload declarations and fixture metadata. +func canonicalSPI1Declarations() (map[performanceKey]spI1CanonicalDeclaration, error) { + repositoryRoot := strings.TrimSpace(commandOutput("git", "rev-parse", "--show-toplevel")) + if repositoryRoot == "" || repositoryRoot == "unknown" { + return nil, fmt.Errorf("locate repository root for frozen SP-I1 declarations") + } + + if corpus, err := loadScaleCorpus(filepath.Join(repositoryRoot, "benchmark", "testdata", "scale")); err != nil { + return nil, fmt.Errorf("load frozen SP-I1 declarations: %w", err) + } else if cohort, err := canonicalSPI1Cohort(); err != nil { + return nil, err + } else { + declarations := make(map[performanceKey]spI1CanonicalDeclaration, len(cohort.keys)) + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, expected := cohort.keys[key]; !expected { + continue + } + if _, duplicate := declarations[key]; duplicate { + return nil, fmt.Errorf("frozen SP-I1 corpus duplicates %s/%s", key.dataset, key.name) + } + if fixture, err := fixtureMetadata("unused", testCase.Dataset); err != nil { + return nil, fmt.Errorf("derive frozen SP-I1 fixture %s: %w", testCase.Dataset, err) + } else { + declarations[key] = spI1CanonicalDeclaration{ + testCase: testCase, + fixture: fixture, + } + } + } + if len(declarations) != len(cohort.keys) { + return nil, fmt.Errorf("frozen SP-I1 corpus omits canonical declarations") + } + + return declarations, nil + } +} + +// canonicalSPI1Cohort builds the immutable training and holdout membership used by qualification. +func canonicalSPI1Cohort() (spI1CanonicalCohort, error) { + cohort := spI1CanonicalCohort{ + keys: map[performanceKey]struct{}{}, + trainingKeys: map[performanceKey]struct{}{}, + holdoutKeys: map[performanceKey]struct{}{}, + trainingCorpusSHA256: spI1TrainingCorpusSHA256, + fullCorpusSHA256: spI1FullCorpusSHA256, + trainingResolvedSHA256: spI1TrainingResolvedSHA, + fullResolvedSHA256: spI1FullResolvedSHA, + } + var full, training, holdout []DeclaredCaseBackend + for _, testCase := range spI1CanonicalCases { + key := performanceKey{ + dataset: testCase.dataset, + name: testCase.name, + backend: ModePostgresSQL, + } + if _, duplicate := cohort.keys[key]; duplicate || !strings.HasPrefix(testCase.dataset, "generated_shortest_paths_v2_") { + return spI1CanonicalCohort{}, fmt.Errorf("frozen SP-I1 cohort contains an invalid declaration") + } + cohort.keys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + item := DeclaredCaseBackend{ + Dataset: key.dataset, + Name: key.name, + Backend: backend, + } + full = append(full, item) + if testCase.split == "training" { + training = append(training, item) + } else if testCase.split == "holdout" { + holdout = append(holdout, item) + } else { + return spI1CanonicalCohort{}, fmt.Errorf("frozen SP-I1 cohort contains an invalid split") + } + } + if testCase.split == "training" { + cohort.trainingKeys[key] = struct{}{} + } else { + cohort.holdoutKeys[key] = struct{}{} + } + } + if len(cohort.trainingKeys) != 4 || len(cohort.holdoutKeys) != 3 || len(cohort.keys) != 7 { + return spI1CanonicalCohort{}, fmt.Errorf("frozen SP-I1 cohort must contain exactly 4 training and 3 holdout cases") + } + cohort.declarationSHA256 = declarationSHA256(full) + cohort.trainingDeclarationSHA256 = declarationSHA256(training) + cohort.holdoutDeclarationSHA256 = declarationSHA256(holdout) + return cohort, nil +} + +// spI1QualificationCaps returns the resource limits enforced for sp i1 qualification. +func spI1QualificationCaps() map[string]int64 { + return map[string]int64{ + "state_limit": 100_000, + "predecessor_limit": 100_000, + "enumeration_limit": 100_000, + "output_bytes_limit": 64 * 1024 * 1024, + } +} + +// spI1TelemetryCaps returns the resource limits enforced for sp i1 telemetry. +func spI1TelemetryCaps() map[string]int64 { + return map[string]int64{ + "state_rows": 100_000, + "predecessor_rows": 100_000, + "output_rows": 100_000, + "output_bytes": 64 * 1024 * 1024, + } +} + +// SPI1QualificationOptions configures spi1 qualification. +type SPI1QualificationOptions struct { + // Seed makes randomized statistical procedures reproducible. + Seed int64 + // Confidence sets the requested statistical confidence level. + Confidence float64 + // BootstrapCount records the number of bootstrap count. + BootstrapCount int + // Protocol identifies the protocol. + Protocol string + // Training evidence paths make confirmation independently recompute the + // discovery decision instead of trusting only a mutable report and freeze. + TrainingBaselinePath string + // TrainingCandidatePath identifies the filesystem training candidate path. + TrainingCandidatePath string + // TrainingResourcePath identifies the filesystem training resource path. + TrainingResourcePath string + // SourceArchiveSHA256 binds the report to git archive HEAD. Report-mode + // callers populate it from the current committed tree; tests may supply a + // synthetic digest without invoking Git. + SourceArchiveSHA256 string + // Freeze supplies the freeze input to the SPI1QualificationOptions contract. + Freeze *SPI1QualificationFreezeManifest + // Discovery supplies the discovery input to the SPI1QualificationOptions contract. + Discovery *SPI1QualificationReport +} + +// SPI1QualificationCase records the evidence and decision for one spi1 qualification workload. +type SPI1QualificationCase struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset"` + // Name identifies the name. + Name string `json:"name"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // Rounds records the number of rounds. + Rounds int `json:"matched_rounds"` + // BaselineSamples supplies the baseline samples input to the SPI1QualificationCase contract. + BaselineSamples int `json:"baseline_samples"` + // CandidateSamples supplies the candidate samples input to the SPI1QualificationCase contract. + CandidateSamples int `json:"candidate_samples"` + // MedianRatio supplies the median ratio input to the SPI1QualificationCase contract. + MedianRatio RatioInterval `json:"median_ratio_to_s4"` + // MedianSaving supplies the median saving input to the SPI1QualificationCase contract. + MedianSaving DurationInterval `json:"median_saving_vs_s4"` + // P95Ratio supplies the p95 ratio input to the SPI1QualificationCase contract. + P95Ratio RatioInterval `json:"p95_ratio_to_s4"` + // Material indicates whether material applies. + Material bool `json:"material"` + // P95Contained indicates whether p95 contained applies. + P95Contained bool `json:"p95_contained"` + // ResourcePassed indicates whether resource passed applies. + ResourcePassed bool `json:"resource_passed"` + // RuntimeBranch supplies the runtime branch input to the SPI1QualificationCase contract. + RuntimeBranch string `json:"runtime_branch"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` +} + +// SPI1QualificationReport records the evidence and outcome produced by spi1 qualification. +type SPI1QualificationReport struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Protocol identifies the protocol. + Protocol string `json:"protocol"` + // Baseline identifies the incumbent execution strategy used for comparison. + Baseline string `json:"baseline"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // Policy identifies the policy. + Policy string `json:"policy"` + // QuerySHA256 binds the referenced query content by SHA-256 digest. + QuerySHA256 string `json:"query_sha256"` + // Seed makes randomized statistical procedures reproducible. + Seed int64 `json:"seed"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // BootstrapCount records the number of bootstrap count. + BootstrapCount int `json:"bootstrap_count"` + // MaterialityRatio supplies the materiality ratio input to the SPI1QualificationReport contract. + MaterialityRatio float64 `json:"materiality_ratio_upper_limit"` + // MaterialityAbsolute supplies the materiality absolute input to the SPI1QualificationReport contract. + MaterialityAbsolute time.Duration `json:"materiality_absolute_lower_limit"` + // P95RatioLimit supplies the p95 ratio limit input to the SPI1QualificationReport contract. + P95RatioLimit float64 `json:"p95_ratio_upper_limit"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // SourceCommit supplies the source commit input to the SPI1QualificationReport contract. + SourceCommit string `json:"source_commit"` + // SourceArchiveSHA256 binds the referenced source archive content by SHA-256 digest. + SourceArchiveSHA256 string `json:"source_archive_sha256"` + // DirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds the referenced corpus content by SHA-256 digest. + CorpusSHA256 string `json:"corpus_sha256"` + // CohortDeclarationSHA256 binds the referenced cohort declaration content by SHA-256 digest. + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + // ResolvedSelectionSHA256 binds the referenced resolved selection content by SHA-256 digest. + ResolvedSelectionSHA256 string `json:"resolved_selection_sha256"` + // TrainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + // HoldoutDeclarationSHA256 binds the referenced holdout declaration content by SHA-256 digest. + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + // FullDeclarationSHA256 binds the referenced full declaration content by SHA-256 digest. + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + // TrainingCorpusSHA256 binds the referenced training corpus content by SHA-256 digest. + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + // FullCorpusSHA256 binds the referenced full corpus content by SHA-256 digest. + FullCorpusSHA256 string `json:"full_corpus_sha256"` + // BaselineArtifactSHA256 binds the referenced baseline artifact content by SHA-256 digest. + BaselineArtifactSHA256 string `json:"baseline_artifact_sha256,omitempty"` + // CandidateArtifactSHA256 binds the referenced candidate artifact content by SHA-256 digest. + CandidateArtifactSHA256 string `json:"candidate_artifact_sha256,omitempty"` + // ResourceReportSHA256 binds the referenced resource report content by SHA-256 digest. + ResourceReportSHA256 string `json:"resource_report_sha256,omitempty"` + // FreezeManifestSHA256 binds the referenced freeze manifest content by SHA-256 digest. + FreezeManifestSHA256 string `json:"freeze_manifest_sha256,omitempty"` + // EvidencePassed indicates whether evidence passed applies. + EvidencePassed bool `json:"evidence_passed"` + // TrainingCases supplies the training cases input to the SPI1QualificationReport contract. + TrainingCases int `json:"training_cases"` + // HoldoutCases supplies the holdout cases input to the SPI1QualificationReport contract. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed indicates whether qualification passed applies. + QualificationPassed bool `json:"qualification_passed"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []SPI1QualificationCase `json:"cases"` +} + +// SPI1QualificationFreezeManifest binds the immutable inputs authorized for spi1 qualification freeze. +type SPI1QualificationFreezeManifest struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Baseline identifies the incumbent execution strategy used for comparison. + Baseline string `json:"baseline"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // Policy identifies the policy. + Policy string `json:"policy"` + // QuerySHA256 binds the referenced query content by SHA-256 digest. + QuerySHA256 string `json:"query_sha256"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // Seed makes randomized statistical procedures reproducible. + Seed int64 `json:"seed"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // BootstrapCount records the number of bootstrap count. + BootstrapCount int `json:"bootstrap_count"` + // SourceCommit supplies the source commit input to the SPI1QualificationFreezeManifest contract. + SourceCommit string `json:"source_commit"` + // SourceArchiveSHA256 binds the referenced source archive content by SHA-256 digest. + SourceArchiveSHA256 string `json:"source_archive_sha256"` + // DirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // TrainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + // HoldoutDeclarationSHA256 binds the referenced holdout declaration content by SHA-256 digest. + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + // FullDeclarationSHA256 binds the referenced full declaration content by SHA-256 digest. + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + // TrainingCorpusSHA256 binds the referenced training corpus content by SHA-256 digest. + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + // FullCorpusSHA256 binds the referenced full corpus content by SHA-256 digest. + FullCorpusSHA256 string `json:"full_corpus_sha256"` + // TrainingResolvedSHA256 binds the referenced training resolved content by SHA-256 digest. + TrainingResolvedSHA256 string `json:"training_resolved_selection_sha256"` + // FullResolvedSHA256 binds the referenced full resolved content by SHA-256 digest. + FullResolvedSHA256 string `json:"full_resolved_selection_sha256"` + // BaselineArtifactSHA256 binds the referenced baseline artifact content by SHA-256 digest. + BaselineArtifactSHA256 string `json:"baseline_artifact_sha256"` + // CandidateArtifactSHA256 binds the referenced candidate artifact content by SHA-256 digest. + CandidateArtifactSHA256 string `json:"candidate_artifact_sha256"` + // ResourceReportSHA256 binds the referenced resource report content by SHA-256 digest. + ResourceReportSHA256 string `json:"resource_report_sha256"` + // DiscoveryReportSHA256 binds the referenced discovery report content by SHA-256 digest. + DiscoveryReportSHA256 string `json:"discovery_report_sha256"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` +} + +// spI1EvidenceIdentity groups state that must remain consistent while processing sp i1 evidence identity. +type spI1EvidenceIdentity struct { + // sourceCommit retains the source commit while spI1EvidenceIdentity is assembled or evaluated. + sourceCommit string + // dirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + dirtyDiffSHA256 string + // binarySHA256 binds the referenced binary content by SHA-256 digest. + binarySHA256 string + // corpusSHA256 binds the referenced corpus content by SHA-256 digest. + corpusSHA256 string + // declarationSHA256 binds the referenced declaration content by SHA-256 digest. + declarationSHA256 string + // resolvedSHA256 binds the referenced resolved content by SHA-256 digest. + resolvedSHA256 string +} + +// sourceArchiveSHA256 supports benchmark evidence processing for source archive sha256. +func sourceArchiveSHA256() (string, error) { + archive, err := exec.Command("git", "archive", "--format=tar", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("archive source commit: %w", err) + } + digest := sha256.Sum256(archive) + return hex.EncodeToString(digest[:]), nil +} + +// equalSPI1Caps returns the resource limits enforced for equal spi1. +func equalSPI1Caps(left, right map[string]int64) bool { + if len(left) != len(right) { + return false + } + for name, value := range left { + if right[name] != value { + return false + } + } + return true +} + +// spI1ProtocolRequirements groups state that must remain consistent while processing sp i1 protocol requirements. +type spI1ProtocolRequirements struct { + // minimumWarmups retains the minimum warmups while spI1ProtocolRequirements is assembled or evaluated. + minimumWarmups int + // minimumRounds records the number of minimum rounds. + minimumRounds int + // maximumRounds records the number of maximum rounds. + maximumRounds int + // minimumSamples retains the minimum samples while spI1ProtocolRequirements is assembled or evaluated. + minimumSamples int + // protectedCount records the number of protected count. + protectedCount int + // protectedSHA retains the protected sha while spI1ProtocolRequirements is assembled or evaluated. + protectedSHA string + // expectedKeys retains the expected keys while spI1ProtocolRequirements is assembled or evaluated. + expectedKeys map[performanceKey]struct{} + // declarationSHA retains the declaration sha while spI1ProtocolRequirements is assembled or evaluated. + declarationSHA string + // corpusSHA retains the corpus sha while spI1ProtocolRequirements is assembled or evaluated. + corpusSHA string + // resolvedSHA retains the resolved sha while spI1ProtocolRequirements is assembled or evaluated. + resolvedSHA string +} + +// spI1QualificationSeries accumulates matched observations used to evaluate sp i1 qualification. +type spI1QualificationSeries struct { + // baseline retains the baseline while spI1QualificationSeries is assembled or evaluated. + baseline roundSamples + // candidate retains the candidate while spI1QualificationSeries is assembled or evaluated. + candidate roundSamples + // runtimeBranch retains the runtime branch while spI1QualificationSeries is assembled or evaluated. + runtimeBranch string + // resourcePassed indicates whether resource passed applies. + resourcePassed bool +} + +// spI1Requirements supports benchmark evidence processing for sp i1 requirements. +func spI1Requirements(protocol string, cohort spI1CanonicalCohort) (spI1ProtocolRequirements, error) { + switch protocol { + case referencePairProtocolDiscovery: + return spI1ProtocolRequirements{ + minimumWarmups: 5, + minimumRounds: 5, + maximumRounds: 20, + minimumSamples: 10, + protectedCount: 2 * len(cohort.holdoutKeys), + protectedSHA: cohort.holdoutDeclarationSHA256, + expectedKeys: cohort.trainingKeys, + declarationSHA: cohort.trainingDeclarationSHA256, + corpusSHA: cohort.trainingCorpusSHA256, + resolvedSHA: cohort.trainingResolvedSHA256, + }, nil + case referencePairProtocolConfirmation: + return spI1ProtocolRequirements{ + minimumWarmups: 20, + minimumRounds: 10, + maximumRounds: 20, + minimumSamples: 50, + expectedKeys: cohort.keys, + declarationSHA: cohort.declarationSHA256, + corpusSHA: cohort.fullCorpusSHA256, + resolvedSHA: cohort.fullResolvedSHA256, + }, nil + default: + return spI1ProtocolRequirements{}, fmt.Errorf("unsupported SP-I1 qualification protocol %q", protocol) + } +} + +// buildSPI1QualificationReport builds spi1 qualification report. +func buildSPI1QualificationReport( + baseline, candidate []CaseResult, + resource ResourceGateReport, + options SPI1QualificationOptions, +) (SPI1QualificationReport, error) { + if options.Confidence != defaultConfidenceLevel || math.IsNaN(options.Confidence) || math.IsInf(options.Confidence, 0) { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 qualification confidence must be the frozen %.4f", defaultConfidenceLevel) + } + if options.Seed != 1 { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 qualification bootstrap seed must be the frozen value 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount != defaultBootstrapCount { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 qualification bootstrap count must be the frozen value %d", defaultBootstrapCount) + } + if options.Protocol == "" { + options.Protocol = referencePairProtocolConfirmation + } + if !lowercaseSHA256(options.SourceArchiveSHA256) { + return SPI1QualificationReport{}, fmt.Errorf("SP-I1 source archive digest is missing or malformed") + } + + cohort, err := canonicalSPI1Cohort() + if err != nil { + return SPI1QualificationReport{}, err + } + requirements, err := spI1Requirements(options.Protocol, cohort) + if err != nil { + return SPI1QualificationReport{}, err + } + identity, err := validateSPI1EvidenceIdentity(baseline, candidate, requirements) + if err != nil { + return SPI1QualificationReport{}, err + } + series, keys, err := collectSPI1QualificationSeries(baseline, candidate, resource, requirements) + if err != nil { + return SPI1QualificationReport{}, err + } + + report := SPI1QualificationReport{ + Version: spI1QualificationVersion, + Protocol: options.Protocol, + Baseline: string(optimize.ShortestPathExecutorS4CanonicalWitness), + Candidate: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Policy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + QuerySHA256: spI1QuerySHA256, + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + MaterialityRatio: 0.95, + MaterialityAbsolute: 100 * time.Microsecond, + P95RatioLimit: 1.05, + Caps: spI1QualificationCaps(), + SourceCommit: identity.sourceCommit, + SourceArchiveSHA256: options.SourceArchiveSHA256, + DirtyDiffSHA256: identity.dirtyDiffSHA256, + BinarySHA256: identity.binarySHA256, + CorpusSHA256: identity.corpusSHA256, + CohortDeclarationSHA256: identity.declarationSHA256, + ResolvedSelectionSHA256: identity.resolvedSHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + EvidencePassed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + if options.Protocol == referencePairProtocolConfirmation { + if err := validateSPI1Freeze(options.Freeze, options.Discovery, report, cohort); err != nil { + return SPI1QualificationReport{}, err + } + } + + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for index, key := range keys { + current := series[key] + baselineRounds, candidateRounds := matchedRounds(current.baseline, current.candidate) + if !slices.Equal(sortedRounds(current.baseline), sortedRounds(current.candidate)) || + len(baselineRounds) != len(current.baseline) || len(candidateRounds) != len(current.candidate) { + return SPI1QualificationReport{}, fmt.Errorf("%s/%s SP-I1 arms do not contain identical nonempty round sets", key.dataset, key.name) + } + rounds := sortedRounds(baselineRounds) + if len(rounds) < requirements.minimumRounds || len(rounds) > requirements.maximumRounds { + return SPI1QualificationReport{}, fmt.Errorf( + "%s/%s requires %d-%d matched SP-I1 rounds, got %d", + key.dataset, key.name, requirements.minimumRounds, requirements.maximumRounds, len(rounds), + ) + } + for _, round := range rounds { + if len(baselineRounds[round]) < requirements.minimumSamples || len(candidateRounds[round]) < requirements.minimumSamples { + return SPI1QualificationReport{}, fmt.Errorf( + "%s/%s round %d requires at least %d warm samples per SP-I1 arm, got %d/%d", + key.dataset, key.name, round, requirements.minimumSamples, + len(baselineRounds[round]), len(candidateRounds[round]), + ) + } + } + if err := validatePairedOrderEvidence(baseline, candidate, key, rounds, requirements.minimumWarmups); err != nil { + return SPI1QualificationReport{}, fmt.Errorf("invalid SP-I1 paired evidence: %w", err) + } + + split := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + split = "holdout" + } + seed := options.Seed + int64(index)*7919 + gateCase := SPI1QualificationCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + Rounds: len(rounds), + BaselineSamples: sampleCount(baselineRounds), + CandidateSamples: sampleCount(candidateRounds), + MedianRatio: bootstrapRoundMedianRatio(baselineRounds, candidateRounds, seed, gateOptions), + MedianSaving: bootstrapRoundMedianSaving(baselineRounds, candidateRounds, seed+1, gateOptions), + P95Ratio: bootstrapStratifiedP95Ratio(baselineRounds, candidateRounds, seed+2, gateOptions), + ResourcePassed: current.resourcePassed, + RuntimeBranch: current.runtimeBranch, + Passed: true, + } + gateCase.Material = gateCase.MedianRatio.Upper <= report.MaterialityRatio || + gateCase.MedianSaving.Lower >= report.MaterialityAbsolute + gateCase.P95Contained = gateCase.P95Ratio.Upper <= report.P95RatioLimit + if !gateCase.Material { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "median improvement is not material: ratio upper %.4f > %.4f and saving lower %s < %s", + gateCase.MedianRatio.Upper, report.MaterialityRatio, + gateCase.MedianSaving.Lower, report.MaterialityAbsolute, + )) + } + if !gateCase.P95Contained { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "p95 ratio upper %.4f exceeds %.4f", gateCase.P95Ratio.Upper, report.P95RatioLimit, + )) + } + if !gateCase.ResourcePassed { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, "candidate resource evidence did not pass") + } + + switch split { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && gateCase.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && gateCase.Passed + } + report.Cases = append(report.Cases, gateCase) + } + report.TrainingPassed = report.TrainingPassed && report.TrainingCases == len(cohort.trainingKeys) + report.HoldoutPassed = report.HoldoutPassed && report.HoldoutCases == len(cohort.holdoutKeys) + if options.Protocol == referencePairProtocolDiscovery { + report.HoldoutPassed = false + } + report.QualificationPassed = report.EvidencePassed && report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +// validateSPI1EvidenceIdentity validates spi1 evidence identity. +func validateSPI1EvidenceIdentity( + baseline, candidate []CaseResult, + requirements spI1ProtocolRequirements, +) (spI1EvidenceIdentity, error) { + if err := validatePerformanceWorkloadIdentity(baseline, candidate); err != nil { + return spI1EvidenceIdentity{}, err + } + baselineHost, err := artifactHostFingerprint(baseline) + if err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 baseline host: %w", err) + } + candidateHost, err := artifactHostFingerprint(candidate) + if err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 candidate host: %w", err) + } + if baselineHost != candidateHost { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 baseline and candidate host identities differ") + } + + identity := spI1EvidenceIdentity{} + for _, artifact := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // records retains the records while anonymous record is assembled or evaluated. + records []CaseResult + }{ + { + name: "baseline", + records: baseline, + }, + { + name: "candidate", + records: candidate, + }, + } { + selection, err := selectionIdentity(artifact.records) + if err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 %s selection: %w", artifact.name, err) + } + if err := validateSPI1Selection(selection, requirements); err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 %s selection: %w", artifact.name, err) + } + currentIdentity := spI1EvidenceIdentity{ + declarationSHA256: selection.DeclarationSHA256, + resolvedSHA256: resolvedSelectionSHA256(selection.Resolved), + } + for _, record := range artifact.records { + if record.Environment == nil || record.PostgresEnvironment == nil { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm lacks source or PostgreSQL environment identity", record.Dataset, record.Name, artifact.name) + } + current := spI1EvidenceIdentity{ + sourceCommit: strings.TrimSpace(record.Environment.SourceCommit), + dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, + binarySHA256: record.Environment.BinarySHA256, + corpusSHA256: record.Environment.CorpusSHA256, + declarationSHA256: selection.DeclarationSHA256, + resolvedSHA256: currentIdentity.resolvedSHA256, + } + if current.sourceCommit == "" || current.sourceCommit == "unknown" || + !lowercaseSHA256(current.dirtyDiffSHA256) || !lowercaseSHA256(current.binarySHA256) || + !lowercaseSHA256(current.corpusSHA256) { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm lacks frozen source, diff, binary, or corpus identity", record.Dataset, record.Name, artifact.name) + } + if current.corpusSHA256 != requirements.corpusSHA { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm corpus digest is not the exact frozen SP-I1 cohort", record.Dataset, record.Name, artifact.name) + } + if identity.sourceCommit == "" { + identity = current + } else if identity != current { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 artifacts mix source, diff, binary, corpus, declaration, or selection identities") + } + } + } + if identity.declarationSHA256 != requirements.declarationSHA || identity.resolvedSHA256 != requirements.resolvedSHA { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 artifacts do not bind the exact frozen declaration and resolved selection") + } + for key := range requirements.expectedKeys { + baselinePostgres, err := postgresTimingEnvironmentSHA256ForKey(baseline, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + candidatePostgres, err := postgresTimingEnvironmentSHA256ForKey(candidate, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + baselineFixture, err := fixtureSHA256ForKey(baseline, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + candidateFixture, err := fixtureSHA256ForKey(candidate, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + if !lowercaseSHA256(baselinePostgres) || baselinePostgres != candidatePostgres { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I1 PostgreSQL timing environments differ between arms", key.dataset, key.name) + } + if !lowercaseSHA256(baselineFixture) || baselineFixture != candidateFixture { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I1 fixture identities differ between arms", key.dataset, key.name) + } + baselineSQL, err := spI1SQLFingerprintForKey(baseline, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + candidateSQL, err := spI1SQLFingerprintForKey(candidate, key) + if err != nil { + return spI1EvidenceIdentity{}, err + } + if baselineSQL == candidateSQL { + return spI1EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I1 arms use the same SQL fingerprint", key.dataset, key.name) + } + if err := validateOrientationExactObservations(key, baseline, candidate); err != nil { + return spI1EvidenceIdentity{}, fmt.Errorf("SP-I1 exact observations: %w", err) + } + } + return identity, nil +} + +// spI1SQLFingerprintForKey derives the lookup key used for sp i1sql fingerprint for. +func spI1SQLFingerprintForKey(records []CaseResult, key performanceKey) (string, error) { + fingerprint := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if fingerprint != "" && fingerprint != record.SQLFingerprint { + return "", fmt.Errorf("%s/%s changes SQL fingerprint within one SP-I1 arm", key.dataset, key.name) + } + fingerprint = record.SQLFingerprint + } + if !lowercaseSHA256(fingerprint) { + return "", fmt.Errorf("%s/%s lacks one stable SP-I1 SQL fingerprint", key.dataset, key.name) + } + return fingerprint, nil +} + +// validateSPI1Selection validates spi1 selection. +func validateSPI1Selection(selection SelectionManifest, requirements spI1ProtocolRequirements) error { + if selection.Version != selectionManifestVersion || !selection.DiagnosticOnly || + selection.SelectedDeclarationCount != 2*len(requirements.expectedKeys) || + selection.FullDeclarationCount != selection.SelectedDeclarationCount+selection.OmittedDeclarationCount || + selection.ProtectedDeclarationCount != requirements.protectedCount || + selection.ProtectedDeclarationSHA256 != requirements.protectedSHA || + len(selection.Resolved) != len(requirements.expectedKeys) || + selection.DeclarationSHA256 != requirements.declarationSHA || + resolvedSelectionSHA256(selection.Resolved) != requirements.resolvedSHA { + return fmt.Errorf("selection manifest does not bind the exact frozen cohort") + } + resolved := make(map[performanceKey]struct{}, len(selection.Resolved)) + for _, item := range selection.Resolved { + if item.Category != "generated_shortest_path_v2" { + return fmt.Errorf("selection contains non-SP-I1 category %q", item.Category) + } + key := performanceKey{ + dataset: item.Dataset, + name: item.Name, + backend: ModePostgresSQL, + } + if _, duplicate := resolved[key]; duplicate { + return fmt.Errorf("selection contains duplicate %s/%s", item.Dataset, item.Name) + } + resolved[key] = struct{}{} + } + if !orientationV2KeySetsEqual(resolved, requirements.expectedKeys) { + return fmt.Errorf("selection does not contain the exact frozen SP-I1 cases") + } + return nil +} + +// collectSPI1QualificationSeries collects spi1 qualification series. +func collectSPI1QualificationSeries( + baseline, candidate []CaseResult, + resource ResourceGateReport, + requirements spI1ProtocolRequirements, +) (map[performanceKey]*spI1QualificationSeries, []performanceKey, error) { + if err := validateSPI1GlobalInvocationIDs(baseline, candidate); err != nil { + return nil, nil, err + } + declarations, err := canonicalSPI1Declarations() + if err != nil { + return nil, nil, err + } + baselineKeys, baselineRounds, err := collectSPI1Artifact("baseline", baseline, requirements, declarations) + if err != nil { + return nil, nil, err + } + candidateKeys, candidateRounds, err := collectSPI1Artifact("candidate", candidate, requirements, declarations) + if err != nil { + return nil, nil, err + } + if !orientationV2KeySetsEqual(baselineKeys, requirements.expectedKeys) || + !orientationV2KeySetsEqual(candidateKeys, requirements.expectedKeys) { + return nil, nil, fmt.Errorf("SP-I1 artifacts do not contain the exact protocol cohort") + } + if err := validateSPI1RunSchedule(baseline, candidate, requirements); err != nil { + return nil, nil, err + } + resourcePassed, err := validateSPI1ResourceCases(resource, candidate, requirements) + if err != nil { + return nil, nil, err + } + + series := make(map[performanceKey]*spI1QualificationSeries, len(requirements.expectedKeys)) + for key := range requirements.expectedKeys { + current := &spI1QualificationSeries{ + baseline: roundSamples{}, + candidate: roundSamples{}, + resourcePassed: resourcePassed[key], + } + series[key] = current + for round, record := range baselineRounds[key] { + appendSPI1WarmSamples(current.baseline, round, record) + } + for round, record := range candidateRounds[key] { + appendSPI1WarmSamples(current.candidate, round, record) + branch := record.TraversalTelemetry.Summary.RuntimeBranch + if current.runtimeBranch != "" && current.runtimeBranch != branch { + return nil, nil, fmt.Errorf("%s/%s changes SP-I1 runtime branch across rounds", key.dataset, key.name) + } + current.runtimeBranch = branch + } + if current.runtimeBranch == "" { + return nil, nil, fmt.Errorf("%s/%s has no attributable SP-I1 candidate runtime", key.dataset, key.name) + } + } + return series, sortedPerformanceKeys(requirements.expectedKeys), nil +} + +// validateSPI1GlobalInvocationIDs prevents one genuine timed receipt from +// being copied into another case, round, or arm. The attestor emits globally +// unique invocation IDs, so the complete paired study must not reuse one. +func validateSPI1GlobalInvocationIDs(artifacts ...[]CaseResult) error { + seen := map[string]struct{}{} + for _, records := range artifacts { + for _, record := range records { + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" { + continue + } + invocationID := strings.TrimSpace(sample.RuntimeInvocationID) + if invocationID == "" { + return fmt.Errorf("%s/%s warm sample lacks a global timed invocation identity", record.Dataset, record.Name) + } + if _, duplicate := seen[invocationID]; duplicate { + return fmt.Errorf("SP-I1 evidence reuses timed invocation identity %q across the paired study", invocationID) + } + seen[invocationID] = struct{}{} + } + } + } + return nil +} + +// spI1InvocationIdentity binds one timed sample to its scheduled run and arm. +type spI1InvocationIdentity struct { + // round identifies the paired benchmark round. + round int + + // block identifies the order-balancing block containing the round. + block int + + // order retains the order while spI1InvocationIdentity is assembled or evaluated. + order int + + // arm identifies the baseline or candidate treatment. + arm string + + // runUUID binds the sample to one benchmark process invocation. + runUUID string + + // startedAt records when timed execution began. + startedAt time.Time + + // endedAt records when timed execution completed. + endedAt time.Time +} + +// validateSPI1RunSchedule validates spi1 run schedule. +func validateSPI1RunSchedule(baseline, candidate []CaseResult, requirements spI1ProtocolRequirements) error { + collect := func(arm string, records []CaseResult) (map[int]spI1InvocationIdentity, error) { + invocations := map[int]spI1InvocationIdentity{} + caseCounts := map[int]int{} + for _, record := range records { + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s %s arm lacks invocation chronology", record.Dataset, record.Name, arm) + } + environment := record.Environment + identity := spI1InvocationIdentity{ + round: environment.Round, + block: environment.Block, + order: environment.ArmOrder, + arm: environment.Arm, + runUUID: environment.RunUUID, + startedAt: environment.StartedAt, + endedAt: environment.EndedAt, + } + if identity.startedAt.IsZero() || identity.endedAt.IsZero() || identity.endedAt.Before(identity.startedAt) { + return nil, fmt.Errorf("SP-I1 %s round %d has malformed invocation timestamps", arm, identity.round) + } + if prior, found := invocations[identity.round]; found && prior != identity { + return nil, fmt.Errorf("SP-I1 %s round %d mixes invocation identities", arm, identity.round) + } + invocations[identity.round] = identity + caseCounts[identity.round]++ + } + for round, count := range caseCounts { + if count != len(requirements.expectedKeys) { + return nil, fmt.Errorf("SP-I1 %s round %d contains %d cases, expected %d", arm, round, count, len(requirements.expectedKeys)) + } + } + return invocations, nil + } + left, err := collect("baseline", baseline) + if err != nil { + return err + } + right, err := collect("candidate", candidate) + if err != nil { + return err + } + if len(left) != len(right) || len(left) < requirements.minimumRounds || len(left) > requirements.maximumRounds { + return fmt.Errorf("SP-I1 artifacts do not contain one complete paired invocation schedule") + } + runUUID := "" + var priorEnded time.Time + for round := 1; round <= len(left); round++ { + baselineInvocation, baselineFound := left[round] + candidateInvocation, candidateFound := right[round] + if !baselineFound || !candidateFound { + return fmt.Errorf("SP-I1 invocation schedule must use contiguous rounds starting at 1") + } + expectedBaselineOrder, expectedCandidateOrder := 1, 2 + if round%2 == 0 { + expectedBaselineOrder, expectedCandidateOrder = 2, 1 + } + if baselineInvocation.block != round || candidateInvocation.block != round || + baselineInvocation.arm != "sp-i1-s4" || candidateInvocation.arm != "sp-i1-candidate" || + baselineInvocation.order != expectedBaselineOrder || candidateInvocation.order != expectedCandidateOrder || + baselineInvocation.runUUID == "" || baselineInvocation.runUUID != candidateInvocation.runUUID { + return fmt.Errorf("SP-I1 round %d does not match the frozen alternating two-arm schedule", round) + } + if runUUID == "" { + runUUID = baselineInvocation.runUUID + } else if runUUID != baselineInvocation.runUUID { + return fmt.Errorf("SP-I1 artifacts mix run UUIDs across rounds") + } + first, second := baselineInvocation, candidateInvocation + if candidateInvocation.order == 1 { + first, second = candidateInvocation, baselineInvocation + } + if first.endedAt.After(second.startedAt) { + return fmt.Errorf("SP-I1 round %d arm timestamps contradict the declared execution order", round) + } + if !priorEnded.IsZero() && priorEnded.After(first.startedAt) { + return fmt.Errorf("SP-I1 round %d overlaps or predates the prior round", round) + } + priorEnded = second.endedAt + } + return nil +} + +// collectSPI1Artifact collects spi1 artifact. +func collectSPI1Artifact( + arm string, + records []CaseResult, + requirements spI1ProtocolRequirements, + declarations map[performanceKey]spI1CanonicalDeclaration, +) (map[performanceKey]struct{}, map[performanceKey]map[int]CaseResult, error) { + if len(records) == 0 { + return nil, nil, fmt.Errorf("SP-I1 %s artifact is empty", arm) + } + keys := map[performanceKey]struct{}{} + rounds := map[performanceKey]map[int]CaseResult{} + for _, record := range records { + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + if _, expected := requirements.expectedKeys[key]; !expected { + return nil, nil, fmt.Errorf("SP-I1 %s artifact contains unexpected case %s/%s", arm, key.dataset, key.name) + } + declaration, found := declarations[key] + if !found { + return nil, nil, fmt.Errorf("SP-I1 %s artifact has no frozen declaration for %s/%s", arm, key.dataset, key.name) + } + if err := validateSPI1Record(record, arm, declaration); err != nil { + return nil, nil, err + } + round, err := orientationV2RecordRound(record) + if err != nil { + return nil, nil, err + } + if rounds[key] == nil { + rounds[key] = map[int]CaseResult{} + } + if _, duplicate := rounds[key][round]; duplicate { + return nil, nil, fmt.Errorf("%s/%s %s artifact duplicates round %d", key.dataset, key.name, arm, round) + } + rounds[key][round] = record + keys[key] = struct{}{} + } + return keys, rounds, nil +} + +// appendSPI1WarmSamples appends spi1 warm samples. +func appendSPI1WarmSamples(series roundSamples, round int, record CaseResult) { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[round] = append(series[round], sample.Duration) + } + } +} + +// validateSPI1Record validates spi1 record. +func validateSPI1Record(record CaseResult, arm string, declaration spI1CanonicalDeclaration) error { + if record.ExecutionMode != ModePostgresSQL || record.Status != StatusOK || + record.Environment == nil || record.PostgresEnvironment == nil || record.Fixture == nil || + record.TraversalTelemetry == nil || record.Optimization == nil || record.PostgresMetrics == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing PostgreSQL record", record.Dataset, record.Name, arm) + } + if record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || + len(record.Environment.Concurrency) != 0 || record.Environment.ExistingGraph || + record.Environment.Protocol != "fixed_confirmation" { + return fmt.Errorf("%s/%s %s arm lacks the schema-v2 single-session fixed-confirmation contract", record.Dataset, record.Name, arm) + } + if record.Fixture.Dataset != record.Dataset || !lowercaseSHA256(record.Fixture.Checksum) || + !record.Fixture.PhysicalValidated || record.Fixture.PhysicalNodeCount != int64(record.Fixture.NodeCount) || + record.Fixture.PhysicalEdgeCount != int64(record.Fixture.EdgeCount) || + record.Fixture.Checksum != declaration.fixture.Checksum || + record.Fixture.NodeCount != declaration.fixture.NodeCount || record.Fixture.EdgeCount != declaration.fixture.EdgeCount || + record.Fixture.Configuration != declaration.fixture.Configuration || + !reflect.DeepEqual(record.Fixture.Shortest, declaration.fixture.Shortest) || + record.Fixture.NodeRelationBytes <= 0 || record.Fixture.EdgeRelationBytes <= 0 { + return fmt.Errorf("%s/%s %s arm lacks one exact physically validated fixture", record.Dataset, record.Name, arm) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") { + return fmt.Errorf("%s/%s %s arm was not measured under Repeatable Read", record.Dataset, record.Name, arm) + } + testCase := declaration.testCase + testCase.Source = record.Source + expectedRecord := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expectedRecord, *record.Fixture) + if filepath.Base(record.Source) != "generated_sp_i1_inbound_v1.json" || + record.Category != testCase.Category || record.Cypher != testCase.Cypher || sqlFingerprint(record.Cypher) != spI1QuerySHA256 || + !lowercaseSHA256(record.WorkloadSHA256) || !lowercaseSHA256(record.SQLFingerprint) || + record.WorkloadSHA256 != expectedRecord.WorkloadSHA256 || + record.SQL == "" || sqlFingerprint(record.SQL) != record.SQLFingerprint || + !reflect.DeepEqual(record.NodeParams, testCase.NodeParams) || + !reflect.DeepEqual(record.NodeListParams, testCase.NodeListParams) || + !reflect.DeepEqual(record.Shape, testCase.Shape) { + return fmt.Errorf("%s/%s %s arm lacks the frozen inbound SP-I1 workload identity", record.Dataset, record.Name, arm) + } + minimumDepth, maximumDepth := 0, 0 + if record.Shape.MinDepth != nil { + minimumDepth = *record.Shape.MinDepth + } + if record.Shape.MaxDepth != nil { + maximumDepth = *record.Shape.MaxDepth + } + if record.Shape.QualificationSplit != "training" && record.Shape.QualificationSplit != "holdout" || + record.Shape.FallbackExpectation != "forbidden" || record.Shape.Direction != "inbound" || + record.Shape.RelationshipKindCount != 1 || !slices.Equal(record.Shape.EdgeKinds, []string{"Traverse"}) || + minimumDepth != 1 || maximumDepth != 64 || !record.Shape.PathMaterializationRequired { + return fmt.Errorf("%s/%s %s arm changes the frozen inbound one-path shape", record.Dataset, record.Name, arm) + } + expectedSplit := testCase.Shape.QualificationSplit + if record.Shape.QualificationSplit != expectedSplit { + return fmt.Errorf("%s/%s %s arm changes the frozen qualification split", record.Dataset, record.Name, arm) + } + expectedRows := *testCase.Expected.RowCount + if !record.StableObservation || record.RowCount != expectedRows || record.ExpectedRowCount == nil || + *record.ExpectedRowCount != expectedRows { + return fmt.Errorf("%s/%s %s arm lacks the exact stable path observation contract", record.Dataset, record.Name, arm) + } + if err := validateExpectedObservations(testCase.Expected, record.ObservedRows); err != nil { + return fmt.Errorf("%s/%s %s arm changes the frozen path observation: %w", record.Dataset, record.Name, arm, err) + } + if len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil || record.Baseline != nil { + return fmt.Errorf("%s/%s %s arm mixes SP-I1 timing with supplemental measurements", record.Dataset, record.Name, arm) + } + if err := ValidateTraversalExecutionTelemetry(record.TraversalTelemetry); err != nil { + return fmt.Errorf("%s/%s %s arm telemetry: %w", record.Dataset, record.Name, arm, err) + } + if err := validateSPI1Runtime(record, arm); err != nil { + return err + } + return nil +} + +// validateSPI1Runtime validates spi1 runtime. +func validateSPI1Runtime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || + summary.Overflow == nil || summary.FallbackExecuted == nil || *summary.Overflow || *summary.FallbackExecuted || + summary.WouldSelectIdentity != "" || summary.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + summary.SchedulerVersion != string(optimize.ShortestPathSchedulerSingleEndedLevel) { + return fmt.Errorf("%s/%s %s arm lacks one non-fallback one-path runtime outcome", record.Dataset, record.Name, arm) + } + outcome, ok := singleTraversalOutcome(record.Optimization.TargetOutcomes) + if !ok || outcome.Family != "SP" { + return fmt.Errorf("%s/%s %s arm lacks one exact SP lowering outcome", record.Dataset, record.Name, arm) + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + plannedIdentities := spI1ShortestPathPlannedIdentities() + outcomeDepthsExact := outcome.MinimumDepth != nil && *outcome.MinimumDepth == 1 && + outcome.MaximumDepth != nil && *outcome.MaximumDepth == 64 + outcomeShapeExact := outcome.Lowering == optimize.LoweringShortestPathExecutor && outcome.TargetKind == "traversal" && + outcome.ObservationMode == string(optimize.ShortestPathObservationOnePath) && outcome.Direction == "inbound" && + outcome.PhysicalExpansion == "end_id" && outcome.RelationshipKindCount == 1 && !outcome.UntypedRelationship && + outcome.TopologyClassification == "physical_inbound_deep" && outcome.SelectionMode == "forced_tool" && + outcome.Scheduler == string(optimize.ShortestPathSchedulerSingleEndedLevel) && outcomeDepthsExact && + outcome.Eligible != nil && *outcome.Eligible && outcome.StaticallyEligible != nil && *outcome.StaticallyEligible + if !outcomeShapeExact { + return fmt.Errorf("%s/%s %s arm changes the frozen SP-I1 lowering shape", record.Dataset, record.Name, arm) + } + switch arm { + case "baseline": + if summary.RequestedIdentity != baseline || summary.EmittedIdentity != baseline || + summary.RuntimeIdentity != baseline || summary.AppliedIdentity != baseline || + !slices.Equal(summary.PlannedIdentities, plannedIdentities) || + summary.SelectorVersion != "sp-tool-v1" || + summary.ExecutionBoundary != optimize.ShortestPathExecutorS4CanonicalWitness.ExecutionBoundary() || + summary.RuntimeBranch != "selected" || + outcome.Candidate != "" || outcome.Selected != baseline || outcome.Applied != baseline || outcome.Fallback != "SP-S0" || + !slices.Equal(outcome.PlannedCandidates, plannedIdentities) || + outcome.ExecutionBoundary != "stored_helper" || outcome.SelectorVersion != "sp-tool-v1" || + outcome.EmittedPolicy != "" || len(outcome.EmittedCandidates) != 0 || + outcome.StateLimit != 100_000 || outcome.FrontierLimit != 100_000 || outcome.PredecessorLimit != 100_000 || + outcome.EnumerationLimit != 100_000 || outcome.OutputBytesLimit != 64*1024*1024 { + return fmt.Errorf("%s/%s baseline arm did not execute exact forced S4", record.Dataset, record.Name) + } + case "candidate": + expectedBranch := "inline_canonical_witness" + if record.RowCount == 0 { + expectedBranch = "inline_canonical_no_path" + } + if summary.RequestedIdentity != candidate || summary.EmittedIdentity != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || + summary.RuntimeIdentity != candidate || summary.AppliedIdentity != candidate || + !slices.Equal(summary.PlannedIdentities, plannedIdentities) || + summary.SelectorVersion != "sp-i1-canonical-tool-v1" || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + summary.RuntimeBranch != expectedBranch || + !equalSPI1Caps(summary.Caps, spI1TelemetryCaps()) || + !slices.Contains(summary.PlannedIdentities, baseline) || !slices.Contains(summary.PlannedIdentities, candidate) || + outcome.Candidate != candidate || outcome.Selected != candidate || outcome.Applied != candidate || + outcome.Fallback != baseline || outcome.EmittedPolicy != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || + !slices.Equal(outcome.PlannedCandidates, plannedIdentities) || + !slices.Equal(outcome.EmittedCandidates, []string{candidate, baseline}) || + outcome.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + outcome.SelectorVersion != "sp-i1-canonical-tool-v1" || + outcome.StateLimit != spI1QualificationCaps()["state_limit"] || + outcome.PredecessorLimit != spI1QualificationCaps()["predecessor_limit"] || + outcome.EnumerationLimit != spI1QualificationCaps()["enumeration_limit"] || + outcome.OutputBytesLimit != spI1QualificationCaps()["output_bytes_limit"] || outcome.FrontierLimit != 0 { + return fmt.Errorf("%s/%s candidate arm did not execute exact guarded canonical I1", record.Dataset, record.Name) + } + diagnostic := record.TraversalTelemetry.Diagnostic + if record.TraversalTelemetry.Level != TraversalTelemetryLevelDiagnostic || diagnostic == nil || + diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete || diagnostic.Counters.InlineShortestPath == nil || + !slices.Contains(diagnostic.RequiredFamilies, TraversalTelemetryFamilySP) || + !slices.Contains(diagnostic.RequiredFamilies, TraversalTelemetryFamilyHydration) { + return fmt.Errorf("%s/%s candidate arm lacks complete typed canonical-I1 resource telemetry", record.Dataset, record.Name) + } + inline := diagnostic.Counters.InlineShortestPath + outputRows, outputPresent := int64(0), false + if diagnostic.PlanReplay != nil { + outputRows, outputPresent = diagnostic.PlanReplay.Counters["asp_i1_output_rows"] + } + if inline.OutputPaths == nil || *inline.OutputPaths != record.RowCount || !outputPresent || outputRows != record.RowCount { + return fmt.Errorf("%s/%s candidate arm runtime branch does not bind the exact output observation", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown SP-I1 arm %q", arm) + } + if err := validateSPI1SampleRuntime(record, arm); err != nil { + return err + } + return nil +} + +// spI1ShortestPathPlannedIdentities mirrors the optimizer's complete SP search +// space. Planned candidates describe every executor considered by lowering; +// emitted candidates and the runtime receipt separately attest the exact +// guarded two-arm statement that executed. +func spI1ShortestPathPlannedIdentities() []string { + return []string{ + string(optimize.ShortestPathExecutorIncumbentWorkspace), + string(optimize.ShortestPathExecutorS0Direct), + string(optimize.ShortestPathExecutorS1ArrayBFS), + string(optimize.ShortestPathExecutorS2TraceRelation), + string(optimize.ShortestPathExecutorS3Unidirectional), + string(optimize.ShortestPathExecutorS3EdgeM0), + string(optimize.ShortestPathExecutorS4CanonicalDistance), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalDistance), + string(optimize.ShortestPathExecutorI1CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorB1AlternatingNodeDistance), + string(optimize.ShortestPathExecutorB1AlternatingNodeWitness), + string(optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance), + string(optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness), + } +} + +// validateSPI1SampleRuntime validates spi1 sample runtime. +func validateSPI1SampleRuntime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + if record.Environment == nil || record.Stats.Iterations < 1 || record.Stats.WarmupIterations != record.Environment.WarmupIterations || + record.Stats.Median <= 0 || record.Stats.P95 <= 0 { + return fmt.Errorf("%s/%s %s arm has malformed iteration or warmup evidence", record.Dataset, record.Name, arm) + } + expectedArm := "sp-i1-s4" + if arm == "candidate" { + expectedArm = "sp-i1-candidate" + } + if record.Environment.Arm != expectedArm || record.Environment.Round < 1 || record.Environment.Block != record.Environment.Round || + record.Environment.ArmOrder < 1 || record.Environment.ArmOrder > 2 || strings.TrimSpace(record.Environment.RunUUID) == "" { + return fmt.Errorf("%s/%s %s arm has malformed frozen run metadata", record.Dataset, record.Name, arm) + } + warmSamples, coldSamples := 0, 0 + iterations := map[int]struct{}{} + invocations := map[string]struct{}{} + for _, sample := range record.Stats.Samples { + if sample.Duration <= 0 || sample.Dataset != record.Dataset || sample.Case != record.Name || sample.Backend != ModePostgresSQL || + sample.Round != record.Environment.Round || sample.Block != record.Environment.Block || sample.Arm != record.Environment.Arm || + sample.ArmOrder != record.Environment.ArmOrder || sample.RunUUID != record.Environment.RunUUID || strings.TrimSpace(sample.ConnectionID) == "" { + return fmt.Errorf("%s/%s %s arm has a sample outside its frozen invocation identity", record.Dataset, record.Name, arm) + } + switch sample.Classification { + case "cold": + if sample.Iteration != 0 { + return fmt.Errorf("%s/%s %s arm cold sample has a nonzero iteration", record.Dataset, record.Name, arm) + } + coldSamples++ + continue + case "warm": + default: + return fmt.Errorf("%s/%s %s arm contains an unexpected sample classification", record.Dataset, record.Name, arm) + } + warmSamples++ + if sample.Iteration < 1 || sample.Iteration > record.Stats.Iterations { + return fmt.Errorf("%s/%s %s arm has an out-of-range warm iteration", record.Dataset, record.Name, arm) + } + if _, duplicate := iterations[sample.Iteration]; duplicate { + return fmt.Errorf("%s/%s %s arm duplicates warm iteration %d", record.Dataset, record.Name, arm, sample.Iteration) + } + iterations[sample.Iteration] = struct{}{} + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || + sample.FallbackExecuted == nil || *sample.FallbackExecuted != *summary.FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample contradicts its runtime summary", record.Dataset, record.Name, arm) + } + if sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s arm warm sample lacks timed-invocation attribution", record.Dataset, record.Name, arm) + } + if strings.TrimSpace(sample.RuntimeInvocationID) == "" { + return fmt.Errorf("%s/%s %s arm warm sample lacks a timed invocation identity", record.Dataset, record.Name, arm) + } + if _, duplicate := invocations[sample.RuntimeInvocationID]; duplicate { + return fmt.Errorf("%s/%s %s arm reuses timed invocation identity %q", record.Dataset, record.Name, arm, sample.RuntimeInvocationID) + } + invocations[sample.RuntimeInvocationID] = struct{}{} + expectedBranch := summary.RuntimeBranch + if arm == "baseline" { + expectedBranch = "compact_workspace_witness" + if record.RowCount == 0 { + expectedBranch = "compact_no_path" + } + } + if sample.RuntimeBranch != expectedBranch || len(sample.RuntimeReceiptEvents) != 1 || + sample.RuntimeReceiptEvents[0].InvocationID != sample.RuntimeInvocationID || sample.RuntimeReceiptEvents[0].FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample has a non-canonical runtime receipt", record.Dataset, record.Name, arm) + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + } + if coldSamples != 1 || warmSamples != record.Stats.Iterations || len(record.Stats.Samples) != record.Stats.Iterations+1 { + return fmt.Errorf("%s/%s %s arm must contain one cold and exactly %d unique warm samples", record.Dataset, record.Name, arm, record.Stats.Iterations) + } + return nil +} + +// validateSPI1ResourceCases validates spi1 resource cases. +func validateSPI1ResourceCases( + report ResourceGateReport, + candidate []CaseResult, + requirements spI1ProtocolRequirements, +) (map[performanceKey]bool, error) { + if report.Version != resourceGateVersion { + return nil, fmt.Errorf("SP-I1 resource report version must be %d", resourceGateVersion) + } + + // recordKey binds resource evidence to an exact scheduled candidate invocation. + type recordKey struct { + // performanceKey identifies the workload and backend. + performanceKey + + // round identifies the paired benchmark round. + round int + + // block identifies the order-balancing block containing the round. + block int + + // order retains the order while recordKey is assembled or evaluated. + order int + + // runUUID binds the record to one benchmark process invocation. + runUUID string + + // arm identifies the treatment that produced the record. + arm string + } + expected := map[recordKey]CaseResult{} + for _, record := range candidate { + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s candidate resource record lacks run identity", record.Dataset, record.Name) + } + key := recordKey{ + performanceKey: performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + }, + round: record.Environment.Round, + block: record.Environment.Block, + order: record.Environment.ArmOrder, + runUUID: record.Environment.RunUUID, + arm: record.Environment.Arm, + } + if _, duplicate := expected[key]; duplicate { + return nil, fmt.Errorf("SP-I1 candidate artifact duplicates a resource record identity") + } + expected[key] = record + } + actual := map[recordKey]struct{}{} + passed := map[performanceKey]bool{} + for key := range requirements.expectedKeys { + passed[key] = true + } + cohort, err := canonicalSPI1Cohort() + if err != nil { + return nil, err + } + allPassed := true + for _, gateCase := range report.Cases { + key := performanceKey{ + dataset: gateCase.Dataset, + name: gateCase.Name, + backend: ModePostgresSQL, + } + if _, expected := requirements.expectedKeys[key]; !expected || gateCase.Reference != "" { + return nil, fmt.Errorf("SP-I1 resource report contains an unexpected production or reference case %s/%s", gateCase.Dataset, gateCase.Name) + } + identity := recordKey{ + performanceKey: key, + round: gateCase.Round, + block: gateCase.Block, + order: gateCase.ArmOrder, + runUUID: gateCase.RunUUID, + arm: gateCase.Arm, + } + record, found := expected[identity] + if !found { + return nil, fmt.Errorf("SP-I1 resource case %s/%s round %d does not bind an exact candidate record", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + if _, duplicate := actual[identity]; duplicate { + return nil, fmt.Errorf("SP-I1 resource report duplicates %s/%s round %d", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + actual[identity] = struct{}{} + recomputed := evaluateProductionResourceGateCase(record) + if !reflect.DeepEqual(gateCase, recomputed) { + return nil, fmt.Errorf("SP-I1 resource case %s/%s round %d differs from the decision recomputed from its candidate record", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + if gateCase.Architecture != string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) || + gateCase.FallbackArchitecture != "" || gateCase.QualificationSplit != expectedSplit || + gateCase.Tier != "normal" || !equalSPI1Caps(gateCase.NumericLimits, spI1TelemetryCaps()) || + gateCase.Passed != (len(gateCase.Reasons) == 0) || + !reflect.DeepEqual(gateCase.RuntimeReceiptChains, runtimeReceiptChains(record.Stats.Samples)) { + return nil, fmt.Errorf("SP-I1 resource case %s/%s does not bind exact guarded-I1 limits and split", gateCase.Dataset, gateCase.Name) + } + observations := traversalNumericObservations(record.TraversalTelemetry.Diagnostic.Counters) + if len(gateCase.NumericObserved) != len(spI1TelemetryCaps()) { + return nil, fmt.Errorf("SP-I1 resource case %s/%s has unexpected numeric observations", gateCase.Dataset, gateCase.Name) + } + for name := range spI1TelemetryCaps() { + observed, found := gateCase.NumericObserved[name] + expectedObserved, expectedFound := observations[name] + if !found || !expectedFound || observed != expectedObserved || observed < 0 { + return nil, fmt.Errorf("SP-I1 resource case %s/%s has invalid %s observation", gateCase.Dataset, gateCase.Name, name) + } + } + passed[key] = passed[key] && gateCase.Passed + allPassed = allPassed && gateCase.Passed + } + if len(actual) != len(expected) { + return nil, fmt.Errorf("SP-I1 resource report has %d exact record decisions, expected %d", len(actual), len(expected)) + } + for key := range requirements.expectedKeys { + if _, found := passed[key]; !found { + return nil, fmt.Errorf("SP-I1 resource report omits %s/%s", key.dataset, key.name) + } + } + if report.Passed != allPassed { + return nil, fmt.Errorf("SP-I1 resource report aggregate disposition contradicts its cases") + } + return passed, nil +} + +// validateSPI1Freeze validates spi1 freeze. +func validateSPI1Freeze( + freeze *SPI1QualificationFreezeManifest, + discovery *SPI1QualificationReport, + report SPI1QualificationReport, + cohort spI1CanonicalCohort, +) error { + if err := validateSPI1FrozenDiscovery(freeze, discovery, cohort); err != nil { + return err + } + if report.Protocol != referencePairProtocolConfirmation || + report.SourceCommit != freeze.SourceCommit || report.SourceArchiveSHA256 != freeze.SourceArchiveSHA256 || + report.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || report.BinarySHA256 != freeze.BinarySHA256 || + report.QuerySHA256 != freeze.QuerySHA256 || report.Policy != freeze.Policy || + report.Baseline != freeze.Baseline || report.Candidate != freeze.Candidate || + report.CohortDeclarationSHA256 != freeze.FullDeclarationSHA256 || + report.CorpusSHA256 != freeze.FullCorpusSHA256 || report.ResolvedSelectionSHA256 != freeze.FullResolvedSHA256 || + report.Seed != freeze.Seed || report.Confidence != freeze.Confidence || report.BootstrapCount != freeze.BootstrapCount || + !equalSPI1Caps(report.Caps, freeze.Caps) { + return fmt.Errorf("SP-I1 confirmation identity differs from the frozen discovery") + } + return nil +} + +// validateSPI1FrozenDiscovery validates spi1 frozen discovery. +func validateSPI1FrozenDiscovery( + freeze *SPI1QualificationFreezeManifest, + discovery *SPI1QualificationReport, + cohort spI1CanonicalCohort, +) error { + if freeze == nil || discovery == nil { + return fmt.Errorf("SP-I1 confirmation requires a discovery report and freeze manifest") + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + if freeze.Version != spI1FreezeVersion || freeze.Baseline != baseline || freeze.Candidate != candidate || + freeze.Policy != optimize.ShortestPathPolicyI1CanonicalGuardedV1 || freeze.QuerySHA256 != spI1QuerySHA256 || + freeze.Seed != 1 || freeze.Confidence != defaultConfidenceLevel || freeze.BootstrapCount != defaultBootstrapCount || + !equalSPI1Caps(freeze.Caps, spI1QualificationCaps()) || + freeze.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + freeze.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + freeze.FullDeclarationSHA256 != cohort.declarationSHA256 || + freeze.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || freeze.FullCorpusSHA256 != cohort.fullCorpusSHA256 || + freeze.TrainingResolvedSHA256 != cohort.trainingResolvedSHA256 || freeze.FullResolvedSHA256 != cohort.fullResolvedSHA256 || + !lowercaseSHA256(freeze.SourceArchiveSHA256) || !lowercaseSHA256(freeze.DirtyDiffSHA256) || + !lowercaseSHA256(freeze.BinarySHA256) || !lowercaseSHA256(freeze.BaselineArtifactSHA256) || + !lowercaseSHA256(freeze.CandidateArtifactSHA256) || !lowercaseSHA256(freeze.ResourceReportSHA256) || + !lowercaseSHA256(freeze.DiscoveryReportSHA256) || strings.TrimSpace(freeze.SourceCommit) == "" { + return fmt.Errorf("SP-I1 freeze manifest does not bind the exact immutable study identity") + } + if freeze.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + return fmt.Errorf("SP-I1 freeze manifest was not created from a clean source tree") + } + if discovery.Version != spI1QualificationVersion || discovery.Protocol != referencePairProtocolDiscovery || + discovery.Baseline != freeze.Baseline || discovery.Candidate != freeze.Candidate || + discovery.Policy != freeze.Policy || discovery.QuerySHA256 != freeze.QuerySHA256 || + discovery.SourceCommit != freeze.SourceCommit || discovery.SourceArchiveSHA256 != freeze.SourceArchiveSHA256 || + discovery.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || discovery.BinarySHA256 != freeze.BinarySHA256 || + discovery.CohortDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + discovery.ResolvedSelectionSHA256 != cohort.trainingResolvedSHA256 || + discovery.CorpusSHA256 != cohort.trainingCorpusSHA256 || + discovery.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + discovery.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + discovery.FullDeclarationSHA256 != cohort.declarationSHA256 || + discovery.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || discovery.FullCorpusSHA256 != cohort.fullCorpusSHA256 || + discovery.BaselineArtifactSHA256 != freeze.BaselineArtifactSHA256 || + discovery.CandidateArtifactSHA256 != freeze.CandidateArtifactSHA256 || + discovery.ResourceReportSHA256 != freeze.ResourceReportSHA256 || + !equalSPI1Caps(discovery.Caps, freeze.Caps) || discovery.Seed != freeze.Seed || + discovery.Confidence != freeze.Confidence || discovery.BootstrapCount != freeze.BootstrapCount || + discovery.MaterialityRatio != 0.95 || discovery.MaterialityAbsolute != 100*time.Microsecond || + discovery.P95RatioLimit != 1.05 || !discovery.EvidencePassed || + discovery.TrainingCases != len(cohort.trainingKeys) || discovery.HoldoutCases != 0 || + discovery.HoldoutPassed || discovery.QualificationPassed || discovery.TrainingPassed != freeze.TrainingPassed { + return fmt.Errorf("SP-I1 discovery report does not prove the exact frozen training identity") + } + seen := map[performanceKey]struct{}{} + for _, entry := range discovery.Cases { + key := performanceKey{ + dataset: entry.Dataset, + name: entry.Name, + backend: ModePostgresSQL, + } + if entry.QualificationSplit != "training" { + return fmt.Errorf("SP-I1 discovery report contains non-training timing") + } + if _, expected := cohort.trainingKeys[key]; !expected { + return fmt.Errorf("SP-I1 discovery report contains unexpected case %s/%s", entry.Dataset, entry.Name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("SP-I1 discovery report duplicates case %s/%s", entry.Dataset, entry.Name) + } + expectedBranch := "inline_canonical_witness" + if strings.HasSuffix(entry.Name, "-disconnected") { + expectedBranch = "inline_canonical_no_path" + } + if !validSPI1RatioInterval(entry.MedianRatio) || !validSPI1RatioInterval(entry.P95Ratio) || + entry.MedianSaving.Lower > entry.MedianSaving.Estimate || entry.MedianSaving.Estimate > entry.MedianSaving.Upper || + entry.Material != (entry.MedianRatio.Upper <= discovery.MaterialityRatio || entry.MedianSaving.Lower >= discovery.MaterialityAbsolute) || + entry.P95Contained != (entry.P95Ratio.Upper <= discovery.P95RatioLimit) || + !entry.Passed || len(entry.Reasons) != 0 || !entry.Material || !entry.P95Contained || !entry.ResourcePassed || + entry.RuntimeBranch != expectedBranch || + entry.Rounds < 5 || entry.Rounds > 20 || entry.BaselineSamples < 50 || entry.CandidateSamples < 50 { + return fmt.Errorf("SP-I1 discovery report case %s/%s did not pass the frozen training gates", entry.Dataset, entry.Name) + } + seen[key] = struct{}{} + } + if !orientationV2KeySetsEqual(seen, cohort.trainingKeys) { + return fmt.Errorf("SP-I1 discovery report omits part of the exact training cohort") + } + if !freeze.TrainingPassed || !discovery.TrainingPassed { + return fmt.Errorf("SP-I1 training discovery did not pass") + } + return nil +} + +// validSPI1RatioInterval reports whether a confidence interval contains finite ordered bounds. +func validSPI1RatioInterval(interval RatioInterval) bool { + return interval.Lower > 0 && interval.Lower <= interval.Estimate && interval.Estimate <= interval.Upper && + !math.IsNaN(interval.Lower) && !math.IsNaN(interval.Estimate) && !math.IsNaN(interval.Upper) && + !math.IsInf(interval.Lower, 0) && !math.IsInf(interval.Estimate, 0) && !math.IsInf(interval.Upper, 0) +} + +// createSPI1QualificationReport loads and evaluates the staged two-arm +// qualification evidence, writes the report even for statistical failures, +// and freezes discovery before any holdout capture is authorized. +func createSPI1QualificationReport( + baselinePath, candidatePath, resourcePath, freezePath, discoveryPath, freezeOutputPath, outputPath string, + options SPI1QualificationOptions, +) (bool, error) { + if err := validateDistinctSPI1Paths(map[string]string{ + "baseline artifact": baselinePath, "candidate artifact": candidatePath, "resource report": resourcePath, + "freeze manifest": freezePath, "discovery report": discoveryPath, "freeze output": freezeOutputPath, "report output": outputPath, + }); err != nil { + return false, err + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return false, fmt.Errorf("read SP-I1 baseline artifact: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return false, fmt.Errorf("read SP-I1 candidate artifact: %w", err) + } + resource, err := loadSPI1ResourceReport(resourcePath) + if err != nil { + return false, err + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return false, err + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return false, err + } + resourceSHA256, err := fileSHA256(resourcePath) + if err != nil { + return false, err + } + if resource.ArtifactSHA256 != candidateSHA256 { + return false, fmt.Errorf("SP-I1 resource report is not bound to the exact candidate artifact") + } + + freezeSHA256 := "" + if freezePath != "" || discoveryPath != "" { + if freezePath == "" || discoveryPath == "" { + return false, fmt.Errorf("SP-I1 confirmation requires both freeze and discovery report paths") + } + freeze, digest, err := loadSPI1FreezeManifest(freezePath) + if err != nil { + return false, fmt.Errorf("read SP-I1 freeze manifest: %w", err) + } + discovery, err := loadSPI1QualificationReport(discoveryPath) + if err != nil { + return false, fmt.Errorf("read SP-I1 discovery report: %w", err) + } + discoverySHA256, err := fileSHA256(discoveryPath) + if err != nil { + return false, err + } + if discoverySHA256 != freeze.DiscoveryReportSHA256 { + return false, fmt.Errorf("SP-I1 discovery report digest does not match freeze manifest") + } + options.Freeze, options.Discovery = freeze, discovery + freezeSHA256 = digest + if err := validateSPI1FrozenTrainingEvidence( + freeze, discovery, + options.TrainingBaselinePath, options.TrainingCandidatePath, options.TrainingResourcePath, + ); err != nil { + return false, err + } + } + options.SourceArchiveSHA256, err = sourceArchiveSHA256() + if err != nil { + return false, err + } + report, err := buildSPI1QualificationReport(baseline, candidate, resource, options) + if err != nil { + return false, err + } + report.BaselineArtifactSHA256 = baselineSHA256 + report.CandidateArtifactSHA256 = candidateSHA256 + report.ResourceReportSHA256 = resourceSHA256 + report.FreezeManifestSHA256 = freezeSHA256 + if err := validateCurrentSPI1Source(report.SourceCommit, report.SourceArchiveSHA256, report.DirtyDiffSHA256, report.BinarySHA256); err != nil { + return false, err + } + if err := writeSPI1QualificationReport(outputPath, report); err != nil { + return false, err + } + if options.Protocol == referencePairProtocolDiscovery { + if err := writeSPI1FreezeManifest(freezeOutputPath, outputPath, report); err != nil { + return false, err + } + return report.TrainingPassed, nil + } + return report.QualificationPassed, nil +} + +// validateSPI1HoldoutCapture authorizes the exact frozen cohort before any +// database setup is allowed to begin. +func validateSPI1HoldoutCapture( + corpus ScaleCorpus, + freezePath, discoveryPath, trainingBaselinePath, trainingCandidatePath, trainingResourcePath string, +) error { + cohort, err := canonicalSPI1Cohort() + if err != nil { + return err + } + if err := validateSPI1Corpus(corpus, cohort); err != nil { + return err + } + freeze, _, err := loadSPI1FreezeManifest(freezePath) + if err != nil { + return fmt.Errorf("read SP-I1 freeze manifest: %w", err) + } + discovery, err := loadSPI1QualificationReport(discoveryPath) + if err != nil { + return fmt.Errorf("read SP-I1 discovery report: %w", err) + } + discoverySHA256, err := fileSHA256(discoveryPath) + if err != nil { + return err + } + if discoverySHA256 != freeze.DiscoveryReportSHA256 { + return fmt.Errorf("SP-I1 discovery report digest does not match freeze manifest") + } + if err := validateSPI1FrozenTrainingEvidence( + freeze, discovery, trainingBaselinePath, trainingCandidatePath, trainingResourcePath, + ); err != nil { + return err + } + if err := validateCurrentSPI1Source(freeze.SourceCommit, freeze.SourceArchiveSHA256, freeze.DirtyDiffSHA256, freeze.BinarySHA256); err != nil { + return err + } + return nil +} + +// validateSPI1FrozenTrainingEvidence reloads and recomputes the exact training +// closure named by the freeze. This prevents an internally consistent but +// hand-edited report/freeze pair from authorizing protected holdout timing. +func validateSPI1FrozenTrainingEvidence( + freeze *SPI1QualificationFreezeManifest, + discovery *SPI1QualificationReport, + baselinePath, candidatePath, resourcePath string, +) error { + cohort, err := canonicalSPI1Cohort() + if err != nil { + return err + } + if err := validateSPI1FrozenDiscovery(freeze, discovery, cohort); err != nil { + return err + } + if baselinePath == "" || candidatePath == "" || resourcePath == "" { + return fmt.Errorf("SP-I1 frozen discovery verification requires the three exact training evidence artifacts") + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I1 training baseline: %w", err) + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I1 training candidate: %w", err) + } + resourceSHA256, err := fileSHA256(resourcePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I1 training resource report: %w", err) + } + if baselineSHA256 != freeze.BaselineArtifactSHA256 || candidateSHA256 != freeze.CandidateArtifactSHA256 || + resourceSHA256 != freeze.ResourceReportSHA256 { + return fmt.Errorf("SP-I1 frozen training evidence digests differ from the discovery freeze") + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return fmt.Errorf("read frozen SP-I1 training baseline: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return fmt.Errorf("read frozen SP-I1 training candidate: %w", err) + } + resource, err := loadSPI1ResourceReport(resourcePath) + if err != nil { + return err + } + if resource.ArtifactSHA256 != candidateSHA256 { + return fmt.Errorf("SP-I1 frozen training resource report is not bound to the candidate artifact") + } + recomputed, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: freeze.Seed, + Confidence: freeze.Confidence, + BootstrapCount: freeze.BootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: freeze.SourceArchiveSHA256, + }) + if err != nil { + return fmt.Errorf("recompute frozen SP-I1 training discovery: %w", err) + } + recomputed.BaselineArtifactSHA256 = baselineSHA256 + recomputed.CandidateArtifactSHA256 = candidateSHA256 + recomputed.ResourceReportSHA256 = resourceSHA256 + if !reflect.DeepEqual(recomputed, *discovery) { + return fmt.Errorf("SP-I1 discovery report differs from its recomputed frozen training evidence") + } + return nil +} + +// validateSPI1Corpus validates spi1 corpus. +func validateSPI1Corpus(corpus ScaleCorpus, cohort spI1CanonicalCohort) error { + if len(corpus.Cases) != len(cohort.keys) { + return fmt.Errorf("SP-I1 holdout capture requires exactly the frozen four-training/three-holdout cohort") + } + seen := map[performanceKey]struct{}{} + resolved := make([]ResolvedCaseSelector, 0, len(corpus.Cases)) + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, expected := cohort.keys[key]; !expected { + return fmt.Errorf("SP-I1 holdout capture contains unexpected case %s/%s", testCase.Dataset, testCase.Name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("SP-I1 holdout capture duplicates case %s/%s", testCase.Dataset, testCase.Name) + } + seen[key] = struct{}{} + if filepath.Base(testCase.Source) != "generated_sp_i1_inbound_v1.json" || + testCase.Category != "generated_shortest_path_v2" || sqlFingerprint(testCase.Cypher) != spI1QuerySHA256 || + testCase.Shape.FallbackExpectation != "forbidden" || testCase.Shape.Direction != "inbound" || + testCase.Shape.RelationshipKindCount != 1 || !slices.Equal(testCase.Shape.EdgeKinds, []string{"Traverse"}) || + testCase.Shape.MinDepth == nil || *testCase.Shape.MinDepth != 1 || + testCase.Shape.MaxDepth == nil || *testCase.Shape.MaxDepth != 64 || + !testCase.Shape.PathMaterializationRequired || + !slices.Equal(testCase.CandidateModes, []ExecutionMode{ModePostgresSQL, ModeNeo4j}) { + return fmt.Errorf("SP-I1 holdout capture changes frozen declaration %s/%s", testCase.Dataset, testCase.Name) + } + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + if testCase.Shape.QualificationSplit != expectedSplit { + return fmt.Errorf("SP-I1 holdout capture changes frozen split for %s/%s", testCase.Dataset, testCase.Name) + } + resolved = append(resolved, ResolvedCaseSelector{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Category: testCase.Category, + }) + } + if !orientationV2KeySetsEqual(seen, cohort.keys) || + declarationSHA256(corpus.DeclaredBackends()) != cohort.declarationSHA256 || + resolvedSelectionSHA256(resolved) != cohort.fullResolvedSHA256 || + corpusIdentity(corpus) != cohort.fullCorpusSHA256 { + return fmt.Errorf("SP-I1 holdout capture does not match the exact frozen declaration, selection, and corpus digests") + } + return nil +} + +// validateCurrentSPI1Source validates current spi1 source. +func validateCurrentSPI1Source(sourceCommit, sourceArchive, dirtyDiff, binary string) error { + currentCommit := strings.TrimSpace(commandOutput("git", "rev-parse", "HEAD")) + currentArchive, err := sourceArchiveSHA256() + if err != nil { + return err + } + currentDiff := workingTreeSHA256() + currentBinary := executableSHA256() + if currentCommit == "" || currentCommit == "unknown" || sourceCommit != currentCommit || + !lowercaseSHA256(sourceArchive) || sourceArchive != currentArchive || + dirtyDiff != cleanWorkingTreeSHA256() || currentDiff != cleanWorkingTreeSHA256() || + !lowercaseSHA256(binary) || binary != currentBinary { + return fmt.Errorf("SP-I1 evidence requires the current clean committed source archive and exact running binary") + } + return nil +} + +// loadSPI1ResourceReport loads spi1 resource report. +func loadSPI1ResourceReport(path string) (ResourceGateReport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return ResourceGateReport{}, fmt.Errorf("read SP-I1 resource report: %w", err) + } + report := ResourceGateReport{} + if err := json.Unmarshal(raw, &report); err != nil { + return ResourceGateReport{}, fmt.Errorf("decode SP-I1 resource report: %w", err) + } + if report.Version != resourceGateVersion || !lowercaseSHA256(report.ArtifactSHA256) { + return ResourceGateReport{}, fmt.Errorf("SP-I1 resource report must be checksummed schema v%d", resourceGateVersion) + } + return report, nil +} + +// loadSPI1QualificationReport loads spi1 qualification report. +func loadSPI1QualificationReport(path string) (*SPI1QualificationReport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + report := &SPI1QualificationReport{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, fmt.Errorf("decode SP-I1 qualification report: %w", err) + } + return report, nil +} + +// loadSPI1FreezeManifest loads spi1 freeze manifest. +func loadSPI1FreezeManifest(path string) (*SPI1QualificationFreezeManifest, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + manifest := &SPI1QualificationFreezeManifest{} + if err := json.Unmarshal(raw, manifest); err != nil { + return nil, "", fmt.Errorf("decode SP-I1 freeze manifest: %w", err) + } + digest := sha256.Sum256(raw) + return manifest, hex.EncodeToString(digest[:]), nil +} + +// writeSPI1QualificationReport writes spi1 qualification report. +func writeSPI1QualificationReport(path string, report SPI1QualificationReport) (err error) { + if path == "" { + return fmt.Errorf("SP-I1 qualification requires an explicit report output path") + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +// writeSPI1FreezeManifest writes spi1 freeze manifest. +func writeSPI1FreezeManifest(path, discoveryReportPath string, report SPI1QualificationReport) (err error) { + if path == "" || discoveryReportPath == "" { + return fmt.Errorf("SP-I1 discovery freeze requires report and manifest output paths") + } + cohort, err := canonicalSPI1Cohort() + if err != nil { + return err + } + if report.Protocol != referencePairProtocolDiscovery || report.CohortDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + report.ResolvedSelectionSHA256 != cohort.trainingResolvedSHA256 || report.CorpusSHA256 != cohort.trainingCorpusSHA256 || + report.TrainingCases != len(cohort.trainingKeys) || report.HoldoutCases != 0 || + report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.BootstrapCount != defaultBootstrapCount || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || !equalSPI1Caps(report.Caps, spI1QualificationCaps()) || + !lowercaseSHA256(report.BaselineArtifactSHA256) || !lowercaseSHA256(report.CandidateArtifactSHA256) || + !lowercaseSHA256(report.ResourceReportSHA256) { + return fmt.Errorf("SP-I1 discovery freeze requires the exact clean training-only report") + } + discoveryReportSHA256, err := fileSHA256(discoveryReportPath) + if err != nil { + return err + } + manifest := SPI1QualificationFreezeManifest{ + Version: spI1FreezeVersion, + Baseline: report.Baseline, + Candidate: report.Candidate, + Policy: report.Policy, + QuerySHA256: report.QuerySHA256, + Caps: report.Caps, + Seed: report.Seed, + Confidence: report.Confidence, + BootstrapCount: report.BootstrapCount, + SourceCommit: report.SourceCommit, + SourceArchiveSHA256: report.SourceArchiveSHA256, + DirtyDiffSHA256: report.DirtyDiffSHA256, + BinarySHA256: report.BinarySHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + TrainingResolvedSHA256: cohort.trainingResolvedSHA256, + FullResolvedSHA256: cohort.fullResolvedSHA256, + BaselineArtifactSHA256: report.BaselineArtifactSHA256, + CandidateArtifactSHA256: report.CandidateArtifactSHA256, + ResourceReportSHA256: report.ResourceReportSHA256, + DiscoveryReportSHA256: discoveryReportSHA256, + TrainingPassed: report.TrainingPassed, + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(manifest) + closeErr := output.Close() + if encodeErr != nil { + return encodeErr + } + return closeErr +} + +// validateDistinctSPI1Paths validates distinct spi1 paths. +func validateDistinctSPI1Paths(paths map[string]string) error { + names := make([]string, 0, len(paths)) + for name, path := range paths { + if path != "" { + names = append(names, name) + } + } + sort.Strings(names) + + // resolvedPath records the canonical filesystem identity of one evidence input. + type resolvedPath struct { + // name retains the name while resolvedPath is assembled or evaluated. + name string + // info retains the info while resolvedPath is assembled or evaluated. + info os.FileInfo + } + resolved := map[string]resolvedPath{} + var existing []resolvedPath + for _, name := range names { + absolute, err := filepath.Abs(filepath.Clean(paths[name])) + if err != nil { + return fmt.Errorf("resolve SP-I1 %s: %w", name, err) + } + if evaluated, err := filepath.EvalSymlinks(absolute); err == nil { + absolute = evaluated + } else if evaluatedParent, parentErr := filepath.EvalSymlinks(filepath.Dir(absolute)); parentErr == nil { + absolute = filepath.Join(evaluatedParent, filepath.Base(absolute)) + } + if prior, duplicate := resolved[absolute]; duplicate { + return fmt.Errorf("SP-I1 %s and %s must use distinct paths", prior.name, name) + } + current := resolvedPath{name: name} + if info, err := os.Stat(paths[name]); err == nil { + current.info = info + for _, prior := range existing { + if prior.info != nil && os.SameFile(prior.info, info) { + return fmt.Errorf("SP-I1 %s and %s must not alias the same file", prior.name, name) + } + } + existing = append(existing, current) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect SP-I1 %s path: %w", name, err) + } + resolved[absolute] = current + } + return nil +} + +// selectedCorpusContainsSPI1Holdout selects ed corpus contains spi1 holdout. +func selectedCorpusContainsSPI1Holdout(corpus ScaleCorpus) bool { + cohort, err := canonicalSPI1Cohort() + if err != nil { + return true + } + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, holdout := cohort.holdoutKeys[key]; holdout { + return true + } + } + return false +} + +// selectRunnableScaleCorpus keeps the protected SP-I1 holdout out of ordinary +// GraphBench selection. The holdout becomes selectable only through its exact +// protocol tag or an exact case name; database capture then passes through the +// freeze checks in main before any target is opened. +func selectRunnableScaleCorpus(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + if err := validateCorpusSelectors(corpus, selectors); err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + includeProtected := slices.Contains(selectors.Tags, spI1HoldoutTag) + if !includeProtected && len(selectors.Cases) > 0 { + protectedNames := make(map[string]struct{}, len(spI1CanonicalCases)) + for _, testCase := range spI1CanonicalCases { + if testCase.split == "holdout" { + protectedNames[testCase.name] = struct{}{} + } + } + for _, name := range selectors.Cases { + if _, protected := protectedNames[name]; protected { + includeProtected = true + break + } + } + } + if includeProtected { + return selectScaleCorpusValidated(corpus, selectors) + } + + cohort, err := canonicalSPI1Cohort() + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + filtered := ScaleCorpus{Cases: make([]ScaleCase, 0, len(corpus.Cases))} + protected := ScaleCorpus{Cases: make([]ScaleCase, 0, len(cohort.holdoutKeys))} + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, isProtected := cohort.holdoutKeys[key]; isProtected { + protected.Cases = append(protected.Cases, testCase) + continue + } + filtered.Cases = append(filtered.Cases, testCase) + } + selected, manifest, err := selectScaleCorpusValidated(filtered, selectors) + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + manifest.FullDeclarationCount = len(corpus.DeclaredBackends()) + manifest.OmittedDeclarationCount = manifest.FullDeclarationCount - manifest.SelectedDeclarationCount + manifest.ProtectedDeclarationCount = len(protected.DeclaredBackends()) + manifest.ProtectedDeclarationSHA256 = declarationSHA256(protected.DeclaredBackends()) + return selected, manifest, nil +} + +// validateSPI1HoldoutCaptureConfig validates spi1 holdout capture config. +func validateSPI1HoldoutCaptureConfig(cfg config) error { + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL || cfg.ExistingGraph || cfg.Discovery { + return fmt.Errorf("SP-I1 holdout capture requires one managed PostgreSQL fixed-confirmation mode") + } + if cfg.Iterations < 50 || cfg.WarmupIterations < 20 || cfg.PoolSize != 1 || len(cfg.Concurrency) != 0 { + return fmt.Errorf("SP-I1 holdout capture requires at least 50 samples, 20 warmups, pool size 1, and no concurrency block") + } + if cfg.Round < 1 || cfg.Round > 20 || cfg.Block != cfg.Round || cfg.ArmOrder < 1 || cfg.ArmOrder > 2 || + strings.TrimSpace(cfg.RunUUID) == "" { + return fmt.Errorf("SP-I1 holdout capture requires rounds 1-20, block equal to round, a two-arm order, and an explicit shared run UUID") + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + expectedArm, expectedOrder := "", 0 + switch cfg.PostgresForceShortest { + case baseline: + expectedArm = "sp-i1-s4" + expectedOrder = 1 + if cfg.Round%2 == 0 { + expectedOrder = 2 + } + case candidate: + expectedArm = "sp-i1-candidate" + expectedOrder = 2 + if cfg.Round%2 == 0 { + expectedOrder = 1 + } + default: + return fmt.Errorf("SP-I1 holdout capture must force exact S4 or guarded canonical I1") + } + if cfg.Arm != expectedArm || cfg.ArmOrder != expectedOrder { + return fmt.Errorf("SP-I1 holdout capture round %d requires arm %q at order %d", cfg.Round, expectedArm, expectedOrder) + } + if !cfg.PostgresRepeatableRead || cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic || + cfg.PostgresProductionManifest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament || + cfg.PostgresReferences || len(cfg.PostgresReferenceArms) != 0 || cfg.Baseline != "" || + cfg.BundleDir != "" || len(cfg.BundleEvidence) != 0 { + return fmt.Errorf("SP-I1 holdout capture requires forced Repeatable Read with diagnostic telemetry and no supplemental PostgreSQL arms") + } + if cfg.OutputJSONL == "" || cfg.Round > 1 && !cfg.AppendJSONL { + return fmt.Errorf("SP-I1 holdout capture requires a JSONL output and append mode after round 1") + } + return validateDistinctSPI1Paths(map[string]string{ + "freeze manifest": cfg.SPI1Freeze, "discovery report": cfg.SPI1DiscoveryReport, + "training baseline artifact": cfg.SPI1TrainingBaseline, + "training candidate artifact": cfg.SPI1TrainingCandidate, + "training resource report": cfg.SPI1TrainingResource, + "capture JSONL": cfg.OutputJSONL, "capture summary": cfg.Summary, "capture JSON summary": cfg.SummaryJSON, + }) +} diff --git a/cmd/graphbench/sp_i1_qualification_test.go b/cmd/graphbench/sp_i1_qualification_test.go new file mode 100644 index 00000000..44aaa49b --- /dev/null +++ b/cmd/graphbench/sp_i1_qualification_test.go @@ -0,0 +1,843 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestSPI1QualificationDiscoveryPassesTrainingWithoutOpeningHoldout verifies spi1 qualification discovery passes training without opening holdout behavior. +func TestSPI1QualificationDiscoveryPassesTrainingWithoutOpeningHoldout(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + report, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.True(t, report.EvidencePassed) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.Equal(t, 4, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.Len(t, report.Cases, 4) + require.Equal(t, spI1QualificationCaps(), report.Caps) + for _, gateCase := range report.Cases { + require.True(t, gateCase.Passed, gateCase.Reasons) + require.Equal(t, "training", gateCase.QualificationSplit) + require.LessOrEqual(t, gateCase.MedianRatio.Upper, 0.95) + require.LessOrEqual(t, gateCase.P95Ratio.Upper, 1.05) + } +} + +// TestTimedRuntimeAttestationIdentityIncludesExactS4Baseline verifies timed runtime attestation identity includes exact s4 baseline behavior. +func TestTimedRuntimeAttestationIdentityIncludesExactS4Baseline(t *testing.T) { + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + translation := translate.Result{Optimization: translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Selected: baseline, + }}}} + require.Equal(t, baseline, timedRuntimeAttestationIdentity(translation)) +} + +// TestSPI1QualificationConfirmationRequiresAndPassesFrozenDiscovery verifies spi1 qualification confirmation requires and passes frozen discovery behavior. +func TestSPI1QualificationConfirmationRequiresAndPassesFrozenDiscovery(t *testing.T) { + trainingBaseline, trainingCandidate, trainingResource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI1QualificationReport(trainingBaseline, trainingCandidate, trainingResource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI1QualificationTestFreeze(t, discovery) + + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolConfirmation) + report, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolConfirmation, + SourceArchiveSHA256: strings.Repeat("a", 64), + Freeze: &freeze, + Discovery: &discovery, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.Equal(t, 4, report.TrainingCases) + require.Equal(t, 3, report.HoldoutCases) + require.Len(t, report.Cases, 7) +} + +// TestSPI1QualificationRejectsUnattestedCandidateAndFreezeMutation verifies spi1 qualification rejects unattested candidate and freeze mutation behavior. +func TestSPI1QualificationRejectsUnattestedCandidateAndFreezeMutation(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + candidate[0].Stats.Samples[1].RuntimeAttestation = "same_case_invocation_local_replay" + candidate[0].Stats.Samples[1].RuntimeReceiptEvents = nil + _, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.ErrorContains(t, err, "timed-invocation attribution") + + trainingBaseline, trainingCandidate, trainingResource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI1QualificationReport(trainingBaseline, trainingCandidate, trainingResource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI1QualificationTestFreeze(t, discovery) + freeze.QuerySHA256 = strings.Repeat("f", 64) + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + require.Error(t, validateSPI1FrozenDiscovery(&freeze, &discovery, cohort)) +} + +// TestSPI1QualificationClassifiesBoundResourceFailure verifies spi1 qualification classifies bound resource failure behavior. +func TestSPI1QualificationClassifiesBoundResourceFailure(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + candidate[0].PostgresMetrics.Buffers.TempWritten = 1 + resource.Cases[0] = evaluateProductionResourceGateCase(candidate[0]) + resource.Passed = false + report, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.False(t, report.TrainingPassed) + require.False(t, report.QualificationPassed) + found := false + for _, gateCase := range report.Cases { + found = found || strings.Contains(strings.Join(gateCase.Reasons, "\n"), "candidate resource evidence did not pass") + } + require.True(t, found) +} + +// TestSPI1QualificationRejectsCanonicalEvidenceAndScheduleTampering verifies spi1 qualification rejects canonical evidence and schedule tampering behavior. +func TestSPI1QualificationRejectsCanonicalEvidenceAndScheduleTampering(t *testing.T) { + tests := map[string]func([]CaseResult, []CaseResult, *ResourceGateReport){ + "canonical observation": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + baseline[0].ObservedRows = []string{`[{"nodes":[],"relationships":[]}]`} + }, + "duplicate warm iteration": func(_ []CaseResult, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].Stats.Samples[2].Iteration = 1 + }, + "duplicate timed invocation": func(_ []CaseResult, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].Stats.Samples[2].RuntimeInvocationID = candidate[0].Stats.Samples[1].RuntimeInvocationID + candidate[0].Stats.Samples[2].RuntimeReceiptEvents[0].InvocationID = candidate[0].Stats.Samples[1].RuntimeInvocationID + }, + "cross-record timed invocation replay": func(baseline, candidate []CaseResult, _ *ResourceGateReport) { + candidate[1].Stats.Samples[1].RuntimeInvocationID = baseline[0].Stats.Samples[1].RuntimeInvocationID + candidate[1].Stats.Samples[1].RuntimeReceiptEvents[0].InvocationID = baseline[0].Stats.Samples[1].RuntimeInvocationID + }, + "contradictory arm chronology": func(baseline, candidate []CaseResult, _ *ResourceGateReport) { + started := baseline[0].Environment.StartedAt.Add(-2 * time.Second) + for index := range candidate { + if candidate[index].Environment.Round == 1 { + candidate[index].Environment.StartedAt = started + candidate[index].Environment.EndedAt = started.Add(time.Second) + } + } + }, + "unbound resource round": func(_, _ []CaseResult, resource *ResourceGateReport) { + resource.Cases[0].Round = 99 + }, + "substituted resource receipt": func(_, _ []CaseResult, resource *ResourceGateReport) { + resource.Cases[0].RuntimeReceiptChains[0][0].RuntimeBranch = "substituted" + }, + "cleared resource spill": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].PostgresMetrics.Buffers.TempWritten = 1 + }, + "reachable relabeled no path": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].TraversalTelemetry.Summary.RuntimeBranch = "inline_canonical_no_path" + for index := range candidate[0].Stats.Samples { + if candidate[0].Stats.Samples[index].Classification == "warm" { + candidate[0].Stats.Samples[index].RuntimeBranch = "inline_canonical_no_path" + candidate[0].Stats.Samples[index].RuntimeReceiptEvents[0].RuntimeBranch = "inline_canonical_no_path" + } + } + }, + "no path relabeled witness": func(_, candidate []CaseResult, _ *ResourceGateReport) { + for recordIndex := range candidate { + if !strings.HasSuffix(candidate[recordIndex].Name, "-disconnected") { + continue + } + candidate[recordIndex].TraversalTelemetry.Summary.RuntimeBranch = "inline_canonical_witness" + for sampleIndex := range candidate[recordIndex].Stats.Samples { + if candidate[recordIndex].Stats.Samples[sampleIndex].Classification == "warm" { + candidate[recordIndex].Stats.Samples[sampleIndex].RuntimeBranch = "inline_canonical_witness" + candidate[recordIndex].Stats.Samples[sampleIndex].RuntimeReceiptEvents[0].RuntimeBranch = "inline_canonical_witness" + } + } + return + } + }, + "output counter differs from observation": func(_, candidate []CaseResult, _ *ResourceGateReport) { + *candidate[0].TraversalTelemetry.Diagnostic.Counters.InlineShortestPath.OutputPaths = 0 + }, + "supplemental planned arm": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].TraversalTelemetry.Summary.PlannedIdentities = append(candidate[0].TraversalTelemetry.Summary.PlannedIdentities, "SP-B1-extra") + }, + "reduced planned search space": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + baseline[0].TraversalTelemetry.Summary.PlannedIdentities = []string{ + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorIncumbentWorkspace), + } + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + mutate(baseline, candidate, &resource) + _, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.Error(t, err) + }) + } +} + +// TestSPI1QualificationFreezesStatisticalPolicyAndDiscoverySemantics verifies spi1 qualification freezes statistical policy and discovery semantics behavior. +func TestSPI1QualificationFreezesStatisticalPolicyAndDiscoverySemantics(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + for _, options := range []SPI1QualificationOptions{ + { + Seed: 2, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + }, + { + Seed: 1, + Confidence: 0.95, + BootstrapCount: defaultBootstrapCount, + }, + { + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: 1, + }, + } { + options.Protocol = referencePairProtocolDiscovery + options.SourceArchiveSHA256 = strings.Repeat("a", 64) + _, err := buildSPI1QualificationReport(baseline, candidate, resource, options) + require.Error(t, err) + } + + discovery, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI1QualificationTestFreeze(t, discovery) + discovery.Cases[0].P95Ratio.Upper = 2 + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + require.Error(t, validateSPI1FrozenDiscovery(&freeze, &discovery, cohort)) +} + +// TestSPI1FrozenTrainingEvidenceIsRecomputedFromNamedArtifacts verifies spi1 frozen training evidence is recomputed from named artifacts behavior. +func TestSPI1FrozenTrainingEvidenceIsRecomputedFromNamedArtifacts(t *testing.T) { + baseline, candidate, resource := spI1QualificationTestArtifacts(t, referencePairProtocolDiscovery) + directory := t.TempDir() + baselinePath := filepath.Join(directory, "s4.jsonl") + candidatePath := filepath.Join(directory, "i1.jsonl") + resourcePath := filepath.Join(directory, "resource.json") + require.NoError(t, writeJSONLFile(baselinePath, baseline)) + require.NoError(t, writeJSONLFile(candidatePath, candidate)) + candidateSHA256, err := fileSHA256(candidatePath) + require.NoError(t, err) + resource.ArtifactSHA256 = candidateSHA256 + resourceRaw, err := json.MarshalIndent(resource, "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(resourcePath, append(resourceRaw, '\n'), 0o600)) + + discovery, err := buildSPI1QualificationReport(baseline, candidate, resource, SPI1QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256, err = fileSHA256(baselinePath) + require.NoError(t, err) + discovery.CandidateArtifactSHA256 = candidateSHA256 + discovery.ResourceReportSHA256, err = fileSHA256(resourcePath) + require.NoError(t, err) + freeze := spI1QualificationTestFreeze(t, discovery) + require.NoError(t, validateSPI1FrozenTrainingEvidence(&freeze, &discovery, baselinePath, candidatePath, resourcePath)) + + forged := discovery + forged.Cases = append([]SPI1QualificationCase(nil), discovery.Cases...) + forged.Cases[0].MedianRatio = RatioInterval{ + Lower: 0.801, + Estimate: 0.801, + Upper: 0.801, + } + require.ErrorContains(t, + validateSPI1FrozenTrainingEvidence(&freeze, &forged, baselinePath, candidatePath, resourcePath), + "differs from its recomputed", + ) +} + +// TestSPI1PathsRejectHardlinkAliases verifies spi1 paths reject hardlink aliases behavior. +func TestSPI1PathsRejectHardlinkAliases(t *testing.T) { + directory := t.TempDir() + input := filepath.Join(directory, "input.json") + alias := filepath.Join(directory, "alias.json") + require.NoError(t, os.WriteFile(input, []byte("{}"), 0o600)) + require.NoError(t, os.Link(input, alias)) + require.Error(t, validateDistinctSPI1Paths(map[string]string{"input": input, "output": alias})) +} + +// TestSPI1HoldoutCaptureProfileAcceptsBothBalancedArmsAndRejectsDrift verifies spi1 holdout capture profile accepts both balanced arms and rejects drift behavior. +func TestSPI1HoldoutCaptureProfileAcceptsBothBalancedArmsAndRejectsDrift(t *testing.T) { + baseline := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidate := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + valid := func(executor, arm string, round, order int) config { + return config{ + Modes: []ExecutionMode{ModePostgresSQL}, + Iterations: 50, + WarmupIterations: 20, + Round: round, + Block: round, + Arm: arm, + ArmOrder: order, + RunUUID: "sp-i1-confirmation", + PoolSize: 1, + PostgresForceShortest: executor, + PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + OutputJSONL: fmt.Sprintf(".coverage/sp-i1-%s-%d.jsonl", arm, round), + AppendJSONL: round > 1, + SPI1Freeze: ".coverage/sp-i1-freeze.json", + SPI1DiscoveryReport: ".coverage/sp-i1-discovery.json", + SPI1TrainingBaseline: ".coverage/sp-i1-training-s4.jsonl", + SPI1TrainingCandidate: ".coverage/sp-i1-training-i1.jsonl", + SPI1TrainingResource: ".coverage/sp-i1-training-resource.json", + } + } + for _, cfg := range []config{ + valid(baseline, "sp-i1-s4", 1, 1), + valid(candidate, "sp-i1-candidate", 1, 2), + valid(baseline, "sp-i1-s4", 2, 2), + valid(candidate, "sp-i1-candidate", 2, 1), + } { + require.NoError(t, validateSPI1HoldoutCaptureConfig(cfg)) + } + + tests := map[string]func(*config){ + "wrong backend": func(cfg *config) { cfg.Modes = []ExecutionMode{ModeNeo4j} }, + "existing graph": func(cfg *config) { cfg.ExistingGraph = true }, + "too few samples": func(cfg *config) { cfg.Iterations = 49 }, + "too few warmups": func(cfg *config) { cfg.WarmupIterations = 19 }, + "pool larger than one": func(cfg *config) { cfg.PoolSize = 2 }, + "concurrency": func(cfg *config) { cfg.Concurrency = []int{2} }, + "round above maximum": func(cfg *config) { cfg.Round, cfg.Block = 21, 21 }, + "mismatched block": func(cfg *config) { cfg.Block = 2 }, + "missing run UUID": func(cfg *config) { cfg.RunUUID = "" }, + "wrong arm label": func(cfg *config) { cfg.Arm = "baseline" }, + "wrong arm order": func(cfg *config) { cfg.ArmOrder = 2 }, + "wrong executor": func(cfg *config) { cfg.PostgresForceShortest = "SP-S3-U-E+MAT-M0" }, + "read committed": func(cfg *config) { cfg.PostgresRepeatableRead = false }, + "summary telemetry": func(cfg *config) { cfg.PostgresTraversalTelemetry = postgresTraversalTelemetrySummary }, + "supplemental references": func(cfg *config) { cfg.PostgresReferences = true }, + "missing output": func(cfg *config) { cfg.OutputJSONL = "" }, + "path alias": func(cfg *config) { cfg.OutputJSONL = cfg.SPI1Freeze }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + cfg := valid(baseline, "sp-i1-s4", 1, 1) + mutate(&cfg) + require.Error(t, validateSPI1HoldoutCaptureConfig(cfg)) + }) + } + t.Run("round after one requires append", func(t *testing.T) { + cfg := valid(baseline, "sp-i1-s4", 2, 2) + cfg.AppendJSONL = false + require.Error(t, validateSPI1HoldoutCaptureConfig(cfg)) + }) +} + +// TestSPI1HoldoutDetectionAndCorpusBindingIgnoreMutableTagAlone verifies spi1 holdout detection and corpus binding ignore mutable tag alone behavior. +func TestSPI1HoldoutDetectionAndCorpusBindingIgnoreMutableTagAlone(t *testing.T) { + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + training, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training"}}) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI1Holdout(training)) + + confirmation, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training", "sp-i1-inbound-v1-holdout"}}) + require.NoError(t, err) + require.True(t, selectedCorpusContainsSPI1Holdout(confirmation)) + for index := range confirmation.Cases { + confirmation.Cases[index].Source = strings.TrimPrefix(confirmation.Cases[index].Source, "../../") + confirmation.Cases[index].Tags = nil + } + require.True(t, selectedCorpusContainsSPI1Holdout(confirmation), "canonical key detection must not depend on tags") + + // Restore exact declarations before checking the complete frozen corpus. + exact, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{"sp-i1-inbound-v1-training", "sp-i1-inbound-v1-holdout"}}) + require.NoError(t, err) + for index := range exact.Cases { + exact.Cases[index].Source = strings.TrimPrefix(exact.Cases[index].Source, "../../") + } + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + require.NoError(t, validateSPI1Corpus(exact, cohort)) + + omitted := ScaleCorpus{Cases: append([]ScaleCase(nil), exact.Cases[:len(exact.Cases)-1]...)} + require.Error(t, validateSPI1Corpus(omitted, cohort)) + mutated := ScaleCorpus{Cases: append([]ScaleCase(nil), exact.Cases...)} + mutated.Cases[0].Cypher += " " + require.Error(t, validateSPI1Corpus(mutated, cohort)) +} + +// TestRunnableCorpusExcludesSPI1HoldoutUntilExactOptIn verifies runnable corpus excludes spi1 holdout until exact opt in behavior. +func TestRunnableCorpusExcludesSPI1HoldoutUntilExactOptIn(t *testing.T) { + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + ordinary, manifest, err := selectRunnableScaleCorpus(full, CorpusSelectors{}) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI1Holdout(ordinary)) + require.False(t, manifest.DiagnosticOnly) + require.Equal(t, manifest.FullDeclarationCount, manifest.SelectedDeclarationCount+manifest.OmittedDeclarationCount) + require.Equal(t, 6, manifest.OmittedDeclarationCount) + require.Equal(t, 6, manifest.ProtectedDeclarationCount) + require.True(t, lowercaseSHA256(manifest.ProtectedDeclarationSHA256)) + require.True(t, selectedCorpusContainsTag(ordinary, spI1TrainingTag)) + + for name, selectors := range map[string]CorpusSelectors{ + "generic holdout tag": {Tags: []string{"holdout"}}, + "broad category": {Categories: []string{"generated_shortest_path_v2"}}, + } { + t.Run(name, func(t *testing.T) { + selected, _, err := selectRunnableScaleCorpus(full, selectors) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI1Holdout(selected)) + }) + } + + exactTag, _, err := selectRunnableScaleCorpus(full, CorpusSelectors{Tags: []string{spI1HoldoutTag}}) + require.NoError(t, err) + require.Len(t, exactTag.Cases, 3) + require.True(t, selectedCorpusContainsSPI1Holdout(exactTag)) + + exactCase, _, err := selectRunnableScaleCorpus(full, CorpusSelectors{Cases: []string{spI1CanonicalCases[4].name}}) + require.NoError(t, err) + require.Len(t, exactCase.Cases, 1) + require.True(t, selectedCorpusContainsSPI1Holdout(exactCase)) +} + +// spI1QualificationTestArtifacts prepares or inspects test evidence for sp i1 qualification test artifacts. +func spI1QualificationTestArtifacts(t *testing.T, protocol string) ([]CaseResult, []CaseResult, ResourceGateReport) { + t.Helper() + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + tags := []string{"sp-i1-inbound-v1-training"} + rounds, samples, warmups := 5, 10, 5 + corpusSHA256 := spI1TrainingCorpusSHA256 + if protocol == referencePairProtocolConfirmation { + tags = append(tags, "sp-i1-inbound-v1-holdout") + rounds, samples, warmups = 10, 50, 20 + corpusSHA256 = spI1FullCorpusSHA256 + } + selected, selection, err := selectRunnableScaleCorpus(full, CorpusSelectors{Tags: tags}) + require.NoError(t, err) + + var baseline, candidate []CaseResult + resource := ResourceGateReport{ + Version: resourceGateVersion, + ArtifactSHA256: strings.Repeat("9", 64), + Passed: true, + } + for _, testCase := range selected.Cases { + fixture, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err) + fixture.PhysicalValidated = true + fixture.PhysicalNodeCount = int64(fixture.NodeCount) + fixture.PhysicalEdgeCount = int64(fixture.EdgeCount) + fixture.NodeRelationBytes = int64(fixture.NodeCount) * 1024 + fixture.EdgeRelationBytes = int64(fixture.EdgeCount) * 1024 + for round := 1; round <= rounds; round++ { + left, right := spI1QualificationTestRecords(t, testCase, fixture, selection, corpusSHA256, round, samples, warmups) + baseline = append(baseline, left) + candidate = append(candidate, right) + allObserved := traversalNumericObservations(right.TraversalTelemetry.Diagnostic.Counters) + observed := make(map[string]int64, len(spI1TelemetryCaps())) + for name := range spI1TelemetryCaps() { + observed[name] = allObserved[name] + } + resource.Cases = append(resource.Cases, ResourceGateCase{ + Dataset: right.Dataset, + Name: right.Name, + Tier: right.Shape.FixtureTier, + Round: right.Environment.Round, + Block: right.Environment.Block, + RunUUID: right.Environment.RunUUID, + Arm: right.Environment.Arm, + ArmOrder: right.Environment.ArmOrder, + QualificationSplit: right.Shape.QualificationSplit, + Architecture: string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + Passed: true, + NumericLimits: spI1TelemetryCaps(), + NumericObserved: observed, + RuntimeReceiptChains: runtimeReceiptChains(right.Stats.Samples), + }) + } + } + return baseline, candidate, resource +} + +// spI1QualificationTestRecords prepares or inspects test evidence for sp i1 qualification test records. +func spI1QualificationTestRecords( + t *testing.T, + testCase ScaleCase, + fixture FixtureMetadata, + selection SelectionManifest, + corpusSHA256 string, + round, samples, warmups int, +) (CaseResult, CaseResult) { + t.Helper() + baselineIdentity := string(optimize.ShortestPathExecutorS4CanonicalWitness) + candidateIdentity := string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + baselineOrder, candidateOrder := 1, 2 + if round%2 == 0 { + baselineOrder, candidateOrder = 2, 1 + } + rowCount := int64(1) + var observed []string + if len(testCase.Expected.PathRows) == 1 { + expectedPath := testCase.Expected.PathRows[0] + path := stablePathObservation{ + Nodes: make([]stableNodeObservation, len(expectedPath.Nodes)), + Relationships: make([]stableRelationshipObservation, len(expectedPath.RelationshipKinds)), + } + for index, identity := range expectedPath.Nodes { + path.Nodes[index].Identity = identity + } + for index, kind := range expectedPath.RelationshipKinds { + path.Relationships[index] = stableRelationshipObservation{ + Identity: expectedPath.RelationshipKeys[index], + Start: expectedPath.Nodes[index], + End: expectedPath.Nodes[index+1], + Kind: kind, + } + } + raw, err := json.Marshal([]any{path}) + require.NoError(t, err) + observed = []string{string(raw)} + } + if strings.HasSuffix(testCase.Name, "-disconnected") { + rowCount, observed = 0, nil + } + falseValue, trueValue := false, true + makeSamples := func(arm string, order int, duration time.Duration, requested, branch, attestation string) []LatencySample { + result := make([]LatencySample, samples+1) + result[0] = LatencySample{ + Round: round, + Block: round, + Arm: arm, + ArmOrder: order, + RunUUID: "sp-i1-test-run", + Iteration: 0, + Case: testCase.Name, + Dataset: testCase.Dataset, + Backend: ModePostgresSQL, + ConnectionID: "101", + Classification: "cold", + Duration: 2 * duration, + } + for index := range samples { + invocationID := fmt.Sprintf("sp-i1-test-%s-%s-%d-%d", arm, testCase.Name, round, index+1) + result[index+1] = LatencySample{ + Round: round, + Block: round, + Arm: arm, + ArmOrder: order, + RunUUID: "sp-i1-test-run", + Iteration: index + 1, + Case: testCase.Name, + Dataset: testCase.Dataset, + Backend: ModePostgresSQL, + ConnectionID: "101", + Classification: "warm", + Duration: duration, + RequestedIdentity: requested, + RuntimeIdentity: requested, + RuntimeBranch: branch, + FallbackExecuted: &falseValue, + RuntimeAttestation: attestation, + RuntimeInvocationID: invocationID, + } + if attestation == "timed_invocation" { + result[index+1].RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + InvocationID: invocationID, + Ordinal: 1, + RuntimeIdentity: requested, + RuntimeBranch: branch, + FallbackExecuted: false, + }} + } + } + return result + } + baseEnvironment := RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: corpusSHA256, + SourceCommit: "deadbeef", + DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: strings.Repeat("b", 64), + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + CgroupMemory: "max", + CPUGovernor: "performance", + RunUUID: "sp-i1-test-run", + Block: round, + Round: round, + WarmupIterations: warmups, + Selection: &selection, + PoolSize: 1, + Protocol: "fixed_confirmation", + } + postgresEnvironment := &PostgresEnvironment{ + Version: "PostgreSQL test", + Database: "dawgs", + PlanCacheMode: "auto", + TransactionIsolation: "repeatable read", + WorkMem: "64MB", + TempFileLimit: "1GB", + GraphPartitionCount: 1, + DatabaseOID: 42, + Autovacuum: "on", + NodeRelationBytes: fixture.NodeRelationBytes, + EdgeRelationBytes: fixture.EdgeRelationBytes, + AnalyzeState: "edge:analyzed,node:analyzed", + SchemaFingerprint: strings.Repeat("c", 64), + IndexFingerprint: strings.Repeat("d", 64), + } + base := newCaseResult(testCase, ModePostgresSQL, nil) + base.RowCount = rowCount + base.ObservedRows = append([]string(nil), observed...) + base.Status = StatusOK + base.WorkloadSHA256 = scaleCaseWorkloadIdentity(testCase, ModePostgresSQL) + attachFixtureMetadata(&base, fixture) + base.PostgresEnvironment = postgresEnvironment + + baseline := base + baseline.Environment = cloneSPI1TestEnvironment(baseEnvironment, "sp-i1-s4", baselineOrder) + firstStarted := time.Unix(1_700_000_000+int64(round)*10, 0) + baselineStarted, candidateStarted := firstStarted, firstStarted.Add(2*time.Second) + if candidateOrder == 1 { + candidateStarted, baselineStarted = firstStarted, firstStarted.Add(2*time.Second) + } + baseline.Environment.StartedAt, baseline.Environment.EndedAt = baselineStarted, baselineStarted.Add(time.Second) + baseline.SQL = "select 's4:' || " + fmt.Sprintf("%q", testCase.Name) + baseline.SQLFingerprint = sqlFingerprint(baseline.SQL) + baselineBranch := "compact_workspace_witness" + if rowCount == 0 { + baselineBranch = "compact_no_path" + } + baseline.Stats = DurationStats{ + Iterations: samples, + WarmupIterations: warmups, + Median: 10 * time.Millisecond, + P95: 10 * time.Millisecond, + Samples: makeSamples("sp-i1-s4", baselineOrder, 10*time.Millisecond, baselineIdentity, baselineBranch, "timed_invocation"), + } + baselineOutcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, + TargetKind: "traversal", + Family: "SP", + Selected: baselineIdentity, + Applied: baselineIdentity, + Fallback: "SP-S0", + PlannedCandidates: spI1ShortestPathPlannedIdentities(), + SelectorVersion: "sp-tool-v1", + ExecutionBoundary: "stored_helper", + ObservationMode: "one_path", + Scheduler: "single_ended_level", + Direction: "inbound", + PhysicalExpansion: "end_id", + RelationshipKindCount: 1, + TopologyClassification: "physical_inbound_deep", + SelectionMode: "forced_tool", + Eligible: &trueValue, + StaticallyEligible: &trueValue, + MinimumDepth: traversalTelemetryPointer(int64(1)), + MaximumDepth: traversalTelemetryPointer(int64(64)), + StateLimit: 100_000, + FrontierLimit: 100_000, + PredecessorLimit: 100_000, + EnumerationLimit: 100_000, + OutputBytesLimit: 64 * 1024 * 1024, + } + baseline.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{baselineOutcome}} + baselineMetrics := PostgresPlanMetrics{Provenance: map[string]string{}} + baseline.PostgresMetrics = &baselineMetrics + baselineTelemetry, err := buildPostgresCaseTraversalTelemetry(*baseline.Optimization, baselineMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + baseline.TraversalTelemetry = baselineTelemetry + + candidate := base + candidate.Environment = cloneSPI1TestEnvironment(baseEnvironment, "sp-i1-candidate", candidateOrder) + candidate.Environment.StartedAt, candidate.Environment.EndedAt = candidateStarted, candidateStarted.Add(time.Second) + candidate.SQL = "select 'i1:' || " + fmt.Sprintf("%q", testCase.Name) + candidate.SQLFingerprint = sqlFingerprint(candidate.SQL) + candidateBranch := "inline_canonical_witness" + if rowCount == 0 { + candidateBranch = "inline_canonical_no_path" + } + candidate.Stats = DurationStats{ + Iterations: samples, + WarmupIterations: warmups, + Median: 8 * time.Millisecond, + P95: 8 * time.Millisecond, + Samples: makeSamples("sp-i1-candidate", candidateOrder, 8*time.Millisecond, candidateIdentity, candidateBranch, "timed_invocation"), + } + candidateOutcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, + TargetKind: "traversal", + Family: "SP", + Candidate: candidateIdentity, + Selected: candidateIdentity, + Applied: candidateIdentity, + Fallback: baselineIdentity, + PlannedCandidates: spI1ShortestPathPlannedIdentities(), + EmittedCandidates: []string{candidateIdentity, baselineIdentity}, + EmittedPolicy: optimize.ShortestPathPolicyI1CanonicalGuardedV1, + SelectorVersion: "sp-i1-canonical-tool-v1", + ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + ObservationMode: "one_path", + Scheduler: "single_ended_level", + Direction: "inbound", + PhysicalExpansion: "end_id", + RelationshipKindCount: 1, + TopologyClassification: "physical_inbound_deep", + SelectionMode: "forced_tool", + Eligible: &trueValue, + StaticallyEligible: &trueValue, + MinimumDepth: traversalTelemetryPointer(int64(1)), + MaximumDepth: traversalTelemetryPointer(int64(64)), + StateLimit: 100_000, + PredecessorLimit: 100_000, + EnumerationLimit: 100_000, + OutputBytesLimit: 64 * 1024 * 1024, + } + candidate.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{candidateOutcome}} + candidateMetrics := PostgresPlanMetrics{ + Provenance: map[string]string{}, + HydrationRows: rowCount, + HydrationLoops: rowCount, + PlanNodes: []PostgresPlanNodeMetric{ + inlinePredecessorPlanNode("asp_i1_distance_bounded", 32, 1), + inlinePredecessorPlanNode("asp_i1_predecessor_bounded", 16, 1), + inlinePredecessorPlanNode("asp_i1_paths_bounded", rowCount, 1), + inlinePredecessorPlanNode("asp_i1_shortest", rowCount, 1), + inlinePredecessorPlanNode("asp_i1_candidate_marker", 1, 1), + inlinePredecessorPlanNode("asp_i1_fallback_marker", 0, 1), + inlinePredecessorPlanNode("asp_i1_candidate_rows", rowCount, 1), + inlinePredecessorPlanNode("asp_i1_fallback_rows", 0, 1), + inlinePredecessorMarkerGateNode("candidate", 1, 1), + inlinePredecessorMarkerGateNode("fallback", 0, 1), + inlinePredecessorExecutorNode("candidate", 1), + inlinePredecessorExecutorNode("fallback", 0), + }, + } + candidate.PostgresMetrics = &candidateMetrics + candidateTelemetry, err := buildPostgresCaseTraversalTelemetry(*candidate.Optimization, candidateMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichInlinePredecessorTraversalTelemetry(candidateTelemetry, candidateMetrics, rowCount, observed) + require.NoError(t, candidateTelemetry.Validate()) + candidate.TraversalTelemetry = candidateTelemetry + return baseline, candidate +} + +// cloneSPI1TestEnvironment returns an independent copy of spi1 test environment. +func cloneSPI1TestEnvironment(environment RunEnvironment, arm string, order int) *RunEnvironment { + copy := environment + copy.Arm = arm + copy.ArmOrder = order + return © +} + +// spI1QualificationTestFreeze prepares or inspects test evidence for sp i1 qualification test freeze. +func spI1QualificationTestFreeze(t *testing.T, discovery SPI1QualificationReport) SPI1QualificationFreezeManifest { + t.Helper() + cohort, err := canonicalSPI1Cohort() + require.NoError(t, err) + return SPI1QualificationFreezeManifest{ + Version: spI1FreezeVersion, + Baseline: discovery.Baseline, + Candidate: discovery.Candidate, + Policy: discovery.Policy, + QuerySHA256: discovery.QuerySHA256, + Caps: discovery.Caps, + Seed: discovery.Seed, + Confidence: discovery.Confidence, + BootstrapCount: discovery.BootstrapCount, + SourceCommit: discovery.SourceCommit, + SourceArchiveSHA256: discovery.SourceArchiveSHA256, + DirtyDiffSHA256: discovery.DirtyDiffSHA256, + BinarySHA256: discovery.BinarySHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + TrainingResolvedSHA256: cohort.trainingResolvedSHA256, + FullResolvedSHA256: cohort.fullResolvedSHA256, + BaselineArtifactSHA256: discovery.BaselineArtifactSHA256, + CandidateArtifactSHA256: discovery.CandidateArtifactSHA256, + ResourceReportSHA256: discovery.ResourceReportSHA256, + DiscoveryReportSHA256: strings.Repeat("4", 64), + TrainingPassed: discovery.TrainingPassed, + } +} diff --git a/cmd/graphbench/sp_i2_cap_contract_test.go b/cmd/graphbench/sp_i2_cap_contract_test.go new file mode 100644 index 00000000..fa5b09ae --- /dev/null +++ b/cmd/graphbench/sp_i2_cap_contract_test.go @@ -0,0 +1,68 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestVerifyPromotionManifestRequiresQualifiedSPI2Caps verifies that final +// promotion evidence cannot authorize cap values outside the qualified study. +func TestVerifyPromotionManifestRequiresQualifiedSPI2Caps(t *testing.T) { + digest := strings.Repeat("a", 64) + base := PromotionManifest{ + Version: promotionManifestVersion, + Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, + ExecutionBoundary: "guarded_dual_arm", + FallbackExecutor: string(optimize.ShortestPathExecutorS4CanonicalDistance), + SourceCommit: "deadbeef", + SourceSHA256: digest, + BinarySHA256: digest, + CorpusSHA256: spI2FullCorpusSHA256, + Caps: spI2PromotionCaps(), + Buckets: []PromotionBucket{{ + Name: "hidden-fan-in-depth32", + QuerySHA256: []string{spI2QuerySHA256}, + Direction: "inbound", + ObservationMode: string(optimize.ShortestPathObservationDistance), + MinimumDepth: 1, + MaximumDepth: 32, + RelationshipKindCount: 1, + QualificationSplit: []string{"training", "holdout"}, + }}, + } + + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, base)) + require.NoError(t, err) + require.True(t, verification.Passed, verification.Reasons) + + for name, test := range map[string]struct { + capName string + value int64 + }{ + "non-qualified state cap": {capName: "state_limit", value: 1000}, + "non-qualified frontier cap": {capName: "frontier_limit", value: 100}, + } { + t.Run(name, func(t *testing.T) { + manifest := base + manifest.Caps = clonePromotionCaps(base.Caps) + manifest.Caps[test.capName] = test.value + + verification, err := verifyPromotionManifest(writePromotionManifestWithPassingEvidence(t, manifest)) + require.NoError(t, err) + require.False(t, verification.Passed) + require.Contains(t, verification.Reasons, fmt.Sprintf( + "SP-I2 distance cap %s must equal %d", + test.capName, + spI2PromotionCaps()[test.capName], + )) + }) + } +} diff --git a/cmd/graphbench/sp_i2_component_authorization_v2.go b/cmd/graphbench/sp_i2_component_authorization_v2.go new file mode 100644 index 00000000..69efdb89 --- /dev/null +++ b/cmd/graphbench/sp_i2_component_authorization_v2.go @@ -0,0 +1,296 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const spI2V2ComponentAuthorizationSchema = "sp-i2-v2-component-authorization-v1" + +type SPI2V2ComponentAuthorization struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + ProtocolDeclarationSHA256 string `json:"protocol_declaration_sha256"` + SourceCommit string `json:"source_commit"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + AuthorizedExecutor string `json:"authorized_executor"` + Components []SPI2V2ComponentAuthorizationCase `json:"components"` + Passed bool `json:"passed"` +} + +type SPI2V2ComponentAuthorizationCase struct { + Executor string `json:"executor"` + ArtifactSHA256 string `json:"artifact_sha256"` + Cases int `json:"cases"` + SemanticPassed bool `json:"semantic_passed"` + PlanPassed bool `json:"plan_passed"` + ResourcePassed bool `json:"resource_passed"` + ReceiptPassed bool `json:"receipt_passed"` + FallbackFree bool `json:"fallback_free"` + CanonicalPlanSeen bool `json:"canonical_plan_seen"` +} + +type spI2V2ComponentSourceIdentity struct { + sourceCommit string + dirtyDiffSHA256 string + binarySHA256 string +} + +func createSPI2V2ComponentAuthorization(corpusRoot, e1dArtifact, e1pArtifact, output string) (bool, error) { + protocolPath := filepath.Join(corpusRoot, "protocols", "sp_i2_distance_v2.json") + _, protocolSHA256, err := loadSPI2ProtocolV2(protocolPath) + if err != nil { + return false, err + } + type componentInput struct { + executor optimize.ShortestPathExecutor + path string + } + inputs := []componentInput{ + {executor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, path: e1dArtifact}, + {executor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, path: e1pArtifact}, + } + authorization := SPI2V2ComponentAuthorization{ + Schema: spI2V2ComponentAuthorizationSchema, + Generation: spI2GenerationV2, + ProtocolDeclarationSHA256: protocolSHA256, + AuthorizedExecutor: string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP), + Passed: true, + } + var source spI2V2ComponentSourceIdentity + for _, input := range inputs { + records, err := readJSONLFile(input.path) + if err != nil { + return false, fmt.Errorf("read %s component artifact: %w", input.executor, err) + } + component, componentSource, err := validateSPI2V2ComponentEvidence(records, input.executor) + if err != nil { + return false, err + } + component.ArtifactSHA256, err = fileSHA256(input.path) + if err != nil { + return false, err + } + if source.sourceCommit == "" { + source = componentSource + } else if source != componentSource { + return false, fmt.Errorf("SP-I2 V2 component artifacts do not share one source and binary identity") + } + authorization.Components = append(authorization.Components, component) + } + authorization.SourceCommit = source.sourceCommit + authorization.DirtyDiffSHA256 = source.dirtyDiffSHA256 + authorization.BinarySHA256 = source.binarySHA256 + if err := validateSPI2V2ComponentAuthorization(authorization, protocolSHA256); err != nil { + return false, err + } + if output == "" { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + if err := encoder.Encode(authorization); err != nil { + return false, err + } + } else if err := writeIndentedJSON(output, authorization); err != nil { + return false, err + } + return true, nil +} + +func validateSPI2V2ComponentEvidence(records []CaseResult, executor optimize.ShortestPathExecutor) (SPI2V2ComponentAuthorizationCase, spI2V2ComponentSourceIdentity, error) { + if executor != optimize.ShortestPathExecutorI2GuardedDistanceV2E1D && executor != optimize.ShortestPathExecutorI2GuardedDistanceV2E1P { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, fmt.Errorf("unsupported SP-I2 V2 component executor %q", executor) + } + cohort, err := canonicalSPI2Cohort() + if err != nil { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, err + } + declarations, err := canonicalSPI2Declarations() + if err != nil { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, err + } + if len(records) != len(cohort.trainingKeys) { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 %s component check contains %d records, expected exactly %d", executor, len(records), len(cohort.trainingKeys)) + } + component := SPI2V2ComponentAuthorizationCase{ + Executor: string(executor), + Cases: len(records), + SemanticPassed: true, + PlanPassed: true, + ResourcePassed: true, + ReceiptPassed: true, + FallbackFree: true, + CanonicalPlanSeen: true, + } + seen := make(map[performanceKey]struct{}, len(records)) + seenInvocations := make(map[string]struct{}, len(records)*2) + var source spI2V2ComponentSourceIdentity + for _, record := range records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + if _, expected := cohort.trainingKeys[key]; !expected { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 %s component check contains unexpected case %s/%s", executor, record.Dataset, record.Name) + } + if _, duplicate := seen[key]; duplicate { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 %s component check duplicates %s/%s", executor, record.Dataset, record.Name) + } + seen[key] = struct{}{} + declaration := declarations[key] + if err := validateSPI2V2ComponentRecord(record, executor, declaration, seenInvocations); err != nil { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, err + } + currentSource := spI2V2ComponentSourceIdentity{ + sourceCommit: record.Environment.SourceCommit, + dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, + binarySHA256: record.Environment.BinarySHA256, + } + if source.sourceCommit == "" { + source = currentSource + } else if source != currentSource { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 %s component check mixes source or binary identities", executor) + } + } + if strings.TrimSpace(source.sourceCommit) == "" || !lowercaseSHA256(source.dirtyDiffSHA256) || !lowercaseSHA256(source.binarySHA256) { + return SPI2V2ComponentAuthorizationCase{}, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 %s component check lacks a complete source and binary identity", executor) + } + return component, source, nil +} + +func validateSPI2V2ComponentRecord(record CaseResult, executor optimize.ShortestPathExecutor, declaration spI2CanonicalDeclaration, seenInvocations map[string]struct{}) error { + if record.ExecutionMode != ModePostgresSQL || record.Status != StatusOK || record.Environment == nil || record.PostgresEnvironment == nil || record.Fixture == nil || + record.PostgresMetrics == nil || record.TraversalTelemetry == nil || record.Optimization == nil || record.Environment.ArtifactSchemaVersion != 2 || + record.Environment.PoolSize != 1 || len(record.Environment.Concurrency) != 0 || record.Environment.ExistingGraph || + record.Environment.WarmupIterations != 1 || record.Stats.WarmupIterations != 1 || record.Stats.Iterations != 1 || len(record.Stats.Samples) != 1 || + record.Environment.Round != 1 || record.Environment.Block != 1 || record.Environment.ArmOrder != 1 || + record.Environment.Arm != string(executor) || strings.TrimSpace(record.Environment.RunUUID) == "" { + return fmt.Errorf("%s/%s lacks the exact SP-I2 V2 %s component-check measurement contract", record.Dataset, record.Name, executor) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") || + len(record.PostgresPlanJSON) == 0 || record.PostgresMetrics.PlanningMS == nil || *record.PostgresMetrics.PlanningMS <= 0 || len(record.PostgresMetrics.PlanNodes) == 0 { + return fmt.Errorf("%s/%s SP-I2 V2 %s component check lacks one canonical planned Repeatable Read observation", record.Dataset, record.Name, executor) + } + testCase := declaration.testCase + testCase.Source = record.Source + expected := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expected, *record.Fixture) + if filepath.Base(record.Source) != "generated_sp_i2_distance_v1.json" || record.Category != testCase.Category || record.Cypher != testCase.Cypher || + record.WorkloadSHA256 != expected.WorkloadSHA256 || !reflect.DeepEqual(record.NodeParams, testCase.NodeParams) || + !reflect.DeepEqual(record.NodeListParams, testCase.NodeListParams) || !reflect.DeepEqual(record.Shape, testCase.Shape) || + !record.StableObservation || record.ExpectedRowCount == nil || testCase.Expected.RowCount == nil || + *record.ExpectedRowCount != *testCase.Expected.RowCount || record.RowCount != *testCase.Expected.RowCount { + return fmt.Errorf("%s/%s SP-I2 V2 %s component check changes the exact open-corpus semantic contract", record.Dataset, record.Name, executor) + } + if err := validateExpectedObservations(testCase.Expected, record.ObservedRows); err != nil { + return fmt.Errorf("%s/%s SP-I2 V2 %s component observation: %w", record.Dataset, record.Name, executor, err) + } + if err := validateSPI2V2DevelopmentSamples(record, executor, seenInvocations); err != nil { + return err + } + telemetry := record.TraversalTelemetry + if err := ValidateTraversalExecutionTelemetry(telemetry); err != nil { + return fmt.Errorf("%s/%s SP-I2 V2 %s telemetry: %w", record.Dataset, record.Name, executor, err) + } + summary := telemetry.Summary + if telemetry.Level != TraversalTelemetryLevelDiagnostic || telemetry.Diagnostic == nil || + telemetry.Diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete || telemetry.Diagnostic.PlanReplay == nil || + summary.RequestedIdentity != string(executor) || summary.RuntimeIdentity != string(executor) || summary.AppliedIdentity != string(executor) || + summary.EmittedIdentity != optimize.ShortestPathPolicyI2DistanceGuardedV2 || summary.FallbackExecuted == nil || *summary.FallbackExecuted || + summary.Overflow == nil || *summary.Overflow || summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable { + return fmt.Errorf("%s/%s SP-I2 V2 %s component check lacks one exact non-fallback runtime receipt", record.Dataset, record.Name, executor) + } + gateCase := evaluateProductionResourceGateCase(record) + if !gateCase.Passed { + return fmt.Errorf("%s/%s SP-I2 V2 %s component resource/plan invariants failed: %s", record.Dataset, record.Name, executor, strings.Join(gateCase.Reasons, "; ")) + } + if executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1D && strings.Contains(record.Name, "cycle-control") { + plan := telemetry.Diagnostic.PlanReplay.Counters + for _, counter := range []string{"recursive_rows", "recursive_loops", "reverse_edge_probe_loops"} { + if _, present := plan[counter]; !present { + return fmt.Errorf("%s/%s SP-I2 V2 E1D direct-cycle plan lacks exact zero counter %s", record.Dataset, record.Name, counter) + } + } + if summary.RuntimeBranch != "inline_direct_distance" || plan["sp_i2_direct_rows"] != 1 || plan["sp_i2_distance_rows"] != 0 || + plan["recursive_rows"] != 0 || plan["recursive_loops"] != 0 || plan["reverse_edge_probe_loops"] != 0 || plan["sp_i2_admission_rows"] != 0 || + plan["sp_i2_admission_loops"] != 0 || plan["sp_i2_fallback_executor_loops"] != 0 { + return fmt.Errorf("%s/%s SP-I2 V2 E1D direct-cycle plan did not suppress recursive, admission, and fallback work", record.Dataset, record.Name) + } + } + if executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1P && + (record.Shape.PathMaterializationRequired || record.PostgresMetrics.HydrationRows != 0 || record.PostgresMetrics.HydrationLoops != 0) { + return fmt.Errorf("%s/%s SP-I2 V2 E1P component check unexpectedly hydrates a scalar result", record.Dataset, record.Name) + } + return nil +} + +func loadSPI2V2ComponentAuthorization(path, protocolSHA256 string) (SPI2V2ComponentAuthorization, error) { + raw, err := os.ReadFile(path) + if err != nil { + return SPI2V2ComponentAuthorization{}, err + } + var authorization SPI2V2ComponentAuthorization + if err := decodePromotionEvidence(raw, &authorization); err != nil { + return SPI2V2ComponentAuthorization{}, fmt.Errorf("decode SP-I2 V2 component authorization: %w", err) + } + if err := validateSPI2V2ComponentAuthorization(authorization, protocolSHA256); err != nil { + return SPI2V2ComponentAuthorization{}, err + } + return authorization, nil +} + +func validateSPI2V2ComponentAuthorization(authorization SPI2V2ComponentAuthorization, protocolSHA256 string) error { + expectedExecutors := []string{ + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P), + } + if authorization.Schema != spI2V2ComponentAuthorizationSchema || authorization.Generation != spI2GenerationV2 || + authorization.ProtocolDeclarationSHA256 != protocolSHA256 || !lowercaseSHA256(protocolSHA256) || + authorization.AuthorizedExecutor != string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP) || !authorization.Passed || + strings.TrimSpace(authorization.SourceCommit) == "" || !lowercaseSHA256(authorization.DirtyDiffSHA256) || !lowercaseSHA256(authorization.BinarySHA256) || + len(authorization.Components) != len(expectedExecutors) { + return fmt.Errorf("SP-I2 V2 component authorization identity is invalid") + } + for index, component := range authorization.Components { + if component.Executor != expectedExecutors[index] || !lowercaseSHA256(component.ArtifactSHA256) || component.Cases != 6 || + !component.SemanticPassed || !component.PlanPassed || !component.ResourcePassed || !component.ReceiptPassed || + !component.FallbackFree || !component.CanonicalPlanSeen { + return fmt.Errorf("SP-I2 V2 component authorization does not prove exact E1D/E1P eligibility") + } + } + return nil +} + +func validateSPI2V2ComponentAuthorizationForCapture(path, corpusRoot string) error { + _, protocolSHA256, err := loadSPI2ProtocolV2(filepath.Join(corpusRoot, "protocols", "sp_i2_distance_v2.json")) + if err != nil { + return err + } + authorization, err := loadSPI2V2ComponentAuthorization(path, protocolSHA256) + if err != nil { + return err + } + if authorization.SourceCommit != commandOutput("git", "rev-parse", "HEAD") || + authorization.DirtyDiffSHA256 != workingTreeSHA256() || authorization.BinarySHA256 != executableSHA256() { + return fmt.Errorf("SP-I2 V2 component authorization does not bind the current source tree and executable") + } + return nil +} + +func spI2V2ComponentExecutors() []optimize.ShortestPathExecutor { + return []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + } +} + +func validSPI2V2ComponentExecutor(executor optimize.ShortestPathExecutor) bool { + return slices.Contains(spI2V2ComponentExecutors(), executor) +} diff --git a/cmd/graphbench/sp_i2_component_authorization_v2_test.go b/cmd/graphbench/sp_i2_component_authorization_v2_test.go new file mode 100644 index 00000000..4659b8c7 --- /dev/null +++ b/cmd/graphbench/sp_i2_component_authorization_v2_test.go @@ -0,0 +1,272 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func TestValidateSPI2V2ComponentAuthorization(t *testing.T) { + protocolSHA256 := strings.Repeat("a", 64) + authorization := validSPI2V2ComponentAuthorization(protocolSHA256) + require.NoError(t, validateSPI2V2ComponentAuthorization(authorization, protocolSHA256)) + + mutations := map[string]func(*SPI2V2ComponentAuthorization){ + "wrong generation": func(value *SPI2V2ComponentAuthorization) { value.Generation = spI2GenerationV1 }, + "wrong protocol": func(value *SPI2V2ComponentAuthorization) { value.ProtocolDeclarationSHA256 = strings.Repeat("b", 64) }, + "wrong combined arm": func(value *SPI2V2ComponentAuthorization) { + value.AuthorizedExecutor = string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1) + }, + "failed authorization": func(value *SPI2V2ComponentAuthorization) { value.Passed = false }, + "missing component": func(value *SPI2V2ComponentAuthorization) { value.Components = value.Components[:1] }, + "reordered component": func(value *SPI2V2ComponentAuthorization) { + value.Components[0], value.Components[1] = value.Components[1], value.Components[0] + }, + "failed semantic check": func(value *SPI2V2ComponentAuthorization) { value.Components[0].SemanticPassed = false }, + "failed plan check": func(value *SPI2V2ComponentAuthorization) { value.Components[1].PlanPassed = false }, + "wrong case count": func(value *SPI2V2ComponentAuthorization) { value.Components[0].Cases = 5 }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + copy := authorization + copy.Components = append([]SPI2V2ComponentAuthorizationCase(nil), authorization.Components...) + mutate(©) + require.Error(t, validateSPI2V2ComponentAuthorization(copy, protocolSHA256)) + }) + } +} + +func TestLoadSPI2V2ComponentAuthorizationStrictlyRejectsTampering(t *testing.T) { + protocolSHA256 := strings.Repeat("a", 64) + authorization := validSPI2V2ComponentAuthorization(protocolSHA256) + path := filepath.Join(t.TempDir(), "authorization.json") + require.NoError(t, writeIndentedJSON(path, authorization)) + _, err := loadSPI2V2ComponentAuthorization(path, protocolSHA256) + require.NoError(t, err) + + raw, err := os.ReadFile(path) + require.NoError(t, err) + for name, mutated := range map[string]string{ + "duplicate": strings.Replace(string(raw), `"schema":`, `"schema": "duplicate", "schema":`, 1), + "unknown": strings.Replace(string(raw), "{", `{"unknown":true,`, 1), + "trailing": string(raw) + `{}`, + } { + t.Run(name, func(t *testing.T) { + mutatedPath := filepath.Join(t.TempDir(), name+".json") + require.NoError(t, os.WriteFile(mutatedPath, []byte(mutated), 0o600)) + _, err := loadSPI2V2ComponentAuthorization(mutatedPath, protocolSHA256) + require.Error(t, err) + }) + } +} + +func TestCreateSPI2V2ComponentAuthorizationFromExactEvidence(t *testing.T) { + e1d := spI2V2ComponentTestRecords(t, optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) + e1p := spI2V2ComponentTestRecords(t, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P) + for executor, records := range map[optimize.ShortestPathExecutor][]CaseResult{ + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D: e1d, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P: e1p, + } { + component, _, err := validateSPI2V2ComponentEvidence(records, executor) + require.NoError(t, err) + require.True(t, component.SemanticPassed) + require.True(t, component.PlanPassed) + } + directory := t.TempDir() + e1dPath := filepath.Join(directory, "e1d.jsonl") + e1pPath := filepath.Join(directory, "e1p.jsonl") + output := filepath.Join(directory, "authorization.json") + require.NoError(t, writeJSONLFile(e1dPath, e1d)) + require.NoError(t, writeJSONLFile(e1pPath, e1p)) + passed, err := createSPI2V2ComponentAuthorization("../../benchmark/testdata/scale", e1dPath, e1pPath, output) + require.NoError(t, err) + require.True(t, passed) + + _, protocolSHA256, err := loadSPI2ProtocolV2("../../benchmark/testdata/scale/protocols/sp_i2_distance_v2.json") + require.NoError(t, err) + authorization, err := loadSPI2V2ComponentAuthorization(output, protocolSHA256) + require.NoError(t, err) + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP), authorization.AuthorizedExecutor) +} + +func TestValidateSPI2V2ComponentEvidenceRejectsSemanticPlanAndReceiptTampering(t *testing.T) { + executor := optimize.ShortestPathExecutorI2GuardedDistanceV2E1D + tests := map[string]func([]CaseResult) []CaseResult{ + "missing case": func(records []CaseResult) []CaseResult { return records[:len(records)-1] }, + "changed observation": func(records []CaseResult) []CaseResult { + records[0].ObservedRows = []string{"[999]"} + return records + }, + "missing canonical plan": func(records []CaseResult) []CaseResult { + records[0].PostgresPlanJSON = nil + return records + }, + "fallback receipt": func(records []CaseResult) []CaseResult { + fallback := true + records[0].TraversalTelemetry.Summary.FallbackExecuted = &fallback + return records + }, + "replayed invocation": func(records []CaseResult) []CaseResult { + first := records[0].Stats.Samples[0].RuntimeInvocationID + records[1].Stats.Samples[0].RuntimeInvocationID = first + records[1].Stats.Samples[0].RuntimeReceiptEvents[0].InvocationID = first + return records + }, + "direct recursion": func(records []CaseResult) []CaseResult { + for index := range records { + if strings.Contains(records[index].Name, "cycle-control") { + records[index].TraversalTelemetry.Diagnostic.PlanReplay.Counters["recursive_rows"] = 1 + break + } + } + return records + }, + "missing direct zero counter": func(records []CaseResult) []CaseResult { + for index := range records { + if strings.Contains(records[index].Name, "cycle-control") { + delete(records[index].TraversalTelemetry.Diagnostic.PlanReplay.Counters, "recursive_loops") + delete(records[index].TraversalTelemetry.Diagnostic.PlanReplay.Provenance, "counters.recursive_loops") + break + } + } + return records + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + records := mutate(spI2V2ComponentTestRecords(t, executor)) + _, _, err := validateSPI2V2ComponentEvidence(records, executor) + require.Error(t, err) + }) + } +} + +func validSPI2V2ComponentAuthorization(protocolSHA256 string) SPI2V2ComponentAuthorization { + component := func(executor, digest string) SPI2V2ComponentAuthorizationCase { + return SPI2V2ComponentAuthorizationCase{ + Executor: executor, ArtifactSHA256: digest, Cases: 6, + SemanticPassed: true, PlanPassed: true, ResourcePassed: true, ReceiptPassed: true, + FallbackFree: true, CanonicalPlanSeen: true, + } + } + return SPI2V2ComponentAuthorization{ + Schema: spI2V2ComponentAuthorizationSchema, + Generation: spI2GenerationV2, + ProtocolDeclarationSHA256: protocolSHA256, + SourceCommit: "deadbeef", + DirtyDiffSHA256: strings.Repeat("d", 64), + BinarySHA256: strings.Repeat("b", 64), + AuthorizedExecutor: string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP), + Components: []SPI2V2ComponentAuthorizationCase{ + component(string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D), strings.Repeat("1", 64)), + component(string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P), strings.Repeat("2", 64)), + }, + Passed: true, + } +} + +func spI2V2ComponentTestRecords(t *testing.T, executor optimize.ShortestPathExecutor) []CaseResult { + t.Helper() + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, selection, err := selectRunnableScaleCorpusWithSPI2Protection(full, CorpusSelectors{Tags: []string{spI2TrainingTag}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 6) + records := make([]CaseResult, 0, len(selected.Cases)) + for _, testCase := range selected.Cases { + fixture, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err) + fixture.PhysicalValidated = true + fixture.PhysicalNodeCount = int64(fixture.NodeCount) + fixture.PhysicalEdgeCount = int64(fixture.EdgeCount) + fixture.NodeRelationBytes = int64(fixture.NodeCount) * 1024 + fixture.EdgeRelationBytes = int64(fixture.EdgeCount) * 1024 + _, record := spI2QualificationTestRecords(t, testCase, fixture, selection, spI2TrainingCorpusSHA256, 1, 1, 1) + record.Environment.Arm = string(executor) + record.Environment.ArmOrder = 1 + record.Environment.RunUUID = "component-check" + record.StableObservation = true + record.PostgresPlanJSON = []byte(`[ {"Plan":{"Node Type":"Result"},"Planning Time":1.0} ]`) + planningMS := 1.0 + record.PostgresMetrics.PlanningMS = &planningMS + outcome := &record.Optimization.TargetOutcomes[0] + outcome.Candidate = string(executor) + outcome.Selected = string(executor) + outcome.Applied = string(executor) + outcome.EmittedPolicy = optimize.ShortestPathPolicyI2DistanceGuardedV2 + outcome.SelectorVersion = optimize.ShortestPathSelectorStaticV9HiddenFanInTail + outcome.EmittedCandidates = []string{string(executor), string(optimize.ShortestPathExecutorS4CanonicalDistance)} + + direct := executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1D + directHit := direct && strings.Contains(testCase.Name, "cycle-control") + admissionRows := int64(1) + if directHit { + admissionRows = 0 + for index := range record.PostgresMetrics.PlanNodes { + node := &record.PostgresMetrics.PlanNodes[index] + switch node.SubplanName { + case "CTE sp_i2_distance_bounded", "CTE sp_i2_target": + node.ActualRows = 0 + } + } + record.PostgresMetrics.RecursiveRows = 0 + record.PostgresMetrics.RecursiveLoops = 0 + } + record.PostgresMetrics.PlanNodes = append(record.PostgresMetrics.PlanNodes, PostgresPlanNodeMetric{ + PlanNodeID: int64(len(record.PostgresMetrics.PlanNodes) + 1), NodeType: "Result", + SubplanName: "CTE sp_i2_admission", ActualRows: admissionRows, ActualLoops: admissionRows, + }) + if direct { + directRows := int64(0) + if directHit { + directRows = 1 + } + record.PostgresMetrics.PlanNodes = append(record.PostgresMetrics.PlanNodes, PostgresPlanNodeMetric{ + PlanNodeID: int64(len(record.PostgresMetrics.PlanNodes) + 1), NodeType: "Result", + SubplanName: "CTE sp_i2_v2_direct", ActualRows: directRows, ActualLoops: 1, + }) + } + telemetry, err := buildPostgresCaseTraversalTelemetry(*record.Optimization, *record.PostgresMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichInlineDistanceTraversalTelemetry(telemetry, record.RowCount) + if directHit { + for _, counter := range []string{"recursive_rows", "recursive_loops", "reverse_edge_probe_loops"} { + telemetry.Diagnostic.PlanReplay.Counters[counter] = 0 + telemetry.Diagnostic.PlanReplay.Provenance["counters."+counter] = "test.plan." + counter + } + } + require.NoError(t, telemetry.Validate()) + record.TraversalTelemetry = telemetry + + branch := telemetry.Summary.RuntimeBranch + invocationID := fmt.Sprintf("component-%s-%s-1", executor, testCase.Name) + fallback := false + record.Stats.Samples = []LatencySample{{ + Round: 1, Block: 1, Arm: string(executor), ArmOrder: 1, RunUUID: "component-check", Iteration: 1, + Case: testCase.Name, Dataset: testCase.Dataset, Backend: ModePostgresSQL, ConnectionID: "101", + Classification: "warm", Duration: time.Millisecond, RequestedIdentity: string(executor), + RuntimeIdentity: string(executor), RuntimeBranch: branch, FallbackExecuted: &fallback, + RuntimeAttestation: "timed_invocation", RuntimeInvocationID: invocationID, + RuntimeReceiptEvents: []RuntimeReceiptEvent{{ + InvocationID: invocationID, Ordinal: 1, RuntimeIdentity: string(executor), RuntimeBranch: branch, + }}, + }} + receiptID := fmt.Sprintf("component-%s-%s-stabilization", executor, testCase.Name) + record.Stats.ReceiptStabilization = &RuntimeStabilizationReceipt{ + InvocationID: receiptID, RequestedIdentity: string(executor), RuntimeIdentity: string(executor), + RuntimeBranch: branch, FallbackExecuted: &fallback, + Events: []RuntimeReceiptEvent{{InvocationID: receiptID, Ordinal: 1, RuntimeIdentity: string(executor), RuntimeBranch: branch}}, + } + records = append(records, record) + } + return records +} diff --git a/cmd/graphbench/sp_i2_development_report_v2.go b/cmd/graphbench/sp_i2_development_report_v2.go new file mode 100644 index 00000000..1cc8b74d --- /dev/null +++ b/cmd/graphbench/sp_i2_development_report_v2.go @@ -0,0 +1,411 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const spI2V2DevelopmentReportSchema = "sp-i2-v2-development-report-v1" + +type SPI2V2DevelopmentReport struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + ProtocolDeclarationSHA256 string `json:"protocol_declaration_sha256"` + ArtifactSHA256 string `json:"artifact_sha256"` + SourceCommit string `json:"source_commit"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + StatisticalImplementation string `json:"statistical_implementation"` + Confidence float64 `json:"confidence_level"` + BootstrapReplicates int `json:"bootstrap_replicates"` + Rounds int `json:"rounds"` + TimedSamplesPerRound int `json:"timed_samples_per_round"` + PromotionEligible bool `json:"promotion_eligible"` + SelectedExecutor string `json:"selected_executor"` + Arms []SPI2V2DevelopmentArmReport `json:"arms"` +} + +type SPI2V2DevelopmentArmReport struct { + Executor string `json:"executor"` + Eligible bool `json:"eligible"` + Reasons []string `json:"reasons,omitempty"` + Ranking *SPI2V2DevelopmentRanking `json:"ranking,omitempty"` + Cases []SPI2V2DevelopmentCaseReport `json:"cases"` +} + +type SPI2V2DevelopmentRanking struct { + PlanNodeScore int `json:"plan_node_score"` + PlanningRatioUpper float64 `json:"maximum_planning_ratio_upper_vs_e0"` + P95RatioUpper float64 `json:"maximum_p95_ratio_upper_vs_e0"` + FixedOrder int `json:"fixed_order"` +} + +type SPI2V2DevelopmentCaseReport struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + MaxPlanNodes int `json:"maximum_plan_node_count"` + Contrasts []SPI2V2DevelopmentCaseContrast `json:"contrasts"` +} + +type SPI2V2DevelopmentCaseContrast struct { + Comparator string `json:"comparator"` + MedianRatio RatioInterval `json:"median_ratio"` + MedianSaving DurationInterval `json:"median_saving"` + P95Ratio RatioInterval `json:"p95_ratio"` + P95Saving DurationInterval `json:"p95_saving"` + PlanningRatio RatioInterval `json:"planning_time_ratio"` +} + +type spI2V2DevelopmentReportOptions struct { + confidence float64 + bootstrapReplicates int +} + +type spI2V2DevelopmentSeries struct { + samples roundSamples + planning roundSamples + maxPlanNodes int +} + +func createSPI2V2DevelopmentReport(corpusRoot, artifact, output string) (SPI2V2DevelopmentReport, error) { + protocolPath := filepath.Join(corpusRoot, "protocols", "sp_i2_distance_v2.json") + if output != "" && (sameCleanPath(output, artifact) || sameCleanPath(output, protocolPath)) { + return SPI2V2DevelopmentReport{}, fmt.Errorf("SP-I2 V2 development report output must not overwrite an input") + } + protocol, protocolSHA256, err := loadSPI2ProtocolV2(protocolPath) + if err != nil { + return SPI2V2DevelopmentReport{}, err + } + records, err := readJSONLFile(artifact) + if err != nil { + return SPI2V2DevelopmentReport{}, fmt.Errorf("read SP-I2 V2 development artifact: %w", err) + } + artifactSHA256, err := fileSHA256(artifact) + if err != nil { + return SPI2V2DevelopmentReport{}, err + } + report, err := buildSPI2V2DevelopmentReport(records, protocolSHA256, artifactSHA256, spI2V2DevelopmentReportOptions{ + confidence: protocol.Design.ConfidenceLevel, bootstrapReplicates: protocol.Design.BootstrapReplicates, + }) + if err != nil { + return SPI2V2DevelopmentReport{}, err + } + if output == "" { + encoder := json.NewEncoder(os.Stdout) + encoder.SetIndent("", " ") + err = encoder.Encode(report) + } else { + err = writeIndentedJSON(output, report) + } + return report, err +} + +func buildSPI2V2DevelopmentReport(records []CaseResult, protocolSHA256, artifactSHA256 string, options spI2V2DevelopmentReportOptions) (SPI2V2DevelopmentReport, error) { + if options.confidence <= 0 || options.confidence >= 1 || options.bootstrapReplicates <= 0 { + return SPI2V2DevelopmentReport{}, fmt.Errorf("invalid SP-I2 V2 development statistical options") + } + if err := validateSPI2V2DevelopmentEvidence(records, spI2V2StudyTournament); err != nil { + return SPI2V2DevelopmentReport{}, err + } + series, source, err := collectSPI2V2DevelopmentSeries(records) + if err != nil { + return SPI2V2DevelopmentReport{}, err + } + report, err := evaluateSPI2V2DevelopmentSeries(series, options) + if err != nil { + return SPI2V2DevelopmentReport{}, err + } + report.ProtocolDeclarationSHA256 = protocolSHA256 + report.ArtifactSHA256 = artifactSHA256 + report.SourceCommit = source.sourceCommit + report.DirtyDiffSHA256 = source.dirtyDiffSHA256 + report.BinarySHA256 = source.binarySHA256 + return report, nil +} + +func collectSPI2V2DevelopmentSeries(records []CaseResult) (map[performanceKey]map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries, spI2V2ComponentSourceIdentity, error) { + declarations, err := canonicalSPI2Declarations() + if err != nil { + return nil, spI2V2ComponentSourceIdentity{}, err + } + series := make(map[performanceKey]map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries) + var source spI2V2ComponentSourceIdentity + for _, record := range records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + arm := optimize.ShortestPathExecutor(record.Environment.Arm) + if err := validateSPI2V2DevelopmentReportRecord(record, arm, declarations[key]); err != nil { + return nil, spI2V2ComponentSourceIdentity{}, err + } + currentSource := spI2V2ComponentSourceIdentity{ + sourceCommit: record.Environment.SourceCommit, dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, binarySHA256: record.Environment.BinarySHA256, + } + if source.sourceCommit == "" { + source = currentSource + } else if source != currentSource { + return nil, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 development artifact mixes source or binary identities") + } + if series[key] == nil { + series[key] = make(map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries) + } + entry := series[key][arm] + if entry == nil { + entry = &spI2V2DevelopmentSeries{samples: roundSamples{}, planning: roundSamples{}} + series[key][arm] = entry + } + round := record.Environment.Round + for _, sample := range record.Stats.Samples { + entry.samples[round] = append(entry.samples[round], sample.Duration) + } + entry.planning[round] = []time.Duration{time.Duration(*record.PostgresMetrics.PlanningMS * float64(time.Millisecond))} + entry.maxPlanNodes = max(entry.maxPlanNodes, len(record.PostgresMetrics.PlanNodes)) + } + if strings.TrimSpace(source.sourceCommit) == "" || !lowercaseSHA256(source.dirtyDiffSHA256) || !lowercaseSHA256(source.binarySHA256) { + return nil, spI2V2ComponentSourceIdentity{}, fmt.Errorf("SP-I2 V2 development artifact lacks a complete source and binary identity") + } + return series, source, nil +} + +func validateSPI2V2DevelopmentReportRecord(record CaseResult, arm optimize.ShortestPathExecutor, declaration spI2CanonicalDeclaration) error { + if record.PostgresEnvironment == nil || record.Fixture == nil || record.PostgresMetrics == nil || record.TraversalTelemetry == nil || record.Optimization == nil || + !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") || len(record.PostgresPlanJSON) == 0 || + record.PostgresMetrics.PlanningMS == nil || *record.PostgresMetrics.PlanningMS <= 0 || len(record.PostgresMetrics.PlanNodes) == 0 { + return fmt.Errorf("%s/%s arm %q lacks its canonical Repeatable Read plan observation", record.Dataset, record.Name, arm) + } + parsedPlan, err := parsePostgresPlanJSONMetrics(record.PostgresPlanJSON) + if err != nil || parsedPlan.PlanningMS == nil || *parsedPlan.PlanningMS != *record.PostgresMetrics.PlanningMS { + return fmt.Errorf("%s/%s arm %q canonical JSON plan contradicts its planning observation", record.Dataset, record.Name, arm) + } + testCase := declaration.testCase + testCase.Source = record.Source + expected := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expected, *record.Fixture) + if filepath.Base(record.Source) != "generated_sp_i2_distance_v1.json" || record.Category != testCase.Category || record.Cypher != testCase.Cypher || + record.WorkloadSHA256 != expected.WorkloadSHA256 || !reflect.DeepEqual(record.NodeParams, testCase.NodeParams) || + !reflect.DeepEqual(record.NodeListParams, testCase.NodeListParams) || !reflect.DeepEqual(record.Shape, testCase.Shape) || !record.StableObservation || + record.ExpectedRowCount == nil || testCase.Expected.RowCount == nil || *record.ExpectedRowCount != *testCase.Expected.RowCount || record.RowCount != *testCase.Expected.RowCount { + return fmt.Errorf("%s/%s arm %q changes the exact open-corpus semantic contract", record.Dataset, record.Name, arm) + } + if err := validateExpectedObservations(testCase.Expected, record.ObservedRows); err != nil { + return fmt.Errorf("%s/%s arm %q observation: %w", record.Dataset, record.Name, arm, err) + } + telemetry := record.TraversalTelemetry + if err := ValidateTraversalExecutionTelemetry(telemetry); err != nil { + return fmt.Errorf("%s/%s arm %q telemetry: %w", record.Dataset, record.Name, arm, err) + } + summary := telemetry.Summary + if telemetry.Level != TraversalTelemetryLevelDiagnostic || telemetry.Diagnostic == nil || telemetry.Diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete || + telemetry.Diagnostic.PlanReplay == nil || summary.RequestedIdentity != string(arm) || summary.RuntimeIdentity != string(arm) || + summary.AppliedIdentity != string(arm) || summary.EmittedIdentity != optimize.ShortestPathPolicyI2DistanceGuardedV2 || + summary.FallbackExecuted == nil || *summary.FallbackExecuted || summary.Overflow == nil || *summary.Overflow || + summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable { + return fmt.Errorf("%s/%s arm %q lacks an exact diagnostic non-fallback runtime receipt", record.Dataset, record.Name, arm) + } + if gate := evaluateProductionResourceGateCase(record); !gate.Passed { + return fmt.Errorf("%s/%s arm %q resource/plan invariants failed: %s", record.Dataset, record.Name, arm, strings.Join(gate.Reasons, "; ")) + } + return nil +} + +func evaluateSPI2V2DevelopmentSeries(series map[performanceKey]map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries, options spI2V2DevelopmentReportOptions) (SPI2V2DevelopmentReport, error) { + report := SPI2V2DevelopmentReport{ + Schema: spI2V2DevelopmentReportSchema, Generation: spI2GenerationV2, StatisticalImplementation: spI2HierBootstrapV2, + Confidence: options.confidence, BootstrapReplicates: options.bootstrapReplicates, Rounds: 10, TimedSamplesPerRound: 100, + PromotionEligible: false, + } + keys := make([]performanceKey, 0, len(series)) + for key := range series { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + return keys[i].name < keys[j].name + }) + if len(keys) == 0 { + return report, fmt.Errorf("SP-I2 V2 development report has no cases") + } + armReports := make(map[optimize.ShortestPathExecutor]*SPI2V2DevelopmentArmReport, len(spI2V2DevelopmentArms)) + contrastIndex := make(map[optimize.ShortestPathExecutor]map[performanceKey]map[optimize.ShortestPathExecutor]SPI2V2DevelopmentCaseContrast) + for _, arm := range spI2V2DevelopmentArms { + entry := &SPI2V2DevelopmentArmReport{Executor: string(arm), Eligible: true} + armReports[arm] = entry + contrastIndex[arm] = make(map[performanceKey]map[optimize.ShortestPathExecutor]SPI2V2DevelopmentCaseContrast) + for _, key := range keys { + armSeries := series[key][arm] + if armSeries == nil { + return report, fmt.Errorf("%s/%s omits arm %q", key.dataset, key.name, arm) + } + caseReport := SPI2V2DevelopmentCaseReport{Dataset: key.dataset, Name: key.name, MaxPlanNodes: armSeries.maxPlanNodes} + for _, comparator := range spI2V2Comparators(arm) { + contrast, err := compareSPI2V2DevelopmentCase(key, comparator, arm, series[key][comparator], armSeries, options) + if err != nil { + return report, err + } + caseReport.Contrasts = append(caseReport.Contrasts, contrast) + if contrastIndex[arm][key] == nil { + contrastIndex[arm][key] = make(map[optimize.ShortestPathExecutor]SPI2V2DevelopmentCaseContrast) + } + contrastIndex[arm][key][comparator] = contrast + } + entry.Cases = append(entry.Cases, caseReport) + } + } + for _, arm := range spI2V2DevelopmentArms[1:] { + entry := armReports[arm] + for _, key := range keys { + baseline := contrastIndex[arm][key][optimize.ShortestPathExecutorI2GuardedDistanceV2E0] + if baseline.MedianRatio.Upper > 1.02 || baseline.P95Ratio.Upper > 1.02 { + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s/%s exceeds the E0 wall-clock non-regression limit", key.dataset, key.name)) + } + } + switch arm { + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1: + applySPI2V2CycleGate(entry, keys, contrastIndex[arm], optimize.ShortestPathExecutorI2GuardedDistanceV2E0, false) + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P: + applySPI2V2CycleGate(entry, keys, contrastIndex[arm], optimize.ShortestPathExecutorI2GuardedDistanceV2E1, true) + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP: + for _, parent := range []optimize.ShortestPathExecutor{optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P} { + for _, key := range keys { + contrast := contrastIndex[arm][key][parent] + if contrast.MedianRatio.Upper > 1.02 || contrast.P95Ratio.Upper > 1.02 || contrast.PlanningRatio.Upper > 1.02 { + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s/%s exceeds a %s parent limit", key.dataset, key.name, parent)) + } + } + } + } + entry.Reasons = slices.Compact(entry.Reasons) + entry.Eligible = len(entry.Reasons) == 0 + } + // E1DP eligibility depends on the final parent decisions. + combined := armReports[optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP] + for _, parent := range []optimize.ShortestPathExecutor{optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P} { + if !armReports[parent].Eligible { + combined.Reasons = append(combined.Reasons, fmt.Sprintf("parent %s is ineligible", parent)) + } + } + combined.Reasons = slices.Compact(combined.Reasons) + combined.Eligible = len(combined.Reasons) == 0 + + var eligible []*SPI2V2DevelopmentArmReport + for fixedOrder, arm := range spI2V2DevelopmentArms[1:] { + entry := armReports[arm] + if entry.Eligible { + ranking := SPI2V2DevelopmentRanking{FixedOrder: fixedOrder + 1} + for index, key := range keys { + ranking.PlanNodeScore += series[key][arm].maxPlanNodes + contrast := contrastIndex[arm][key][optimize.ShortestPathExecutorI2GuardedDistanceV2E0] + if index == 0 || contrast.PlanningRatio.Upper > ranking.PlanningRatioUpper { + ranking.PlanningRatioUpper = contrast.PlanningRatio.Upper + } + if index == 0 || contrast.P95Ratio.Upper > ranking.P95RatioUpper { + ranking.P95RatioUpper = contrast.P95Ratio.Upper + } + } + entry.Ranking = &ranking + eligible = append(eligible, entry) + } + } + sort.SliceStable(eligible, func(i, j int) bool { return lessSPI2V2DevelopmentRanking(*eligible[i].Ranking, *eligible[j].Ranking) }) + selected := armReports[optimize.ShortestPathExecutorI2GuardedDistanceV2E0] + if len(eligible) > 0 { + selected = eligible[0] + } + report.SelectedExecutor = selected.Executor + for _, arm := range spI2V2DevelopmentArms { + report.Arms = append(report.Arms, *armReports[arm]) + } + return report, nil +} + +func sameCleanPath(left, right string) bool { + leftAbsolute, leftErr := filepath.Abs(left) + rightAbsolute, rightErr := filepath.Abs(right) + return leftErr == nil && rightErr == nil && filepath.Clean(leftAbsolute) == filepath.Clean(rightAbsolute) +} + +func spI2V2Comparators(arm optimize.ShortestPathExecutor) []optimize.ShortestPathExecutor { + switch arm { + case optimize.ShortestPathExecutorI2GuardedDistanceV2E0: + return nil + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1: + return []optimize.ShortestPathExecutor{optimize.ShortestPathExecutorI2GuardedDistanceV2E0} + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P: + return []optimize.ShortestPathExecutor{optimize.ShortestPathExecutorI2GuardedDistanceV2E0, optimize.ShortestPathExecutorI2GuardedDistanceV2E1} + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP: + return []optimize.ShortestPathExecutor{optimize.ShortestPathExecutorI2GuardedDistanceV2E0, optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P} + default: + return nil + } +} + +func compareSPI2V2DevelopmentCase(key performanceKey, comparator, candidate optimize.ShortestPathExecutor, baseline, treatment *spI2V2DevelopmentSeries, options spI2V2DevelopmentReportOptions) (SPI2V2DevelopmentCaseContrast, error) { + if baseline == nil || treatment == nil { + return SPI2V2DevelopmentCaseContrast{}, fmt.Errorf("%s/%s lacks %s/%s contrast series", key.dataset, key.name, comparator, candidate) + } + domain := string(comparator) + "-vs-" + string(candidate) + medianRatio, medianSaving, err := bootstrapSPI2RoundMedianV2(baseline.samples, treatment.samples, key.dataset, key.name, domain+"-median", options.confidence, options.bootstrapReplicates) + if err != nil { + return SPI2V2DevelopmentCaseContrast{}, err + } + tail, err := bootstrapSPI2HierarchicalTailV2(baseline.samples, treatment.samples, key.dataset, key.name, domain+"-p95", .95, options.confidence, options.bootstrapReplicates) + if err != nil { + return SPI2V2DevelopmentCaseContrast{}, err + } + planningRatio, _, err := bootstrapSPI2RoundMedianV2(baseline.planning, treatment.planning, key.dataset, key.name, domain+"-planning", options.confidence, options.bootstrapReplicates) + if err != nil { + return SPI2V2DevelopmentCaseContrast{}, err + } + return SPI2V2DevelopmentCaseContrast{ + Comparator: string(comparator), MedianRatio: medianRatio, MedianSaving: medianSaving, P95Ratio: tail.Ratio, + P95Saving: DurationInterval{Estimate: -tail.Change.Estimate, Lower: -tail.Change.Upper, Upper: -tail.Change.Lower}, PlanningRatio: planningRatio, + }, nil +} + +func applySPI2V2CycleGate(entry *SPI2V2DevelopmentArmReport, keys []performanceKey, contrasts map[performanceKey]map[optimize.ShortestPathExecutor]SPI2V2DevelopmentCaseContrast, comparator optimize.ShortestPathExecutor, planningGate bool) { + cycleSeen := false + for _, key := range keys { + contrast := contrasts[key][comparator] + if contrast.MedianRatio.Upper > 1.02 || contrast.P95Ratio.Upper > 1.02 { + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s/%s exceeds the %s wall-clock non-regression limit", key.dataset, key.name, comparator)) + } + if planningGate && contrast.PlanningRatio.Upper > 1.02 { + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s/%s exceeds the %s planning-time limit", key.dataset, key.name, comparator)) + } + if strings.Contains(key.name, "cycle-control") { + cycleSeen = true + if contrast.P95Ratio.Upper > .95 && contrast.P95Saving.Lower < 50*time.Microsecond { + entry.Reasons = append(entry.Reasons, fmt.Sprintf("%s/%s misses the %s cycle-control gain", key.dataset, key.name, comparator)) + } + } + } + if !cycleSeen { + entry.Reasons = append(entry.Reasons, "cycle-control case is missing") + } +} + +func lessSPI2V2DevelopmentRanking(left, right SPI2V2DevelopmentRanking) bool { + if left.PlanNodeScore != right.PlanNodeScore { + return left.PlanNodeScore < right.PlanNodeScore + } + if math.Float64bits(left.PlanningRatioUpper) != math.Float64bits(right.PlanningRatioUpper) { + return left.PlanningRatioUpper < right.PlanningRatioUpper + } + if math.Float64bits(left.P95RatioUpper) != math.Float64bits(right.P95RatioUpper) { + return left.P95RatioUpper < right.P95RatioUpper + } + return left.FixedOrder < right.FixedOrder +} diff --git a/cmd/graphbench/sp_i2_development_report_v2_test.go b/cmd/graphbench/sp_i2_development_report_v2_test.go new file mode 100644 index 00000000..eec94e70 --- /dev/null +++ b/cmd/graphbench/sp_i2_development_report_v2_test.go @@ -0,0 +1,127 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func TestCreateSPI2V2DevelopmentReportRejectsInputOverwrite(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "tournament.jsonl") + _, err := createSPI2V2DevelopmentReport("../../benchmark/testdata/scale", artifact, artifact) + require.ErrorContains(t, err, "must not overwrite") +} + +func TestEvaluateSPI2V2DevelopmentSeriesSelectsEligibleArmByDeclaredRanking(t *testing.T) { + series := spI2V2DevelopmentReportTestSeries() + report, err := evaluateSPI2V2DevelopmentSeries(series, spI2V2DevelopmentReportOptions{confidence: .975, bootstrapReplicates: 100}) + require.NoError(t, err) + require.False(t, report.PromotionEligible) + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP), report.SelectedExecutor) + require.Len(t, report.Arms, 5) + for _, arm := range report.Arms[1:] { + require.True(t, arm.Eligible, arm.Reasons) + require.NotNil(t, arm.Ranking) + require.Len(t, arm.Cases, 6) + } + require.Len(t, report.Arms[1].Cases[0].Contrasts, 1) + require.Len(t, report.Arms[2].Cases[0].Contrasts, 2) + require.Len(t, report.Arms[3].Cases[0].Contrasts, 2) + require.Len(t, report.Arms[4].Cases[0].Contrasts, 3) +} + +func TestEvaluateSPI2V2DevelopmentSeriesRejectsPlanningRegressionAndCombinedParent(t *testing.T) { + series := spI2V2DevelopmentReportTestSeries() + e1d := optimize.ShortestPathExecutorI2GuardedDistanceV2E1D + for key := range series { + series[key][e1d].planning = constantSPI2V2DevelopmentRounds(1100*time.Microsecond, 1) + } + report, err := evaluateSPI2V2DevelopmentSeries(series, spI2V2DevelopmentReportOptions{confidence: .975, bootstrapReplicates: 100}) + require.NoError(t, err) + require.False(t, report.Arms[2].Eligible) + require.Contains(t, strings.Join(report.Arms[2].Reasons, " "), "planning-time limit") + require.False(t, report.Arms[4].Eligible) + require.Contains(t, strings.Join(report.Arms[4].Reasons, " "), "parent") + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P), report.SelectedExecutor) +} + +func TestEvaluateSPI2V2DevelopmentSeriesFallsBackToE0(t *testing.T) { + series := spI2V2DevelopmentReportTestSeries() + for key := range series { + for _, arm := range spI2V2DevelopmentArms[1:] { + series[key][arm].samples = constantSPI2V2DevelopmentRounds(1100*time.Microsecond, 100) + } + } + report, err := evaluateSPI2V2DevelopmentSeries(series, spI2V2DevelopmentReportOptions{confidence: .975, bootstrapReplicates: 100}) + require.NoError(t, err) + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0), report.SelectedExecutor) +} + +func TestLessSPI2V2DevelopmentRankingUsesEveryDeclaredTieBreaker(t *testing.T) { + base := SPI2V2DevelopmentRanking{PlanNodeScore: 10, PlanningRatioUpper: 1, P95RatioUpper: 1, FixedOrder: 2} + require.True(t, lessSPI2V2DevelopmentRanking(SPI2V2DevelopmentRanking{PlanNodeScore: 9}, base)) + require.True(t, lessSPI2V2DevelopmentRanking(SPI2V2DevelopmentRanking{PlanNodeScore: 10, PlanningRatioUpper: .99}, base)) + require.True(t, lessSPI2V2DevelopmentRanking(SPI2V2DevelopmentRanking{PlanNodeScore: 10, PlanningRatioUpper: 1, P95RatioUpper: .99}, base)) + require.True(t, lessSPI2V2DevelopmentRanking(SPI2V2DevelopmentRanking{PlanNodeScore: 10, PlanningRatioUpper: 1, P95RatioUpper: 1, FixedOrder: 1}, base)) +} + +func TestValidateSPI2V2DevelopmentReportRecordRejectsPlanningAndPlanTampering(t *testing.T) { + records := spI2V2ComponentTestRecords(t, optimize.ShortestPathExecutorI2GuardedDistanceV2E1D) + record := records[0] + declarations, err := canonicalSPI2Declarations() + require.NoError(t, err) + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + require.NoError(t, validateSPI2V2DevelopmentReportRecord(record, optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, declarations[key])) + + planningMS := *record.PostgresMetrics.PlanningMS + record.PostgresMetrics.PlanningMS = nil + require.ErrorContains(t, validateSPI2V2DevelopmentReportRecord(record, optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, declarations[key]), "canonical") + + record = records[0] + record.PostgresMetrics.PlanningMS = &planningMS + record.PostgresPlanJSON = []byte(`[{"Planning Time":1.0}]`) + require.ErrorContains(t, validateSPI2V2DevelopmentReportRecord(record, optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, declarations[key]), "contradicts") +} + +func spI2V2DevelopmentReportTestSeries() map[performanceKey]map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries { + series := make(map[performanceKey]map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries) + caseNames := []string{"cycle-control", "case-2", "case-3", "case-4", "case-5", "case-6"} + for _, name := range caseNames { + key := performanceKey{dataset: "fixture", name: name, backend: ModePostgresSQL} + series[key] = make(map[optimize.ShortestPathExecutor]*spI2V2DevelopmentSeries) + for armIndex, arm := range spI2V2DevelopmentArms { + latency := 1000 * time.Microsecond + if name == "cycle-control" { + switch arm { + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1: + latency = 900 * time.Microsecond + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP: + latency = 800 * time.Microsecond + } + } + series[key][arm] = &spI2V2DevelopmentSeries{ + samples: constantSPI2V2DevelopmentRounds(latency, 100), planning: constantSPI2V2DevelopmentRounds(time.Millisecond, 1), + maxPlanNodes: 5 - armIndex, + } + } + } + return series +} + +func constantSPI2V2DevelopmentRounds(value time.Duration, samples int) roundSamples { + result := make(roundSamples, 10) + for round := 1; round <= 10; round++ { + result[round] = make([]time.Duration, samples) + for index := range result[round] { + result[round][index] = value + } + } + return result +} diff --git a/cmd/graphbench/sp_i2_development_v2.go b/cmd/graphbench/sp_i2_development_v2.go new file mode 100644 index 00000000..285ea456 --- /dev/null +++ b/cmd/graphbench/sp_i2_development_v2.go @@ -0,0 +1,387 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "slices" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +type spI2V2DevelopmentStudy string + +const ( + spI2V2StudyReadiness spI2V2DevelopmentStudy = "readiness" + spI2V2StudyTournament spI2V2DevelopmentStudy = "tournament" +) + +var spI2V2DevelopmentArms = []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorI2GuardedDistanceV2E0, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP, +} + +var spI2V2ReadinessArms = []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorS4CanonicalDistance, + optimize.ShortestPathExecutorI2GuardedDistanceV2E0, +} + +// spI2V2DevelopmentOrder returns the preregistered doubled five-arm Williams +// row. Across ten rounds every arm occupies every physical position twice. +func spI2V2DevelopmentOrder(round int) ([]optimize.ShortestPathExecutor, error) { + if round < 1 || round > 10 { + return nil, fmt.Errorf("SP-I2 V2 development round must be in 1..10") + } + schedule := [10][5]int{ + {0, 1, 4, 2, 3}, {1, 2, 0, 3, 4}, {2, 3, 1, 4, 0}, {3, 4, 2, 0, 1}, {4, 0, 3, 1, 2}, + {3, 2, 4, 1, 0}, {4, 3, 0, 2, 1}, {0, 4, 1, 3, 2}, {1, 0, 2, 4, 3}, {2, 1, 3, 0, 4}, + } + row := schedule[round-1] + ordered := make([]optimize.ShortestPathExecutor, len(row)) + for position, arm := range row { + ordered[position] = spI2V2DevelopmentArms[arm] + } + return ordered, nil +} + +// spI2V2ReadinessOrder alternates the supplemental control pair so each arm +// occupies each physical position five times across the fixed ten rounds. +func spI2V2ReadinessOrder(round int) ([]optimize.ShortestPathExecutor, error) { + if round < 1 || round > 10 { + return nil, fmt.Errorf("SP-I2 V2 readiness round must be in 1..10") + } + if round%2 == 1 { + return slices.Clone(spI2V2ReadinessArms), nil + } + return []optimize.ShortestPathExecutor{spI2V2ReadinessArms[1], spI2V2ReadinessArms[0]}, nil +} + +// validateSPI2V2DevelopmentCaptureConfig freezes one invocation's position in +// the open-corpus component tournament before any database setup occurs. +func validateSPI2V2DevelopmentCaptureConfig(cfg config) error { + if cfg.SPI2V2ReadinessComparison || cfg.SPI2V2ComponentCheck { + return fmt.Errorf("SP-I2 V2 development tournament and readiness comparison are mutually exclusive") + } + if cfg.SPI2Generation != spI2GenerationV2 { + return fmt.Errorf("SP-I2 V2 development tournament requires generation %q", spI2GenerationV2) + } + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL || cfg.ExistingGraph || cfg.Discovery { + return fmt.Errorf("SP-I2 V2 development tournament requires one managed PostgreSQL mode") + } + if cfg.Iterations != 100 || cfg.WarmupIterations != 25 || cfg.PoolSize != 1 || len(cfg.Concurrency) != 0 { + return fmt.Errorf("SP-I2 V2 development tournament requires exactly 100 samples, 25 warmups, pool size 1, and no concurrency block") + } + if cfg.Block != cfg.Round || cfg.ArmOrder < 1 || cfg.ArmOrder > 5 || strings.TrimSpace(cfg.RunUUID) == "" { + return fmt.Errorf("SP-I2 V2 development tournament requires block equal to round, a five-arm order, and an explicit shared run UUID") + } + if len(cfg.Tags) != 1 || cfg.Tags[0] != spI2TrainingTag || len(cfg.Cases) != 0 || len(cfg.Datasets) != 0 || len(cfg.Categories) != 0 { + return fmt.Errorf("SP-I2 V2 development tournament is restricted to the six open V1 training cases") + } + executor := optimize.ShortestPathExecutor(cfg.PostgresForceShortest) + if !slices.Contains(spI2V2DevelopmentArms, executor) { + return fmt.Errorf("SP-I2 V2 development tournament must force a declared E0/E1 component arm") + } + order, err := spI2V2DevelopmentOrder(cfg.Round) + if err != nil { + return err + } + expectedOrder := slices.Index(order, executor) + 1 + if cfg.Arm != string(executor) || cfg.ArmOrder != expectedOrder { + return fmt.Errorf("SP-I2 V2 development round %d requires arm %q at order %d", cfg.Round, executor, expectedOrder) + } + if executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP { + if strings.TrimSpace(cfg.SPI2V2ComponentAuthorization) == "" { + return fmt.Errorf("SP-I2 V2 E1DP capture requires an exact E1D/E1P component authorization") + } + if err := validateSPI2V2ComponentAuthorizationForCapture(cfg.SPI2V2ComponentAuthorization, cfg.CorpusRoot); err != nil { + return err + } + } else if cfg.SPI2V2ComponentAuthorization != "" { + return fmt.Errorf("SP-I2 V2 component authorization may be supplied only for E1DP capture") + } + if !cfg.PostgresRepeatableRead || cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic || + cfg.PostgresProductionManifest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament || + cfg.PostgresReferences || len(cfg.PostgresReferenceArms) != 0 || cfg.Baseline != "" || + cfg.BundleDir != "" || len(cfg.BundleEvidence) != 0 || cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "" { + return fmt.Errorf("SP-I2 V2 development tournament requires forced Repeatable Read with diagnostic telemetry and no supplemental or protected-evidence arms") + } + if cfg.OutputJSONL == "" || cfg.Round > 1 && !cfg.AppendJSONL { + return fmt.Errorf("SP-I2 V2 development tournament requires a JSONL output and append mode after round 1") + } + return nil +} + +// validateSPI2V2ReadinessCaptureConfig freezes one invocation's position in +// the supplemental open-corpus E0/S4 comparison before database setup. +func validateSPI2V2ReadinessCaptureConfig(cfg config) error { + if cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ComponentCheck { + return fmt.Errorf("SP-I2 V2 readiness comparison and development tournament are mutually exclusive") + } + if cfg.SPI2Generation != spI2GenerationV2 { + return fmt.Errorf("SP-I2 V2 readiness comparison requires generation %q", spI2GenerationV2) + } + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL || cfg.ExistingGraph || cfg.Discovery { + return fmt.Errorf("SP-I2 V2 readiness comparison requires one managed PostgreSQL mode") + } + if cfg.Iterations != 100 || cfg.WarmupIterations != 25 || cfg.PoolSize != 1 || len(cfg.Concurrency) != 0 { + return fmt.Errorf("SP-I2 V2 readiness comparison requires exactly 100 samples, 25 warmups, pool size 1, and no concurrency block") + } + if cfg.Block != cfg.Round || cfg.ArmOrder < 1 || cfg.ArmOrder > 2 || strings.TrimSpace(cfg.RunUUID) == "" { + return fmt.Errorf("SP-I2 V2 readiness comparison requires block equal to round, a two-arm order, and an explicit shared run UUID") + } + if len(cfg.Tags) != 1 || cfg.Tags[0] != spI2TrainingTag || len(cfg.Cases) != 0 || len(cfg.Datasets) != 0 || len(cfg.Categories) != 0 { + return fmt.Errorf("SP-I2 V2 readiness comparison is restricted to the six open V1 training cases") + } + executor := optimize.ShortestPathExecutor(cfg.PostgresForceShortest) + if !slices.Contains(spI2V2ReadinessArms, executor) { + return fmt.Errorf("SP-I2 V2 readiness comparison must force exact S4 distance or E0") + } + order, err := spI2V2ReadinessOrder(cfg.Round) + if err != nil { + return err + } + expectedOrder := slices.Index(order, executor) + 1 + if cfg.Arm != string(executor) || cfg.ArmOrder != expectedOrder { + return fmt.Errorf("SP-I2 V2 readiness round %d requires arm %q at order %d", cfg.Round, executor, expectedOrder) + } + if !cfg.PostgresRepeatableRead || cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic || + cfg.PostgresProductionManifest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament || + cfg.PostgresReferences || len(cfg.PostgresReferenceArms) != 0 || cfg.Baseline != "" || + cfg.BundleDir != "" || len(cfg.BundleEvidence) != 0 || cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "" { + return fmt.Errorf("SP-I2 V2 readiness comparison requires forced Repeatable Read with diagnostic telemetry and no supplemental or protected-evidence arms") + } + if cfg.OutputJSONL == "" || cfg.Round > 1 && !cfg.AppendJSONL { + return fmt.Errorf("SP-I2 V2 readiness comparison requires a JSONL output and append mode after round 1") + } + return nil +} + +// validateSPI2V2ComponentCheckCaptureConfig freezes the diagnostic semantic +// and plan-invariant capture used to authorize the combined E1DP arm. +func validateSPI2V2ComponentCheckCaptureConfig(cfg config) error { + if cfg.SPI2V2DevelopmentTournament || cfg.SPI2V2ReadinessComparison { + return fmt.Errorf("SP-I2 V2 component check cannot be combined with development timing captures") + } + if cfg.SPI2Generation != spI2GenerationV2 { + return fmt.Errorf("SP-I2 V2 component check requires generation %q", spI2GenerationV2) + } + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL || cfg.ExistingGraph || cfg.Discovery { + return fmt.Errorf("SP-I2 V2 component check requires one managed PostgreSQL mode") + } + if cfg.Iterations != 1 || cfg.WarmupIterations != 1 || cfg.PoolSize != 1 || len(cfg.Concurrency) != 0 { + return fmt.Errorf("SP-I2 V2 component check requires exactly one sample, one warmup, pool size 1, and no concurrency block") + } + if cfg.Round != 1 || cfg.Block != 1 || cfg.ArmOrder != 1 || strings.TrimSpace(cfg.RunUUID) == "" { + return fmt.Errorf("SP-I2 V2 component check requires round, block, and arm order 1 with an explicit run UUID") + } + if len(cfg.Tags) != 1 || cfg.Tags[0] != spI2TrainingTag || len(cfg.Cases) != 0 || len(cfg.Datasets) != 0 || len(cfg.Categories) != 0 { + return fmt.Errorf("SP-I2 V2 component check is restricted to the six open V1 training cases") + } + executor := optimize.ShortestPathExecutor(cfg.PostgresForceShortest) + if !validSPI2V2ComponentExecutor(executor) || cfg.Arm != string(executor) { + return fmt.Errorf("SP-I2 V2 component check must force and label exact E1D or E1P") + } + if !cfg.PostgresRepeatableRead || cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic || + cfg.PostgresProductionManifest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament || + cfg.PostgresReferences || len(cfg.PostgresReferenceArms) != 0 || cfg.Baseline != "" || + cfg.BundleDir != "" || len(cfg.BundleEvidence) != 0 || cfg.SPI2Freeze != "" || cfg.SPI2DiscoveryReport != "" || + cfg.SPI2V2ComponentAuthorization != "" { + return fmt.Errorf("SP-I2 V2 component check requires Repeatable Read with diagnostic telemetry and no supplemental or protected evidence") + } + if cfg.OutputJSONL == "" || cfg.AppendJSONL { + return fmt.Errorf("SP-I2 V2 component check requires one fresh JSONL output") + } + return nil +} + +type spI2V2DevelopmentRecordKey struct { + caseKey performanceKey + round int + arm optimize.ShortestPathExecutor +} + +type spI2V2DevelopmentInvocation struct { + order int + startedAt time.Time + endedAt time.Time +} + +// validateSPI2V2DevelopmentEvidence rejects partial, relabeled, replayed, or +// out-of-order raw development artifacts before statistical interpretation. +func validateSPI2V2DevelopmentEvidence(records []CaseResult, study spI2V2DevelopmentStudy) error { + cohort, err := canonicalSPI2Cohort() + if err != nil { + return err + } + var arms []optimize.ShortestPathExecutor + switch study { + case spI2V2StudyReadiness: + arms = spI2V2ReadinessArms + case spI2V2StudyTournament: + arms = spI2V2DevelopmentArms + default: + return fmt.Errorf("unknown SP-I2 V2 development study %q", study) + } + expectedRecords := len(cohort.trainingKeys) * 10 * len(arms) + if len(records) != expectedRecords { + return fmt.Errorf("SP-I2 V2 %s artifact contains %d records, expected exactly %d", study, len(records), expectedRecords) + } + seenRecords := make(map[spI2V2DevelopmentRecordKey]struct{}, expectedRecords) + seenInvocations := make(map[string]struct{}, expectedRecords*100) + invocations := make(map[int]map[optimize.ShortestPathExecutor]spI2V2DevelopmentInvocation, 10) + runUUID := "" + for _, record := range records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + if _, expected := cohort.trainingKeys[key]; !expected { + return fmt.Errorf("SP-I2 V2 %s artifact contains unexpected case %s/%s", study, record.Dataset, record.Name) + } + if record.ExecutionMode != ModePostgresSQL || record.Status != StatusOK || record.Environment == nil || + record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || + len(record.Environment.Concurrency) != 0 || record.Environment.ExistingGraph || + record.Environment.WarmupIterations != 25 || record.Stats.WarmupIterations != 25 || + record.Stats.Iterations != 100 || len(record.Stats.Samples) != 100 || + len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil || record.Baseline != nil { + return fmt.Errorf("%s/%s lacks the exact single-session SP-I2 V2 %s measurement contract", record.Dataset, record.Name, study) + } + environment := record.Environment + arm := optimize.ShortestPathExecutor(environment.Arm) + if !slices.Contains(arms, arm) || environment.Block != environment.Round || environment.Round < 1 || environment.Round > 10 || + strings.TrimSpace(environment.RunUUID) == "" || environment.StartedAt.IsZero() || environment.EndedAt.Before(environment.StartedAt) { + return fmt.Errorf("%s/%s has malformed SP-I2 V2 %s invocation metadata", record.Dataset, record.Name, study) + } + order, err := spI2V2StudyOrder(study, environment.Round) + if err != nil { + return err + } + expectedOrder := slices.Index(order, arm) + 1 + if environment.ArmOrder != expectedOrder { + return fmt.Errorf("SP-I2 V2 %s round %d requires arm %q at order %d", study, environment.Round, arm, expectedOrder) + } + if runUUID == "" { + runUUID = environment.RunUUID + } else if runUUID != environment.RunUUID { + return fmt.Errorf("SP-I2 V2 %s artifact mixes run UUIDs", study) + } + recordKey := spI2V2DevelopmentRecordKey{caseKey: key, round: environment.Round, arm: arm} + if _, duplicate := seenRecords[recordKey]; duplicate { + return fmt.Errorf("SP-I2 V2 %s artifact duplicates %s/%s round %d arm %q", study, record.Dataset, record.Name, environment.Round, arm) + } + seenRecords[recordKey] = struct{}{} + if invocations[environment.Round] == nil { + invocations[environment.Round] = map[optimize.ShortestPathExecutor]spI2V2DevelopmentInvocation{} + } + invocation := spI2V2DevelopmentInvocation{order: environment.ArmOrder, startedAt: environment.StartedAt, endedAt: environment.EndedAt} + if prior, found := invocations[environment.Round][arm]; found && prior != invocation { + return fmt.Errorf("SP-I2 V2 %s round %d arm %q mixes invocation chronology", study, environment.Round, arm) + } + invocations[environment.Round][arm] = invocation + if err := validateSPI2V2DevelopmentSamples(record, arm, seenInvocations); err != nil { + return err + } + } + var priorRoundEnded time.Time + for round := 1; round <= 10; round++ { + order, err := spI2V2StudyOrder(study, round) + if err != nil { + return err + } + var priorEnded time.Time + for position, arm := range order { + invocation, found := invocations[round][arm] + if !found || invocation.order != position+1 { + return fmt.Errorf("SP-I2 V2 %s round %d omits scheduled arm %q", study, round, arm) + } + if !priorEnded.IsZero() && priorEnded.After(invocation.startedAt) { + return fmt.Errorf("SP-I2 V2 %s round %d arm chronology contradicts the fixed order", study, round) + } + if position == 0 && !priorRoundEnded.IsZero() && priorRoundEnded.After(invocation.startedAt) { + return fmt.Errorf("SP-I2 V2 %s round %d overlaps or predates the prior round", study, round) + } + priorEnded = invocation.endedAt + } + priorRoundEnded = priorEnded + } + return nil +} + +func validateSPI2V2DevelopmentArtifact(path string, study spI2V2DevelopmentStudy) error { + records, err := readJSONLFile(path) + if err != nil { + return fmt.Errorf("read artifact: %w", err) + } + return validateSPI2V2DevelopmentEvidence(records, study) +} + +func spI2V2StudyOrder(study spI2V2DevelopmentStudy, round int) ([]optimize.ShortestPathExecutor, error) { + if study == spI2V2StudyReadiness { + return spI2V2ReadinessOrder(round) + } + if study == spI2V2StudyTournament { + return spI2V2DevelopmentOrder(round) + } + return nil, fmt.Errorf("unknown SP-I2 V2 development study %q", study) +} + +func validateSPI2V2DevelopmentSamples(record CaseResult, arm optimize.ShortestPathExecutor, seenInvocations map[string]struct{}) error { + environment := record.Environment + receipt := record.Stats.ReceiptStabilization + if receipt == nil || strings.TrimSpace(receipt.InvocationID) == "" || receipt.RequestedIdentity != string(arm) || + receipt.RuntimeIdentity != string(arm) || strings.TrimSpace(receipt.RuntimeBranch) == "" || + receipt.FallbackExecuted == nil || *receipt.FallbackExecuted || len(receipt.Events) == 0 { + return fmt.Errorf("%s/%s arm %q lacks one exact excluded stabilization receipt", record.Dataset, record.Name, arm) + } + if _, duplicate := seenInvocations[receipt.InvocationID]; duplicate { + return fmt.Errorf("SP-I2 V2 development evidence reuses invocation identity %q", receipt.InvocationID) + } + seenInvocations[receipt.InvocationID] = struct{}{} + if err := validateSPI2V2ReceiptEvents(receipt.InvocationID, receipt.RuntimeIdentity, receipt.RuntimeBranch, receipt.Events); err != nil { + return fmt.Errorf("%s/%s stabilization receipt: %w", record.Dataset, record.Name, err) + } + iterations := make(map[int]struct{}, 100) + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 || sample.Iteration < 1 || sample.Iteration > 100 || + sample.Dataset != record.Dataset || sample.Case != record.Name || sample.Backend != ModePostgresSQL || + sample.Round != environment.Round || sample.Block != environment.Block || sample.Arm != environment.Arm || + sample.ArmOrder != environment.ArmOrder || sample.RunUUID != environment.RunUUID || strings.TrimSpace(sample.ConnectionID) == "" || + sample.RequestedIdentity != string(arm) || sample.RuntimeIdentity != string(arm) || strings.TrimSpace(sample.RuntimeBranch) == "" || + sample.FallbackExecuted == nil || *sample.FallbackExecuted || strings.TrimSpace(sample.RuntimeAttestation) == "" || + strings.TrimSpace(sample.RuntimeInvocationID) == "" || len(sample.RuntimeReceiptEvents) == 0 { + return fmt.Errorf("%s/%s arm %q contains a sample outside its exact invocation identity", record.Dataset, record.Name, arm) + } + if _, duplicate := iterations[sample.Iteration]; duplicate { + return fmt.Errorf("%s/%s arm %q duplicates timed iteration %d", record.Dataset, record.Name, arm, sample.Iteration) + } + iterations[sample.Iteration] = struct{}{} + if _, duplicate := seenInvocations[sample.RuntimeInvocationID]; duplicate { + return fmt.Errorf("SP-I2 V2 development evidence reuses invocation identity %q", sample.RuntimeInvocationID) + } + seenInvocations[sample.RuntimeInvocationID] = struct{}{} + if err := validateSPI2V2ReceiptEvents(sample.RuntimeInvocationID, sample.RuntimeIdentity, sample.RuntimeBranch, sample.RuntimeReceiptEvents); err != nil { + return fmt.Errorf("%s/%s timed iteration %d: %w", record.Dataset, record.Name, sample.Iteration, err) + } + } + return nil +} + +func validateSPI2V2ReceiptEvents(invocationID, runtimeIdentity, runtimeBranch string, events []RuntimeReceiptEvent) error { + for index, event := range events { + if event.InvocationID != invocationID || event.Ordinal != index+1 || strings.TrimSpace(event.RuntimeIdentity) == "" || strings.TrimSpace(event.RuntimeBranch) == "" { + return fmt.Errorf("runtime receipt event chain is malformed") + } + } + last := events[len(events)-1] + if last.RuntimeIdentity != runtimeIdentity || last.RuntimeBranch != runtimeBranch || last.FallbackExecuted { + return fmt.Errorf("runtime receipt terminal event contradicts the invocation outcome") + } + return nil +} diff --git a/cmd/graphbench/sp_i2_development_v2_test.go b/cmd/graphbench/sp_i2_development_v2_test.go new file mode 100644 index 00000000..5291ed85 --- /dev/null +++ b/cmd/graphbench/sp_i2_development_v2_test.go @@ -0,0 +1,390 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func TestSPI2V2DevelopmentOrderBalancesEveryArmAndPosition(t *testing.T) { + positionCounts := make(map[optimize.ShortestPathExecutor]map[int]int, len(spI2V2DevelopmentArms)) + for round := 1; round <= 10; round++ { + order, err := spI2V2DevelopmentOrder(round) + require.NoError(t, err) + require.Len(t, order, 5) + require.ElementsMatch(t, spI2V2DevelopmentArms, order) + for position, arm := range order { + if positionCounts[arm] == nil { + positionCounts[arm] = map[int]int{} + } + positionCounts[arm][position+1]++ + } + } + for _, arm := range spI2V2DevelopmentArms { + for position := 1; position <= 5; position++ { + require.Equal(t, 2, positionCounts[arm][position], "%s position %d", arm, position) + } + } + _, err := spI2V2DevelopmentOrder(0) + require.Error(t, err) + _, err = spI2V2DevelopmentOrder(11) + require.Error(t, err) +} + +func TestSPI2V2ReadinessOrderBalancesEveryArmAndPosition(t *testing.T) { + positionCounts := make(map[optimize.ShortestPathExecutor]map[int]int, len(spI2V2ReadinessArms)) + for round := 1; round <= 10; round++ { + order, err := spI2V2ReadinessOrder(round) + require.NoError(t, err) + require.Len(t, order, 2) + require.ElementsMatch(t, spI2V2ReadinessArms, order) + for position, arm := range order { + if positionCounts[arm] == nil { + positionCounts[arm] = map[int]int{} + } + positionCounts[arm][position+1]++ + } + } + for _, arm := range spI2V2ReadinessArms { + require.Equal(t, 5, positionCounts[arm][1], arm) + require.Equal(t, 5, positionCounts[arm][2], arm) + } + _, err := spI2V2ReadinessOrder(0) + require.Error(t, err) + _, err = spI2V2ReadinessOrder(11) + require.Error(t, err) +} + +func TestValidateSPI2V2DevelopmentCaptureConfig(t *testing.T) { + order, err := spI2V2DevelopmentOrder(4) + require.NoError(t, err) + executor := optimize.ShortestPathExecutorI2GuardedDistanceV2E1D + expectedOrder := 0 + for position, arm := range order { + if arm == executor { + expectedOrder = position + 1 + } + } + cfg := config{ + SPI2Generation: spI2GenerationV2, + SPI2V2DevelopmentTournament: true, + Modes: []ExecutionMode{ModePostgresSQL}, + Iterations: 100, + WarmupIterations: 25, + PoolSize: 1, + Round: 4, + Block: 4, + Arm: string(executor), + ArmOrder: expectedOrder, + RunUUID: "development-series", + Tags: []string{spI2TrainingTag}, + OutputJSONL: "development.jsonl", + AppendJSONL: true, + PostgresForceShortest: string(executor), + PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + } + require.NoError(t, validateSPI2V2DevelopmentCaptureConfig(cfg)) + + mutations := []func(*config){ + func(cfg *config) { cfg.SPI2Generation = spI2GenerationV1 }, + func(cfg *config) { cfg.Iterations = 99 }, + func(cfg *config) { cfg.WarmupIterations = 24 }, + func(cfg *config) { cfg.ArmOrder = expectedOrder%5 + 1 }, + func(cfg *config) { cfg.Arm = "alias" }, + func(cfg *config) { cfg.Tags = []string{spI2HoldoutTag} }, + func(cfg *config) { + cfg.PostgresForceShortest = string(optimize.ShortestPathExecutorI2GuardedDistanceV2) + }, + func(cfg *config) { cfg.PostgresRepeatableRead = false }, + func(cfg *config) { cfg.PostgresReferences = true }, + func(cfg *config) { cfg.SPI2Freeze = "freeze.json" }, + } + for _, mutate := range mutations { + copy := cfg + mutate(©) + require.Error(t, validateSPI2V2DevelopmentCaptureConfig(copy)) + } +} + +func TestValidateSPI2V2ReadinessCaptureConfig(t *testing.T) { + executor := optimize.ShortestPathExecutorS4CanonicalDistance + cfg := config{ + SPI2Generation: spI2GenerationV2, + SPI2V2ReadinessComparison: true, + Modes: []ExecutionMode{ModePostgresSQL}, + Iterations: 100, + WarmupIterations: 25, + PoolSize: 1, + Round: 3, + Block: 3, + Arm: string(executor), + ArmOrder: 1, + RunUUID: "readiness-series", + Tags: []string{spI2TrainingTag}, + OutputJSONL: "readiness.jsonl", + AppendJSONL: true, + PostgresForceShortest: string(executor), + PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + } + require.NoError(t, validateSPI2V2ReadinessCaptureConfig(cfg)) + + mutations := []func(*config){ + func(cfg *config) { cfg.SPI2Generation = spI2GenerationV1 }, + func(cfg *config) { cfg.Iterations = 101 }, + func(cfg *config) { cfg.Round = 11; cfg.Block = 11 }, + func(cfg *config) { cfg.ArmOrder = 2 }, + func(cfg *config) { cfg.Arm = "alias" }, + func(cfg *config) { cfg.Tags = []string{spI2HoldoutTag} }, + func(cfg *config) { + cfg.PostgresForceShortest = string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1) + }, + func(cfg *config) { cfg.PostgresRepeatableRead = false }, + func(cfg *config) { cfg.PostgresReferences = true }, + func(cfg *config) { cfg.SPI2V2DevelopmentTournament = true }, + } + for _, mutate := range mutations { + copy := cfg + mutate(©) + require.Error(t, validateSPI2V2ReadinessCaptureConfig(copy)) + } +} + +func TestValidateSPI2V2ComponentCheckCaptureConfig(t *testing.T) { + executor := optimize.ShortestPathExecutorI2GuardedDistanceV2E1D + cfg := config{ + SPI2Generation: spI2GenerationV2, + SPI2V2ComponentCheck: true, + Modes: []ExecutionMode{ModePostgresSQL}, + Iterations: 1, + WarmupIterations: 1, + PoolSize: 1, + Round: 1, + Block: 1, + Arm: string(executor), + ArmOrder: 1, + RunUUID: "component-check", + Tags: []string{spI2TrainingTag}, + OutputJSONL: "component.jsonl", + PostgresForceShortest: string(executor), + PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + } + require.NoError(t, validateSPI2V2ComponentCheckCaptureConfig(cfg)) + + for _, mutate := range []func(*config){ + func(cfg *config) { cfg.Iterations = 2 }, + func(cfg *config) { cfg.WarmupIterations = 0 }, + func(cfg *config) { cfg.Tags = []string{spI2HoldoutTag} }, + func(cfg *config) { + cfg.PostgresForceShortest = string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1) + }, + func(cfg *config) { cfg.Arm = "alias" }, + func(cfg *config) { cfg.PostgresRepeatableRead = false }, + func(cfg *config) { cfg.SPI2V2DevelopmentTournament = true }, + func(cfg *config) { cfg.SPI2V2ComponentAuthorization = "authorization.json" }, + } { + copy := cfg + mutate(©) + require.Error(t, validateSPI2V2ComponentCheckCaptureConfig(copy)) + } +} + +func TestValidateSPI2V2DevelopmentCaptureRequiresAuthorizationForCombinedArm(t *testing.T) { + order, err := spI2V2DevelopmentOrder(1) + require.NoError(t, err) + executor := optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP + cfg := config{ + SPI2Generation: spI2GenerationV2, + SPI2V2DevelopmentTournament: true, + Modes: []ExecutionMode{ModePostgresSQL}, + Iterations: 100, + WarmupIterations: 25, + PoolSize: 1, + Round: 1, + Block: 1, + Arm: string(executor), + ArmOrder: slices.Index(order, executor) + 1, + RunUUID: "development-series", + Tags: []string{spI2TrainingTag}, + OutputJSONL: "development.jsonl", + PostgresForceShortest: string(executor), + PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + } + require.ErrorContains(t, validateSPI2V2DevelopmentCaptureConfig(cfg), "requires an exact E1D/E1P component authorization") + + _, protocolSHA256, err := loadSPI2ProtocolV2("../../benchmark/testdata/scale/protocols/sp_i2_distance_v2.json") + require.NoError(t, err) + authorization := validSPI2V2ComponentAuthorization(protocolSHA256) + authorization.SourceCommit = commandOutput("git", "rev-parse", "HEAD") + authorization.DirtyDiffSHA256 = workingTreeSHA256() + authorization.BinarySHA256 = executableSHA256() + authorizationPath := filepath.Join(t.TempDir(), "authorization.json") + require.NoError(t, writeIndentedJSON(authorizationPath, authorization)) + cfg.CorpusRoot = "../../benchmark/testdata/scale" + cfg.SPI2V2ComponentAuthorization = authorizationPath + require.NoError(t, validateSPI2V2DevelopmentCaptureConfig(cfg)) + + authorization.BinarySHA256 = strings.Repeat("f", 64) + require.NoError(t, writeIndentedJSON(authorizationPath, authorization)) + require.ErrorContains(t, validateSPI2V2DevelopmentCaptureConfig(cfg), "does not bind the current source tree and executable") +} + +func TestValidateSPI2V2DevelopmentEvidenceAcceptsCompleteStudies(t *testing.T) { + require.NoError(t, validateSPI2V2DevelopmentEvidence(spI2V2DevelopmentTestRecords(t, spI2V2StudyReadiness), spI2V2StudyReadiness)) + require.NoError(t, validateSPI2V2DevelopmentEvidence(spI2V2DevelopmentTestRecords(t, spI2V2StudyTournament), spI2V2StudyTournament)) +} + +func TestValidateSPI2V2DevelopmentArtifact(t *testing.T) { + artifact := filepath.Join(t.TempDir(), "readiness.jsonl") + require.NoError(t, writeJSONLFile(artifact, spI2V2DevelopmentTestRecords(t, spI2V2StudyReadiness))) + require.NoError(t, validateSPI2V2DevelopmentArtifact(artifact, spI2V2StudyReadiness)) +} + +func TestValidateSPI2V2DevelopmentEvidenceRejectsTampering(t *testing.T) { + tests := map[string]func([]CaseResult) []CaseResult{ + "missing record": func(records []CaseResult) []CaseResult { + return records[:len(records)-1] + }, + "wrong sample count": func(records []CaseResult) []CaseResult { + records[0].Stats.Samples = records[0].Stats.Samples[:99] + return records + }, + "wrong arm order": func(records []CaseResult) []CaseResult { + records[0].Environment.ArmOrder = 2 + return records + }, + "mixed run UUID": func(records []CaseResult) []CaseResult { + records[0].Environment.RunUUID = "other" + return records + }, + "relabeled requested identity": func(records []CaseResult) []CaseResult { + records[0].Stats.Samples[0].RequestedIdentity = "other" + return records + }, + "missing stabilization": func(records []CaseResult) []CaseResult { + records[0].Stats.ReceiptStabilization = nil + return records + }, + "replayed timed invocation": func(records []CaseResult) []CaseResult { + first := records[0].Stats.Samples[0].RuntimeInvocationID + records[0].Stats.Samples[1].RuntimeInvocationID = first + records[0].Stats.Samples[1].RuntimeReceiptEvents[0].InvocationID = first + return records + }, + "unexpected case": func(records []CaseResult) []CaseResult { + records[0].Name = "holdout" + return records + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + records := mutate(spI2V2DevelopmentTestRecords(t, spI2V2StudyReadiness)) + require.Error(t, validateSPI2V2DevelopmentEvidence(records, spI2V2StudyReadiness)) + }) + } +} + +func spI2V2DevelopmentTestRecords(t *testing.T, study spI2V2DevelopmentStudy) []CaseResult { + t.Helper() + cohort, err := canonicalSPI2Cohort() + require.NoError(t, err) + arms := spI2V2DevelopmentArms + if study == spI2V2StudyReadiness { + arms = spI2V2ReadinessArms + } + records := make([]CaseResult, 0, len(cohort.trainingKeys)*10*len(arms)) + base := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + fallback := false + for round := 1; round <= 10; round++ { + order, err := spI2V2StudyOrder(study, round) + require.NoError(t, err) + for position, arm := range order { + startedAt := base.Add(time.Duration(round)*time.Hour + time.Duration(position)*2*time.Minute) + endedAt := startedAt.Add(time.Minute) + for key := range cohort.trainingKeys { + prefix := fmt.Sprintf("%s-%d-%s-%s", study, round, arm, key.name) + receiptID := prefix + "-stabilization" + record := CaseResult{ + Dataset: key.dataset, + Name: key.name, + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + Environment: &RunEnvironment{ + ArtifactSchemaVersion: 2, + RunUUID: "development-series", + Arm: string(arm), + ArmOrder: position + 1, + Block: round, + Round: round, + StartedAt: startedAt, + EndedAt: endedAt, + WarmupIterations: 25, + PoolSize: 1, + }, + Stats: DurationStats{ + Iterations: 100, + WarmupIterations: 25, + ReceiptStabilization: &RuntimeStabilizationReceipt{ + InvocationID: receiptID, + RequestedIdentity: string(arm), + RuntimeIdentity: string(arm), + RuntimeBranch: "selected", + FallbackExecuted: &fallback, + Events: []RuntimeReceiptEvent{{ + InvocationID: receiptID, + Ordinal: 1, + RuntimeIdentity: string(arm), + RuntimeBranch: "selected", + FallbackExecuted: false, + }}, + }, + }, + } + for iteration := 1; iteration <= 100; iteration++ { + invocationID := fmt.Sprintf("%s-%d", prefix, iteration) + record.Stats.Samples = append(record.Stats.Samples, LatencySample{ + Round: round, + Block: round, + Arm: string(arm), + ArmOrder: position + 1, + RunUUID: "development-series", + Iteration: iteration, + Case: key.name, + Dataset: key.dataset, + Backend: ModePostgresSQL, + ConnectionID: "connection-1", + Classification: "warm", + Duration: time.Millisecond, + RequestedIdentity: string(arm), + RuntimeIdentity: string(arm), + RuntimeBranch: "selected", + FallbackExecuted: &fallback, + RuntimeAttestation: "receipt", + RuntimeInvocationID: invocationID, + RuntimeReceiptEvents: []RuntimeReceiptEvent{{ + InvocationID: invocationID, + Ordinal: 1, + RuntimeIdentity: string(arm), + RuntimeBranch: "selected", + FallbackExecuted: false, + }}, + }) + } + records = append(records, record) + } + } + } + return records +} diff --git a/cmd/graphbench/sp_i2_distance_integration_test.go b/cmd/graphbench/sp_i2_distance_integration_test.go new file mode 100644 index 00000000..31dc0fbd --- /dev/null +++ b/cmd/graphbench/sp_i2_distance_integration_test.go @@ -0,0 +1,301 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package main + +import ( + "context" + "net/url" + "os" + "slices" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestPostgreSQLSPI2GuardedDistancePlanAttributionAndFallback executes only +// already-open SP-I2 training cases. It proves reachable and no-path candidate +// receipts, then lowers diagnostic caps to force the same-statement exact S4 +// arm and compares every public observation with an explicit S4 run. +func TestPostgreSQLSPI2GuardedDistancePlanAttributionAndFallback(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + const ( + reachableCase = "GSP-I2-V1-TRAIN-D03-RI064-FI032-full" + overflowCase = "GSP-I2-V1-TRAIN-D16-RI256-FI512-full" + noPathCase = "GSP-I2-V1-TRAIN-D16-RI256-FI512-disconnected" + cycleCase = "GSP-I2-V1-TRAIN-cycle-control" + ) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{reachableCase, overflowCase, noPathCase, cycleCase}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 4) + for _, testCase := range selected.Cases { + require.Equal(t, "training", testCase.Shape.QualificationSplit) + require.NotContains(t, testCase.Tags, spI2HoldoutTag) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + + run := func(corpus ScaleCorpus, options translate.ToolOptions) []CaseResult { + runner.toolOptions = options + records, err := runner.Run(ctx, 0, 1, corpus) + require.NoError(t, err) + for _, record := range records { + require.Equal(t, StatusOK, record.Status, record.Error) + } + return records + } + byName := func(records []CaseResult) map[string]CaseResult { + indexed := make(map[string]CaseResult, len(records)) + for _, record := range records { + indexed[record.Name] = record + } + return indexed + } + + baseline := byName(run(selected, translate.ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS4CanonicalDistance, + })) + candidate := run(selected, translate.ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistance, + }) + require.Len(t, candidate, 4) + for _, record := range candidate { + exact := baseline[record.Name] + require.Equal(t, exact.RowCount, record.RowCount) + require.True(t, slices.Equal(exact.ObservedRows, record.ObservedRows)) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + + summary := record.TraversalTelemetry.Summary + require.Equal(t, optimize.ShortestPathPolicyI2DistanceGuardedV1, summary.EmittedIdentity) + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistance), summary.RuntimeIdentity) + require.False(t, *summary.Overflow) + require.False(t, *summary.FallbackExecuted) + if record.Name == noPathCase { + require.Equal(t, "inline_canonical_distance_no_path", summary.RuntimeBranch) + } else { + require.Equal(t, "inline_canonical_distance", summary.RuntimeBranch) + } + + diagnostic := record.TraversalTelemetry.Diagnostic + require.Equal(t, TraversalTelemetryCounterStatusComplete, diagnostic.CounterStatus) + require.NotNil(t, diagnostic.PlanReplay) + counters := diagnostic.PlanReplay.Counters + if record.Name == cycleCase { + require.Equal(t, int64(2), counters["sp_i2_distance_rows"]) + } + require.Equal(t, int64(1), counters["sp_i2_candidate_marker_rows"]) + require.Zero(t, counters["sp_i2_fallback_marker_rows"]) + require.Equal(t, int64(1), counters["sp_i2_candidate_executor_loops"]) + require.Zero(t, counters["sp_i2_fallback_executor_loops"]) + require.Zero(t, counters["sp_i2_fallback_branch_rows"]) + require.Equal(t, record.RowCount, counters["sp_i2_output_rows"]) + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, record.TraversalTelemetry) + require.Empty(t, gateCase.Reasons) + } + + reduced, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{overflowCase}}) + require.NoError(t, err) + require.Len(t, reduced.Cases, 1) + fallback := run(reduced, translate.ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistance, + GuardedDistanceStateLimit: 10, + GuardedDistanceFrontierLimit: 10, + }) + require.Len(t, fallback, 1) + record := fallback[0] + exact := baseline[record.Name] + require.Equal(t, exact.RowCount, record.RowCount) + require.True(t, slices.Equal(exact.ObservedRows, record.ObservedRows)) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + + summary := record.TraversalTelemetry.Summary + require.Equal(t, optimize.ShortestPathPolicyI2DistanceGuardedV1, summary.EmittedIdentity) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalDistance), summary.RuntimeIdentity) + require.Equal(t, "exact_s4_distance_fallback", summary.RuntimeBranch) + require.True(t, *summary.Overflow) + require.True(t, *summary.FallbackExecuted) + diagnostic := record.TraversalTelemetry.Diagnostic + require.Equal(t, TraversalTelemetryCounterStatusComplete, diagnostic.CounterStatus) + counters := diagnostic.PlanReplay.Counters + require.Zero(t, counters["sp_i2_candidate_marker_rows"]) + require.Equal(t, int64(1), counters["sp_i2_fallback_marker_rows"]) + require.Zero(t, counters["sp_i2_candidate_executor_loops"]) + require.Equal(t, int64(1), counters["sp_i2_fallback_executor_loops"]) + require.Zero(t, counters["sp_i2_candidate_branch_rows"]) + require.Equal(t, record.RowCount, counters["sp_i2_output_rows"]) + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, record.TraversalTelemetry) + require.Empty(t, gateCase.Reasons) + + chains := runtimeReceiptChains(record.Stats.Samples) + require.Len(t, chains, 1) + warm := operationalWarmSamples(record) + require.Len(t, warm, 1) + require.NoError(t, validateRuntimeReceiptEvents(chains[0], warm[0].RuntimeIdentity, warm[0].RuntimeBranch, warm[0].FallbackExecuted)) + require.True(t, receiptChainContainsIdentity(chains[0], string(optimize.ShortestPathExecutorS4CanonicalDistance), true)) +} + +// TestPostgreSQLSPI2V2DevelopmentArms compares every component arm with exact +// S4 on an open direct hit, recursive hit, and no-path case. The direct arms +// additionally prove that a depth-one result leaves recursive admission and +// target bodies unexecuted. +func TestPostgreSQLSPI2V2DevelopmentArms(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + const ( + directCase = "GSP-D01-F001_distance" + recursiveCase = "GSP-I2-V1-TRAIN-D03-RI064-FI032-full" + noPathCase = "GSP-I2-V1-TRAIN-D16-RI256-FI512-disconnected" + ) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{directCase, recursiveCase, noPathCase}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 3) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + + run := func(executor optimize.ShortestPathExecutor) map[string]CaseResult { + runner.toolOptions = translate.ToolOptions{ForceShortestPathExecutor: executor} + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + indexed := make(map[string]CaseResult, len(records)) + for _, record := range records { + require.Equal(t, StatusOK, record.Status, record.Error) + indexed[record.Name] = record + } + return indexed + } + + baseline := run(optimize.ShortestPathExecutorS4CanonicalDistance) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP, + } { + t.Run(string(executor), func(t *testing.T) { + candidate := run(executor) + for name, record := range candidate { + exact := baseline[name] + require.Equal(t, exact.RowCount, record.RowCount) + require.True(t, slices.Equal(exact.ObservedRows, record.ObservedRows)) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + gateCase := &ResourceGateCase{} + appendInlineDistanceAttributionReasons(gateCase, record.TraversalTelemetry) + require.Empty(t, gateCase.Reasons) + } + + if executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1D || executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP { + direct := candidate[directCase] + require.Equal(t, "inline_direct_distance", direct.TraversalTelemetry.Summary.RuntimeBranch) + counters := direct.TraversalTelemetry.Diagnostic.PlanReplay.Counters + require.Equal(t, int64(1), counters["sp_i2_direct_rows"]) + require.Zero(t, counters["sp_i2_distance_rows"]) + require.Zero(t, counters["sp_i2_admission_rows"]) + require.Zero(t, counters["sp_i2_target_rows"]) + require.Zero(t, counters["sp_i2_fallback_executor_loops"]) + } + }) + } +} + +// TestSPI2V2FormalTrainingSemantics executes only the fresh open training +// cohort on the backend selected by CONNECTION_STRING. Holdouts remain +// unreachable through this test. +func TestSPI2V2FormalTrainingSemantics(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectRunnableScaleCorpusWithSPI2Protection(full, CorpusSelectors{Tags: []string{spI2V2TrainingTag}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 8) + for _, testCase := range selected.Cases { + require.Equal(t, "training", testCase.Shape.QualificationSplit) + require.NotContains(t, testCase.Tags, spI2V2HoldoutTag) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + var records []CaseResult + switch connectionURL.Scheme { + case "postgres", "postgresql": + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions = translate.ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1} + records, err = runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + case "neo4j", "neo4j+s", "neo4j+ssc": + runner, err := newNeo4jRunner(ctx, "../../integration/testdata", connection, selected) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(ctx)) }) + records, err = runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + default: + t.Skip("CONNECTION_STRING does not select PostgreSQL or Neo4j") + } + require.Len(t, records, 8) + for _, record := range records { + require.Equal(t, StatusOK, record.Status, record.Error) + require.True(t, record.StableObservation, record.Name) + require.NoError(t, validateExpectedObservations(findScaleCase(t, selected, record.Name).Expected, record.ObservedRows), record.Name) + } +} + +func findScaleCase(t *testing.T, corpus ScaleCorpus, name string) ScaleCase { + t.Helper() + for _, testCase := range corpus.Cases { + if testCase.Name == name { + return testCase + } + } + t.Fatalf("missing scale case %s", name) + return ScaleCase{} +} diff --git a/cmd/graphbench/sp_i2_formal_v2.go b/cmd/graphbench/sp_i2_formal_v2.go new file mode 100644 index 00000000..fb8b50ec --- /dev/null +++ b/cmd/graphbench/sp_i2_formal_v2.go @@ -0,0 +1,139 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "slices" +) + +const ( + spI2V2TrainingTag = "sp-i2-distance-v2-training" + spI2V2HoldoutTag = "sp-i2-distance-v2-holdout" + + spI2V2TrainingCorpusSHA256 = "b57e77369d9686123a847e24fd4037d7ee4bc4c9b3d6f73b67eca7a956b0493b" + spI2V2HoldoutCorpusSHA256 = "16a4f8663cdc1c537c99a85571931c3e7d3f9f71500cb1311375aa9930f8c201" + spI2V2FullCorpusSHA256 = "f057a779bd1587ff08596459de42ef51f7befcc0365a9bf86f894a77e0e06d0e" + spI2V2TrainingDeclarationSHA256 = "5d704f62c70fea909565ae0541d8a74a925c6cc14587a49a9a6422d5aa077133" + spI2V2HoldoutDeclarationSHA256 = "009101538c650a213e807189bd45dede5ab6785dd5ac9e93dc3f1ad328b3fcfa" + spI2V2FullDeclarationSHA256 = "1721f48e724b227e0bf4d9a1e03b0471f10fc23ef7402e47536b664af6b96a69" + spI2V2TrainingResolvedSHA256 = "75802b0d76034fac1b2c144c125069f8b971180997540b6b2bf46b89523fbacc" + spI2V2HoldoutResolvedSHA256 = "d08d149fcaf29e91750fde1e1eae1f3b2f2a6819608a073558ee4d6f13d82ce9" + spI2V2FullResolvedSHA256 = "fa1abc601d60d295add2095c0ff343c47d013165fe23b9a09eeca429254318c2" +) + +type spI2V2FormalCase struct { + dataset string + name string + split string + role string +} + +var spI2V2FormalCases = []spI2V2FormalCase{ + {"generated_shortest_paths_v2_d1_o0_r4_fo0_fi0_l0_k0_t0_w0_x1_p0_c0_s0", "GSP-I2-V2-TRAIN-direct-acyclic-shallow", "training", "adverse_control"}, + {"generated_shortest_paths_v2_d7_o0_r0_fo0_fi0_l0_k0_t0_w0_x7_p0_c1_s0", "GSP-I2-V2-TRAIN-direct-cycle-control", "training", "adverse_control"}, + {"generated_shortest_paths_v2_d5_o0_r19_fo0_fi11_l3_k0_t0_w0_x5_p0_c1_s0", "GSP-I2-V2-TRAIN-D02-post-target-cycle", "training", "adverse_control"}, + {"generated_shortest_paths_v2_d4_o0_r37_fo0_fi73_l2_k0_t0_w0_x4_p0_c0_s0", "GSP-I2-V2-TRAIN-D02-hidden-intermediate-fanin", "training", "efficacy_target"}, + {"generated_shortest_paths_v2_d3_o0_r83_fo0_fi41_l2_k0_t0_w0_x3_p0_c0_s0", "GSP-I2-V2-TRAIN-D03-hidden-root-fanin", "training", "efficacy_target"}, + {"generated_shortest_paths_v2_d8_o0_r149_fo0_fi79_l4_k0_t0_w0_x8_p0_c0_s0", "GSP-I2-V2-TRAIN-D08-mixed-fanin", "training", "efficacy_target"}, + {"generated_shortest_paths_v2_d16_o0_r263_fo0_fi521_l8_k0_t0_w0_x16_p0_c0_s0", "GSP-I2-V2-TRAIN-D16-high-fanin", "training", "efficacy_target"}, + {"generated_shortest_paths_v2_d16_o0_r271_fo0_fi527_l8_k0_t0_w0_x17_p0_c1_s0", "GSP-I2-V2-TRAIN-D16-disconnected-cyclic-exhaustion", "training", "efficacy_target"}, + {"generated_shortest_paths_v2_d1_o0_r17_fo0_fi0_l0_k3_t2_w0_x1_p0_c1_s0", "GSP-I2-V2-HOLDOUT-direct-parallel-asymmetric-cycle", "holdout", "adverse_control"}, + {"generated_shortest_paths_v2_d6_o0_r43_fo0_fi29_l3_k0_t0_w0_x6_p0_c1_s1", "GSP-I2-V2-HOLDOUT-D02-longer-competing-cycle", "holdout", "adverse_control"}, + {"generated_shortest_paths_v2_d9_o0_r173_fo0_fi97_l4_k0_t0_w0_x9_p0_c0_s1", "GSP-I2-V2-HOLDOUT-D03-irrelevant-high-fanout", "holdout", "efficacy_target"}, + {"generated_shortest_paths_v2_d11_o0_r197_fo0_fi211_l6_k0_t0_w0_x11_p0_c0_s0", "GSP-I2-V2-HOLDOUT-D11-medium-fanin", "holdout", "efficacy_target"}, + {"generated_shortest_paths_v2_d23_o0_r307_fo0_fi601_l12_k0_t0_w0_x23_p0_c0_s0", "GSP-I2-V2-HOLDOUT-D23-deep-fanin", "holdout", "efficacy_target"}, + {"generated_shortest_paths_v2_d27_o0_r313_fo0_fi607_l14_k0_t0_w0_x29_p0_c1_s1", "GSP-I2-V2-HOLDOUT-D27-disconnected-cycles", "holdout", "efficacy_target"}, +} + +type spI2V2FormalCohort struct { + trainingKeys map[performanceKey]string + holdoutKeys map[performanceKey]string +} + +func canonicalSPI2V2FormalCohort() (spI2V2FormalCohort, error) { + cohort := spI2V2FormalCohort{trainingKeys: map[performanceKey]string{}, holdoutKeys: map[performanceKey]string{}} + for _, declaration := range spI2V2FormalCases { + key := performanceKey{dataset: declaration.dataset, name: declaration.name, backend: ModePostgresSQL} + if declaration.role != "adverse_control" && declaration.role != "efficacy_target" { + return spI2V2FormalCohort{}, fmt.Errorf("SP-I2 V2 formal case %s has invalid role", declaration.name) + } + switch declaration.split { + case "training": + if _, duplicate := cohort.trainingKeys[key]; duplicate { + return spI2V2FormalCohort{}, fmt.Errorf("SP-I2 V2 formal training case %s is duplicated", declaration.name) + } + cohort.trainingKeys[key] = declaration.role + case "holdout": + if _, duplicate := cohort.holdoutKeys[key]; duplicate { + return spI2V2FormalCohort{}, fmt.Errorf("SP-I2 V2 formal holdout case %s is duplicated", declaration.name) + } + cohort.holdoutKeys[key] = declaration.role + default: + return spI2V2FormalCohort{}, fmt.Errorf("SP-I2 V2 formal case %s has invalid split", declaration.name) + } + } + if len(cohort.trainingKeys) != 8 || len(cohort.holdoutKeys) != 6 { + return spI2V2FormalCohort{}, fmt.Errorf("SP-I2 V2 formal cohort requires exactly eight training and six holdout cases") + } + return cohort, nil +} + +func spI2V2FormalProtocolSelection(selectors CorpusSelectors) bool { + if slices.Contains(selectors.Tags, spI2V2TrainingTag) || slices.Contains(selectors.Tags, spI2V2HoldoutTag) { + return true + } + for _, selected := range selectors.Cases { + for _, declaration := range spI2V2FormalCases { + if selected == declaration.name { + return true + } + } + } + return false +} + +func spI2V2FormalHoldoutSelected(selectors CorpusSelectors) bool { + if slices.Contains(selectors.Tags, spI2V2HoldoutTag) { + return true + } + for _, selected := range selectors.Cases { + for _, declaration := range spI2V2FormalCases { + if declaration.split == "holdout" && selected == declaration.name { + return true + } + } + } + return false +} + +func selectedCorpusContainsSPI2V2FormalHoldout(corpus ScaleCorpus) bool { + cohort, err := canonicalSPI2V2FormalCohort() + if err != nil { + return true + } + for _, testCase := range corpus.Cases { + if _, found := cohort.holdoutKeys[performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL}]; found { + return true + } + } + return false +} + +func selectedCorpusContainsSPI2V2FormalCase(corpus ScaleCorpus) bool { + cohort, err := canonicalSPI2V2FormalCohort() + if err != nil { + return true + } + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + if _, found := cohort.trainingKeys[key]; found { + return true + } + if _, found := cohort.holdoutKeys[key]; found { + return true + } + } + return false +} diff --git a/cmd/graphbench/sp_i2_formal_v2_test.go b/cmd/graphbench/sp_i2_formal_v2_test.go new file mode 100644 index 00000000..8e717a8e --- /dev/null +++ b/cmd/graphbench/sp_i2_formal_v2_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSPI2V2FormalCorpusFreezesRolesAndDisjointCohorts(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + cohort, err := canonicalSPI2V2FormalCohort() + require.NoError(t, err) + + seenTraining, seenHoldout := map[performanceKey]bool{}, map[performanceKey]bool{} + v1HoldoutDatasets := map[string]bool{} + for _, declaration := range spI2CanonicalCases { + if declaration.split == "holdout" { + v1HoldoutDatasets[declaration.dataset] = true + } + } + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + role, training := cohort.trainingKeys[key] + if !training { + role, _ = cohort.holdoutKeys[key] + } + if role == "" { + continue + } + require.Equal(t, role, testCase.Shape.QualificationRole, testCase.Name) + require.False(t, testCase.Shape.PathMaterializationRequired, testCase.Name) + require.Equal(t, "forbidden", testCase.Shape.FallbackExpectation, testCase.Name) + require.Equal(t, []ExecutionMode{ModePostgresSQL, ModeNeo4j}, testCase.CandidateModes, testCase.Name) + require.NotNil(t, testCase.Expected.RowCount, testCase.Name) + _, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + if training { + require.Equal(t, "training", testCase.Shape.QualificationSplit) + require.True(t, slices.Contains(testCase.Tags, spI2V2TrainingTag)) + seenTraining[key] = true + } else { + require.Equal(t, "holdout", testCase.Shape.QualificationSplit) + require.True(t, slices.Contains(testCase.Tags, spI2V2HoldoutTag)) + require.False(t, v1HoldoutDatasets[testCase.Dataset], testCase.Name) + seenHoldout[key] = true + } + } + require.Len(t, seenTraining, 8) + require.Len(t, seenHoldout, 6) +} + +func TestSPI2V2FormalHoldoutsAreProtectedAsASeparateGeneration(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + cohort, err := canonicalSPI2V2FormalCohort() + require.NoError(t, err) + + ordinary, _, err := selectRunnableScaleCorpusWithSPI2Protection(corpus, CorpusSelectors{}) + require.NoError(t, err) + for _, testCase := range ordinary.Cases { + _, protected := cohort.holdoutKeys[performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL}] + require.False(t, protected, testCase.Name) + } + + holdout, holdoutSelection, err := selectRunnableScaleCorpusWithSPI2Protection(corpus, CorpusSelectors{Tags: []string{spI2V2HoldoutTag}}) + require.NoError(t, err) + require.Len(t, holdout.Cases, 6) + require.True(t, selectedCorpusContainsSPI2V2FormalCase(holdout)) + training, trainingSelection, err := selectRunnableScaleCorpusWithSPI2Protection(corpus, CorpusSelectors{Tags: []string{spI2V2TrainingTag}}) + require.NoError(t, err) + require.Len(t, training.Cases, 8) + require.True(t, selectedCorpusContainsSPI2V2FormalCase(training)) + require.Equal(t, spI2V2TrainingCorpusSHA256, corpusIdentity(training)) + require.Equal(t, spI2V2TrainingDeclarationSHA256, trainingSelection.DeclarationSHA256) + require.Equal(t, spI2V2TrainingResolvedSHA256, resolvedSelectionSHA256(trainingSelection.Resolved)) + require.Equal(t, spI2V2HoldoutCorpusSHA256, corpusIdentity(holdout)) + require.Equal(t, spI2V2HoldoutDeclarationSHA256, holdoutSelection.DeclarationSHA256) + require.Equal(t, spI2V2HoldoutResolvedSHA256, resolvedSelectionSHA256(holdoutSelection.Resolved)) + fullSelectionCases, fullSelection, err := selectRunnableScaleCorpusWithSPI2Protection(corpus, CorpusSelectors{Tags: []string{spI2V2TrainingTag, spI2V2HoldoutTag}}) + require.NoError(t, err) + require.Len(t, fullSelectionCases.Cases, 14) + require.Equal(t, spI2V2FullCorpusSHA256, corpusIdentity(fullSelectionCases)) + require.Equal(t, spI2V2FullDeclarationSHA256, fullSelection.DeclarationSHA256) + require.Equal(t, spI2V2FullResolvedSHA256, resolvedSelectionSHA256(fullSelection.Resolved)) + + _, _, err = selectRunnableScaleCorpusWithSPI2Protection(corpus, CorpusSelectors{Tags: []string{spI2TrainingTag, spI2V2TrainingTag}}) + require.ErrorContains(t, err, "cannot be mixed") +} diff --git a/cmd/graphbench/sp_i2_hier_bootstrap_v2.go b/cmd/graphbench/sp_i2_hier_bootstrap_v2.go new file mode 100644 index 00000000..50ca555c --- /dev/null +++ b/cmd/graphbench/sp_i2_hier_bootstrap_v2.go @@ -0,0 +1,218 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "fmt" + "math" + randv2 "math/rand/v2" + "slices" + "sort" + "time" +) + +const spI2HierBootstrapV2 = "sp-i2-hier-bootstrap-v2/chacha8-sha256" + +// spI2TailIntervalV2 contains the conjunctive relative and absolute tail +// estimands produced from the same deterministic hierarchical draws. +type spI2TailIntervalV2 struct { + Ratio RatioInterval + Change DurationInterval +} + +// spI2BootstrapSeedV2 derives a metric-local deterministic ChaCha8 stream. +// Keeping dataset, case, and metric in the domain separation makes report +// iteration order irrelevant. +func spI2BootstrapSeedV2(dataset, caseName, metric string) [32]byte { + return sha256.Sum256([]byte("sp-i2-tail-bootstrap-v2\x00" + "1" + "\x00" + dataset + "\x00" + caseName + "\x00" + metric)) +} + +// bootstrapSPI2HierarchicalTailV2 resamples round pairs together, then samples +// each arm independently within every selected round occurrence. Ratios are +// intervalled on the log scale; changes are candidate minus baseline. +func bootstrapSPI2HierarchicalTailV2( + baseline, candidate roundSamples, + dataset, caseName, metric string, + probability, confidence float64, + replicates int, +) (spI2TailIntervalV2, error) { + rounds, err := validateSPI2HierarchicalInputs(baseline, candidate, probability, confidence, replicates) + if err != nil { + return spI2TailIntervalV2{}, err + } + baselinePoint := durationQuantile(flattenSamples(baseline, rounds), probability) + candidatePoint := durationQuantile(flattenSamples(candidate, rounds), probability) + if baselinePoint <= 0 || candidatePoint <= 0 { + return spI2TailIntervalV2{}, fmt.Errorf("SP-I2 hierarchical ratio requires positive quantiles") + } + + baselineIndexed := indexSPI2RoundSamples(baseline, rounds) + candidateIndexed := indexSPI2RoundSamples(candidate, rounds) + baselineCounts := make([]int, len(baselineIndexed.values)) + candidateCounts := make([]int, len(candidateIndexed.values)) + rng := randv2.New(randv2.NewChaCha8(spI2BootstrapSeedV2(dataset, caseName, metric))) + logRatios := make([]float64, replicates) + changes := make([]float64, replicates) + for iteration := range replicates { + clear(baselineCounts) + clear(candidateCounts) + baselineDrawCount, candidateDrawCount := 0, 0 + for range rounds { + selected := rng.Uint64N(uint64(len(rounds))) + baselineValues := baselineIndexed.rounds[selected] + candidateValues := candidateIndexed.rounds[selected] + for range baselineValues { + baselineCounts[baselineValues[rng.Uint64N(uint64(len(baselineValues)))]]++ + baselineDrawCount++ + } + for range candidateValues { + candidateCounts[candidateValues[rng.Uint64N(uint64(len(candidateValues)))]]++ + candidateDrawCount++ + } + } + baselineDraw := countedSPI2Quantile(baselineIndexed.values, baselineCounts, baselineDrawCount, probability) + candidateDraw := countedSPI2Quantile(candidateIndexed.values, candidateCounts, candidateDrawCount, probability) + if baselineDraw <= 0 || candidateDraw <= 0 { + return spI2TailIntervalV2{}, fmt.Errorf("SP-I2 hierarchical draw %d produced a non-positive quantile", iteration) + } + logRatios[iteration] = math.Log(candidateDraw / baselineDraw) + changes[iteration] = candidateDraw - baselineDraw + } + + alpha := (1 - confidence) / 2 + return spI2TailIntervalV2{ + Ratio: RatioInterval{ + Estimate: candidatePoint / baselinePoint, + Lower: math.Exp(quantile(logRatios, alpha)), + Upper: math.Exp(quantile(logRatios, 1-alpha)), + }, + Change: DurationInterval{ + Estimate: time.Duration(candidatePoint - baselinePoint), + Lower: time.Duration(quantile(changes, alpha)), + Upper: time.Duration(quantile(changes, 1-alpha)), + }, + }, nil +} + +type spI2IndexedRoundSamples struct { + values []time.Duration + rounds [][]int +} + +// indexSPI2RoundSamples converts native durations to stable sorted-value +// indexes. Bootstrap draws then increment counts and scan to the nearest-rank +// quantile instead of allocating and sorting pooled samples 100,000 times. +func indexSPI2RoundSamples(samples roundSamples, rounds []int) spI2IndexedRoundSamples { + var values []time.Duration + for _, round := range rounds { + values = append(values, samples[round]...) + } + sort.Slice(values, func(left, right int) bool { return values[left] < values[right] }) + values = slices.Compact(values) + valueIndex := make(map[time.Duration]int, len(values)) + for index, value := range values { + valueIndex[value] = index + } + indexed := spI2IndexedRoundSamples{values: values, rounds: make([][]int, len(rounds))} + for roundIndex, round := range rounds { + indexed.rounds[roundIndex] = make([]int, len(samples[round])) + for sampleIndex, value := range samples[round] { + indexed.rounds[roundIndex][sampleIndex] = valueIndex[value] + } + } + return indexed +} + +func countedSPI2Quantile(values []time.Duration, counts []int, total int, probability float64) float64 { + rank := int(math.Ceil(probability * float64(total))) + if rank < 1 { + rank = 1 + } + seen := 0 + for index, count := range counts { + seen += count + if seen >= rank { + return float64(values[index]) + } + } + return math.NaN() +} + +// bootstrapSPI2RoundMedianV2 uses only paired round-median resampling, as +// preregistered. Saving is baseline minus candidate. +func bootstrapSPI2RoundMedianV2( + baseline, candidate roundSamples, + dataset, caseName, metric string, + confidence float64, + replicates int, +) (RatioInterval, DurationInterval, error) { + rounds, err := validateSPI2HierarchicalInputs(baseline, candidate, 0.5, confidence, replicates) + if err != nil { + return RatioInterval{}, DurationInterval{}, err + } + baselineMedians := make([]float64, len(rounds)) + candidateMedians := make([]float64, len(rounds)) + for index, round := range rounds { + baselineMedians[index] = durationQuantile(baseline[round], 0.5) + candidateMedians[index] = durationQuantile(candidate[round], 0.5) + if baselineMedians[index] <= 0 || candidateMedians[index] <= 0 { + return RatioInterval{}, DurationInterval{}, fmt.Errorf("SP-I2 round medians must be positive") + } + } + baselinePoint := quantile(baselineMedians, 0.5) + candidatePoint := quantile(candidateMedians, 0.5) + rng := randv2.New(randv2.NewChaCha8(spI2BootstrapSeedV2(dataset, caseName, metric))) + logRatios := make([]float64, replicates) + savings := make([]float64, replicates) + resampledBaseline := make([]float64, len(rounds)) + resampledCandidate := make([]float64, len(rounds)) + for iteration := range replicates { + for index := range rounds { + selected := rng.Uint64N(uint64(len(rounds))) + resampledBaseline[index] = baselineMedians[selected] + resampledCandidate[index] = candidateMedians[selected] + } + baselineDraw := quantile(resampledBaseline, 0.5) + candidateDraw := quantile(resampledCandidate, 0.5) + logRatios[iteration] = math.Log(candidateDraw / baselineDraw) + savings[iteration] = baselineDraw - candidateDraw + } + alpha := (1 - confidence) / 2 + return RatioInterval{ + Estimate: candidatePoint / baselinePoint, + Lower: math.Exp(quantile(logRatios, alpha)), + Upper: math.Exp(quantile(logRatios, 1-alpha)), + }, DurationInterval{ + Estimate: time.Duration(baselinePoint - candidatePoint), + Lower: time.Duration(quantile(savings, alpha)), + Upper: time.Duration(quantile(savings, 1-alpha)), + }, nil +} + +func validateSPI2HierarchicalInputs(baseline, candidate roundSamples, probability, confidence float64, replicates int) ([]int, error) { + if probability <= 0 || probability > 1 || math.IsNaN(probability) { + return nil, fmt.Errorf("SP-I2 hierarchical probability must be in (0,1]") + } + if confidence <= 0 || confidence >= 1 || math.IsNaN(confidence) { + return nil, fmt.Errorf("SP-I2 hierarchical confidence must be in (0,1)") + } + if replicates <= 0 { + return nil, fmt.Errorf("SP-I2 hierarchical bootstrap count must be positive") + } + baselineRounds := sortedRounds(baseline) + candidateRounds := sortedRounds(candidate) + if len(baselineRounds) == 0 || !slices.Equal(baselineRounds, candidateRounds) { + return nil, fmt.Errorf("SP-I2 hierarchical arms require identical nonempty round sets") + } + for _, round := range baselineRounds { + if len(baseline[round]) == 0 || len(candidate[round]) == 0 { + return nil, fmt.Errorf("SP-I2 hierarchical round %d contains an empty arm", round) + } + if len(baseline[round]) != len(candidate[round]) { + return nil, fmt.Errorf("SP-I2 hierarchical round %d contains unequal arm sample counts", round) + } + } + return baselineRounds, nil +} diff --git a/cmd/graphbench/sp_i2_hier_bootstrap_v2_test.go b/cmd/graphbench/sp_i2_hier_bootstrap_v2_test.go new file mode 100644 index 00000000..c13bce23 --- /dev/null +++ b/cmd/graphbench/sp_i2_hier_bootstrap_v2_test.go @@ -0,0 +1,67 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/hex" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSPI2HierarchicalBootstrapV2DeterministicGoldenVector(t *testing.T) { + baseline := roundSamples{ + 1: {900 * time.Microsecond, 1 * time.Millisecond, 2 * time.Millisecond, 2200 * time.Microsecond}, + 2: {1 * time.Millisecond, 1100 * time.Microsecond, 2100 * time.Microsecond, 2400 * time.Microsecond}, + 3: {950 * time.Microsecond, 1050 * time.Microsecond, 2300 * time.Microsecond, 2500 * time.Microsecond}, + } + candidate := roundSamples{ + 1: {850 * time.Microsecond, 950 * time.Microsecond, 1800 * time.Microsecond, 2100 * time.Microsecond}, + 2: {900 * time.Microsecond, 1 * time.Millisecond, 1900 * time.Microsecond, 2200 * time.Microsecond}, + 3: {875 * time.Microsecond, 975 * time.Microsecond, 2 * time.Millisecond, 2300 * time.Microsecond}, + } + seed := spI2BootstrapSeedV2("dataset-a", "case-a", "p95") + require.Equal(t, "20c91adbc448b55ac3d9dff2b91f60378f144fec7c2df53df6f2b702afc457d4", hex.EncodeToString(seed[:])) + + first, err := bootstrapSPI2HierarchicalTailV2(baseline, candidate, "dataset-a", "case-a", "p95", 0.95, 0.975, 1000) + require.NoError(t, err) + second, err := bootstrapSPI2HierarchicalTailV2(baseline, candidate, "dataset-a", "case-a", "p95", 0.95, 0.975, 1000) + require.NoError(t, err) + require.Equal(t, first, second) + require.Equal(t, RatioInterval{Estimate: 0.92, Lower: 0.8, Upper: 1.0454545454545454}, first.Ratio) + require.Equal(t, DurationInterval{Estimate: -200 * time.Microsecond, Lower: -500 * time.Microsecond, Upper: 100 * time.Microsecond}, first.Change) +} + +func TestSPI2HierarchicalBootstrapV2RejectsMissingAndUnequalSamples(t *testing.T) { + baseline := roundSamples{1: {time.Millisecond, 2 * time.Millisecond}, 2: {time.Millisecond, 2 * time.Millisecond}} + _, err := bootstrapSPI2HierarchicalTailV2(baseline, roundSamples{1: {time.Millisecond, 2 * time.Millisecond}}, "d", "c", "p95", 0.95, 0.975, 10) + require.ErrorContains(t, err, "identical nonempty round sets") + _, err = bootstrapSPI2HierarchicalTailV2(baseline, roundSamples{1: {time.Millisecond}, 2: {time.Millisecond}}, "d", "c", "p95", 0.95, 0.975, 10) + require.ErrorContains(t, err, "unequal arm sample counts") +} + +func TestSPI2RoundMedianBootstrapDoesNotResampleWithinRounds(t *testing.T) { + baseline := roundSamples{1: {time.Millisecond, 100 * time.Millisecond}, 2: {2 * time.Millisecond, 200 * time.Millisecond}} + candidate := roundSamples{1: {900 * time.Microsecond, 90 * time.Millisecond}, 2: {1800 * time.Microsecond, 180 * time.Millisecond}} + ratio, saving, err := bootstrapSPI2RoundMedianV2(baseline, candidate, "d", "c", "median", 0.975, 100) + require.NoError(t, err) + require.InDelta(t, 0.9, ratio.Estimate, 0.000001) + require.Equal(t, 100*time.Microsecond, saving.Estimate) +} + +func BenchmarkSPI2HierarchicalBootstrapV2_40x100x100000(b *testing.B) { + baseline, candidate := roundSamples{}, roundSamples{} + for round := 1; round <= 40; round++ { + for sample := range 100 { + baseline[round] = append(baseline[round], time.Duration(900+round*3+sample*11)*time.Microsecond) + candidate[round] = append(candidate[round], time.Duration(875+round*3+sample*10)*time.Microsecond) + } + } + b.ResetTimer() + for range b.N { + _, err := bootstrapSPI2HierarchicalTailV2(baseline, candidate, "benchmark", "representative", "p95", 0.95, 0.975, 100_000) + require.NoError(b, err) + } +} diff --git a/cmd/graphbench/sp_i2_power_simulation_v2.go b/cmd/graphbench/sp_i2_power_simulation_v2.go new file mode 100644 index 00000000..7cf09306 --- /dev/null +++ b/cmd/graphbench/sp_i2_power_simulation_v2.go @@ -0,0 +1,222 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/hex" + "fmt" + "math" + randv2 "math/rand/v2" +) + +const spI2PowerSimulationV2 = "sp-i2-power-simulation-v2/chacha8-sha256-normal-pivot" + +type SPI2PowerSimulationReportV2 struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + Implementation string `json:"implementation"` + ProtocolSHA256 string `json:"protocol_sha256"` + Passed bool `json:"passed"` + Scenarios []SPI2PowerSimulationScenarioReportV2 `json:"scenarios"` +} + +type SPI2PowerSimulationScenarioReportV2 struct { + Name string `json:"name"` + Kind string `json:"kind"` + Seed string `json:"seed"` + Runs int `json:"runs"` + SuccessfulDecisions int `json:"successful_decisions"` + DecisionRate float64 `json:"decision_rate"` + DecisionWilson RatioInterval `json:"decision_wilson_95"` + CoveredIntervals int `json:"covered_intervals"` + TotalIntervals int `json:"total_intervals"` + CoverageRate float64 `json:"coverage_rate"` + CoverageWilson RatioInterval `json:"coverage_wilson_95"` + RequiredDecisionResult string `json:"required_decision_result"` + Passed bool `json:"passed"` +} + +type spI2SimulationIntervalV2 struct { + lower float64 + upper float64 + trueValue float64 +} + +// buildSPI2PowerSimulationReportV2 executes the immutable prospective +// calibration matrix. The normal pivots are fitted to the archived V1/open +// traces, while the paired empirical drift vectors are resampled at the fixed +// 40-round design. Formal evidence continues to use the exact 100,000-draw +// hierarchical bootstrap; this simulation calibrates that frozen design +// without substituting a cheaper estimator in a formal report. +func buildSPI2PowerSimulationReportV2(protocol spI2ProtocolV2, protocolSHA256 string) (SPI2PowerSimulationReportV2, error) { + if err := validateSPI2SimulationProtocolV2(protocol.Simulation); err != nil { + return SPI2PowerSimulationReportV2{}, err + } + report := SPI2PowerSimulationReportV2{ + Schema: "sp-i2-power-simulation-report-v2", + Generation: spI2GenerationV2, + Implementation: spI2PowerSimulationV2, + ProtocolSHA256: protocolSHA256, + Passed: true, + } + for _, scenario := range protocol.Simulation.Scenarios { + result, err := simulateSPI2ScenarioV2(protocol, scenario) + if err != nil { + return SPI2PowerSimulationReportV2{}, err + } + report.Scenarios = append(report.Scenarios, result) + report.Passed = report.Passed && result.Passed + } + return report, nil +} + +func simulateSPI2ScenarioV2(protocol spI2ProtocolV2, scenario spI2SimulationScenarioV2) (SPI2PowerSimulationScenarioReportV2, error) { + seedBytes, err := hex.DecodeString(scenario.Seed) + if err != nil || len(seedBytes) != 32 { + return SPI2PowerSimulationScenarioReportV2{}, fmt.Errorf("decode SP-I2 simulation seed for %s", scenario.Name) + } + var seed [32]byte + copy(seed[:], seedBytes) + rng := randv2.New(randv2.NewChaCha8(seed)) + runs := protocol.Simulation.RunsPerScenario + successes, covered, intervals := 0, 0, 0 + for range runs { + passed, nextCovered, nextIntervals := simulateSPI2StudyV2(protocol, scenario, rng) + if passed { + successes++ + } + covered += nextCovered + intervals += nextIntervals + } + decisionWilson := spI2WilsonIntervalV2(successes, runs) + coverageWilson := spI2WilsonIntervalV2(covered, intervals) + result := SPI2PowerSimulationScenarioReportV2{ + Name: scenario.Name, + Kind: scenario.Kind, + Seed: scenario.Seed, + Runs: runs, + SuccessfulDecisions: successes, + DecisionRate: float64(successes) / float64(runs), + DecisionWilson: decisionWilson, + CoveredIntervals: covered, + TotalIntervals: intervals, + CoverageRate: float64(covered) / float64(intervals), + CoverageWilson: coverageWilson, + } + coveragePass := coverageWilson.Lower <= protocol.Simulation.RequiredCoverage && coverageWilson.Upper >= protocol.Simulation.RequiredCoverage + switch scenario.Kind { + case "aa_power", "aa_order_power", "target_power", "control_power": + result.RequiredDecisionResult = "wilson_lower>=0.90" + result.Passed = decisionWilson.Lower >= protocol.Simulation.RequiredPowerLower && coveragePass + case "aa_boundary", "aa_order_boundary": + result.RequiredDecisionResult = "false_pass_rate<=0.015" + result.Passed = result.DecisionRate <= protocol.Simulation.P95BoundaryFalsePassUpper && coveragePass + case "target_boundary", "control_boundary": + result.RequiredDecisionResult = "false_pass_rate<=0.0275" + result.Passed = result.DecisionRate <= protocol.Simulation.DecisionFalsePassUpper && coveragePass + default: + return SPI2PowerSimulationScenarioReportV2{}, fmt.Errorf("unsupported SP-I2 simulation kind %q", scenario.Kind) + } + return result, nil +} + +func simulateSPI2StudyV2(protocol spI2ProtocolV2, scenario spI2SimulationScenarioV2, rng *randv2.Rand) (bool, int, int) { + z := 2.241402727604947 + logSE := protocol.Simulation.LogStandardErrors + absSE := protocol.Simulation.AbsoluteStandardErrorsUS + p50Drift := meanResampledSPI2DriftV2(rng, protocol.Simulation.P50RoundDrift, protocol.Design.Rounds) + p95Drift := meanResampledSPI2DriftV2(rng, protocol.Simulation.P95RoundDrift, protocol.Design.Rounds) + + pooledP50 := simulatedSPI2RatioIntervalV2(rng, scenario.CandidateP50US/scenario.BaselineP50US, logSE.Pooled, z) + pooledP95 := simulatedSPI2RatioIntervalV2(rng, scenario.CandidateP95US/scenario.BaselineP95US, logSE.Pooled, z) + pooledP50Change := simulatedSPI2AbsoluteIntervalV2(rng, (scenario.CandidateP50US-scenario.BaselineP50US)*p50Drift, absSE.Pooled, z) + pooledP95Change := simulatedSPI2AbsoluteIntervalV2(rng, (scenario.CandidateP95US-scenario.BaselineP95US)*p95Drift, absSE.Pooled, z) + all := []spI2SimulationIntervalV2{pooledP50, pooledP95, pooledP50Change, pooledP95Change} + + if scenario.Kind == "target_power" || scenario.Kind == "target_boundary" { + pass := (pooledP50.upper <= protocol.Gates.TargetMedianRatioUpper || -pooledP50Change.upper >= float64(protocol.Gates.TargetMedianSavingLowerUS)) && + pooledP95.upper <= protocol.Gates.P95RatioUpper + covered, total := coveredSPI2IntervalsV2(all) + return pass, covered, total + } + if scenario.Kind == "control_power" || scenario.Kind == "control_boundary" { + pass := (pooledP50.upper <= protocol.Gates.ControlMedianRatioUpper || pooledP50Change.upper <= float64(protocol.Gates.ControlMedianOverheadUpperUS)) && + pooledP95.upper <= protocol.Gates.P95RatioUpper && pooledP95Change.upper <= float64(protocol.Gates.ControlP95OverheadUpperUS) + covered, total := coveredSPI2IntervalsV2(all) + return pass, covered, total + } + + oddP50 := simulatedSPI2RatioIntervalV2(rng, scenario.CandidateP50US*scenario.OddCandidateMultiplier/scenario.BaselineP50US, logSE.OrderStratum, z) + oddP95 := simulatedSPI2RatioIntervalV2(rng, scenario.CandidateP95US*scenario.OddCandidateMultiplier/scenario.BaselineP95US, logSE.OrderStratum, z) + evenP50 := simulatedSPI2RatioIntervalV2(rng, scenario.CandidateP50US*scenario.EvenCandidateMultiplier/scenario.BaselineP50US, logSE.OrderStratum, z) + evenP95 := simulatedSPI2RatioIntervalV2(rng, scenario.CandidateP95US*scenario.EvenCandidateMultiplier/scenario.BaselineP95US, logSE.OrderStratum, z) + oddChange := simulatedSPI2AbsoluteIntervalV2(rng, (scenario.CandidateP95US*scenario.OddCandidateMultiplier-scenario.BaselineP95US)*p95Drift, absSE.OrderStratum, z) + evenChange := simulatedSPI2AbsoluteIntervalV2(rng, (scenario.CandidateP95US*scenario.EvenCandidateMultiplier-scenario.BaselineP95US)*p95Drift, absSE.OrderStratum, z) + all = append(all, oddP50, oddP95, evenP50, evenP95, oddChange, evenChange) + firstPass := true + for range 2 { + firstRatio := simulatedSPI2RatioIntervalV2(rng, 1, logSE.FirstPosition, z) + firstChange := simulatedSPI2AbsoluteIntervalV2(rng, 0, absSE.FirstPosition, z) + all = append(all, firstRatio, firstChange) + firstPass = firstPass && firstRatio.upper <= protocol.Gates.AAFirstPositionRatioUpper && firstChange.upper <= float64(protocol.Gates.AAFirstPositionOverheadUpperUS) + } + equivalence := protocol.Gates.AAEquivalenceRatio + lower := 1 / equivalence + pass := intervalInsideSPI2V2(pooledP50, lower, equivalence) && intervalInsideSPI2V2(pooledP95, lower, equivalence) && intervalInsideSPI2V2(pooledP95Change, -100, 100) && + intervalInsideSPI2V2(oddP50, lower, equivalence) && intervalInsideSPI2V2(oddP95, lower, equivalence) && intervalInsideSPI2V2(oddChange, -100, 100) && + intervalInsideSPI2V2(evenP50, lower, equivalence) && intervalInsideSPI2V2(evenP95, lower, equivalence) && intervalInsideSPI2V2(evenChange, -100, 100) && firstPass + covered, total := coveredSPI2IntervalsV2(all) + return pass, covered, total +} + +func simulatedSPI2RatioIntervalV2(rng *randv2.Rand, truth, standardError, z float64) spI2SimulationIntervalV2 { + estimate := math.Log(truth) + standardError*standardNormalSPI2V2(rng) + return spI2SimulationIntervalV2{lower: math.Exp(estimate - z*standardError), upper: math.Exp(estimate + z*standardError), trueValue: truth} +} + +func simulatedSPI2AbsoluteIntervalV2(rng *randv2.Rand, truth, standardError, z float64) spI2SimulationIntervalV2 { + estimate := truth + standardError*standardNormalSPI2V2(rng) + return spI2SimulationIntervalV2{lower: estimate - z*standardError, upper: estimate + z*standardError, trueValue: truth} +} + +func standardNormalSPI2V2(rng *randv2.Rand) float64 { + u1 := (float64(rng.Uint64()>>11) + 0.5) / (1 << 53) + u2 := (float64(rng.Uint64()>>11) + 0.5) / (1 << 53) + return math.Sqrt(-2*math.Log(u1)) * math.Cos(2*math.Pi*u2) +} + +func meanResampledSPI2DriftV2(rng *randv2.Rand, drift []float64, blocks int) float64 { + total := 0.0 + for range blocks { + total += drift[rng.Uint64N(uint64(len(drift)))] + } + return total / float64(blocks) +} + +func intervalInsideSPI2V2(interval spI2SimulationIntervalV2, lower, upper float64) bool { + return interval.lower >= lower && interval.upper <= upper +} + +func coveredSPI2IntervalsV2(intervals []spI2SimulationIntervalV2) (int, int) { + covered := 0 + for _, interval := range intervals { + if interval.lower <= interval.trueValue && interval.upper >= interval.trueValue { + covered++ + } + } + return covered, len(intervals) +} + +func spI2WilsonIntervalV2(successes, total int) RatioInterval { + if total <= 0 { + return RatioInterval{} + } + z := 1.959963984540054 + n := float64(total) + p := float64(successes) / n + denominator := 1 + z*z/n + center := (p + z*z/(2*n)) / denominator + half := z * math.Sqrt(p*(1-p)/n+z*z/(4*n*n)) / denominator + return RatioInterval{Estimate: p, Lower: center - half, Upper: center + half} +} diff --git a/cmd/graphbench/sp_i2_power_simulation_v2_test.go b/cmd/graphbench/sp_i2_power_simulation_v2_test.go new file mode 100644 index 00000000..17ce69de --- /dev/null +++ b/cmd/graphbench/sp_i2_power_simulation_v2_test.go @@ -0,0 +1,45 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSPI2PowerSimulationV2FrozenMatrixTerminatesInadequateDesign(t *testing.T) { + path := filepath.Join("..", "..", "benchmark", "testdata", "scale", "protocols", "sp_i2_distance_v2.json") + protocol, digest, err := loadSPI2ProtocolV2(path) + require.NoError(t, err) + + report, err := buildSPI2PowerSimulationReportV2(protocol, digest) + require.NoError(t, err) + require.False(t, report.Passed) + require.Len(t, report.Scenarios, 11) + expected := map[string][2]int{ + "aa_identity": {0, 272905}, "aa_upper_margin": {0, 273129}, "aa_lower_margin": {0, 273008}, + "target_power": {9588, 78059}, "target_boundary": {8, 78033}, "control_power": {10246, 78052}, + "control_boundary": {0, 78028}, "aa_order_odd_high": {0, 273035}, "aa_order_even_high": {0, 272934}, + "aa_order_upper_margin": {0, 272969}, "aa_order_lower_margin": {0, 273010}, + } + failures := 0 + for _, scenario := range report.Scenarios { + require.Equal(t, 20_000, scenario.Runs) + require.Equal(t, expected[scenario.Name][0], scenario.SuccessfulDecisions) + require.Equal(t, expected[scenario.Name][1], scenario.CoveredIntervals) + if !scenario.Passed { + failures++ + } + } + require.Greater(t, failures, 0) +} + +func TestSPI2WilsonIntervalV2KnownVector(t *testing.T) { + interval := spI2WilsonIntervalV2(18_000, 20_000) + require.InDelta(t, 0.9, interval.Estimate, 1e-12) + require.InDelta(t, 0.895765, interval.Lower, 0.000001) + require.InDelta(t, 0.904081, interval.Upper, 0.000001) +} diff --git a/cmd/graphbench/sp_i2_protocol_v2.go b/cmd/graphbench/sp_i2_protocol_v2.go new file mode 100644 index 00000000..4405c437 --- /dev/null +++ b/cmd/graphbench/sp_i2_protocol_v2.go @@ -0,0 +1,400 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "maps" + "os" + "slices" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const ( + spI2GenerationV1 = "sp-i2-distance-v1" + spI2GenerationV2 = "sp-i2-distance-v2" + // spI2V2TerminalStatus retires V2 after its frozen prospective study + // showed that normal measurement variation was too wide to distinguish + // the required improvements reliably. A successor must use a new protocol; + // V2 may be audited, but it may not enter formal timing or production. + spI2V2TerminalStatus = "terminated_inadequate_power" +) + +type spI2ProtocolV2 struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + Status string `json:"status"` + ProductionDefault string `json:"production_default"` + Identities spI2ProtocolIdentitiesV2 `json:"identities"` + DevelopmentExecutors []string `json:"development_executors"` + SelectedArchitecture string `json:"selected_architecture"` + Limits spI2ProtocolLimitsV2 `json:"limits"` + Design spI2ProtocolDesignV2 `json:"design"` + Gates spI2ProtocolGatesV2 `json:"gates"` + OperationalDesign spI2OperationalDesignV2 `json:"operational_design"` + Bootstrap spI2ProtocolBootstrapV2 `json:"bootstrap"` + HostAdmission spI2HostAdmissionV2 `json:"host_admission"` + Corpus spI2ProtocolCorpusV2 `json:"corpus"` + Simulation spI2ProtocolSimulationV2 `json:"simulation"` + MultiplicityRule string `json:"multiplicity_rule"` + V1EvidenceReuse bool `json:"v1_evidence_reuse"` + HoldoutAuthorizationBeforeDBSetup bool `json:"holdout_authorization_required_before_database_setup"` +} + +type spI2ProtocolSimulationV2 struct { + Implementation string `json:"implementation"` + RunsPerScenario int `json:"runs_per_scenario"` + WilsonConfidence float64 `json:"wilson_confidence"` + RequiredPowerLower float64 `json:"required_power_lower"` + RequiredCoverage float64 `json:"required_coverage"` + P95BoundaryFalsePassUpper float64 `json:"p95_boundary_false_pass_upper"` + DecisionFalsePassUpper float64 `json:"decision_false_pass_upper"` + TraceRescalingTransform string `json:"trace_rescaling_transform"` + SourceCommit string `json:"source_commit"` + BaselineTraceSHA256 string `json:"baseline_trace_sha256"` + CandidateTraceSHA256 string `json:"candidate_trace_sha256"` + P50RoundDrift []float64 `json:"p50_round_drift"` + P95RoundDrift []float64 `json:"p95_round_drift"` + LogStandardErrors spI2SimulationErrorsV2 `json:"log_standard_errors"` + AbsoluteStandardErrorsUS spI2SimulationErrorsV2 `json:"absolute_standard_errors_us"` + Scenarios []spI2SimulationScenarioV2 `json:"scenarios"` +} + +type spI2SimulationErrorsV2 struct { + Pooled float64 `json:"pooled"` + OrderStratum float64 `json:"order_stratum"` + FirstPosition float64 `json:"first_position"` +} + +type spI2SimulationScenarioV2 struct { + Name string `json:"name"` + Kind string `json:"kind"` + BaselineP50US float64 `json:"baseline_p50_us"` + BaselineP95US float64 `json:"baseline_p95_us"` + CandidateP50US float64 `json:"candidate_p50_us"` + CandidateP95US float64 `json:"candidate_p95_us"` + OddCandidateMultiplier float64 `json:"odd_candidate_multiplier"` + EvenCandidateMultiplier float64 `json:"even_candidate_multiplier"` + Seed string `json:"seed"` +} + +type spI2ProtocolCorpusV2 struct { + Source string `json:"source"` + TrainingCases int `json:"training_cases"` + HoldoutCases int `json:"holdout_cases"` + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + HoldoutCorpusSHA256 string `json:"holdout_corpus_sha256"` + FullCorpusSHA256 string `json:"full_corpus_sha256"` + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + TrainingResolvedSHA256 string `json:"training_resolved_sha256"` + HoldoutResolvedSHA256 string `json:"holdout_resolved_sha256"` + FullResolvedSHA256 string `json:"full_resolved_sha256"` +} + +type spI2V1Rejection struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + Executor string `json:"executor"` + Policy string `json:"policy"` + Selector string `json:"selector"` + SourceCommit string `json:"source_commit"` + DiscoveryReportSHA256 string `json:"discovery_report_sha256"` + FailedGate spI2V1RejectedGate `json:"failed_gate"` + FreezeCreated bool `json:"freeze_created"` + HoldoutOpened bool `json:"holdout_opened"` + Terminal bool `json:"terminal"` +} + +type spI2V1RejectedGate struct { + Metric string `json:"metric"` + Observed float64 `json:"observed"` + Limit float64 `json:"limit"` +} + +type spI2V2Rejection struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + SourceCommit string `json:"source_commit"` + ProtocolSHA256 string `json:"protocol_sha256"` + SimulationReportSHA256 string `json:"simulation_report_sha256"` + SimulationImplementation string `json:"simulation_implementation"` + RunsPerScenario int `json:"runs_per_scenario"` + FailedGates []spI2V2RejectedGate `json:"failed_gates"` + CoverageCalibrated bool `json:"coverage_calibrated"` + FormalAAStarted bool `json:"formal_aa_started"` + CapturePlanCreated bool `json:"capture_plan_created"` + SealedPreregistrationCreated bool `json:"sealed_preregistration_created"` + HoldoutOpened bool `json:"holdout_opened"` + ProductionActivated bool `json:"production_activated"` + SuccessorProtocolRequired bool `json:"successor_protocol_required"` + Terminal bool `json:"terminal"` +} + +type spI2V2RejectedGate struct { + Scenario string `json:"scenario"` + Metric string `json:"metric"` + Observed float64 `json:"observed"` + Required float64 `json:"required"` +} + +type spI2ProtocolIdentitiesV2 struct { + Executor string `json:"executor"` + Policy string `json:"policy"` + Selector string `json:"selector"` + FallbackExecutor string `json:"fallback_executor"` + FallbackInternalExecutor string `json:"fallback_internal_executor"` + TrainingTag string `json:"training_tag"` + HoldoutTag string `json:"holdout_tag"` + DevelopmentTag string `json:"development_tag"` + QualificationSchema string `json:"qualification_schema"` + FreezeSchema string `json:"freeze_schema"` + AASchema string `json:"aa_schema"` + PromotionManifestSchema int `json:"promotion_manifest_schema"` + RollbackSwitch string `json:"rollback_switch"` + StatisticalImplementation string `json:"statistical_implementation"` +} + +type spI2ProtocolLimitsV2 struct { + StateRows int64 `json:"state_rows"` + FrontierRows int64 `json:"frontier_rows"` + MaximumDepth int64 `json:"maximum_depth"` + MinimumDepth int64 `json:"minimum_depth"` +} + +type spI2ProtocolDesignV2 struct { + Seed int `json:"seed"` + ConfidenceLevel float64 `json:"confidence_level"` + BootstrapReplicates int `json:"bootstrap_replicates"` + Rounds int `json:"rounds"` + OrdinaryWarmups int `json:"ordinary_warmups"` + AttestedStabilizations int `json:"attested_stabilizations"` + TimedSamplesPerRound int `json:"timed_samples_per_round"` + PoolSize int `json:"pool_size"` + Isolation string `json:"isolation"` + ArmOrder string `json:"arm_order"` +} + +type spI2ProtocolGatesV2 struct { + TargetMedianRatioUpper float64 `json:"target_median_ratio_upper"` + TargetMedianSavingLowerUS int64 `json:"target_median_saving_lower_us"` + ControlMedianRatioUpper float64 `json:"control_median_ratio_upper"` + ControlMedianOverheadUpperUS int64 `json:"control_median_overhead_upper_us"` + P95RatioUpper float64 `json:"p95_ratio_upper"` + ControlP95OverheadUpperUS int64 `json:"control_p95_overhead_upper_us"` + AAEquivalenceRatio float64 `json:"aa_equivalence_ratio"` + AAFirstPositionRatioUpper float64 `json:"aa_first_position_ratio_upper"` + AAFirstPositionOverheadUpperUS int64 `json:"aa_first_position_overhead_upper_us"` + SessionFirstP95RatioUpper float64 `json:"session_first_p95_ratio_upper"` + SessionFirstP95OverheadUpperUS int64 `json:"session_first_p95_overhead_upper_us"` +} + +type spI2OperationalDesignV2 struct { + Blocks int `json:"blocks"` + FreshSessionsPerArmCaseBlock int `json:"fresh_sessions_per_arm_case_block"` + SamplesPerArmCase int `json:"samples_per_arm_case"` + PlanCacheMode string `json:"plan_cache_mode"` +} + +type spI2ProtocolBootstrapV2 struct { + Domain string `json:"domain"` + CaseOrder []string `json:"case_order"` + RatioScale string `json:"ratio_scale"` + LowerPercentile float64 `json:"lower_percentile"` + UpperPercentile float64 `json:"upper_percentile"` + Quantile string `json:"quantile"` + RoundResampling string `json:"round_resampling"` + WithinRoundResampling string `json:"within_round_resampling"` +} + +type spI2HostAdmissionV2 struct { + Sequence []string `json:"sequence"` + MaximumS4Remediations int `json:"maximum_s4_remediations"` + CandidateEpochLockedOnFirstInvocation bool `json:"candidate_epoch_locked_on_first_invocation"` + MachineThresholds spI2HostMachineThresholdsV2 `json:"machine_thresholds"` + RemediableCauses []string `json:"remediable_causes"` +} + +type spI2HostMachineThresholdsV2 struct { + RunnerProcessOverlapCount int `json:"runner_process_overlap_count"` + ThermalThrottleEvents int `json:"thermal_throttle_events"` + MaximumStealTimePercent float64 `json:"maximum_steal_time_percent"` + CPUGovernor string `json:"cpu_governor"` + PostgreSQLSessionSettings string `json:"postgresql_session_settings"` +} + +func loadSPI2ProtocolV2(path string) (spI2ProtocolV2, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return spI2ProtocolV2{}, "", fmt.Errorf("read SP-I2 V2 protocol: %w", err) + } + var protocol spI2ProtocolV2 + if err := decodePromotionEvidence(raw, &protocol); err != nil { + return spI2ProtocolV2{}, "", fmt.Errorf("decode SP-I2 V2 protocol: %w", err) + } + if err := validateSPI2ProtocolV2(protocol); err != nil { + return spI2ProtocolV2{}, "", err + } + digest := sha256.Sum256(raw) + return protocol, hex.EncodeToString(digest[:]), nil +} + +func loadSPI2V1Rejection(path string) (spI2V1Rejection, error) { + raw, err := os.ReadFile(path) + if err != nil { + return spI2V1Rejection{}, fmt.Errorf("read SP-I2 V1 rejection: %w", err) + } + var rejection spI2V1Rejection + if err := decodePromotionEvidence(raw, &rejection); err != nil { + return spI2V1Rejection{}, fmt.Errorf("decode SP-I2 V1 rejection: %w", err) + } + if rejection.Schema != "sp-i2-terminal-rejection-v1" || rejection.Generation != spI2GenerationV1 || + rejection.Executor != string(optimize.ShortestPathExecutorI2GuardedDistance) || rejection.Policy != optimize.ShortestPathPolicyI2DistanceGuardedV1 || + rejection.Selector != optimize.ShortestPathSelectorStaticV8HiddenFanIn || rejection.SourceCommit != "3865cbc57758b7b20b7ffe431f27235873422eed" || + rejection.DiscoveryReportSHA256 != "f80b0f54624de79e9161673f7c9971662bcd5286bf70829176febc6de2681309" || + rejection.FailedGate.Metric != "p95_ratio_upper" || rejection.FailedGate.Observed != 1.2528773826285173 || rejection.FailedGate.Limit != 1.05 || + rejection.FreezeCreated || rejection.HoldoutOpened || !rejection.Terminal { + return spI2V1Rejection{}, fmt.Errorf("SP-I2 V1 terminal rejection declaration is invalid") + } + return rejection, nil +} + +func loadSPI2V2Rejection(path string) (spI2V2Rejection, error) { + raw, err := os.ReadFile(path) + if err != nil { + return spI2V2Rejection{}, fmt.Errorf("read SP-I2 V2 rejection: %w", err) + } + var rejection spI2V2Rejection + if err := decodePromotionEvidence(raw, &rejection); err != nil { + return spI2V2Rejection{}, fmt.Errorf("decode SP-I2 V2 rejection: %w", err) + } + expected := []spI2V2RejectedGate{ + {Scenario: "aa_identity", Metric: "admission_power_wilson_lower", Observed: 0, Required: 0.9}, + {Scenario: "target_power", Metric: "full_decision_power_wilson_lower", Observed: 0.4724809842358317, Required: 0.9}, + {Scenario: "control_power", Metric: "full_decision_power_wilson_lower", Observed: 0.5053708806725798, Required: 0.9}, + {Scenario: "aa_order_odd_high", Metric: "admission_power_wilson_lower", Observed: 0, Required: 0.9}, + {Scenario: "aa_order_even_high", Metric: "admission_power_wilson_lower", Observed: 0, Required: 0.9}, + } + if rejection.Schema != "sp-i2-terminal-rejection-v2" || rejection.Generation != spI2GenerationV2 || + rejection.SourceCommit != "5df040c2992dd92cf0480beed887c4068c3052b2" || + rejection.ProtocolSHA256 != "17cddc5100bc4f523122b0664ec63d3b4954ae2c01000f04864f10fdd00e1e89" || + rejection.SimulationReportSHA256 != "cbf4fc593a0adfa72ead23f4f391d530790a474a292a9cc47788a18048b17875" || + rejection.SimulationImplementation != spI2PowerSimulationV2 || rejection.RunsPerScenario != 20_000 || + !slices.Equal(rejection.FailedGates, expected) || !rejection.CoverageCalibrated || rejection.FormalAAStarted || + rejection.CapturePlanCreated || rejection.SealedPreregistrationCreated || rejection.HoldoutOpened || rejection.ProductionActivated || + !rejection.SuccessorProtocolRequired || !rejection.Terminal { + return spI2V2Rejection{}, fmt.Errorf("SP-I2 V2 terminal rejection declaration is invalid") + } + return rejection, nil +} + +func validateSPI2ProtocolV2(protocol spI2ProtocolV2) error { + expectedDevelopment := []string{ + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E0), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1D), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1P), + string(optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP), + } + if protocol.Schema != "sp-i2-tail-protocol-v2" || protocol.Generation != spI2GenerationV2 || protocol.Status != spI2V2TerminalStatus || protocol.ProductionDefault != "off" { + return fmt.Errorf("SP-I2 V2 protocol identity is invalid") + } + identities := protocol.Identities + if identities.Executor != string(optimize.ShortestPathExecutorI2GuardedDistanceV2) || + identities.Policy != optimize.ShortestPathPolicyI2DistanceGuardedV2 || + identities.Selector != optimize.ShortestPathSelectorStaticV9HiddenFanInTail || + identities.FallbackExecutor != string(optimize.ShortestPathExecutorS4CanonicalDistance) || + identities.StatisticalImplementation != spI2HierBootstrapV2 || + identities.PromotionManifestSchema != 3 || identities.RollbackSwitch != "DisableInlineSPDistance" { + return fmt.Errorf("SP-I2 V2 protocol compiled identities do not match the declaration") + } + if identities.Executor == string(optimize.ShortestPathExecutorI2GuardedDistance) || + identities.Policy == optimize.ShortestPathPolicyI2DistanceGuardedV1 || + identities.Selector == optimize.ShortestPathSelectorStaticV8HiddenFanIn { + return fmt.Errorf("SP-I2 V2 protocol collides with V1 identity") + } + if !slices.Equal(protocol.DevelopmentExecutors, expectedDevelopment) || slices.Contains(protocol.DevelopmentExecutors, identities.Executor) { + return fmt.Errorf("SP-I2 V2 development executor registry is invalid") + } + if protocol.SelectedArchitecture != "E1" || protocol.Limits.StateRows != optimize.ShortestPathI2QualifiedStateLimit || + protocol.Limits.FrontierRows != optimize.ShortestPathI2QualifiedFrontierLimit || protocol.Limits.MinimumDepth != 1 || protocol.Limits.MaximumDepth != 64 { + return fmt.Errorf("SP-I2 V2 architecture or cap contract is invalid") + } + corpus := protocol.Corpus + if corpus.Source != "cases/generated_sp_i2_distance_v2.json" || corpus.TrainingCases != 8 || corpus.HoldoutCases != 6 || + corpus.TrainingCorpusSHA256 != spI2V2TrainingCorpusSHA256 || corpus.HoldoutCorpusSHA256 != spI2V2HoldoutCorpusSHA256 || + corpus.FullCorpusSHA256 != spI2V2FullCorpusSHA256 || corpus.TrainingDeclarationSHA256 != spI2V2TrainingDeclarationSHA256 || + corpus.HoldoutDeclarationSHA256 != spI2V2HoldoutDeclarationSHA256 || corpus.FullDeclarationSHA256 != spI2V2FullDeclarationSHA256 || + corpus.TrainingResolvedSHA256 != spI2V2TrainingResolvedSHA256 || corpus.HoldoutResolvedSHA256 != spI2V2HoldoutResolvedSHA256 || + corpus.FullResolvedSHA256 != spI2V2FullResolvedSHA256 { + return fmt.Errorf("SP-I2 V2 formal corpus contract is invalid") + } + if protocol.Design.Seed != 1 || protocol.Design.ConfidenceLevel != 0.975 || protocol.Design.BootstrapReplicates != 100_000 || + protocol.Design.Rounds != 40 || protocol.Design.OrdinaryWarmups != 25 || protocol.Design.AttestedStabilizations != 1 || + protocol.Design.TimedSamplesPerRound != 100 || protocol.Design.PoolSize != 1 || protocol.Design.Isolation != "repeatable_read" { + return fmt.Errorf("SP-I2 V2 fixed capture design is invalid") + } + if protocol.Bootstrap.Domain != "sp-i2-tail-bootstrap-v2" || !slices.Equal(protocol.Bootstrap.CaseOrder, []string{"dataset", "case"}) || + protocol.Bootstrap.RatioScale != "log" || protocol.Bootstrap.LowerPercentile != 0.0125 || protocol.Bootstrap.UpperPercentile != 0.9875 || + protocol.Bootstrap.Quantile != "nearest_rank" || protocol.Bootstrap.RoundResampling != "paired" || + protocol.Bootstrap.WithinRoundResampling != "independent_by_arm" { + return fmt.Errorf("SP-I2 V2 bootstrap declaration is invalid") + } + if err := validateSPI2SimulationProtocolV2(protocol.Simulation); err != nil { + return err + } + thresholds := protocol.HostAdmission.MachineThresholds + if !slices.Equal(protocol.HostAdmission.Sequence, []string{"S4/S4", "V2/V2", "S4/V2"}) || protocol.HostAdmission.MaximumS4Remediations != 1 || + !protocol.HostAdmission.CandidateEpochLockedOnFirstInvocation || thresholds.RunnerProcessOverlapCount != 0 || thresholds.ThermalThrottleEvents != 0 || + thresholds.MaximumStealTimePercent != 1 || thresholds.CPUGovernor != "performance" || thresholds.PostgreSQLSessionSettings != "exact_match" { + return fmt.Errorf("SP-I2 V2 host admission contract is invalid") + } + if protocol.V1EvidenceReuse || !protocol.HoldoutAuthorizationBeforeDBSetup || protocol.MultiplicityRule != "intersection_union_all_cases_must_pass" { + return fmt.Errorf("SP-I2 V2 evidence isolation contract is invalid") + } + return nil +} + +func validateSPI2SimulationProtocolV2(simulation spI2ProtocolSimulationV2) error { + if simulation.Implementation != spI2PowerSimulationV2 || simulation.RunsPerScenario != 20_000 || + simulation.WilsonConfidence != 0.95 || simulation.RequiredPowerLower != 0.90 || simulation.RequiredCoverage != 0.975 || + simulation.P95BoundaryFalsePassUpper != 0.015 || simulation.DecisionFalsePassUpper != 0.0275 || + simulation.TraceRescalingTransform != "piecewise_log_quantile_anchor_then_paired_empirical_round_drift" || + simulation.SourceCommit != "3865cbc57758b7b20b7ffe431f27235873422eed" || + simulation.BaselineTraceSHA256 != "ac3ceb27ee92e3f4e21e3994ff9ee82d483b8081e9d44ddcef8e695ffdb1b6d0" || + simulation.CandidateTraceSHA256 != "f6d79e81bdaafedaa95568d57140c14e0808fbb6fc261387abc916081137785a" || + len(simulation.P50RoundDrift) != 20 || len(simulation.P95RoundDrift) != 20 || len(simulation.Scenarios) != 11 { + return fmt.Errorf("SP-I2 V2 simulation declaration is invalid") + } + if simulation.LogStandardErrors != (spI2SimulationErrorsV2{Pooled: 0.025959, OrderStratum: 0.036712, FirstPosition: 0.036712}) || + simulation.AbsoluteStandardErrorsUS != (spI2SimulationErrorsV2{Pooled: 59.338, OrderStratum: 83.917, FirstPosition: 83.917}) { + return fmt.Errorf("SP-I2 V2 simulation error calibration is invalid") + } + expectedKinds := map[string]int{"aa_power": 1, "aa_boundary": 2, "target_power": 1, "target_boundary": 1, "control_power": 1, "control_boundary": 1, "aa_order_power": 2, "aa_order_boundary": 2} + observedKinds := map[string]int{} + seen := map[string]struct{}{} + for _, scenario := range simulation.Scenarios { + if scenario.Name == "" || scenario.Seed == "" || scenario.BaselineP50US <= 0 || scenario.BaselineP95US <= scenario.BaselineP50US || + scenario.CandidateP50US <= 0 || scenario.CandidateP95US <= scenario.CandidateP50US { + return fmt.Errorf("SP-I2 V2 simulation scenario is invalid") + } + if _, duplicate := seen[scenario.Name]; duplicate { + return fmt.Errorf("SP-I2 V2 simulation scenario %q is duplicated", scenario.Name) + } + seen[scenario.Name] = struct{}{} + observedKinds[scenario.Kind]++ + seed := sha256.Sum256([]byte("sp-i2-power-simulation-v2\x00" + scenario.Name)) + if scenario.Seed != hex.EncodeToString(seed[:]) { + return fmt.Errorf("SP-I2 V2 simulation scenario %q seed is invalid", scenario.Name) + } + } + if !maps.Equal(observedKinds, expectedKinds) { + return fmt.Errorf("SP-I2 V2 simulation matrix is incomplete") + } + return nil +} diff --git a/cmd/graphbench/sp_i2_protocol_v2_test.go b/cmd/graphbench/sp_i2_protocol_v2_test.go new file mode 100644 index 00000000..a7c8c703 --- /dev/null +++ b/cmd/graphbench/sp_i2_protocol_v2_test.go @@ -0,0 +1,65 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCheckedInSPI2ProtocolV2MatchesCompiledContract(t *testing.T) { + path := filepath.Join("..", "..", "benchmark", "testdata", "scale", "protocols", "sp_i2_distance_v2.json") + protocol, digest, err := loadSPI2ProtocolV2(path) + require.NoError(t, err) + require.Len(t, digest, 64) + require.Equal(t, spI2GenerationV2, protocol.Generation) + require.Equal(t, spI2HierBootstrapV2, protocol.Identities.StatisticalImplementation) +} + +func TestCheckedInSPI2V1TerminalRejection(t *testing.T) { + path := filepath.Join("..", "..", "benchmark", "testdata", "scale", "protocols", "sp_i2_distance_v1_rejection.json") + rejection, err := loadSPI2V1Rejection(path) + require.NoError(t, err) + require.True(t, rejection.Terminal) + require.False(t, rejection.FreezeCreated) + require.False(t, rejection.HoldoutOpened) +} + +func TestCheckedInSPI2V2TerminalRejection(t *testing.T) { + path := filepath.Join("..", "..", "benchmark", "testdata", "scale", "protocols", "sp_i2_distance_v2_rejection.json") + rejection, err := loadSPI2V2Rejection(path) + require.NoError(t, err) + require.True(t, rejection.Terminal) + require.True(t, rejection.SuccessorProtocolRequired) + require.False(t, rejection.FormalAAStarted) + require.False(t, rejection.HoldoutOpened) +} + +func TestSPI2V1CannotCreateFreezeOrAuthorizeHoldout(t *testing.T) { + _, err := createSPI2QualificationReport("baseline", "candidate", "resource", "", "", "freeze.json", "report.json", SPI2QualificationOptions{}) + require.ErrorContains(t, err, "terminally rejected") + require.True(t, spI2V1TerminallyRejected()) +} + +func TestSPI2ProtocolV2StrictlyRejectsDuplicateUnknownAndTrailingData(t *testing.T) { + path := filepath.Join("..", "..", "benchmark", "testdata", "scale", "protocols", "sp_i2_distance_v2.json") + raw, err := os.ReadFile(path) + require.NoError(t, err) + + for name, mutated := range map[string][]byte{ + "duplicate": append([]byte(`{"schema":"collision",`), raw[1:]...), + "unknown": append([]byte(`{"unknown":true,`), raw[1:]...), + "trailing": append(append([]byte(nil), raw...), []byte(`{}`)...), + } { + t.Run(name, func(t *testing.T) { + mutatedPath := filepath.Join(t.TempDir(), "protocol.json") + require.NoError(t, os.WriteFile(mutatedPath, mutated, 0o600)) + _, _, err := loadSPI2ProtocolV2(mutatedPath) + require.Error(t, err) + }) + } +} diff --git a/cmd/graphbench/sp_i2_qualification.go b/cmd/graphbench/sp_i2_qualification.go new file mode 100644 index 00000000..45ee3cb8 --- /dev/null +++ b/cmd/graphbench/sp_i2_qualification.go @@ -0,0 +1,2377 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const ( + // spI2QualificationVersion reserves the stable protocol value used to recognize sp i2 qualification version across artifacts and executions. + spI2QualificationVersion = 1 + + // spI2FreezeVersion reserves the stable protocol value used to recognize sp i2 freeze version across artifacts and executions. + spI2FreezeVersion = 1 + + // spI2TrainingTag reserves the stable protocol value used to recognize sp i2 training tag across artifacts and executions. + spI2TrainingTag = "sp-i2-distance-v1-training" + + // spI2HoldoutTag reserves the stable protocol value used to recognize sp i2 holdout tag across artifacts and executions. + spI2HoldoutTag = "sp-i2-distance-v1-holdout" + + // spI2QuerySHA256 reserves the stable protocol value used to recognize sp i2 query sha256 across artifacts and executions. + spI2QuerySHA256 = "69c1d7778963a742dbac8adeff01213850b60d1ed61858832a8315aa5184b3db" + + // spI2TrainingCorpusSHA256 reserves the stable protocol value used to recognize sp i2 training corpus sha256 across artifacts and executions. + spI2TrainingCorpusSHA256 = "33294507fdf87e5fed07e702f7c8c00d5abc7c0d19b9b9472b740b244531e9f9" + + // spI2FullCorpusSHA256 reserves the stable protocol value used to recognize sp i2 full corpus sha256 across artifacts and executions. + spI2FullCorpusSHA256 = "eca42bb762acc379673edffac130e88432dcf089261f1295451e03ea7f1fa35a" + + // spI2TrainingResolvedSHA reserves the stable protocol value used to recognize sp i2 training resolved sha across artifacts and executions. + spI2TrainingResolvedSHA = "3c05a0f65efed8d08d79953d8398c013f6ea52b2a53f9900af7d7f68626cf4fb" + + // spI2FullResolvedSHA reserves the stable protocol value used to recognize sp i2 full resolved sha across artifacts and executions. + spI2FullResolvedSHA = "2d7acf8e3d9904b9cf5fefb5c83f2740a5b184c6dfcd682d5115250cc5b19fe5" +) + +// spI2CanonicalCases freezes the training and holdout workloads admitted to SP-I2 qualification. +var spI2CanonicalCases = []struct { + // dataset identifies the generated fixture containing the workload. + dataset string + + // name identifies the workload within the fixture dataset. + name string + + // split assigns the workload to training or unopened holdout evidence. + split string +}{ + { + dataset: "generated_shortest_paths_v2_d3_o0_r64_fo0_fi32_l2_k0_t0_w0_x3_p0_c0_s0", + name: "GSP-I2-V1-TRAIN-D03-RI064-FI032-full", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d8_o0_r128_fo0_fi64_l4_k0_t0_w0_x8_p0_c0_s0", + name: "GSP-I2-V1-TRAIN-D08-RI128-FI064-early-d02", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d8_o0_r128_fo0_fi64_l4_k0_t0_w0_x8_p0_c0_s0", + name: "GSP-I2-V1-TRAIN-D08-RI128-FI064-full", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d16_o0_r256_fo0_fi512_l8_k0_t0_w0_x16_p0_c0_s0", + name: "GSP-I2-V1-TRAIN-D16-RI256-FI512-full", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d16_o0_r256_fo0_fi512_l8_k0_t0_w0_x16_p0_c0_s0", + name: "GSP-I2-V1-TRAIN-D16-RI256-FI512-disconnected", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d6_o0_r0_fo0_fi0_l0_k0_t0_w0_x6_p0_c1_s0", + name: "GSP-I2-V1-TRAIN-cycle-control", + split: "training", + }, + { + dataset: "generated_shortest_paths_v2_d5_o0_r47_fo0_fi23_l2_k0_t0_w0_x5_p0_c0_s0", + name: "GSP-I2-V1-HOLDOUT-D05-RI047-FI023-full", + split: "holdout", + }, + { + dataset: "generated_shortest_paths_v2_d13_o0_r191_fo0_fi383_l7_k0_t0_w0_x13_p0_c0_s0", + name: "GSP-I2-V1-HOLDOUT-D13-RI191-FI383-full", + split: "holdout", + }, + { + dataset: "generated_shortest_paths_v2_d13_o0_r191_fo0_fi383_l7_k0_t0_w0_x13_p0_c0_s0", + name: "GSP-I2-V1-HOLDOUT-D13-RI191-FI383-early-d03", + split: "holdout", + }, + { + dataset: "generated_shortest_paths_v2_d21_o0_r127_fo0_fi255_l11_k0_t0_w0_x21_p0_c0_s0", + name: "GSP-I2-V1-HOLDOUT-D21-RI127-FI255-disconnected", + split: "holdout", + }, +} + +// spI2CanonicalCohort groups state that must remain consistent while processing sp i2 canonical cohort. +type spI2CanonicalCohort struct { + // keys retains the keys while spI2CanonicalCohort is assembled or evaluated. + keys map[performanceKey]struct{} + // trainingKeys retains the training keys while spI2CanonicalCohort is assembled or evaluated. + trainingKeys map[performanceKey]struct{} + // holdoutKeys retains the holdout keys while spI2CanonicalCohort is assembled or evaluated. + holdoutKeys map[performanceKey]struct{} + // declarationSHA256 binds the referenced declaration content by SHA-256 digest. + declarationSHA256 string + // trainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + trainingDeclarationSHA256 string + // holdoutDeclarationSHA256 binds the referenced holdout declaration content by SHA-256 digest. + holdoutDeclarationSHA256 string + // trainingCorpusSHA256 binds the referenced training corpus content by SHA-256 digest. + trainingCorpusSHA256 string + // fullCorpusSHA256 binds the referenced full corpus content by SHA-256 digest. + fullCorpusSHA256 string + // trainingResolvedSHA256 binds the referenced training resolved content by SHA-256 digest. + trainingResolvedSHA256 string + // fullResolvedSHA256 binds the referenced full resolved content by SHA-256 digest. + fullResolvedSHA256 string +} + +// spI2CanonicalDeclaration groups state that must remain consistent while processing sp i2 canonical declaration. +type spI2CanonicalDeclaration struct { + // testCase retains the test case while spI2CanonicalDeclaration is assembled or evaluated. + testCase ScaleCase + // fixture retains the fixture while spI2CanonicalDeclaration is assembled or evaluated. + fixture FixtureMetadata +} + +// canonicalSPI2Declarations resolves the frozen SP-I2 workload declarations and fixture metadata. +func canonicalSPI2Declarations() (map[performanceKey]spI2CanonicalDeclaration, error) { + repositoryRoot := strings.TrimSpace(commandOutput("git", "rev-parse", "--show-toplevel")) + if repositoryRoot == "" || repositoryRoot == "unknown" { + return nil, fmt.Errorf("locate repository root for frozen SP-I2 declarations") + } + + if corpus, err := loadScaleCorpus(filepath.Join(repositoryRoot, "benchmark", "testdata", "scale")); err != nil { + return nil, fmt.Errorf("load frozen SP-I2 declarations: %w", err) + } else if cohort, err := canonicalSPI2Cohort(); err != nil { + return nil, err + } else { + declarations := make(map[performanceKey]spI2CanonicalDeclaration, len(cohort.keys)) + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, expected := cohort.keys[key]; !expected { + continue + } + if _, duplicate := declarations[key]; duplicate { + return nil, fmt.Errorf("frozen SP-I2 corpus duplicates %s/%s", key.dataset, key.name) + } + if fixture, err := fixtureMetadata("unused", testCase.Dataset); err != nil { + return nil, fmt.Errorf("derive frozen SP-I2 fixture %s: %w", testCase.Dataset, err) + } else { + declarations[key] = spI2CanonicalDeclaration{ + testCase: testCase, + fixture: fixture, + } + } + } + if len(declarations) != len(cohort.keys) { + return nil, fmt.Errorf("frozen SP-I2 corpus omits canonical declarations") + } + + return declarations, nil + } +} + +// canonicalSPI2Cohort builds the immutable training and holdout membership used by qualification. +func canonicalSPI2Cohort() (spI2CanonicalCohort, error) { + cohort := spI2CanonicalCohort{ + keys: map[performanceKey]struct{}{}, + trainingKeys: map[performanceKey]struct{}{}, + holdoutKeys: map[performanceKey]struct{}{}, + trainingCorpusSHA256: spI2TrainingCorpusSHA256, + fullCorpusSHA256: spI2FullCorpusSHA256, + trainingResolvedSHA256: spI2TrainingResolvedSHA, + fullResolvedSHA256: spI2FullResolvedSHA, + } + var full, training, holdout []DeclaredCaseBackend + for _, testCase := range spI2CanonicalCases { + key := performanceKey{ + dataset: testCase.dataset, + name: testCase.name, + backend: ModePostgresSQL, + } + if _, duplicate := cohort.keys[key]; duplicate || !strings.HasPrefix(testCase.dataset, "generated_shortest_paths_v2_") { + return spI2CanonicalCohort{}, fmt.Errorf("frozen SP-I2 cohort contains an invalid declaration") + } + cohort.keys[key] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + item := DeclaredCaseBackend{ + Dataset: key.dataset, + Name: key.name, + Backend: backend, + } + full = append(full, item) + if testCase.split == "training" { + training = append(training, item) + } else if testCase.split == "holdout" { + holdout = append(holdout, item) + } else { + return spI2CanonicalCohort{}, fmt.Errorf("frozen SP-I2 cohort contains an invalid split") + } + } + if testCase.split == "training" { + cohort.trainingKeys[key] = struct{}{} + } else { + cohort.holdoutKeys[key] = struct{}{} + } + } + if len(cohort.trainingKeys) != 6 || len(cohort.holdoutKeys) != 4 || len(cohort.keys) != 10 { + return spI2CanonicalCohort{}, fmt.Errorf("frozen SP-I2 cohort must contain exactly 6 training and 4 holdout cases") + } + cohort.declarationSHA256 = declarationSHA256(full) + cohort.trainingDeclarationSHA256 = declarationSHA256(training) + cohort.holdoutDeclarationSHA256 = declarationSHA256(holdout) + return cohort, nil +} + +// spI2QualificationCaps returns the resource limits enforced for sp i2 qualification. +func spI2QualificationCaps() map[string]int64 { + return spI2PromotionCaps() +} + +// spI2TelemetryCaps returns the resource limits enforced for sp i2 telemetry. +func spI2TelemetryCaps() map[string]int64 { + return map[string]int64{ + "state_rows": optimize.ShortestPathI2QualifiedStateLimit, + "frontier_rows": optimize.ShortestPathI2QualifiedFrontierLimit, + "queue_rows": optimize.ShortestPathI2QualifiedFrontierLimit, + } +} + +// spI2SummaryCaps freezes the translator's conservative queue alias as part +// of the emitted runtime identity while the resource report keeps only the +// independently enforced state and frontier dimensions. +func spI2SummaryCaps() map[string]int64 { + return map[string]int64{ + "state_rows": optimize.ShortestPathI2QualifiedStateLimit, + "frontier_rows": optimize.ShortestPathI2QualifiedFrontierLimit, + "queue_rows": optimize.ShortestPathI2QualifiedFrontierLimit, + } +} + +// SPI2QualificationOptions configures spi2 qualification. +type SPI2QualificationOptions struct { + // Seed makes randomized statistical procedures reproducible. + Seed int64 + // Confidence sets the requested statistical confidence level. + Confidence float64 + // BootstrapCount records the number of bootstrap count. + BootstrapCount int + // Protocol identifies the protocol. + Protocol string + // Training evidence paths make confirmation independently recompute the + // discovery decision instead of trusting only a mutable report and freeze. + TrainingBaselinePath string + // TrainingCandidatePath identifies the filesystem training candidate path. + TrainingCandidatePath string + // TrainingResourcePath identifies the filesystem training resource path. + TrainingResourcePath string + // SourceArchiveSHA256 binds the report to git archive HEAD. Report-mode + // callers populate it from the current committed tree; tests may supply a + // synthetic digest without invoking Git. + SourceArchiveSHA256 string + // Freeze supplies the freeze input to the SPI2QualificationOptions contract. + Freeze *SPI2QualificationFreezeManifest + // Discovery supplies the discovery input to the SPI2QualificationOptions contract. + Discovery *SPI2QualificationReport +} + +// SPI2QualificationCase records the evidence and decision for one spi2 qualification workload. +type SPI2QualificationCase struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset"` + // Name identifies the name. + Name string `json:"name"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit string `json:"qualification_split"` + // QualificationRole distinguishes improvement targets from preregistered adverse controls. + QualificationRole string `json:"qualification_role"` + // Rounds records the number of rounds. + Rounds int `json:"matched_rounds"` + // BaselineSamples supplies the baseline samples input to the SPI2QualificationCase contract. + BaselineSamples int `json:"baseline_samples"` + // CandidateSamples supplies the candidate samples input to the SPI2QualificationCase contract. + CandidateSamples int `json:"candidate_samples"` + // MedianRatio supplies the median ratio input to the SPI2QualificationCase contract. + MedianRatio RatioInterval `json:"median_ratio_to_s4"` + // MedianSaving supplies the median saving input to the SPI2QualificationCase contract. + MedianSaving DurationInterval `json:"median_saving_vs_s4"` + // P95Ratio supplies the p95 ratio input to the SPI2QualificationCase contract. + P95Ratio RatioInterval `json:"p95_ratio_to_s4"` + // Material indicates whether material applies. + Material bool `json:"material"` + // P95Contained indicates whether p95 contained applies. + P95Contained bool `json:"p95_contained"` + // ResourcePassed indicates whether resource passed applies. + ResourcePassed bool `json:"resource_passed"` + // RuntimeBranch supplies the runtime branch input to the SPI2QualificationCase contract. + RuntimeBranch string `json:"runtime_branch"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` + // Reasons explains each failed or inapplicable validation gate. + Reasons []string `json:"reasons,omitempty"` +} + +// SPI2QualificationReport records the evidence and outcome produced by spi2 qualification. +type SPI2QualificationReport struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Protocol identifies the protocol. + Protocol string `json:"protocol"` + // Baseline identifies the incumbent execution strategy used for comparison. + Baseline string `json:"baseline"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // Policy identifies the policy. + Policy string `json:"policy"` + // QuerySHA256 binds the referenced query content by SHA-256 digest. + QuerySHA256 string `json:"query_sha256"` + // Seed makes randomized statistical procedures reproducible. + Seed int64 `json:"seed"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // BootstrapCount records the number of bootstrap count. + BootstrapCount int `json:"bootstrap_count"` + // MaterialityRatio supplies the materiality ratio input to the SPI2QualificationReport contract. + MaterialityRatio float64 `json:"materiality_ratio_upper_limit"` + // MaterialityAbsolute supplies the materiality absolute input to the SPI2QualificationReport contract. + MaterialityAbsolute time.Duration `json:"materiality_absolute_lower_limit"` + // P95RatioLimit supplies the p95 ratio limit input to the SPI2QualificationReport contract. + P95RatioLimit float64 `json:"p95_ratio_upper_limit"` + // AdverseRatioLimit caps relative overhead for preregistered adverse controls. + AdverseRatioLimit float64 `json:"adverse_ratio_upper_limit"` + // AdverseAbsoluteLimit caps absolute overhead for preregistered adverse controls. + AdverseAbsoluteLimit time.Duration `json:"adverse_absolute_upper_limit"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // SourceCommit supplies the source commit input to the SPI2QualificationReport contract. + SourceCommit string `json:"source_commit"` + // SourceArchiveSHA256 binds the referenced source archive content by SHA-256 digest. + SourceArchiveSHA256 string `json:"source_archive_sha256"` + // DirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds the referenced corpus content by SHA-256 digest. + CorpusSHA256 string `json:"corpus_sha256"` + // CohortDeclarationSHA256 binds the referenced cohort declaration content by SHA-256 digest. + CohortDeclarationSHA256 string `json:"cohort_declaration_sha256"` + // ResolvedSelectionSHA256 binds the referenced resolved selection content by SHA-256 digest. + ResolvedSelectionSHA256 string `json:"resolved_selection_sha256"` + // TrainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + // HoldoutDeclarationSHA256 binds the referenced holdout declaration content by SHA-256 digest. + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + // FullDeclarationSHA256 binds the referenced full declaration content by SHA-256 digest. + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + // TrainingCorpusSHA256 binds the referenced training corpus content by SHA-256 digest. + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + // FullCorpusSHA256 binds the referenced full corpus content by SHA-256 digest. + FullCorpusSHA256 string `json:"full_corpus_sha256"` + // BaselineArtifactSHA256 binds the referenced baseline artifact content by SHA-256 digest. + BaselineArtifactSHA256 string `json:"baseline_artifact_sha256,omitempty"` + // CandidateArtifactSHA256 binds the referenced candidate artifact content by SHA-256 digest. + CandidateArtifactSHA256 string `json:"candidate_artifact_sha256,omitempty"` + // ResourceReportSHA256 binds the referenced resource report content by SHA-256 digest. + ResourceReportSHA256 string `json:"resource_report_sha256,omitempty"` + // FreezeManifestSHA256 binds the referenced freeze manifest content by SHA-256 digest. + FreezeManifestSHA256 string `json:"freeze_manifest_sha256,omitempty"` + // EvidencePassed indicates whether evidence passed applies. + EvidencePassed bool `json:"evidence_passed"` + // TrainingCases supplies the training cases input to the SPI2QualificationReport contract. + TrainingCases int `json:"training_cases"` + // HoldoutCases supplies the holdout cases input to the SPI2QualificationReport contract. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // QualificationPassed indicates whether qualification passed applies. + QualificationPassed bool `json:"qualification_passed"` + // Cases contains the per-workload evidence underlying the aggregate decision. + Cases []SPI2QualificationCase `json:"cases"` +} + +// SPI2QualificationFreezeManifest binds the immutable inputs authorized for spi2 qualification freeze. +type SPI2QualificationFreezeManifest struct { + // Version identifies the schema version for version. + Version int `json:"version"` + // Baseline identifies the incumbent execution strategy used for comparison. + Baseline string `json:"baseline"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // Policy identifies the policy. + Policy string `json:"policy"` + // QuerySHA256 binds the referenced query content by SHA-256 digest. + QuerySHA256 string `json:"query_sha256"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // Seed makes randomized statistical procedures reproducible. + Seed int64 `json:"seed"` + // Confidence sets the requested statistical confidence level. + Confidence float64 `json:"confidence_level"` + // BootstrapCount records the number of bootstrap count. + BootstrapCount int `json:"bootstrap_count"` + // SourceCommit supplies the source commit input to the SPI2QualificationFreezeManifest contract. + SourceCommit string `json:"source_commit"` + // SourceArchiveSHA256 binds the referenced source archive content by SHA-256 digest. + SourceArchiveSHA256 string `json:"source_archive_sha256"` + // DirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // TrainingDeclarationSHA256 binds the referenced training declaration content by SHA-256 digest. + TrainingDeclarationSHA256 string `json:"training_declaration_sha256"` + // HoldoutDeclarationSHA256 binds the referenced holdout declaration content by SHA-256 digest. + HoldoutDeclarationSHA256 string `json:"holdout_declaration_sha256"` + // FullDeclarationSHA256 binds the referenced full declaration content by SHA-256 digest. + FullDeclarationSHA256 string `json:"full_declaration_sha256"` + // TrainingCorpusSHA256 binds the referenced training corpus content by SHA-256 digest. + TrainingCorpusSHA256 string `json:"training_corpus_sha256"` + // FullCorpusSHA256 binds the referenced full corpus content by SHA-256 digest. + FullCorpusSHA256 string `json:"full_corpus_sha256"` + // TrainingResolvedSHA256 binds the referenced training resolved content by SHA-256 digest. + TrainingResolvedSHA256 string `json:"training_resolved_selection_sha256"` + // FullResolvedSHA256 binds the referenced full resolved content by SHA-256 digest. + FullResolvedSHA256 string `json:"full_resolved_selection_sha256"` + // BaselineArtifactSHA256 binds the referenced baseline artifact content by SHA-256 digest. + BaselineArtifactSHA256 string `json:"baseline_artifact_sha256"` + // CandidateArtifactSHA256 binds the referenced candidate artifact content by SHA-256 digest. + CandidateArtifactSHA256 string `json:"candidate_artifact_sha256"` + // ResourceReportSHA256 binds the referenced resource report content by SHA-256 digest. + ResourceReportSHA256 string `json:"resource_report_sha256"` + // DiscoveryReportSHA256 binds the referenced discovery report content by SHA-256 digest. + DiscoveryReportSHA256 string `json:"discovery_report_sha256"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` +} + +// spI2EvidenceIdentity groups state that must remain consistent while processing sp i2 evidence identity. +type spI2EvidenceIdentity struct { + // sourceCommit retains the source commit while spI2EvidenceIdentity is assembled or evaluated. + sourceCommit string + // dirtyDiffSHA256 binds the referenced dirty diff content by SHA-256 digest. + dirtyDiffSHA256 string + // binarySHA256 binds the referenced binary content by SHA-256 digest. + binarySHA256 string + // corpusSHA256 binds the referenced corpus content by SHA-256 digest. + corpusSHA256 string + // declarationSHA256 binds the referenced declaration content by SHA-256 digest. + declarationSHA256 string + // resolvedSHA256 binds the referenced resolved content by SHA-256 digest. + resolvedSHA256 string +} + +// sourceArchiveSHA256 supports benchmark evidence processing for source archive sha256. +func spI2SourceArchiveSHA256() (string, error) { + archive, err := exec.Command("git", "archive", "--format=tar", "HEAD").Output() + if err != nil { + return "", fmt.Errorf("archive source commit: %w", err) + } + digest := sha256.Sum256(archive) + return hex.EncodeToString(digest[:]), nil +} + +// equalSPI2Caps returns the resource limits enforced for equal spi2. +func equalSPI2Caps(left, right map[string]int64) bool { + if len(left) != len(right) { + return false + } + for name, value := range left { + if right[name] != value { + return false + } + } + return true +} + +// spI2ProtocolRequirements groups state that must remain consistent while processing sp i2 protocol requirements. +type spI2ProtocolRequirements struct { + // minimumWarmups retains the minimum warmups while spI2ProtocolRequirements is assembled or evaluated. + minimumWarmups int + // minimumRounds records the number of minimum rounds. + minimumRounds int + // maximumRounds records the number of maximum rounds. + maximumRounds int + // minimumSamples retains the minimum samples while spI2ProtocolRequirements is assembled or evaluated. + minimumSamples int + // protectedCount records the number of protected count. + protectedCount int + // protectedSHA retains the protected sha while spI2ProtocolRequirements is assembled or evaluated. + protectedSHA string + // expectedKeys retains the expected keys while spI2ProtocolRequirements is assembled or evaluated. + expectedKeys map[performanceKey]struct{} + // declarationSHA retains the declaration sha while spI2ProtocolRequirements is assembled or evaluated. + declarationSHA string + // corpusSHA retains the corpus sha while spI2ProtocolRequirements is assembled or evaluated. + corpusSHA string + // resolvedSHA retains the resolved sha while spI2ProtocolRequirements is assembled or evaluated. + resolvedSHA string +} + +// spI2QualificationSeries accumulates matched observations used to evaluate sp i2 qualification. +type spI2QualificationSeries struct { + // baseline retains the baseline while spI2QualificationSeries is assembled or evaluated. + baseline roundSamples + // candidate retains the candidate while spI2QualificationSeries is assembled or evaluated. + candidate roundSamples + // runtimeBranch retains the runtime branch while spI2QualificationSeries is assembled or evaluated. + runtimeBranch string + // resourcePassed indicates whether resource passed applies. + resourcePassed bool +} + +// spI2Requirements supports benchmark evidence processing for sp i2 requirements. +func spI2Requirements(protocol string, cohort spI2CanonicalCohort) (spI2ProtocolRequirements, error) { + switch protocol { + case referencePairProtocolDiscovery: + return spI2ProtocolRequirements{ + minimumWarmups: 5, + minimumRounds: 5, + maximumRounds: 20, + minimumSamples: 10, + protectedCount: 2 * len(cohort.holdoutKeys), + protectedSHA: cohort.holdoutDeclarationSHA256, + expectedKeys: cohort.trainingKeys, + declarationSHA: cohort.trainingDeclarationSHA256, + corpusSHA: cohort.trainingCorpusSHA256, + resolvedSHA: cohort.trainingResolvedSHA256, + }, nil + case referencePairProtocolConfirmation: + return spI2ProtocolRequirements{ + minimumWarmups: 20, + minimumRounds: 10, + maximumRounds: 20, + minimumSamples: 50, + expectedKeys: cohort.keys, + declarationSHA: cohort.declarationSHA256, + corpusSHA: cohort.fullCorpusSHA256, + resolvedSHA: cohort.fullResolvedSHA256, + }, nil + default: + return spI2ProtocolRequirements{}, fmt.Errorf("unsupported SP-I2 qualification protocol %q", protocol) + } +} + +// buildSPI2QualificationReport builds spi2 qualification report. +func buildSPI2QualificationReport( + baseline, candidate []CaseResult, + resource ResourceGateReport, + options SPI2QualificationOptions, +) (SPI2QualificationReport, error) { + if options.Confidence != defaultConfidenceLevel || math.IsNaN(options.Confidence) || math.IsInf(options.Confidence, 0) { + return SPI2QualificationReport{}, fmt.Errorf("SP-I2 qualification confidence must be the frozen %.4f", defaultConfidenceLevel) + } + if options.Seed != 1 { + return SPI2QualificationReport{}, fmt.Errorf("SP-I2 qualification bootstrap seed must be the frozen value 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount != defaultBootstrapCount { + return SPI2QualificationReport{}, fmt.Errorf("SP-I2 qualification bootstrap count must be the frozen value %d", defaultBootstrapCount) + } + if options.Protocol == "" { + options.Protocol = referencePairProtocolConfirmation + } + if !lowercaseSHA256(options.SourceArchiveSHA256) { + return SPI2QualificationReport{}, fmt.Errorf("SP-I2 source archive digest is missing or malformed") + } + + cohort, err := canonicalSPI2Cohort() + if err != nil { + return SPI2QualificationReport{}, err + } + requirements, err := spI2Requirements(options.Protocol, cohort) + if err != nil { + return SPI2QualificationReport{}, err + } + identity, err := validateSPI2EvidenceIdentity(baseline, candidate, requirements) + if err != nil { + return SPI2QualificationReport{}, err + } + series, keys, err := collectSPI2QualificationSeries(baseline, candidate, resource, requirements) + if err != nil { + return SPI2QualificationReport{}, err + } + + report := SPI2QualificationReport{ + Version: spI2QualificationVersion, + Protocol: options.Protocol, + Baseline: string(optimize.ShortestPathExecutorS4CanonicalDistance), + Candidate: string(optimize.ShortestPathExecutorI2GuardedDistance), + Policy: optimize.ShortestPathPolicyI2DistanceGuardedV1, + QuerySHA256: spI2QuerySHA256, + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + MaterialityRatio: 0.95, + MaterialityAbsolute: 100 * time.Microsecond, + P95RatioLimit: 1.05, + AdverseRatioLimit: 1.10, + AdverseAbsoluteLimit: 100 * time.Microsecond, + Caps: spI2QualificationCaps(), + SourceCommit: identity.sourceCommit, + SourceArchiveSHA256: options.SourceArchiveSHA256, + DirtyDiffSHA256: identity.dirtyDiffSHA256, + BinarySHA256: identity.binarySHA256, + CorpusSHA256: identity.corpusSHA256, + CohortDeclarationSHA256: identity.declarationSHA256, + ResolvedSelectionSHA256: identity.resolvedSHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + EvidencePassed: true, + TrainingPassed: true, + HoldoutPassed: true, + } + if options.Protocol == referencePairProtocolConfirmation { + if err := validateSPI2Freeze(options.Freeze, options.Discovery, report, cohort); err != nil { + return SPI2QualificationReport{}, err + } + } + + gateOptions := PerfGateOptions{ + Seed: options.Seed, + Confidence: options.Confidence, + BootstrapCount: options.BootstrapCount, + } + for index, key := range keys { + current := series[key] + baselineRounds, candidateRounds := matchedRounds(current.baseline, current.candidate) + if !slices.Equal(sortedRounds(current.baseline), sortedRounds(current.candidate)) || + len(baselineRounds) != len(current.baseline) || len(candidateRounds) != len(current.candidate) { + return SPI2QualificationReport{}, fmt.Errorf("%s/%s SP-I2 arms do not contain identical nonempty round sets", key.dataset, key.name) + } + rounds := sortedRounds(baselineRounds) + if len(rounds) < requirements.minimumRounds || len(rounds) > requirements.maximumRounds { + return SPI2QualificationReport{}, fmt.Errorf( + "%s/%s requires %d-%d matched SP-I2 rounds, got %d", + key.dataset, key.name, requirements.minimumRounds, requirements.maximumRounds, len(rounds), + ) + } + for _, round := range rounds { + if len(baselineRounds[round]) < requirements.minimumSamples || len(candidateRounds[round]) < requirements.minimumSamples { + return SPI2QualificationReport{}, fmt.Errorf( + "%s/%s round %d requires at least %d warm samples per SP-I2 arm, got %d/%d", + key.dataset, key.name, round, requirements.minimumSamples, + len(baselineRounds[round]), len(candidateRounds[round]), + ) + } + } + if err := validatePairedOrderEvidence(baseline, candidate, key, rounds, requirements.minimumWarmups); err != nil { + return SPI2QualificationReport{}, fmt.Errorf("invalid SP-I2 paired evidence: %w", err) + } + + split := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + split = "holdout" + } + seed := options.Seed + int64(index)*7919 + gateCase := SPI2QualificationCase{ + Dataset: key.dataset, + Name: key.name, + QualificationSplit: split, + QualificationRole: "target", + Rounds: len(rounds), + BaselineSamples: sampleCount(baselineRounds), + CandidateSamples: sampleCount(candidateRounds), + MedianRatio: bootstrapRoundMedianRatio(baselineRounds, candidateRounds, seed, gateOptions), + MedianSaving: bootstrapRoundMedianSaving(baselineRounds, candidateRounds, seed+1, gateOptions), + P95Ratio: bootstrapStratifiedP95Ratio(baselineRounds, candidateRounds, seed+2, gateOptions), + ResourcePassed: current.resourcePassed, + RuntimeBranch: current.runtimeBranch, + Passed: true, + } + if strings.Contains(gateCase.Name, "cycle-control") { + gateCase.QualificationRole = "adverse_control" + gateCase.Material = gateCase.MedianRatio.Upper <= report.AdverseRatioLimit || + gateCase.MedianSaving.Lower >= -report.AdverseAbsoluteLimit + } else { + gateCase.Material = gateCase.MedianRatio.Upper <= report.MaterialityRatio || + gateCase.MedianSaving.Lower >= report.MaterialityAbsolute + } + gateCase.P95Contained = gateCase.P95Ratio.Upper <= report.P95RatioLimit + if !gateCase.Material { + gateCase.Passed = false + if gateCase.QualificationRole == "adverse_control" { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "adverse-control overhead is not contained: ratio upper %.4f > %.4f and overhead upper %s > %s", + gateCase.MedianRatio.Upper, report.AdverseRatioLimit, + -gateCase.MedianSaving.Lower, report.AdverseAbsoluteLimit, + )) + } else { + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "median improvement is not material: ratio upper %.4f > %.4f and saving lower %s < %s", + gateCase.MedianRatio.Upper, report.MaterialityRatio, + gateCase.MedianSaving.Lower, report.MaterialityAbsolute, + )) + } + } + if !gateCase.P95Contained { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, fmt.Sprintf( + "p95 ratio upper %.4f exceeds %.4f", gateCase.P95Ratio.Upper, report.P95RatioLimit, + )) + } + if !gateCase.ResourcePassed { + gateCase.Passed = false + gateCase.Reasons = append(gateCase.Reasons, "candidate resource evidence did not pass") + } + + switch split { + case "training": + report.TrainingCases++ + report.TrainingPassed = report.TrainingPassed && gateCase.Passed + case "holdout": + report.HoldoutCases++ + report.HoldoutPassed = report.HoldoutPassed && gateCase.Passed + } + report.Cases = append(report.Cases, gateCase) + } + report.TrainingPassed = report.TrainingPassed && report.TrainingCases == len(cohort.trainingKeys) + report.HoldoutPassed = report.HoldoutPassed && report.HoldoutCases == len(cohort.holdoutKeys) + if options.Protocol == referencePairProtocolDiscovery { + report.HoldoutPassed = false + } + report.QualificationPassed = report.EvidencePassed && report.TrainingPassed && report.HoldoutPassed + return report, nil +} + +// validateSPI2EvidenceIdentity validates spi2 evidence identity. +func validateSPI2EvidenceIdentity( + baseline, candidate []CaseResult, + requirements spI2ProtocolRequirements, +) (spI2EvidenceIdentity, error) { + if err := validatePerformanceWorkloadIdentity(baseline, candidate); err != nil { + return spI2EvidenceIdentity{}, err + } + baselineHost, err := artifactHostFingerprint(baseline) + if err != nil { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 baseline host: %w", err) + } + candidateHost, err := artifactHostFingerprint(candidate) + if err != nil { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 candidate host: %w", err) + } + if baselineHost != candidateHost { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 baseline and candidate host identities differ") + } + + identity := spI2EvidenceIdentity{} + for _, artifact := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // records retains the records while anonymous record is assembled or evaluated. + records []CaseResult + }{ + { + name: "baseline", + records: baseline, + }, + { + name: "candidate", + records: candidate, + }, + } { + selection, err := selectionIdentity(artifact.records) + if err != nil { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 %s selection: %w", artifact.name, err) + } + if err := validateSPI2Selection(selection, requirements); err != nil { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 %s selection: %w", artifact.name, err) + } + currentIdentity := spI2EvidenceIdentity{ + declarationSHA256: selection.DeclarationSHA256, + resolvedSHA256: resolvedSelectionSHA256(selection.Resolved), + } + for _, record := range artifact.records { + if record.Environment == nil || record.PostgresEnvironment == nil { + return spI2EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm lacks source or PostgreSQL environment identity", record.Dataset, record.Name, artifact.name) + } + current := spI2EvidenceIdentity{ + sourceCommit: strings.TrimSpace(record.Environment.SourceCommit), + dirtyDiffSHA256: record.Environment.DirtyDiffSHA256, + binarySHA256: record.Environment.BinarySHA256, + corpusSHA256: record.Environment.CorpusSHA256, + declarationSHA256: selection.DeclarationSHA256, + resolvedSHA256: currentIdentity.resolvedSHA256, + } + if current.sourceCommit == "" || current.sourceCommit == "unknown" || + !lowercaseSHA256(current.dirtyDiffSHA256) || !lowercaseSHA256(current.binarySHA256) || + !lowercaseSHA256(current.corpusSHA256) { + return spI2EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm lacks frozen source, diff, binary, or corpus identity", record.Dataset, record.Name, artifact.name) + } + if current.corpusSHA256 != requirements.corpusSHA { + return spI2EvidenceIdentity{}, fmt.Errorf("%s/%s %s arm corpus digest is not the exact frozen SP-I2 cohort", record.Dataset, record.Name, artifact.name) + } + if identity.sourceCommit == "" { + identity = current + } else if identity != current { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 artifacts mix source, diff, binary, corpus, declaration, or selection identities") + } + } + } + if identity.declarationSHA256 != requirements.declarationSHA || identity.resolvedSHA256 != requirements.resolvedSHA { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 artifacts do not bind the exact frozen declaration and resolved selection") + } + for key := range requirements.expectedKeys { + baselinePostgres, err := postgresTimingEnvironmentSHA256ForKey(baseline, key) + if err != nil { + return spI2EvidenceIdentity{}, err + } + candidatePostgres, err := postgresTimingEnvironmentSHA256ForKey(candidate, key) + if err != nil { + return spI2EvidenceIdentity{}, err + } + baselineFixture, err := fixtureSHA256ForKey(baseline, key) + if err != nil { + return spI2EvidenceIdentity{}, err + } + candidateFixture, err := fixtureSHA256ForKey(candidate, key) + if err != nil { + return spI2EvidenceIdentity{}, err + } + if !lowercaseSHA256(baselinePostgres) || baselinePostgres != candidatePostgres { + return spI2EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I2 PostgreSQL timing environments differ between arms", key.dataset, key.name) + } + if !lowercaseSHA256(baselineFixture) || baselineFixture != candidateFixture { + return spI2EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I2 fixture identities differ between arms", key.dataset, key.name) + } + baselineSQL, err := spI2SQLFingerprintForKey(baseline, key) + if err != nil { + return spI2EvidenceIdentity{}, err + } + candidateSQL, err := spI2SQLFingerprintForKey(candidate, key) + if err != nil { + return spI2EvidenceIdentity{}, err + } + if baselineSQL == candidateSQL { + return spI2EvidenceIdentity{}, fmt.Errorf("%s/%s SP-I2 arms use the same SQL fingerprint", key.dataset, key.name) + } + if err := validateOrientationExactObservations(key, baseline, candidate); err != nil { + return spI2EvidenceIdentity{}, fmt.Errorf("SP-I2 exact observations: %w", err) + } + } + return identity, nil +} + +// spI2SQLFingerprintForKey derives the lookup key used for sp i2sql fingerprint for. +func spI2SQLFingerprintForKey(records []CaseResult, key performanceKey) (string, error) { + fingerprint := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if fingerprint != "" && fingerprint != record.SQLFingerprint { + return "", fmt.Errorf("%s/%s changes SQL fingerprint within one SP-I2 arm", key.dataset, key.name) + } + fingerprint = record.SQLFingerprint + } + if !lowercaseSHA256(fingerprint) { + return "", fmt.Errorf("%s/%s lacks one stable SP-I2 SQL fingerprint", key.dataset, key.name) + } + return fingerprint, nil +} + +// validateSPI2Selection validates spi2 selection. +func validateSPI2Selection(selection SelectionManifest, requirements spI2ProtocolRequirements) error { + if selection.Version != selectionManifestVersion || !selection.DiagnosticOnly || + selection.SelectedDeclarationCount != 2*len(requirements.expectedKeys) || + selection.FullDeclarationCount != selection.SelectedDeclarationCount+selection.OmittedDeclarationCount || + selection.ProtectedDeclarationCount != requirements.protectedCount || + selection.ProtectedDeclarationSHA256 != requirements.protectedSHA || + len(selection.Resolved) != len(requirements.expectedKeys) || + selection.DeclarationSHA256 != requirements.declarationSHA || + resolvedSelectionSHA256(selection.Resolved) != requirements.resolvedSHA { + return fmt.Errorf("selection manifest does not bind the exact frozen cohort") + } + resolved := make(map[performanceKey]struct{}, len(selection.Resolved)) + for _, item := range selection.Resolved { + if item.Category != "generated_shortest_path_v2" { + return fmt.Errorf("selection contains non-SP-I2 category %q", item.Category) + } + key := performanceKey{ + dataset: item.Dataset, + name: item.Name, + backend: ModePostgresSQL, + } + if _, duplicate := resolved[key]; duplicate { + return fmt.Errorf("selection contains duplicate %s/%s", item.Dataset, item.Name) + } + resolved[key] = struct{}{} + } + if !orientationV2KeySetsEqual(resolved, requirements.expectedKeys) { + return fmt.Errorf("selection does not contain the exact frozen SP-I2 cases") + } + return nil +} + +// collectSPI2QualificationSeries collects spi2 qualification series. +func collectSPI2QualificationSeries( + baseline, candidate []CaseResult, + resource ResourceGateReport, + requirements spI2ProtocolRequirements, +) (map[performanceKey]*spI2QualificationSeries, []performanceKey, error) { + if err := validateSPI2GlobalInvocationIDs(baseline, candidate); err != nil { + return nil, nil, err + } + declarations, err := canonicalSPI2Declarations() + if err != nil { + return nil, nil, err + } + baselineKeys, baselineRounds, err := collectSPI2Artifact("baseline", baseline, requirements, declarations) + if err != nil { + return nil, nil, err + } + candidateKeys, candidateRounds, err := collectSPI2Artifact("candidate", candidate, requirements, declarations) + if err != nil { + return nil, nil, err + } + if !orientationV2KeySetsEqual(baselineKeys, requirements.expectedKeys) || + !orientationV2KeySetsEqual(candidateKeys, requirements.expectedKeys) { + return nil, nil, fmt.Errorf("SP-I2 artifacts do not contain the exact protocol cohort") + } + if err := validateSPI2RunSchedule(baseline, candidate, requirements); err != nil { + return nil, nil, err + } + resourcePassed, err := validateSPI2ResourceCases(resource, candidate, requirements) + if err != nil { + return nil, nil, err + } + + series := make(map[performanceKey]*spI2QualificationSeries, len(requirements.expectedKeys)) + for key := range requirements.expectedKeys { + current := &spI2QualificationSeries{ + baseline: roundSamples{}, + candidate: roundSamples{}, + resourcePassed: resourcePassed[key], + } + series[key] = current + for round, record := range baselineRounds[key] { + appendSPI2WarmSamples(current.baseline, round, record) + } + for round, record := range candidateRounds[key] { + appendSPI2WarmSamples(current.candidate, round, record) + branch := record.TraversalTelemetry.Summary.RuntimeBranch + if current.runtimeBranch != "" && current.runtimeBranch != branch { + return nil, nil, fmt.Errorf("%s/%s changes SP-I2 runtime branch across rounds", key.dataset, key.name) + } + current.runtimeBranch = branch + } + if current.runtimeBranch == "" { + return nil, nil, fmt.Errorf("%s/%s has no attributable SP-I2 candidate runtime", key.dataset, key.name) + } + } + return series, sortedPerformanceKeys(requirements.expectedKeys), nil +} + +// validateSPI2GlobalInvocationIDs prevents one genuine timed receipt from +// being copied into another case, round, or arm. The attestor emits globally +// unique invocation IDs, so the complete paired study must not reuse one. +func validateSPI2GlobalInvocationIDs(artifacts ...[]CaseResult) error { + seen := map[string]struct{}{} + for _, records := range artifacts { + for _, record := range records { + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" { + continue + } + invocationID := strings.TrimSpace(sample.RuntimeInvocationID) + if invocationID == "" { + return fmt.Errorf("%s/%s warm sample lacks a global timed invocation identity", record.Dataset, record.Name) + } + if _, duplicate := seen[invocationID]; duplicate { + return fmt.Errorf("SP-I2 evidence reuses timed invocation identity %q across the paired study", invocationID) + } + seen[invocationID] = struct{}{} + } + } + } + return nil +} + +// spI2InvocationIdentity binds one timed sample to its scheduled run and arm. +type spI2InvocationIdentity struct { + // round identifies the paired benchmark round. + round int + + // block identifies the order-balancing block containing the round. + block int + + // order retains the order while spI2InvocationIdentity is assembled or evaluated. + order int + + // arm identifies the baseline or candidate treatment. + arm string + + // runUUID binds the sample to one benchmark process invocation. + runUUID string + + // startedAt records when timed execution began. + startedAt time.Time + + // endedAt records when timed execution completed. + endedAt time.Time +} + +// validateSPI2RunSchedule validates spi2 run schedule. +func validateSPI2RunSchedule(baseline, candidate []CaseResult, requirements spI2ProtocolRequirements) error { + collect := func(arm string, records []CaseResult) (map[int]spI2InvocationIdentity, error) { + invocations := map[int]spI2InvocationIdentity{} + caseCounts := map[int]int{} + for _, record := range records { + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s %s arm lacks invocation chronology", record.Dataset, record.Name, arm) + } + environment := record.Environment + identity := spI2InvocationIdentity{ + round: environment.Round, + block: environment.Block, + order: environment.ArmOrder, + arm: environment.Arm, + runUUID: environment.RunUUID, + startedAt: environment.StartedAt, + endedAt: environment.EndedAt, + } + if identity.startedAt.IsZero() || identity.endedAt.IsZero() || identity.endedAt.Before(identity.startedAt) { + return nil, fmt.Errorf("SP-I2 %s round %d has malformed invocation timestamps", arm, identity.round) + } + if prior, found := invocations[identity.round]; found && prior != identity { + return nil, fmt.Errorf("SP-I2 %s round %d mixes invocation identities", arm, identity.round) + } + invocations[identity.round] = identity + caseCounts[identity.round]++ + } + for round, count := range caseCounts { + if count != len(requirements.expectedKeys) { + return nil, fmt.Errorf("SP-I2 %s round %d contains %d cases, expected %d", arm, round, count, len(requirements.expectedKeys)) + } + } + return invocations, nil + } + left, err := collect("baseline", baseline) + if err != nil { + return err + } + right, err := collect("candidate", candidate) + if err != nil { + return err + } + if len(left) != len(right) || len(left) < requirements.minimumRounds || len(left) > requirements.maximumRounds { + return fmt.Errorf("SP-I2 artifacts do not contain one complete paired invocation schedule") + } + runUUID := "" + var priorEnded time.Time + for round := 1; round <= len(left); round++ { + baselineInvocation, baselineFound := left[round] + candidateInvocation, candidateFound := right[round] + if !baselineFound || !candidateFound { + return fmt.Errorf("SP-I2 invocation schedule must use contiguous rounds starting at 1") + } + expectedBaselineOrder, expectedCandidateOrder := 1, 2 + if round%2 == 0 { + expectedBaselineOrder, expectedCandidateOrder = 2, 1 + } + if baselineInvocation.block != round || candidateInvocation.block != round || + baselineInvocation.arm != "sp-i2-s4" || candidateInvocation.arm != "sp-i2-candidate" || + baselineInvocation.order != expectedBaselineOrder || candidateInvocation.order != expectedCandidateOrder || + baselineInvocation.runUUID == "" || baselineInvocation.runUUID != candidateInvocation.runUUID { + return fmt.Errorf("SP-I2 round %d does not match the frozen alternating two-arm schedule", round) + } + if runUUID == "" { + runUUID = baselineInvocation.runUUID + } else if runUUID != baselineInvocation.runUUID { + return fmt.Errorf("SP-I2 artifacts mix run UUIDs across rounds") + } + first, second := baselineInvocation, candidateInvocation + if candidateInvocation.order == 1 { + first, second = candidateInvocation, baselineInvocation + } + if first.endedAt.After(second.startedAt) { + return fmt.Errorf("SP-I2 round %d arm timestamps contradict the declared execution order", round) + } + if !priorEnded.IsZero() && priorEnded.After(first.startedAt) { + return fmt.Errorf("SP-I2 round %d overlaps or predates the prior round", round) + } + priorEnded = second.endedAt + } + return nil +} + +// collectSPI2Artifact collects spi2 artifact. +func collectSPI2Artifact( + arm string, + records []CaseResult, + requirements spI2ProtocolRequirements, + declarations map[performanceKey]spI2CanonicalDeclaration, +) (map[performanceKey]struct{}, map[performanceKey]map[int]CaseResult, error) { + if len(records) == 0 { + return nil, nil, fmt.Errorf("SP-I2 %s artifact is empty", arm) + } + keys := map[performanceKey]struct{}{} + rounds := map[performanceKey]map[int]CaseResult{} + for _, record := range records { + key := performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: record.ExecutionMode, + } + if _, expected := requirements.expectedKeys[key]; !expected { + return nil, nil, fmt.Errorf("SP-I2 %s artifact contains unexpected case %s/%s", arm, key.dataset, key.name) + } + declaration, found := declarations[key] + if !found { + return nil, nil, fmt.Errorf("SP-I2 %s artifact has no frozen declaration for %s/%s", arm, key.dataset, key.name) + } + if err := validateSPI2Record(record, arm, declaration); err != nil { + return nil, nil, err + } + round, err := orientationV2RecordRound(record) + if err != nil { + return nil, nil, err + } + if rounds[key] == nil { + rounds[key] = map[int]CaseResult{} + } + if _, duplicate := rounds[key][round]; duplicate { + return nil, nil, fmt.Errorf("%s/%s %s artifact duplicates round %d", key.dataset, key.name, arm, round) + } + rounds[key][round] = record + keys[key] = struct{}{} + } + return keys, rounds, nil +} + +// appendSPI2WarmSamples appends spi2 warm samples. +func appendSPI2WarmSamples(series roundSamples, round int, record CaseResult) { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + series[round] = append(series[round], sample.Duration) + } + } +} + +// validateSPI2Record validates spi2 record. +func validateSPI2Record(record CaseResult, arm string, declaration spI2CanonicalDeclaration) error { + if record.ExecutionMode != ModePostgresSQL || record.Status != StatusOK || + record.Environment == nil || record.PostgresEnvironment == nil || record.Fixture == nil || + record.TraversalTelemetry == nil || record.Optimization == nil || record.PostgresMetrics == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing PostgreSQL record", record.Dataset, record.Name, arm) + } + if record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || + len(record.Environment.Concurrency) != 0 || record.Environment.ExistingGraph || + record.Environment.Protocol != "fixed_confirmation" { + return fmt.Errorf("%s/%s %s arm lacks the schema-v2 single-session fixed-confirmation contract", record.Dataset, record.Name, arm) + } + if record.Fixture.Dataset != record.Dataset || !lowercaseSHA256(record.Fixture.Checksum) || + !record.Fixture.PhysicalValidated || record.Fixture.PhysicalNodeCount != int64(record.Fixture.NodeCount) || + record.Fixture.PhysicalEdgeCount != int64(record.Fixture.EdgeCount) || + record.Fixture.Checksum != declaration.fixture.Checksum || + record.Fixture.NodeCount != declaration.fixture.NodeCount || record.Fixture.EdgeCount != declaration.fixture.EdgeCount || + record.Fixture.Configuration != declaration.fixture.Configuration || + !reflect.DeepEqual(record.Fixture.Shortest, declaration.fixture.Shortest) || + record.Fixture.NodeRelationBytes <= 0 || record.Fixture.EdgeRelationBytes <= 0 { + return fmt.Errorf("%s/%s %s arm lacks one exact physically validated fixture", record.Dataset, record.Name, arm) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") { + return fmt.Errorf("%s/%s %s arm was not measured under Repeatable Read", record.Dataset, record.Name, arm) + } + testCase := declaration.testCase + testCase.Source = record.Source + expectedRecord := newCaseResult(testCase, ModePostgresSQL, nil) + attachFixtureMetadata(&expectedRecord, *record.Fixture) + if filepath.Base(record.Source) != "generated_sp_i2_distance_v1.json" || + record.Category != testCase.Category || record.Cypher != testCase.Cypher || sqlFingerprint(record.Cypher) != spI2QuerySHA256 || + !lowercaseSHA256(record.WorkloadSHA256) || !lowercaseSHA256(record.SQLFingerprint) || + record.WorkloadSHA256 != expectedRecord.WorkloadSHA256 || + record.SQL == "" || sqlFingerprint(record.SQL) != record.SQLFingerprint || + !reflect.DeepEqual(record.NodeParams, testCase.NodeParams) || + !reflect.DeepEqual(record.NodeListParams, testCase.NodeListParams) || + !reflect.DeepEqual(record.Shape, testCase.Shape) { + return fmt.Errorf("%s/%s %s arm lacks the frozen inbound SP-I2 workload identity", record.Dataset, record.Name, arm) + } + minimumDepth, maximumDepth := 0, 0 + if record.Shape.MinDepth != nil { + minimumDepth = *record.Shape.MinDepth + } + if record.Shape.MaxDepth != nil { + maximumDepth = *record.Shape.MaxDepth + } + if record.Shape.QualificationSplit != "training" && record.Shape.QualificationSplit != "holdout" || + record.Shape.FallbackExpectation != "forbidden" || record.Shape.Direction != "inbound" || + record.Shape.RelationshipKindCount != 1 || !slices.Equal(record.Shape.EdgeKinds, []string{"Traverse"}) || + minimumDepth != 1 || maximumDepth != 64 || record.Shape.PathMaterializationRequired { + return fmt.Errorf("%s/%s %s arm changes the frozen inbound distance shape", record.Dataset, record.Name, arm) + } + expectedSplit := testCase.Shape.QualificationSplit + if record.Shape.QualificationSplit != expectedSplit { + return fmt.Errorf("%s/%s %s arm changes the frozen qualification split", record.Dataset, record.Name, arm) + } + expectedRows := *testCase.Expected.RowCount + if !record.StableObservation || record.RowCount != expectedRows || record.ExpectedRowCount == nil || + *record.ExpectedRowCount != expectedRows { + return fmt.Errorf("%s/%s %s arm lacks the exact stable distance observation contract", record.Dataset, record.Name, arm) + } + if err := validateExpectedObservations(testCase.Expected, record.ObservedRows); err != nil { + return fmt.Errorf("%s/%s %s arm changes the frozen distance observation: %w", record.Dataset, record.Name, arm, err) + } + if len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil || record.Baseline != nil { + return fmt.Errorf("%s/%s %s arm mixes SP-I2 timing with supplemental measurements", record.Dataset, record.Name, arm) + } + if err := ValidateTraversalExecutionTelemetry(record.TraversalTelemetry); err != nil { + return fmt.Errorf("%s/%s %s arm telemetry: %w", record.Dataset, record.Name, arm, err) + } + if err := validateSPI2Runtime(record, arm); err != nil { + return err + } + return nil +} + +// validateSPI2Runtime validates spi2 runtime. +func validateSPI2Runtime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || + summary.Overflow == nil || summary.FallbackExecuted == nil || *summary.Overflow || *summary.FallbackExecuted || + summary.WouldSelectIdentity != "" || summary.ObservationMode != string(optimize.ShortestPathObservationDistance) || + summary.SchedulerVersion != string(optimize.ShortestPathSchedulerSingleEndedLevel) { + return fmt.Errorf("%s/%s %s arm lacks one non-fallback distance runtime outcome", record.Dataset, record.Name, arm) + } + outcome, ok := singleTraversalOutcome(record.Optimization.TargetOutcomes) + if !ok || outcome.Family != "SP" { + return fmt.Errorf("%s/%s %s arm lacks one exact SP lowering outcome", record.Dataset, record.Name, arm) + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalDistance) + candidate := string(optimize.ShortestPathExecutorI2GuardedDistance) + plannedIdentities := spI2ShortestPathPlannedIdentities() + outcomeDepthsExact := outcome.MinimumDepth != nil && *outcome.MinimumDepth == 1 && + outcome.MaximumDepth != nil && *outcome.MaximumDepth == 64 + outcomeShapeExact := outcome.Lowering == optimize.LoweringShortestPathExecutor && outcome.TargetKind == "traversal" && + outcome.ObservationMode == string(optimize.ShortestPathObservationDistance) && outcome.Direction == "inbound" && + outcome.PhysicalExpansion == "end_id" && outcome.RelationshipKindCount == 1 && !outcome.UntypedRelationship && + outcome.TopologyClassification == "physical_inbound_deep" && outcome.SelectionMode == "forced_tool" && + outcome.Scheduler == string(optimize.ShortestPathSchedulerSingleEndedLevel) && outcomeDepthsExact && + outcome.Eligible != nil && *outcome.Eligible && outcome.StaticallyEligible != nil && *outcome.StaticallyEligible + if !outcomeShapeExact { + return fmt.Errorf("%s/%s %s arm changes the frozen SP-I2 lowering shape", record.Dataset, record.Name, arm) + } + switch arm { + case "baseline": + if summary.RequestedIdentity != baseline || summary.EmittedIdentity != baseline || + summary.RuntimeIdentity != baseline || summary.AppliedIdentity != baseline || + !slices.Equal(summary.PlannedIdentities, plannedIdentities) || + summary.SelectorVersion != "sp-tool-v1" || + summary.ExecutionBoundary != optimize.ShortestPathExecutorS4CanonicalDistance.ExecutionBoundary() || + summary.RuntimeBranch != "selected" || + outcome.Candidate != "" || outcome.Selected != baseline || outcome.Applied != baseline || outcome.Fallback != "SP-S0" || + !slices.Equal(outcome.PlannedCandidates, plannedIdentities) || + outcome.ExecutionBoundary != "stored_helper" || outcome.SelectorVersion != "sp-tool-v1" || + outcome.EmittedPolicy != "" || len(outcome.EmittedCandidates) != 0 || + outcome.StateLimit != 100_000 || outcome.FrontierLimit != 100_000 || outcome.PredecessorLimit != 100_000 || + outcome.EnumerationLimit != 100_000 || outcome.OutputBytesLimit != 64*1024*1024 { + return fmt.Errorf("%s/%s baseline arm did not execute exact forced S4", record.Dataset, record.Name) + } + case "candidate": + expectedBranch := "inline_canonical_distance" + if record.RowCount == 0 { + expectedBranch = "inline_canonical_distance_no_path" + } + if summary.RequestedIdentity != candidate || summary.EmittedIdentity != optimize.ShortestPathPolicyI2DistanceGuardedV1 || + summary.RuntimeIdentity != candidate || summary.AppliedIdentity != candidate || + !slices.Equal(summary.PlannedIdentities, plannedIdentities) || + summary.SelectorVersion != optimize.ShortestPathSelectorStaticV8HiddenFanIn || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + summary.RuntimeBranch != expectedBranch || + !equalSPI2Caps(summary.Caps, spI2SummaryCaps()) || + !slices.Contains(summary.PlannedIdentities, baseline) || !slices.Contains(summary.PlannedIdentities, candidate) || + outcome.Candidate != candidate || outcome.Selected != candidate || outcome.Applied != candidate || + outcome.Fallback != baseline || outcome.EmittedPolicy != optimize.ShortestPathPolicyI2DistanceGuardedV1 || + !slices.Equal(outcome.PlannedCandidates, plannedIdentities) || + !slices.Equal(outcome.EmittedCandidates, []string{candidate, baseline}) || + outcome.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + outcome.SelectorVersion != optimize.ShortestPathSelectorStaticV8HiddenFanIn || + outcome.StateLimit != spI2QualificationCaps()["state_limit"] || + outcome.FrontierLimit != spI2QualificationCaps()["frontier_limit"] || + outcome.PredecessorLimit != 0 || outcome.EnumerationLimit != 0 || outcome.OutputBytesLimit != 0 { + return fmt.Errorf("%s/%s candidate arm did not execute exact guarded SP-I2 distance", record.Dataset, record.Name) + } + diagnostic := record.TraversalTelemetry.Diagnostic + if record.TraversalTelemetry.Level != TraversalTelemetryLevelDiagnostic || diagnostic == nil || + diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete || diagnostic.Counters.InlineShortestDistance == nil || + !slices.Contains(diagnostic.RequiredFamilies, TraversalTelemetryFamilySP) { + return fmt.Errorf("%s/%s candidate arm lacks complete typed SP-I2 distance resource telemetry", record.Dataset, record.Name) + } + inline := diagnostic.Counters.InlineShortestDistance + outputRows, outputPresent := int64(0), false + if diagnostic.PlanReplay != nil { + outputRows, outputPresent = diagnostic.PlanReplay.Counters["sp_i2_output_rows"] + } + if inline.OutputRows == nil || *inline.OutputRows != record.RowCount || !outputPresent || outputRows != record.RowCount { + return fmt.Errorf("%s/%s candidate arm runtime branch does not bind the exact output observation", record.Dataset, record.Name) + } + default: + return fmt.Errorf("unknown SP-I2 arm %q", arm) + } + if err := validateSPI2SampleRuntime(record, arm); err != nil { + return err + } + return nil +} + +// spI2ShortestPathPlannedIdentities mirrors the optimizer's complete SP search +// space. Planned candidates describe every executor considered by lowering; +// emitted candidates and the runtime receipt separately attest the exact +// guarded two-arm statement that executed. +func spI2ShortestPathPlannedIdentities() []string { + return []string{ + string(optimize.ShortestPathExecutorIncumbentWorkspace), + string(optimize.ShortestPathExecutorS0Direct), + string(optimize.ShortestPathExecutorS1ArrayBFS), + string(optimize.ShortestPathExecutorS2TraceRelation), + string(optimize.ShortestPathExecutorS3Unidirectional), + string(optimize.ShortestPathExecutorS3EdgeM0), + string(optimize.ShortestPathExecutorS4CanonicalDistance), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalDistance), + string(optimize.ShortestPathExecutorI2GuardedDistance), + string(optimize.ShortestPathExecutorI1CanonicalWitness), + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorB1AlternatingNodeDistance), + string(optimize.ShortestPathExecutorB1AlternatingNodeWitness), + string(optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance), + string(optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness), + } +} + +// validateSPI2SampleRuntime validates spi2 sample runtime. +func validateSPI2SampleRuntime(record CaseResult, arm string) error { + summary := record.TraversalTelemetry.Summary + if record.Environment == nil || record.Stats.Iterations < 1 || record.Stats.WarmupIterations != record.Environment.WarmupIterations || + record.Stats.Median <= 0 || record.Stats.P95 <= 0 { + return fmt.Errorf("%s/%s %s arm has malformed iteration or warmup evidence", record.Dataset, record.Name, arm) + } + expectedArm := "sp-i2-s4" + if arm == "candidate" { + expectedArm = "sp-i2-candidate" + } + if record.Environment.Arm != expectedArm || record.Environment.Round < 1 || record.Environment.Block != record.Environment.Round || + record.Environment.ArmOrder < 1 || record.Environment.ArmOrder > 2 || strings.TrimSpace(record.Environment.RunUUID) == "" { + return fmt.Errorf("%s/%s %s arm has malformed frozen run metadata", record.Dataset, record.Name, arm) + } + warmSamples, coldSamples := 0, 0 + iterations := map[int]struct{}{} + invocations := map[string]struct{}{} + for _, sample := range record.Stats.Samples { + if sample.Duration <= 0 || sample.Dataset != record.Dataset || sample.Case != record.Name || sample.Backend != ModePostgresSQL || + sample.Round != record.Environment.Round || sample.Block != record.Environment.Block || sample.Arm != record.Environment.Arm || + sample.ArmOrder != record.Environment.ArmOrder || sample.RunUUID != record.Environment.RunUUID || strings.TrimSpace(sample.ConnectionID) == "" { + return fmt.Errorf("%s/%s %s arm has a sample outside its frozen invocation identity", record.Dataset, record.Name, arm) + } + switch sample.Classification { + case "cold": + if sample.Iteration != 0 { + return fmt.Errorf("%s/%s %s arm cold sample has a nonzero iteration", record.Dataset, record.Name, arm) + } + coldSamples++ + continue + case "warm": + default: + return fmt.Errorf("%s/%s %s arm contains an unexpected sample classification", record.Dataset, record.Name, arm) + } + warmSamples++ + if sample.Iteration < 1 || sample.Iteration > record.Stats.Iterations { + return fmt.Errorf("%s/%s %s arm has an out-of-range warm iteration", record.Dataset, record.Name, arm) + } + if _, duplicate := iterations[sample.Iteration]; duplicate { + return fmt.Errorf("%s/%s %s arm duplicates warm iteration %d", record.Dataset, record.Name, arm, sample.Iteration) + } + iterations[sample.Iteration] = struct{}{} + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || + sample.FallbackExecuted == nil || *sample.FallbackExecuted != *summary.FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample contradicts its runtime summary", record.Dataset, record.Name, arm) + } + if sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("%s/%s %s arm warm sample lacks timed-invocation attribution", record.Dataset, record.Name, arm) + } + if strings.TrimSpace(sample.RuntimeInvocationID) == "" { + return fmt.Errorf("%s/%s %s arm warm sample lacks a timed invocation identity", record.Dataset, record.Name, arm) + } + if _, duplicate := invocations[sample.RuntimeInvocationID]; duplicate { + return fmt.Errorf("%s/%s %s arm reuses timed invocation identity %q", record.Dataset, record.Name, arm, sample.RuntimeInvocationID) + } + invocations[sample.RuntimeInvocationID] = struct{}{} + expectedBranch := summary.RuntimeBranch + if arm == "baseline" { + expectedBranch = "compact_workspace_witness" + if record.RowCount == 0 { + expectedBranch = "compact_no_path" + } else if strings.Contains(record.Name, "cycle-control") { + expectedBranch = "one_hop_preflight" + } else if strings.Contains(record.Name, "early-d02") { + expectedBranch = "two_hop_preflight" + } + } + if sample.RuntimeBranch != expectedBranch || len(sample.RuntimeReceiptEvents) != 1 || + sample.RuntimeReceiptEvents[0].InvocationID != sample.RuntimeInvocationID || sample.RuntimeReceiptEvents[0].FallbackExecuted { + return fmt.Errorf("%s/%s %s arm warm sample has a non-canonical runtime receipt", record.Dataset, record.Name, arm) + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("%s/%s %s arm warm sample receipt: %w", record.Dataset, record.Name, arm, err) + } + } + if coldSamples != 1 || warmSamples != record.Stats.Iterations || len(record.Stats.Samples) != record.Stats.Iterations+1 { + return fmt.Errorf("%s/%s %s arm must contain one cold and exactly %d unique warm samples", record.Dataset, record.Name, arm, record.Stats.Iterations) + } + return nil +} + +// validateSPI2ResourceCases validates spi2 resource cases. +func validateSPI2ResourceCases( + report ResourceGateReport, + candidate []CaseResult, + requirements spI2ProtocolRequirements, +) (map[performanceKey]bool, error) { + if report.Version != resourceGateVersion { + return nil, fmt.Errorf("SP-I2 resource report version must be %d", resourceGateVersion) + } + + // recordKey binds resource evidence to an exact scheduled candidate invocation. + type recordKey struct { + // performanceKey identifies the workload and backend. + performanceKey + + // round identifies the paired benchmark round. + round int + + // block identifies the order-balancing block containing the round. + block int + + // order retains the order while recordKey is assembled or evaluated. + order int + + // runUUID binds the record to one benchmark process invocation. + runUUID string + + // arm identifies the treatment that produced the record. + arm string + } + expected := map[recordKey]CaseResult{} + for _, record := range candidate { + if record.Environment == nil { + return nil, fmt.Errorf("%s/%s candidate resource record lacks run identity", record.Dataset, record.Name) + } + key := recordKey{ + performanceKey: performanceKey{ + dataset: record.Dataset, + name: record.Name, + backend: ModePostgresSQL, + }, + round: record.Environment.Round, + block: record.Environment.Block, + order: record.Environment.ArmOrder, + runUUID: record.Environment.RunUUID, + arm: record.Environment.Arm, + } + if _, duplicate := expected[key]; duplicate { + return nil, fmt.Errorf("SP-I2 candidate artifact duplicates a resource record identity") + } + expected[key] = record + } + actual := map[recordKey]struct{}{} + passed := map[performanceKey]bool{} + for key := range requirements.expectedKeys { + passed[key] = true + } + cohort, err := canonicalSPI2Cohort() + if err != nil { + return nil, err + } + allPassed := true + for _, gateCase := range report.Cases { + key := performanceKey{ + dataset: gateCase.Dataset, + name: gateCase.Name, + backend: ModePostgresSQL, + } + if _, expected := requirements.expectedKeys[key]; !expected || gateCase.Reference != "" { + return nil, fmt.Errorf("SP-I2 resource report contains an unexpected production or reference case %s/%s", gateCase.Dataset, gateCase.Name) + } + identity := recordKey{ + performanceKey: key, + round: gateCase.Round, + block: gateCase.Block, + order: gateCase.ArmOrder, + runUUID: gateCase.RunUUID, + arm: gateCase.Arm, + } + record, found := expected[identity] + if !found { + return nil, fmt.Errorf("SP-I2 resource case %s/%s round %d does not bind an exact candidate record", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + if _, duplicate := actual[identity]; duplicate { + return nil, fmt.Errorf("SP-I2 resource report duplicates %s/%s round %d", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + actual[identity] = struct{}{} + recomputed := evaluateProductionResourceGateCase(record) + if !reflect.DeepEqual(gateCase, recomputed) { + return nil, fmt.Errorf("SP-I2 resource case %s/%s round %d differs from the decision recomputed from its candidate record", gateCase.Dataset, gateCase.Name, gateCase.Round) + } + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + if gateCase.Architecture != string(optimize.ShortestPathExecutorI2GuardedDistance) || + gateCase.FallbackArchitecture != "" || gateCase.QualificationSplit != expectedSplit || + gateCase.Tier != "normal" || !equalSPI2Caps(gateCase.NumericLimits, spI2TelemetryCaps()) || + gateCase.Passed != (len(gateCase.Reasons) == 0) || + !reflect.DeepEqual(gateCase.RuntimeReceiptChains, runtimeReceiptChains(record.Stats.Samples)) { + return nil, fmt.Errorf("SP-I2 resource case %s/%s does not bind exact guarded-distance limits and split", gateCase.Dataset, gateCase.Name) + } + observations := traversalNumericObservations(record.TraversalTelemetry.Diagnostic.Counters) + if len(gateCase.NumericObserved) != len(spI2TelemetryCaps()) { + return nil, fmt.Errorf("SP-I2 resource case %s/%s has unexpected numeric observations", gateCase.Dataset, gateCase.Name) + } + for name := range spI2TelemetryCaps() { + observed, found := gateCase.NumericObserved[name] + expectedObserved, expectedFound := observations[name] + if !found || !expectedFound || observed != expectedObserved || observed < 0 { + return nil, fmt.Errorf("SP-I2 resource case %s/%s has invalid %s observation", gateCase.Dataset, gateCase.Name, name) + } + } + passed[key] = passed[key] && gateCase.Passed + allPassed = allPassed && gateCase.Passed + } + if len(actual) != len(expected) { + return nil, fmt.Errorf("SP-I2 resource report has %d exact record decisions, expected %d", len(actual), len(expected)) + } + for key := range requirements.expectedKeys { + if _, found := passed[key]; !found { + return nil, fmt.Errorf("SP-I2 resource report omits %s/%s", key.dataset, key.name) + } + } + if report.Passed != allPassed { + return nil, fmt.Errorf("SP-I2 resource report aggregate disposition contradicts its cases") + } + return passed, nil +} + +// validateSPI2Freeze validates spi2 freeze. +func validateSPI2Freeze( + freeze *SPI2QualificationFreezeManifest, + discovery *SPI2QualificationReport, + report SPI2QualificationReport, + cohort spI2CanonicalCohort, +) error { + if err := validateSPI2FrozenDiscovery(freeze, discovery, cohort); err != nil { + return err + } + if report.Protocol != referencePairProtocolConfirmation || + report.SourceCommit != freeze.SourceCommit || report.SourceArchiveSHA256 != freeze.SourceArchiveSHA256 || + report.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || report.BinarySHA256 != freeze.BinarySHA256 || + report.QuerySHA256 != freeze.QuerySHA256 || report.Policy != freeze.Policy || + report.Baseline != freeze.Baseline || report.Candidate != freeze.Candidate || + report.CohortDeclarationSHA256 != freeze.FullDeclarationSHA256 || + report.CorpusSHA256 != freeze.FullCorpusSHA256 || report.ResolvedSelectionSHA256 != freeze.FullResolvedSHA256 || + report.Seed != freeze.Seed || report.Confidence != freeze.Confidence || report.BootstrapCount != freeze.BootstrapCount || + !equalSPI2Caps(report.Caps, freeze.Caps) { + return fmt.Errorf("SP-I2 confirmation identity differs from the frozen discovery") + } + return nil +} + +// validateSPI2FrozenDiscovery validates spi2 frozen discovery. +func validateSPI2FrozenDiscovery( + freeze *SPI2QualificationFreezeManifest, + discovery *SPI2QualificationReport, + cohort spI2CanonicalCohort, +) error { + if freeze == nil || discovery == nil { + return fmt.Errorf("SP-I2 confirmation requires a discovery report and freeze manifest") + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalDistance) + candidate := string(optimize.ShortestPathExecutorI2GuardedDistance) + if freeze.Version != spI2FreezeVersion || freeze.Baseline != baseline || freeze.Candidate != candidate || + freeze.Policy != optimize.ShortestPathPolicyI2DistanceGuardedV1 || freeze.QuerySHA256 != spI2QuerySHA256 || + freeze.Seed != 1 || freeze.Confidence != defaultConfidenceLevel || freeze.BootstrapCount != defaultBootstrapCount || + !equalSPI2Caps(freeze.Caps, spI2QualificationCaps()) || + freeze.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + freeze.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + freeze.FullDeclarationSHA256 != cohort.declarationSHA256 || + freeze.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || freeze.FullCorpusSHA256 != cohort.fullCorpusSHA256 || + freeze.TrainingResolvedSHA256 != cohort.trainingResolvedSHA256 || freeze.FullResolvedSHA256 != cohort.fullResolvedSHA256 || + !lowercaseSHA256(freeze.SourceArchiveSHA256) || !lowercaseSHA256(freeze.DirtyDiffSHA256) || + !lowercaseSHA256(freeze.BinarySHA256) || !lowercaseSHA256(freeze.BaselineArtifactSHA256) || + !lowercaseSHA256(freeze.CandidateArtifactSHA256) || !lowercaseSHA256(freeze.ResourceReportSHA256) || + !lowercaseSHA256(freeze.DiscoveryReportSHA256) || strings.TrimSpace(freeze.SourceCommit) == "" { + return fmt.Errorf("SP-I2 freeze manifest does not bind the exact immutable study identity") + } + if freeze.DirtyDiffSHA256 != cleanWorkingTreeSHA256() { + return fmt.Errorf("SP-I2 freeze manifest was not created from a clean source tree") + } + if discovery.Version != spI2QualificationVersion || discovery.Protocol != referencePairProtocolDiscovery || + discovery.Baseline != freeze.Baseline || discovery.Candidate != freeze.Candidate || + discovery.Policy != freeze.Policy || discovery.QuerySHA256 != freeze.QuerySHA256 || + discovery.SourceCommit != freeze.SourceCommit || discovery.SourceArchiveSHA256 != freeze.SourceArchiveSHA256 || + discovery.DirtyDiffSHA256 != freeze.DirtyDiffSHA256 || discovery.BinarySHA256 != freeze.BinarySHA256 || + discovery.CohortDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + discovery.ResolvedSelectionSHA256 != cohort.trainingResolvedSHA256 || + discovery.CorpusSHA256 != cohort.trainingCorpusSHA256 || + discovery.TrainingDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + discovery.HoldoutDeclarationSHA256 != cohort.holdoutDeclarationSHA256 || + discovery.FullDeclarationSHA256 != cohort.declarationSHA256 || + discovery.TrainingCorpusSHA256 != cohort.trainingCorpusSHA256 || discovery.FullCorpusSHA256 != cohort.fullCorpusSHA256 || + discovery.BaselineArtifactSHA256 != freeze.BaselineArtifactSHA256 || + discovery.CandidateArtifactSHA256 != freeze.CandidateArtifactSHA256 || + discovery.ResourceReportSHA256 != freeze.ResourceReportSHA256 || + !equalSPI2Caps(discovery.Caps, freeze.Caps) || discovery.Seed != freeze.Seed || + discovery.Confidence != freeze.Confidence || discovery.BootstrapCount != freeze.BootstrapCount || + discovery.MaterialityRatio != 0.95 || discovery.MaterialityAbsolute != 100*time.Microsecond || + discovery.P95RatioLimit != 1.05 || discovery.AdverseRatioLimit != 1.10 || + discovery.AdverseAbsoluteLimit != 100*time.Microsecond || !discovery.EvidencePassed || + discovery.TrainingCases != len(cohort.trainingKeys) || discovery.HoldoutCases != 0 || + discovery.HoldoutPassed || discovery.QualificationPassed || discovery.TrainingPassed != freeze.TrainingPassed { + return fmt.Errorf("SP-I2 discovery report does not prove the exact frozen training identity") + } + seen := map[performanceKey]struct{}{} + for _, entry := range discovery.Cases { + key := performanceKey{ + dataset: entry.Dataset, + name: entry.Name, + backend: ModePostgresSQL, + } + if entry.QualificationSplit != "training" { + return fmt.Errorf("SP-I2 discovery report contains non-training timing") + } + if _, expected := cohort.trainingKeys[key]; !expected { + return fmt.Errorf("SP-I2 discovery report contains unexpected case %s/%s", entry.Dataset, entry.Name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("SP-I2 discovery report duplicates case %s/%s", entry.Dataset, entry.Name) + } + expectedBranch := "inline_canonical_distance" + if strings.HasSuffix(entry.Name, "-disconnected") { + expectedBranch = "inline_canonical_distance_no_path" + } + expectedRole := "target" + expectedPerformanceGate := entry.MedianRatio.Upper <= discovery.MaterialityRatio || entry.MedianSaving.Lower >= discovery.MaterialityAbsolute + if strings.Contains(entry.Name, "cycle-control") { + expectedRole = "adverse_control" + expectedPerformanceGate = entry.MedianRatio.Upper <= discovery.AdverseRatioLimit || entry.MedianSaving.Lower >= -discovery.AdverseAbsoluteLimit + } + if !validSPI2RatioInterval(entry.MedianRatio) || !validSPI2RatioInterval(entry.P95Ratio) || + entry.MedianSaving.Lower > entry.MedianSaving.Estimate || entry.MedianSaving.Estimate > entry.MedianSaving.Upper || + entry.QualificationRole != expectedRole || entry.Material != expectedPerformanceGate || + entry.P95Contained != (entry.P95Ratio.Upper <= discovery.P95RatioLimit) || + !entry.Passed || len(entry.Reasons) != 0 || !entry.Material || !entry.P95Contained || !entry.ResourcePassed || + entry.RuntimeBranch != expectedBranch || + entry.Rounds < 5 || entry.Rounds > 20 || entry.BaselineSamples < 50 || entry.CandidateSamples < 50 { + return fmt.Errorf("SP-I2 discovery report case %s/%s did not pass the frozen training gates", entry.Dataset, entry.Name) + } + seen[key] = struct{}{} + } + if !orientationV2KeySetsEqual(seen, cohort.trainingKeys) { + return fmt.Errorf("SP-I2 discovery report omits part of the exact training cohort") + } + if !freeze.TrainingPassed || !discovery.TrainingPassed { + return fmt.Errorf("SP-I2 training discovery did not pass") + } + return nil +} + +// validSPI2RatioInterval reports whether a confidence interval contains finite ordered bounds. +func validSPI2RatioInterval(interval RatioInterval) bool { + return interval.Lower > 0 && interval.Lower <= interval.Estimate && interval.Estimate <= interval.Upper && + !math.IsNaN(interval.Lower) && !math.IsNaN(interval.Estimate) && !math.IsNaN(interval.Upper) && + !math.IsInf(interval.Lower, 0) && !math.IsInf(interval.Estimate, 0) && !math.IsInf(interval.Upper, 0) +} + +// createSPI2QualificationReport loads and evaluates the staged two-arm +// qualification evidence, writes the report even for statistical failures, +// and freezes discovery before any holdout capture is authorized. +func createSPI2QualificationReport( + baselinePath, candidatePath, resourcePath, freezePath, discoveryPath, freezeOutputPath, outputPath string, + options SPI2QualificationOptions, +) (bool, error) { + if freezeOutputPath != "" { + return false, fmt.Errorf("SP-I2 V1 is terminally rejected and cannot create a new freeze") + } + if err := validateDistinctSPI2Paths(map[string]string{ + "baseline artifact": baselinePath, "candidate artifact": candidatePath, "resource report": resourcePath, + "freeze manifest": freezePath, "discovery report": discoveryPath, "freeze output": freezeOutputPath, "report output": outputPath, + }); err != nil { + return false, err + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return false, fmt.Errorf("read SP-I2 baseline artifact: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return false, fmt.Errorf("read SP-I2 candidate artifact: %w", err) + } + resource, err := loadSPI2ResourceReport(resourcePath) + if err != nil { + return false, err + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return false, err + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return false, err + } + resourceSHA256, err := fileSHA256(resourcePath) + if err != nil { + return false, err + } + if resource.ArtifactSHA256 != candidateSHA256 { + return false, fmt.Errorf("SP-I2 resource report is not bound to the exact candidate artifact") + } + + freezeSHA256 := "" + if freezePath != "" || discoveryPath != "" { + if freezePath == "" || discoveryPath == "" { + return false, fmt.Errorf("SP-I2 confirmation requires both freeze and discovery report paths") + } + freeze, digest, err := loadSPI2FreezeManifest(freezePath) + if err != nil { + return false, fmt.Errorf("read SP-I2 freeze manifest: %w", err) + } + discovery, err := loadSPI2QualificationReport(discoveryPath) + if err != nil { + return false, fmt.Errorf("read SP-I2 discovery report: %w", err) + } + discoverySHA256, err := fileSHA256(discoveryPath) + if err != nil { + return false, err + } + if discoverySHA256 != freeze.DiscoveryReportSHA256 { + return false, fmt.Errorf("SP-I2 discovery report digest does not match freeze manifest") + } + options.Freeze, options.Discovery = freeze, discovery + freezeSHA256 = digest + if err := validateSPI2FrozenTrainingEvidence( + freeze, discovery, + options.TrainingBaselinePath, options.TrainingCandidatePath, options.TrainingResourcePath, + ); err != nil { + return false, err + } + } + options.SourceArchiveSHA256, err = spI2SourceArchiveSHA256() + if err != nil { + return false, err + } + report, err := buildSPI2QualificationReport(baseline, candidate, resource, options) + if err != nil { + return false, err + } + report.BaselineArtifactSHA256 = baselineSHA256 + report.CandidateArtifactSHA256 = candidateSHA256 + report.ResourceReportSHA256 = resourceSHA256 + report.FreezeManifestSHA256 = freezeSHA256 + if err := validateCurrentSPI2Source(report.SourceCommit, report.SourceArchiveSHA256, report.DirtyDiffSHA256, report.BinarySHA256); err != nil { + return false, err + } + if err := writeSPI2QualificationReport(outputPath, report); err != nil { + return false, err + } + if options.Protocol == referencePairProtocolDiscovery { + if err := writeSPI2FreezeManifest(freezeOutputPath, outputPath, report); err != nil { + return false, err + } + return report.TrainingPassed, nil + } + return report.QualificationPassed, nil +} + +// validateSPI2HoldoutCapture authorizes the exact frozen cohort before any +// database setup is allowed to begin. +func validateSPI2HoldoutCapture( + corpus ScaleCorpus, + freezePath, discoveryPath, trainingBaselinePath, trainingCandidatePath, trainingResourcePath string, +) error { + if spI2V1TerminallyRejected() { + return fmt.Errorf("SP-I2 V1 is terminally rejected and cannot authorize holdout capture") + } + cohort, err := canonicalSPI2Cohort() + if err != nil { + return err + } + if err := validateSPI2Corpus(corpus, cohort); err != nil { + return err + } + freeze, _, err := loadSPI2FreezeManifest(freezePath) + if err != nil { + return fmt.Errorf("read SP-I2 freeze manifest: %w", err) + } + discovery, err := loadSPI2QualificationReport(discoveryPath) + if err != nil { + return fmt.Errorf("read SP-I2 discovery report: %w", err) + } + discoverySHA256, err := fileSHA256(discoveryPath) + if err != nil { + return err + } + if discoverySHA256 != freeze.DiscoveryReportSHA256 { + return fmt.Errorf("SP-I2 discovery report digest does not match freeze manifest") + } + if err := validateSPI2FrozenTrainingEvidence( + freeze, discovery, trainingBaselinePath, trainingCandidatePath, trainingResourcePath, + ); err != nil { + return err + } + if err := validateCurrentSPI2Source(freeze.SourceCommit, freeze.SourceArchiveSHA256, freeze.DirtyDiffSHA256, freeze.BinarySHA256); err != nil { + return err + } + return nil +} + +// spI2V1TerminallyRejected is a function rather than a mutable switch so +// archived verification code remains compilable without exposing an activation +// seam that production or tooling could override. +func spI2V1TerminallyRejected() bool { return true } + +// validateSPI2FrozenTrainingEvidence reloads and recomputes the exact training +// closure named by the freeze. This prevents an internally consistent but +// hand-edited report/freeze pair from authorizing protected holdout timing. +func validateSPI2FrozenTrainingEvidence( + freeze *SPI2QualificationFreezeManifest, + discovery *SPI2QualificationReport, + baselinePath, candidatePath, resourcePath string, +) error { + cohort, err := canonicalSPI2Cohort() + if err != nil { + return err + } + if err := validateSPI2FrozenDiscovery(freeze, discovery, cohort); err != nil { + return err + } + if baselinePath == "" || candidatePath == "" || resourcePath == "" { + return fmt.Errorf("SP-I2 frozen discovery verification requires the three exact training evidence artifacts") + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I2 training baseline: %w", err) + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I2 training candidate: %w", err) + } + resourceSHA256, err := fileSHA256(resourcePath) + if err != nil { + return fmt.Errorf("hash frozen SP-I2 training resource report: %w", err) + } + if baselineSHA256 != freeze.BaselineArtifactSHA256 || candidateSHA256 != freeze.CandidateArtifactSHA256 || + resourceSHA256 != freeze.ResourceReportSHA256 { + return fmt.Errorf("SP-I2 frozen training evidence digests differ from the discovery freeze") + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return fmt.Errorf("read frozen SP-I2 training baseline: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return fmt.Errorf("read frozen SP-I2 training candidate: %w", err) + } + resource, err := loadSPI2ResourceReport(resourcePath) + if err != nil { + return err + } + if resource.ArtifactSHA256 != candidateSHA256 { + return fmt.Errorf("SP-I2 frozen training resource report is not bound to the candidate artifact") + } + recomputed, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: freeze.Seed, + Confidence: freeze.Confidence, + BootstrapCount: freeze.BootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: freeze.SourceArchiveSHA256, + }) + if err != nil { + return fmt.Errorf("recompute frozen SP-I2 training discovery: %w", err) + } + recomputed.BaselineArtifactSHA256 = baselineSHA256 + recomputed.CandidateArtifactSHA256 = candidateSHA256 + recomputed.ResourceReportSHA256 = resourceSHA256 + if !reflect.DeepEqual(recomputed, *discovery) { + return fmt.Errorf("SP-I2 discovery report differs from its recomputed frozen training evidence") + } + return nil +} + +// validateSPI2Corpus validates spi2 corpus. +func validateSPI2Corpus(corpus ScaleCorpus, cohort spI2CanonicalCohort) error { + if len(corpus.Cases) != len(cohort.keys) { + return fmt.Errorf("SP-I2 holdout capture requires exactly the frozen six-training/four-holdout cohort") + } + seen := map[performanceKey]struct{}{} + resolved := make([]ResolvedCaseSelector, 0, len(corpus.Cases)) + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, expected := cohort.keys[key]; !expected { + return fmt.Errorf("SP-I2 holdout capture contains unexpected case %s/%s", testCase.Dataset, testCase.Name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("SP-I2 holdout capture duplicates case %s/%s", testCase.Dataset, testCase.Name) + } + seen[key] = struct{}{} + if filepath.Base(testCase.Source) != "generated_sp_i2_distance_v1.json" || + testCase.Category != "generated_shortest_path_v2" || sqlFingerprint(testCase.Cypher) != spI2QuerySHA256 || + testCase.Shape.FallbackExpectation != "forbidden" || testCase.Shape.Direction != "inbound" || + testCase.Shape.RelationshipKindCount != 1 || !slices.Equal(testCase.Shape.EdgeKinds, []string{"Traverse"}) || + testCase.Shape.MinDepth == nil || *testCase.Shape.MinDepth != 1 || + testCase.Shape.MaxDepth == nil || *testCase.Shape.MaxDepth != 64 || + testCase.Shape.PathMaterializationRequired || + !slices.Equal(testCase.CandidateModes, []ExecutionMode{ModePostgresSQL, ModeNeo4j}) { + return fmt.Errorf("SP-I2 holdout capture changes frozen declaration %s/%s", testCase.Dataset, testCase.Name) + } + expectedSplit := "training" + if _, holdout := cohort.holdoutKeys[key]; holdout { + expectedSplit = "holdout" + } + if testCase.Shape.QualificationSplit != expectedSplit { + return fmt.Errorf("SP-I2 holdout capture changes frozen split for %s/%s", testCase.Dataset, testCase.Name) + } + resolved = append(resolved, ResolvedCaseSelector{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Category: testCase.Category, + }) + } + if !orientationV2KeySetsEqual(seen, cohort.keys) || + declarationSHA256(corpus.DeclaredBackends()) != cohort.declarationSHA256 || + resolvedSelectionSHA256(resolved) != cohort.fullResolvedSHA256 || + corpusIdentity(corpus) != cohort.fullCorpusSHA256 { + return fmt.Errorf("SP-I2 holdout capture does not match the exact frozen declaration, selection, and corpus digests") + } + return nil +} + +// validateCurrentSPI2Source validates current spi2 source. +func validateCurrentSPI2Source(sourceCommit, sourceArchive, dirtyDiff, binary string) error { + currentCommit := strings.TrimSpace(commandOutput("git", "rev-parse", "HEAD")) + currentArchive, err := spI2SourceArchiveSHA256() + if err != nil { + return err + } + currentDiff := workingTreeSHA256() + currentBinary := executableSHA256() + if currentCommit == "" || currentCommit == "unknown" || sourceCommit != currentCommit || + !lowercaseSHA256(sourceArchive) || sourceArchive != currentArchive || + dirtyDiff != cleanWorkingTreeSHA256() || currentDiff != cleanWorkingTreeSHA256() || + !lowercaseSHA256(binary) || binary != currentBinary { + return fmt.Errorf("SP-I2 evidence requires the current clean committed source archive and exact running binary") + } + return nil +} + +// loadSPI2ResourceReport loads spi2 resource report. +func loadSPI2ResourceReport(path string) (ResourceGateReport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return ResourceGateReport{}, fmt.Errorf("read SP-I2 resource report: %w", err) + } + report := ResourceGateReport{} + if err := json.Unmarshal(raw, &report); err != nil { + return ResourceGateReport{}, fmt.Errorf("decode SP-I2 resource report: %w", err) + } + if report.Version != resourceGateVersion || !lowercaseSHA256(report.ArtifactSHA256) { + return ResourceGateReport{}, fmt.Errorf("SP-I2 resource report must be checksummed schema v%d", resourceGateVersion) + } + return report, nil +} + +// loadSPI2QualificationReport loads spi2 qualification report. +func loadSPI2QualificationReport(path string) (*SPI2QualificationReport, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + report := &SPI2QualificationReport{} + if err := json.Unmarshal(raw, report); err != nil { + return nil, fmt.Errorf("decode SP-I2 qualification report: %w", err) + } + return report, nil +} + +// loadSPI2FreezeManifest loads spi2 freeze manifest. +func loadSPI2FreezeManifest(path string) (*SPI2QualificationFreezeManifest, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, "", err + } + manifest := &SPI2QualificationFreezeManifest{} + if err := json.Unmarshal(raw, manifest); err != nil { + return nil, "", fmt.Errorf("decode SP-I2 freeze manifest: %w", err) + } + digest := sha256.Sum256(raw) + return manifest, hex.EncodeToString(digest[:]), nil +} + +// writeSPI2QualificationReport writes spi2 qualification report. +func writeSPI2QualificationReport(path string, report SPI2QualificationReport) (err error) { + if path == "" { + return fmt.Errorf("SP-I2 qualification requires an explicit report output path") + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} + +// writeSPI2FreezeManifest writes spi2 freeze manifest. +func writeSPI2FreezeManifest(path, discoveryReportPath string, report SPI2QualificationReport) (err error) { + if path == "" || discoveryReportPath == "" { + return fmt.Errorf("SP-I2 discovery freeze requires report and manifest output paths") + } + cohort, err := canonicalSPI2Cohort() + if err != nil { + return err + } + if report.Protocol != referencePairProtocolDiscovery || report.CohortDeclarationSHA256 != cohort.trainingDeclarationSHA256 || + report.ResolvedSelectionSHA256 != cohort.trainingResolvedSHA256 || report.CorpusSHA256 != cohort.trainingCorpusSHA256 || + report.TrainingCases != len(cohort.trainingKeys) || report.HoldoutCases != 0 || + !report.EvidencePassed || !report.TrainingPassed || + report.Seed != 1 || report.Confidence != defaultConfidenceLevel || report.BootstrapCount != defaultBootstrapCount || + report.DirtyDiffSHA256 != cleanWorkingTreeSHA256() || !equalSPI2Caps(report.Caps, spI2QualificationCaps()) || + !lowercaseSHA256(report.BaselineArtifactSHA256) || !lowercaseSHA256(report.CandidateArtifactSHA256) || + !lowercaseSHA256(report.ResourceReportSHA256) { + return fmt.Errorf("SP-I2 discovery freeze requires the exact passing clean training-only report") + } + discoveryReportSHA256, err := fileSHA256(discoveryReportPath) + if err != nil { + return err + } + manifest := SPI2QualificationFreezeManifest{ + Version: spI2FreezeVersion, + Baseline: report.Baseline, + Candidate: report.Candidate, + Policy: report.Policy, + QuerySHA256: report.QuerySHA256, + Caps: report.Caps, + Seed: report.Seed, + Confidence: report.Confidence, + BootstrapCount: report.BootstrapCount, + SourceCommit: report.SourceCommit, + SourceArchiveSHA256: report.SourceArchiveSHA256, + DirtyDiffSHA256: report.DirtyDiffSHA256, + BinarySHA256: report.BinarySHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + TrainingResolvedSHA256: cohort.trainingResolvedSHA256, + FullResolvedSHA256: cohort.fullResolvedSHA256, + BaselineArtifactSHA256: report.BaselineArtifactSHA256, + CandidateArtifactSHA256: report.CandidateArtifactSHA256, + ResourceReportSHA256: report.ResourceReportSHA256, + DiscoveryReportSHA256: discoveryReportSHA256, + TrainingPassed: report.TrainingPassed, + } + if err := ensureOutputDir(path); err != nil { + return err + } + output, err := os.Create(path) + if err != nil { + return err + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + encodeErr := encoder.Encode(manifest) + closeErr := output.Close() + if encodeErr != nil { + return encodeErr + } + return closeErr +} + +// validateDistinctSPI2Paths validates distinct spi2 paths. +func validateDistinctSPI2Paths(paths map[string]string) error { + names := make([]string, 0, len(paths)) + for name, path := range paths { + if path != "" { + names = append(names, name) + } + } + sort.Strings(names) + + // resolvedPath records the canonical filesystem identity of one evidence input. + type resolvedPath struct { + // name retains the name while resolvedPath is assembled or evaluated. + name string + // info retains the info while resolvedPath is assembled or evaluated. + info os.FileInfo + } + resolved := map[string]resolvedPath{} + var existing []resolvedPath + for _, name := range names { + absolute, err := filepath.Abs(filepath.Clean(paths[name])) + if err != nil { + return fmt.Errorf("resolve SP-I2 %s: %w", name, err) + } + if evaluated, err := filepath.EvalSymlinks(absolute); err == nil { + absolute = evaluated + } else if evaluatedParent, parentErr := filepath.EvalSymlinks(filepath.Dir(absolute)); parentErr == nil { + absolute = filepath.Join(evaluatedParent, filepath.Base(absolute)) + } + if prior, duplicate := resolved[absolute]; duplicate { + return fmt.Errorf("SP-I2 %s and %s must use distinct paths", prior.name, name) + } + current := resolvedPath{name: name} + if info, err := os.Stat(paths[name]); err == nil { + current.info = info + for _, prior := range existing { + if prior.info != nil && os.SameFile(prior.info, info) { + return fmt.Errorf("SP-I2 %s and %s must not alias the same file", prior.name, name) + } + } + existing = append(existing, current) + } else if !os.IsNotExist(err) { + return fmt.Errorf("inspect SP-I2 %s path: %w", name, err) + } + resolved[absolute] = current + } + return nil +} + +// selectedCorpusContainsSPI2Holdout selects ed corpus contains spi2 holdout. +func selectedCorpusContainsSPI2Holdout(corpus ScaleCorpus) bool { + cohort, err := canonicalSPI2Cohort() + if err != nil { + return true + } + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, holdout := cohort.holdoutKeys[key]; holdout { + return true + } + } + return false +} + +// selectRunnableScaleCorpus keeps the protected SP-I2 holdout out of ordinary +// GraphBench selection. The holdout becomes selectable only through its exact +// protocol tag or an exact case name; database capture then passes through the +// freeze checks in main before any target is opened. +func selectRunnableScaleCorpusWithSPI2Protection(corpus ScaleCorpus, selectors CorpusSelectors) (ScaleCorpus, SelectionManifest, error) { + if err := validateCorpusSelectors(corpus, selectors); err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + // Preserve SP-I1's independently frozen selection identity when its exact + // protocol selectors are in use. The two qualification studies must never + // change one another's protected-declaration digest. + spI1ProtocolSelection := slices.Contains(selectors.Tags, spI1TrainingTag) || slices.Contains(selectors.Tags, spI1HoldoutTag) + if !spI1ProtocolSelection { + for _, selectedName := range selectors.Cases { + for _, testCase := range spI1CanonicalCases { + if selectedName == testCase.name { + spI1ProtocolSelection = true + break + } + } + } + } + if spI1ProtocolSelection { + return selectRunnableScaleCorpus(withoutSPI2V2FormalCases(corpus), selectors) + } + spI2ProtocolSelection := slices.Contains(selectors.Tags, spI2TrainingTag) || slices.Contains(selectors.Tags, spI2HoldoutTag) + if !spI2ProtocolSelection { + for _, selectedName := range selectors.Cases { + for _, testCase := range spI2CanonicalCases { + if selectedName == testCase.name { + spI2ProtocolSelection = true + break + } + } + } + } + spI2V2ProtocolSelection := spI2V2FormalProtocolSelection(selectors) + if spI2ProtocolSelection && spI2V2ProtocolSelection { + return ScaleCorpus{}, SelectionManifest{}, fmt.Errorf("SP-I2 V1 and V2 protocol selectors cannot be mixed") + } + if spI2ProtocolSelection { + corpus = withoutSPI2V2FormalCases(corpus) + } + includeProtected := slices.Contains(selectors.Tags, spI2HoldoutTag) + if !includeProtected && len(selectors.Cases) > 0 { + protectedNames := make(map[string]struct{}, len(spI2CanonicalCases)) + for _, testCase := range spI2CanonicalCases { + if testCase.split == "holdout" { + protectedNames[testCase.name] = struct{}{} + } + } + for _, name := range selectors.Cases { + if _, protected := protectedNames[name]; protected { + includeProtected = true + break + } + } + } + if includeProtected { + return selectScaleCorpusValidated(corpus, selectors) + } + includeV2Protected := spI2V2FormalHoldoutSelected(selectors) + if includeV2Protected { + return selectScaleCorpusValidated(corpus, selectors) + } + + cohort, err := canonicalSPI2Cohort() + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + filtered := ScaleCorpus{Cases: make([]ScaleCase, 0, len(corpus.Cases))} + protected := ScaleCorpus{Cases: make([]ScaleCase, 0, len(cohort.holdoutKeys))} + formalV2, err := canonicalSPI2V2FormalCohort() + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + for _, testCase := range corpus.Cases { + key := performanceKey{ + dataset: testCase.Dataset, + name: testCase.Name, + backend: ModePostgresSQL, + } + if _, isProtected := cohort.holdoutKeys[key]; isProtected { + protected.Cases = append(protected.Cases, testCase) + continue + } + if _, isProtected := formalV2.holdoutKeys[key]; isProtected { + protected.Cases = append(protected.Cases, testCase) + continue + } + filtered.Cases = append(filtered.Cases, testCase) + } + var selected ScaleCorpus + var manifest SelectionManifest + if spI2ProtocolSelection || spI2V2ProtocolSelection { + selected, manifest, err = selectScaleCorpusValidated(filtered, selectors) + } else { + selected, manifest, err = selectRunnableScaleCorpus(filtered, selectors) + } + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + manifest.FullDeclarationCount = len(corpus.DeclaredBackends()) + manifest.OmittedDeclarationCount = manifest.FullDeclarationCount - manifest.SelectedDeclarationCount + if !spI2ProtocolSelection && manifest.ProtectedDeclarationCount > 0 { + spI1Cohort, err := canonicalSPI1Cohort() + if err != nil { + return ScaleCorpus{}, SelectionManifest{}, err + } + for _, testCase := range corpus.Cases { + key := performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL} + if _, isProtected := spI1Cohort.holdoutKeys[key]; isProtected { + protected.Cases = append(protected.Cases, testCase) + } + } + } + manifest.ProtectedDeclarationCount = len(protected.DeclaredBackends()) + manifest.ProtectedDeclarationSHA256 = declarationSHA256(protected.DeclaredBackends()) + return selected, manifest, nil +} + +func withoutSPI2V2FormalCases(corpus ScaleCorpus) ScaleCorpus { + formal := make(map[performanceKey]struct{}, len(spI2V2FormalCases)) + for _, declaration := range spI2V2FormalCases { + formal[performanceKey{dataset: declaration.dataset, name: declaration.name, backend: ModePostgresSQL}] = struct{}{} + } + filtered := ScaleCorpus{Cases: make([]ScaleCase, 0, len(corpus.Cases))} + for _, testCase := range corpus.Cases { + if _, found := formal[performanceKey{dataset: testCase.Dataset, name: testCase.Name, backend: ModePostgresSQL}]; !found { + filtered.Cases = append(filtered.Cases, testCase) + } + } + return filtered +} + +// validateSPI2HoldoutCaptureConfig validates spi2 holdout capture config. +func validateSPI2HoldoutCaptureConfig(cfg config) error { + if len(cfg.Modes) != 1 || cfg.Modes[0] != ModePostgresSQL || cfg.ExistingGraph || cfg.Discovery { + return fmt.Errorf("SP-I2 holdout capture requires one managed PostgreSQL fixed-confirmation mode") + } + if cfg.Iterations < 50 || cfg.WarmupIterations < 20 || cfg.PoolSize != 1 || len(cfg.Concurrency) != 0 { + return fmt.Errorf("SP-I2 holdout capture requires at least 50 samples, 20 warmups, pool size 1, and no concurrency block") + } + if cfg.Round < 1 || cfg.Round > 20 || cfg.Block != cfg.Round || cfg.ArmOrder < 1 || cfg.ArmOrder > 2 || + strings.TrimSpace(cfg.RunUUID) == "" { + return fmt.Errorf("SP-I2 holdout capture requires rounds 1-20, block equal to round, a two-arm order, and an explicit shared run UUID") + } + baseline := string(optimize.ShortestPathExecutorS4CanonicalDistance) + candidate := string(optimize.ShortestPathExecutorI2GuardedDistance) + expectedArm, expectedOrder := "", 0 + switch cfg.PostgresForceShortest { + case baseline: + expectedArm = "sp-i2-s4" + expectedOrder = 1 + if cfg.Round%2 == 0 { + expectedOrder = 2 + } + case candidate: + expectedArm = "sp-i2-candidate" + expectedOrder = 2 + if cfg.Round%2 == 0 { + expectedOrder = 1 + } + default: + return fmt.Errorf("SP-I2 holdout capture must force exact S4 distance or guarded SP-I2 distance") + } + if cfg.Arm != expectedArm || cfg.ArmOrder != expectedOrder { + return fmt.Errorf("SP-I2 holdout capture round %d requires arm %q at order %d", cfg.Round, expectedArm, expectedOrder) + } + if !cfg.PostgresRepeatableRead || cfg.PostgresTraversalTelemetry != postgresTraversalTelemetryDiagnostic || + cfg.PostgresProductionManifest != "" || cfg.PostgresForceExpansion != "" || + cfg.PostgresExpansionOrientationShadow || cfg.PostgresExpansionOrientationTournament || + cfg.PostgresReferences || len(cfg.PostgresReferenceArms) != 0 || cfg.Baseline != "" || + cfg.BundleDir != "" || len(cfg.BundleEvidence) != 0 { + return fmt.Errorf("SP-I2 holdout capture requires forced Repeatable Read with diagnostic telemetry and no supplemental PostgreSQL arms") + } + if cfg.OutputJSONL == "" || cfg.Round > 1 && !cfg.AppendJSONL { + return fmt.Errorf("SP-I2 holdout capture requires a JSONL output and append mode after round 1") + } + return validateDistinctSPI2Paths(map[string]string{ + "freeze manifest": cfg.SPI2Freeze, "discovery report": cfg.SPI2DiscoveryReport, + "training baseline artifact": cfg.SPI2TrainingBaseline, + "training candidate artifact": cfg.SPI2TrainingCandidate, + "training resource report": cfg.SPI2TrainingResource, + "capture JSONL": cfg.OutputJSONL, "capture summary": cfg.Summary, "capture JSON summary": cfg.SummaryJSON, + }) +} diff --git a/cmd/graphbench/sp_i2_qualification_test.go b/cmd/graphbench/sp_i2_qualification_test.go new file mode 100644 index 00000000..96f82069 --- /dev/null +++ b/cmd/graphbench/sp_i2_qualification_test.go @@ -0,0 +1,901 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestSPI2QualificationDiscoveryPassesTrainingWithoutOpeningHoldout verifies spi2 qualification discovery passes training without opening holdout behavior. +func TestSPI2QualificationDiscoveryPassesTrainingWithoutOpeningHoldout(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + report, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.True(t, report.EvidencePassed) + require.True(t, report.TrainingPassed) + require.False(t, report.HoldoutPassed) + require.False(t, report.QualificationPassed) + require.Equal(t, 6, report.TrainingCases) + require.Zero(t, report.HoldoutCases) + require.Len(t, report.Cases, 6) + require.Equal(t, spI2QualificationCaps(), report.Caps) + for _, gateCase := range report.Cases { + require.True(t, gateCase.Passed, gateCase.Reasons) + require.Equal(t, "training", gateCase.QualificationSplit) + require.LessOrEqual(t, gateCase.MedianRatio.Upper, 0.95) + require.LessOrEqual(t, gateCase.P95Ratio.Upper, 1.05) + } +} + +// TestTimedRuntimeAttestationIdentityIncludesExactS4Baseline verifies timed runtime attestation identity includes exact s4 baseline behavior. +func TestTimedRuntimeAttestationIdentityIncludesExactS4DistanceBaseline(t *testing.T) { + baseline := string(optimize.ShortestPathExecutorS4CanonicalDistance) + translation := translate.Result{Optimization: translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{{ + Family: "SP", + Selected: baseline, + }}}} + require.Equal(t, baseline, timedRuntimeAttestationIdentity(translation)) +} + +// TestSPI2QualificationConfirmationRequiresAndPassesFrozenDiscovery verifies spi2 qualification confirmation requires and passes frozen discovery behavior. +func TestSPI2QualificationConfirmationRequiresAndPassesFrozenDiscovery(t *testing.T) { + trainingBaseline, trainingCandidate, trainingResource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI2QualificationReport(trainingBaseline, trainingCandidate, trainingResource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI2QualificationTestFreeze(t, discovery) + + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolConfirmation) + report, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolConfirmation, + SourceArchiveSHA256: strings.Repeat("a", 64), + Freeze: &freeze, + Discovery: &discovery, + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + require.True(t, report.HoldoutPassed) + require.True(t, report.QualificationPassed) + require.Equal(t, 6, report.TrainingCases) + require.Equal(t, 4, report.HoldoutCases) + require.Len(t, report.Cases, 10) +} + +// TestSPI2QualificationRejectsUnattestedCandidateAndFreezeMutation verifies spi2 qualification rejects unattested candidate and freeze mutation behavior. +func TestSPI2QualificationRejectsUnattestedCandidateAndFreezeMutation(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + candidate[0].Stats.Samples[1].RuntimeAttestation = "same_case_invocation_local_replay" + candidate[0].Stats.Samples[1].RuntimeReceiptEvents = nil + _, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.ErrorContains(t, err, "timed-invocation attribution") + + trainingBaseline, trainingCandidate, trainingResource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI2QualificationReport(trainingBaseline, trainingCandidate, trainingResource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI2QualificationTestFreeze(t, discovery) + freeze.QuerySHA256 = strings.Repeat("f", 64) + cohort, err := canonicalSPI2Cohort() + require.NoError(t, err) + require.Error(t, validateSPI2FrozenDiscovery(&freeze, &discovery, cohort)) +} + +// TestSPI2QualificationClassifiesBoundResourceFailure verifies spi2 qualification classifies bound resource failure behavior. +func TestSPI2QualificationClassifiesBoundResourceFailure(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + candidate[0].PostgresMetrics.Buffers.TempWritten = 1 + resource.Cases[0] = evaluateProductionResourceGateCase(candidate[0]) + resource.Passed = false + report, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.False(t, report.TrainingPassed) + require.False(t, report.QualificationPassed) + found := false + for _, gateCase := range report.Cases { + found = found || strings.Contains(strings.Join(gateCase.Reasons, "\n"), "candidate resource evidence did not pass") + } + require.True(t, found) +} + +func TestSPI2QualificationAppliesPreregisteredAdverseControlGate(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + for index := range candidate { + if !strings.Contains(candidate[index].Name, "cycle-control") { + continue + } + candidate[index].Stats.Median = 10500 * time.Microsecond + candidate[index].Stats.P95 = 10500 * time.Microsecond + for sampleIndex := range candidate[index].Stats.Samples { + if candidate[index].Stats.Samples[sampleIndex].Classification == "warm" { + candidate[index].Stats.Samples[sampleIndex].Duration = 10500 * time.Microsecond + } + } + } + report, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, Confidence: defaultConfidenceLevel, BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + require.True(t, report.TrainingPassed) + for _, gateCase := range report.Cases { + if strings.Contains(gateCase.Name, "cycle-control") { + require.Equal(t, "adverse_control", gateCase.QualificationRole) + require.True(t, gateCase.Material) + require.Greater(t, gateCase.MedianRatio.Upper, report.MaterialityRatio) + require.LessOrEqual(t, gateCase.MedianRatio.Upper, report.AdverseRatioLimit) + } else { + require.Equal(t, "target", gateCase.QualificationRole) + } + } +} + +// TestSPI2QualificationRejectsCanonicalEvidenceAndScheduleTampering verifies spi2 qualification rejects canonical evidence and schedule tampering behavior. +func TestSPI2QualificationRejectsCanonicalEvidenceAndScheduleTampering(t *testing.T) { + tests := map[string]func([]CaseResult, []CaseResult, *ResourceGateReport){ + "canonical observation": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + baseline[0].ObservedRows = []string{`[{"nodes":[],"relationships":[]}]`} + }, + "duplicate warm iteration": func(_ []CaseResult, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].Stats.Samples[2].Iteration = 1 + }, + "duplicate timed invocation": func(_ []CaseResult, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].Stats.Samples[2].RuntimeInvocationID = candidate[0].Stats.Samples[1].RuntimeInvocationID + candidate[0].Stats.Samples[2].RuntimeReceiptEvents[0].InvocationID = candidate[0].Stats.Samples[1].RuntimeInvocationID + }, + "cross-record timed invocation replay": func(baseline, candidate []CaseResult, _ *ResourceGateReport) { + candidate[1].Stats.Samples[1].RuntimeInvocationID = baseline[0].Stats.Samples[1].RuntimeInvocationID + candidate[1].Stats.Samples[1].RuntimeReceiptEvents[0].InvocationID = baseline[0].Stats.Samples[1].RuntimeInvocationID + }, + "contradictory arm chronology": func(baseline, candidate []CaseResult, _ *ResourceGateReport) { + started := baseline[0].Environment.StartedAt.Add(-2 * time.Second) + for index := range candidate { + if candidate[index].Environment.Round == 1 { + candidate[index].Environment.StartedAt = started + candidate[index].Environment.EndedAt = started.Add(time.Second) + } + } + }, + "block differs from round": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + for index := range baseline { + if baseline[index].Environment.Round == 2 { + baseline[index].Environment.Block = 1 + } + } + }, + "unbound resource round": func(_, _ []CaseResult, resource *ResourceGateReport) { + resource.Cases[0].Round = 99 + }, + "substituted resource receipt": func(_, _ []CaseResult, resource *ResourceGateReport) { + resource.Cases[0].RuntimeReceiptChains[0][0].RuntimeBranch = "substituted" + }, + "cleared resource spill": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].PostgresMetrics.Buffers.TempWritten = 1 + }, + "reachable relabeled no path": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].TraversalTelemetry.Summary.RuntimeBranch = "inline_canonical_distance_no_path" + for index := range candidate[0].Stats.Samples { + if candidate[0].Stats.Samples[index].Classification == "warm" { + candidate[0].Stats.Samples[index].RuntimeBranch = "inline_canonical_distance_no_path" + candidate[0].Stats.Samples[index].RuntimeReceiptEvents[0].RuntimeBranch = "inline_canonical_distance_no_path" + } + } + }, + "no path relabeled witness": func(_, candidate []CaseResult, _ *ResourceGateReport) { + for recordIndex := range candidate { + if !strings.HasSuffix(candidate[recordIndex].Name, "-disconnected") { + continue + } + candidate[recordIndex].TraversalTelemetry.Summary.RuntimeBranch = "inline_canonical_distance" + for sampleIndex := range candidate[recordIndex].Stats.Samples { + if candidate[recordIndex].Stats.Samples[sampleIndex].Classification == "warm" { + candidate[recordIndex].Stats.Samples[sampleIndex].RuntimeBranch = "inline_canonical_distance" + candidate[recordIndex].Stats.Samples[sampleIndex].RuntimeReceiptEvents[0].RuntimeBranch = "inline_canonical_distance" + } + } + return + } + }, + "output counter differs from observation": func(_, candidate []CaseResult, _ *ResourceGateReport) { + *candidate[0].TraversalTelemetry.Diagnostic.Counters.InlineShortestDistance.OutputRows = 0 + }, + "supplemental planned arm": func(_, candidate []CaseResult, _ *ResourceGateReport) { + candidate[0].TraversalTelemetry.Summary.PlannedIdentities = append(candidate[0].TraversalTelemetry.Summary.PlannedIdentities, "SP-B1-extra") + }, + "reduced planned search space": func(baseline, _ []CaseResult, _ *ResourceGateReport) { + baseline[0].TraversalTelemetry.Summary.PlannedIdentities = []string{ + string(optimize.ShortestPathExecutorS4CanonicalDistance), + string(optimize.ShortestPathExecutorIncumbentWorkspace), + } + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + mutate(baseline, candidate, &resource) + _, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.Error(t, err) + }) + } +} + +// TestSPI2QualificationFreezesStatisticalPolicyAndDiscoverySemantics verifies spi2 qualification freezes statistical policy and discovery semantics behavior. +func TestSPI2QualificationFreezesStatisticalPolicyAndDiscoverySemantics(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + for _, options := range []SPI2QualificationOptions{ + { + Seed: 2, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + }, + { + Seed: 1, + Confidence: 0.95, + BootstrapCount: defaultBootstrapCount, + }, + { + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: 1, + }, + } { + options.Protocol = referencePairProtocolDiscovery + options.SourceArchiveSHA256 = strings.Repeat("a", 64) + _, err := buildSPI2QualificationReport(baseline, candidate, resource, options) + require.Error(t, err) + } + + discovery, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + freeze := spI2QualificationTestFreeze(t, discovery) + discovery.Cases[0].P95Ratio.Upper = 2 + cohort, err := canonicalSPI2Cohort() + require.NoError(t, err) + require.Error(t, validateSPI2FrozenDiscovery(&freeze, &discovery, cohort)) +} + +// TestSPI2FrozenTrainingEvidenceIsRecomputedFromNamedArtifacts verifies spi2 frozen training evidence is recomputed from named artifacts behavior. +func TestSPI2FrozenTrainingEvidenceIsRecomputedFromNamedArtifacts(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + directory := t.TempDir() + baselinePath := filepath.Join(directory, "s4.jsonl") + candidatePath := filepath.Join(directory, "i1.jsonl") + resourcePath := filepath.Join(directory, "resource.json") + require.NoError(t, writeJSONLFile(baselinePath, baseline)) + require.NoError(t, writeJSONLFile(candidatePath, candidate)) + candidateSHA256, err := fileSHA256(candidatePath) + require.NoError(t, err) + resource.ArtifactSHA256 = candidateSHA256 + resourceRaw, err := json.MarshalIndent(resource, "", " ") + require.NoError(t, err) + require.NoError(t, os.WriteFile(resourcePath, append(resourceRaw, '\n'), 0o600)) + + discovery, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256, err = fileSHA256(baselinePath) + require.NoError(t, err) + discovery.CandidateArtifactSHA256 = candidateSHA256 + discovery.ResourceReportSHA256, err = fileSHA256(resourcePath) + require.NoError(t, err) + freeze := spI2QualificationTestFreeze(t, discovery) + require.NoError(t, validateSPI2FrozenTrainingEvidence(&freeze, &discovery, baselinePath, candidatePath, resourcePath)) + + forged := discovery + forged.Cases = append([]SPI2QualificationCase(nil), discovery.Cases...) + forged.Cases[0].MedianRatio = RatioInterval{ + Lower: 0.801, + Estimate: 0.801, + Upper: 0.801, + } + require.ErrorContains(t, + validateSPI2FrozenTrainingEvidence(&freeze, &forged, baselinePath, candidatePath, resourcePath), + "differs from its recomputed", + ) +} + +// TestSPI2FreezeManifestRequiresPassingDiscovery verifies that failed discovery +// evidence cannot leave behind a holdout-authorization artifact. +func TestSPI2FreezeManifestRequiresPassingDiscovery(t *testing.T) { + baseline, candidate, resource := spI2QualificationTestArtifacts(t, referencePairProtocolDiscovery) + discovery, err := buildSPI2QualificationReport(baseline, candidate, resource, SPI2QualificationOptions{ + Seed: 1, + Confidence: defaultConfidenceLevel, + BootstrapCount: defaultBootstrapCount, + Protocol: referencePairProtocolDiscovery, + SourceArchiveSHA256: strings.Repeat("a", 64), + }) + require.NoError(t, err) + discovery.BaselineArtifactSHA256 = strings.Repeat("1", 64) + discovery.CandidateArtifactSHA256 = strings.Repeat("2", 64) + discovery.ResourceReportSHA256 = strings.Repeat("3", 64) + + for name, mutate := range map[string]func(*SPI2QualificationReport){ + "failed evidence": func(report *SPI2QualificationReport) { + report.EvidencePassed = false + }, + "failed training": func(report *SPI2QualificationReport) { + report.TrainingPassed = false + }, + } { + t.Run(name, func(t *testing.T) { + report := discovery + mutate(&report) + directory := t.TempDir() + reportPath := filepath.Join(directory, "discovery.json") + freezePath := filepath.Join(directory, "freeze.json") + require.NoError(t, writeSPI2QualificationReport(reportPath, report)) + require.ErrorContains(t, writeSPI2FreezeManifest(freezePath, reportPath, report), "exact passing clean training-only report") + _, statErr := os.Stat(freezePath) + require.ErrorIs(t, statErr, os.ErrNotExist) + }) + } + + t.Run("passing discovery", func(t *testing.T) { + directory := t.TempDir() + reportPath := filepath.Join(directory, "discovery.json") + freezePath := filepath.Join(directory, "freeze.json") + require.NoError(t, writeSPI2QualificationReport(reportPath, discovery)) + require.NoError(t, writeSPI2FreezeManifest(freezePath, reportPath, discovery)) + freeze, _, err := loadSPI2FreezeManifest(freezePath) + require.NoError(t, err) + require.True(t, freeze.TrainingPassed) + require.Equal(t, discovery.BaselineArtifactSHA256, freeze.BaselineArtifactSHA256) + require.Equal(t, discovery.CandidateArtifactSHA256, freeze.CandidateArtifactSHA256) + require.Equal(t, discovery.ResourceReportSHA256, freeze.ResourceReportSHA256) + }) +} + +// TestSPI2PathsRejectHardlinkAliases verifies spi2 paths reject hardlink aliases behavior. +func TestSPI2PathsRejectHardlinkAliases(t *testing.T) { + directory := t.TempDir() + input := filepath.Join(directory, "input.json") + alias := filepath.Join(directory, "alias.json") + require.NoError(t, os.WriteFile(input, []byte("{}"), 0o600)) + require.NoError(t, os.Link(input, alias)) + require.Error(t, validateDistinctSPI2Paths(map[string]string{"input": input, "output": alias})) +} + +// TestSPI2HoldoutCaptureProfileAcceptsBothBalancedArmsAndRejectsDrift verifies spi2 holdout capture profile accepts both balanced arms and rejects drift behavior. +func TestSPI2HoldoutCaptureProfileAcceptsBothBalancedArmsAndRejectsDrift(t *testing.T) { + baseline := string(optimize.ShortestPathExecutorS4CanonicalDistance) + candidate := string(optimize.ShortestPathExecutorI2GuardedDistance) + valid := func(executor, arm string, round, order int) config { + return config{ + Modes: []ExecutionMode{ModePostgresSQL}, + Iterations: 50, + WarmupIterations: 20, + Round: round, + Block: round, + Arm: arm, + ArmOrder: order, + RunUUID: "sp-i2-confirmation", + PoolSize: 1, + PostgresForceShortest: executor, + PostgresRepeatableRead: true, + PostgresTraversalTelemetry: postgresTraversalTelemetryDiagnostic, + OutputJSONL: fmt.Sprintf(".coverage/sp-i2-%s-%d.jsonl", arm, round), + AppendJSONL: round > 1, + SPI2Freeze: ".coverage/sp-i2-freeze.json", + SPI2DiscoveryReport: ".coverage/sp-i2-discovery.json", + SPI2TrainingBaseline: ".coverage/sp-i2-training-s4.jsonl", + SPI2TrainingCandidate: ".coverage/sp-i2-training-i2.jsonl", + SPI2TrainingResource: ".coverage/sp-i2-training-resource.json", + } + } + for _, cfg := range []config{ + valid(baseline, "sp-i2-s4", 1, 1), + valid(candidate, "sp-i2-candidate", 1, 2), + valid(baseline, "sp-i2-s4", 2, 2), + valid(candidate, "sp-i2-candidate", 2, 1), + } { + require.NoError(t, validateSPI2HoldoutCaptureConfig(cfg)) + } + + tests := map[string]func(*config){ + "wrong backend": func(cfg *config) { cfg.Modes = []ExecutionMode{ModeNeo4j} }, + "existing graph": func(cfg *config) { cfg.ExistingGraph = true }, + "too few samples": func(cfg *config) { cfg.Iterations = 49 }, + "too few warmups": func(cfg *config) { cfg.WarmupIterations = 19 }, + "pool larger than one": func(cfg *config) { cfg.PoolSize = 2 }, + "concurrency": func(cfg *config) { cfg.Concurrency = []int{2} }, + "round above maximum": func(cfg *config) { cfg.Round, cfg.Block = 21, 21 }, + "mismatched block": func(cfg *config) { cfg.Block = 2 }, + "missing run UUID": func(cfg *config) { cfg.RunUUID = "" }, + "wrong arm label": func(cfg *config) { cfg.Arm = "baseline" }, + "wrong arm order": func(cfg *config) { cfg.ArmOrder = 2 }, + "wrong executor": func(cfg *config) { cfg.PostgresForceShortest = "SP-S3-U-E+MAT-M0" }, + "read committed": func(cfg *config) { cfg.PostgresRepeatableRead = false }, + "summary telemetry": func(cfg *config) { cfg.PostgresTraversalTelemetry = postgresTraversalTelemetrySummary }, + "supplemental references": func(cfg *config) { cfg.PostgresReferences = true }, + "missing output": func(cfg *config) { cfg.OutputJSONL = "" }, + "path alias": func(cfg *config) { cfg.OutputJSONL = cfg.SPI2Freeze }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + cfg := valid(baseline, "sp-i2-s4", 1, 1) + mutate(&cfg) + require.Error(t, validateSPI2HoldoutCaptureConfig(cfg)) + }) + } + t.Run("round after one requires append", func(t *testing.T) { + cfg := valid(baseline, "sp-i2-s4", 2, 2) + cfg.AppendJSONL = false + require.Error(t, validateSPI2HoldoutCaptureConfig(cfg)) + }) +} + +// TestSPI2HoldoutDetectionAndCorpusBindingIgnoreMutableTagAlone verifies spi2 holdout detection and corpus binding ignore mutable tag alone behavior. +func TestSPI2HoldoutDetectionAndCorpusBindingIgnoreMutableTagAlone(t *testing.T) { + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + training, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{spI2TrainingTag}}) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI2Holdout(training)) + + confirmation, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{spI2TrainingTag, spI2HoldoutTag}}) + require.NoError(t, err) + require.True(t, selectedCorpusContainsSPI2Holdout(confirmation)) + for index := range confirmation.Cases { + confirmation.Cases[index].Source = strings.TrimPrefix(confirmation.Cases[index].Source, "../../") + confirmation.Cases[index].Tags = nil + } + require.True(t, selectedCorpusContainsSPI2Holdout(confirmation), "canonical key detection must not depend on tags") + + // Restore exact declarations before checking the complete frozen corpus. + exact, _, err := selectScaleCorpus(full, CorpusSelectors{Tags: []string{spI2TrainingTag, spI2HoldoutTag}}) + require.NoError(t, err) + for index := range exact.Cases { + exact.Cases[index].Source = strings.TrimPrefix(exact.Cases[index].Source, "../../") + } + cohort, err := canonicalSPI2Cohort() + require.NoError(t, err) + require.NoError(t, validateSPI2Corpus(exact, cohort)) + + omitted := ScaleCorpus{Cases: append([]ScaleCase(nil), exact.Cases[:len(exact.Cases)-1]...)} + require.Error(t, validateSPI2Corpus(omitted, cohort)) + mutated := ScaleCorpus{Cases: append([]ScaleCase(nil), exact.Cases...)} + mutated.Cases[0].Cypher += " " + require.Error(t, validateSPI2Corpus(mutated, cohort)) +} + +// TestRunnableCorpusExcludesSPI2HoldoutUntilExactOptIn verifies runnable corpus excludes spi2 holdout until exact opt in behavior. +func TestRunnableCorpusExcludesSPI2HoldoutUntilExactOptIn(t *testing.T) { + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + + ordinary, manifest, err := selectRunnableScaleCorpusWithSPI2Protection(full, CorpusSelectors{}) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI2Holdout(ordinary)) + require.False(t, manifest.DiagnosticOnly) + require.Equal(t, manifest.FullDeclarationCount, manifest.SelectedDeclarationCount+manifest.OmittedDeclarationCount) + require.Equal(t, 26, manifest.OmittedDeclarationCount) + require.Equal(t, 26, manifest.ProtectedDeclarationCount) + require.True(t, lowercaseSHA256(manifest.ProtectedDeclarationSHA256)) + require.True(t, selectedCorpusContainsTag(ordinary, spI2TrainingTag)) + + for name, selectors := range map[string]CorpusSelectors{ + "generic holdout tag": {Tags: []string{"holdout"}}, + "broad category": {Categories: []string{"generated_shortest_path_v2"}}, + } { + t.Run(name, func(t *testing.T) { + selected, _, err := selectRunnableScaleCorpusWithSPI2Protection(full, selectors) + require.NoError(t, err) + require.False(t, selectedCorpusContainsSPI2Holdout(selected)) + }) + } + + exactTag, _, err := selectRunnableScaleCorpusWithSPI2Protection(full, CorpusSelectors{Tags: []string{spI2HoldoutTag}}) + require.NoError(t, err) + require.Len(t, exactTag.Cases, 4) + require.True(t, selectedCorpusContainsSPI2Holdout(exactTag)) + + exactCase, _, err := selectRunnableScaleCorpusWithSPI2Protection(full, CorpusSelectors{Cases: []string{spI2CanonicalCases[6].name}}) + require.NoError(t, err) + require.Len(t, exactCase.Cases, 1) + require.True(t, selectedCorpusContainsSPI2Holdout(exactCase)) +} + +// spI2QualificationTestArtifacts prepares or inspects test evidence for sp i2 qualification test artifacts. +func spI2QualificationTestArtifacts(t *testing.T, protocol string) ([]CaseResult, []CaseResult, ResourceGateReport) { + t.Helper() + full, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + tags := []string{spI2TrainingTag} + rounds, samples, warmups := 5, 10, 5 + corpusSHA256 := spI2TrainingCorpusSHA256 + if protocol == referencePairProtocolConfirmation { + tags = append(tags, spI2HoldoutTag) + rounds, samples, warmups = 10, 50, 20 + corpusSHA256 = spI2FullCorpusSHA256 + } + selected, selection, err := selectRunnableScaleCorpusWithSPI2Protection(full, CorpusSelectors{Tags: tags}) + require.NoError(t, err) + + var baseline, candidate []CaseResult + resource := ResourceGateReport{ + Version: resourceGateVersion, + ArtifactSHA256: strings.Repeat("9", 64), + Passed: true, + } + for _, testCase := range selected.Cases { + fixture, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err) + fixture.PhysicalValidated = true + fixture.PhysicalNodeCount = int64(fixture.NodeCount) + fixture.PhysicalEdgeCount = int64(fixture.EdgeCount) + fixture.NodeRelationBytes = int64(fixture.NodeCount) * 1024 + fixture.EdgeRelationBytes = int64(fixture.EdgeCount) * 1024 + for round := 1; round <= rounds; round++ { + left, right := spI2QualificationTestRecords(t, testCase, fixture, selection, corpusSHA256, round, samples, warmups) + baseline = append(baseline, left) + candidate = append(candidate, right) + resource.Cases = append(resource.Cases, evaluateProductionResourceGateCase(right)) + } + } + return baseline, candidate, resource +} + +// spI2QualificationTestRecords prepares or inspects test evidence for sp i2 qualification test records. +func spI2QualificationTestRecords( + t *testing.T, + testCase ScaleCase, + fixture FixtureMetadata, + selection SelectionManifest, + corpusSHA256 string, + round, samples, warmups int, +) (CaseResult, CaseResult) { + t.Helper() + baselineIdentity := string(optimize.ShortestPathExecutorS4CanonicalDistance) + candidateIdentity := string(optimize.ShortestPathExecutorI2GuardedDistance) + baselineOrder, candidateOrder := 1, 2 + if round%2 == 0 { + baselineOrder, candidateOrder = 2, 1 + } + rowCount := int64(1) + var observed []string + if testCase.Expected.ScalarInt != nil { + observed = []string{fmt.Sprintf("[%d]", *testCase.Expected.ScalarInt)} + } + if strings.HasSuffix(testCase.Name, "-disconnected") { + rowCount, observed = 0, nil + } + falseValue, trueValue := false, true + makeSamples := func(arm string, order int, duration time.Duration, requested, branch, attestation string) []LatencySample { + result := make([]LatencySample, samples+1) + result[0] = LatencySample{ + Round: round, + Block: round, + Arm: arm, + ArmOrder: order, + RunUUID: "sp-i2-test-run", + Iteration: 0, + Case: testCase.Name, + Dataset: testCase.Dataset, + Backend: ModePostgresSQL, + ConnectionID: "101", + Classification: "cold", + Duration: 2 * duration, + } + for index := range samples { + invocationID := fmt.Sprintf("sp-i2-test-%s-%s-%d-%d", arm, testCase.Name, round, index+1) + result[index+1] = LatencySample{ + Round: round, + Block: round, + Arm: arm, + ArmOrder: order, + RunUUID: "sp-i2-test-run", + Iteration: index + 1, + Case: testCase.Name, + Dataset: testCase.Dataset, + Backend: ModePostgresSQL, + ConnectionID: "101", + Classification: "warm", + Duration: duration, + RequestedIdentity: requested, + RuntimeIdentity: requested, + RuntimeBranch: branch, + FallbackExecuted: &falseValue, + RuntimeAttestation: attestation, + RuntimeInvocationID: invocationID, + } + if attestation == "timed_invocation" { + result[index+1].RuntimeReceiptEvents = []RuntimeReceiptEvent{{ + InvocationID: invocationID, + Ordinal: 1, + RuntimeIdentity: requested, + RuntimeBranch: branch, + FallbackExecuted: false, + }} + } + } + return result + } + baseEnvironment := RunEnvironment{ + ArtifactSchemaVersion: 2, + CorpusSHA256: corpusSHA256, + SourceCommit: "deadbeef", + DirtyDiffSHA256: cleanWorkingTreeSHA256(), + BinarySHA256: strings.Repeat("b", 64), + GOOS: "linux", + GOARCH: "amd64", + CPUCount: 8, + CPUModel: "test-cpu", + Kernel: "test-kernel", + CgroupCPU: "max 100000", + CgroupMemory: "max", + CPUGovernor: "performance", + RunUUID: "sp-i2-test-run", + Block: round, + Round: round, + WarmupIterations: warmups, + Selection: &selection, + PoolSize: 1, + Protocol: "fixed_confirmation", + } + postgresEnvironment := &PostgresEnvironment{ + Version: "PostgreSQL test", + Database: "dawgs", + PlanCacheMode: "auto", + TransactionIsolation: "repeatable read", + WorkMem: "64MB", + TempFileLimit: "1GB", + GraphPartitionCount: 1, + DatabaseOID: 42, + Autovacuum: "on", + NodeRelationBytes: fixture.NodeRelationBytes, + EdgeRelationBytes: fixture.EdgeRelationBytes, + AnalyzeState: "edge:analyzed,node:analyzed", + SchemaFingerprint: strings.Repeat("c", 64), + IndexFingerprint: strings.Repeat("d", 64), + } + base := newCaseResult(testCase, ModePostgresSQL, nil) + base.RowCount = rowCount + base.ObservedRows = append([]string(nil), observed...) + base.Status = StatusOK + base.WorkloadSHA256 = scaleCaseWorkloadIdentity(testCase, ModePostgresSQL) + attachFixtureMetadata(&base, fixture) + base.PostgresEnvironment = postgresEnvironment + + baseline := base + baseline.Environment = cloneSPI2TestEnvironment(baseEnvironment, "sp-i2-s4", baselineOrder) + firstStarted := time.Unix(1_700_000_000+int64(round)*10, 0) + baselineStarted, candidateStarted := firstStarted, firstStarted.Add(2*time.Second) + if candidateOrder == 1 { + candidateStarted, baselineStarted = firstStarted, firstStarted.Add(2*time.Second) + } + baseline.Environment.StartedAt, baseline.Environment.EndedAt = baselineStarted, baselineStarted.Add(time.Second) + baseline.SQL = "select 's4:' || " + fmt.Sprintf("%q", testCase.Name) + baseline.SQLFingerprint = sqlFingerprint(baseline.SQL) + baselineBranch := "compact_workspace_witness" + if rowCount == 0 { + baselineBranch = "compact_no_path" + } else if strings.Contains(testCase.Name, "cycle-control") { + baselineBranch = "one_hop_preflight" + } else if strings.Contains(testCase.Name, "early-d02") { + baselineBranch = "two_hop_preflight" + } + baseline.Stats = DurationStats{ + Iterations: samples, + WarmupIterations: warmups, + Median: 10 * time.Millisecond, + P95: 10 * time.Millisecond, + Samples: makeSamples("sp-i2-s4", baselineOrder, 10*time.Millisecond, baselineIdentity, baselineBranch, "timed_invocation"), + } + baselineOutcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, + TargetKind: "traversal", + Family: "SP", + Selected: baselineIdentity, + Applied: baselineIdentity, + Fallback: "SP-S0", + PlannedCandidates: spI2ShortestPathPlannedIdentities(), + SelectorVersion: "sp-tool-v1", + ExecutionBoundary: "stored_helper", + ObservationMode: "distance", + Scheduler: "single_ended_level", + Direction: "inbound", + PhysicalExpansion: "end_id", + RelationshipKindCount: 1, + TopologyClassification: "physical_inbound_deep", + SelectionMode: "forced_tool", + Eligible: &trueValue, + StaticallyEligible: &trueValue, + MinimumDepth: traversalTelemetryPointer(int64(1)), + MaximumDepth: traversalTelemetryPointer(int64(64)), + StateLimit: 100_000, + FrontierLimit: 100_000, + PredecessorLimit: 100_000, + EnumerationLimit: 100_000, + OutputBytesLimit: 64 * 1024 * 1024, + } + baseline.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{baselineOutcome}} + baselineMetrics := PostgresPlanMetrics{Provenance: map[string]string{}} + baseline.PostgresMetrics = &baselineMetrics + baselineTelemetry, err := buildPostgresCaseTraversalTelemetry(*baseline.Optimization, baselineMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + baseline.TraversalTelemetry = baselineTelemetry + + candidate := base + candidate.Environment = cloneSPI2TestEnvironment(baseEnvironment, "sp-i2-candidate", candidateOrder) + candidate.Environment.StartedAt, candidate.Environment.EndedAt = candidateStarted, candidateStarted.Add(time.Second) + candidate.SQL = "select 'i2:' || " + fmt.Sprintf("%q", testCase.Name) + candidate.SQLFingerprint = sqlFingerprint(candidate.SQL) + candidateBranch := "inline_canonical_distance" + if rowCount == 0 { + candidateBranch = "inline_canonical_distance_no_path" + } + candidate.Stats = DurationStats{ + Iterations: samples, + WarmupIterations: warmups, + Median: 8 * time.Millisecond, + P95: 8 * time.Millisecond, + Samples: makeSamples("sp-i2-candidate", candidateOrder, 8*time.Millisecond, candidateIdentity, candidateBranch, "timed_invocation"), + } + candidateOutcome := translate.TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, + TargetKind: "traversal", + Family: "SP", + Candidate: candidateIdentity, + Selected: candidateIdentity, + Applied: candidateIdentity, + Fallback: baselineIdentity, + PlannedCandidates: spI2ShortestPathPlannedIdentities(), + EmittedCandidates: []string{candidateIdentity, baselineIdentity}, + EmittedPolicy: optimize.ShortestPathPolicyI2DistanceGuardedV1, + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, + ExecutionBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + ObservationMode: "distance", + Scheduler: "single_ended_level", + Direction: "inbound", + PhysicalExpansion: "end_id", + RelationshipKindCount: 1, + TopologyClassification: "physical_inbound_deep", + SelectionMode: "forced_tool", + Eligible: &trueValue, + StaticallyEligible: &trueValue, + MinimumDepth: traversalTelemetryPointer(int64(1)), + MaximumDepth: traversalTelemetryPointer(int64(64)), + StateLimit: 100_000, + FrontierLimit: 100_000, + } + candidate.Optimization = &translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{candidateOutcome}} + ids := map[string]int64{ + "sp_i2_distance_bounded": 1, "sp_i2_target": 2, + "sp_i2_candidate_marker": 3, "sp_i2_fallback_marker": 4, + "sp_i2_candidate_rows": 5, "sp_i2_fallback_rows": 6, + } + planNode := func(name string, rows int64) PostgresPlanNodeMetric { + return PostgresPlanNodeMetric{PlanNodeID: ids[name], NodeType: "Result", SubplanName: "CTE " + name, ActualRows: rows, ActualLoops: 1} + } + markerGate := func(branch string, rows int64) PostgresPlanNodeMetric { + body := ids["sp_i2_"+branch+"_rows"] + return PostgresPlanNodeMetric{PlanNodeID: body + 100, ParentPlanNodeID: body, ParentRelationship: "Outer", NodeType: "CTE Scan", CTEName: "sp_i2_" + branch + "_marker", ActualRows: rows, ActualLoops: 1} + } + executor := func(branch string, loops int64) PostgresPlanNodeMetric { + body := ids["sp_i2_"+branch+"_rows"] + return PostgresPlanNodeMetric{PlanNodeID: body + 200, ParentPlanNodeID: body, ParentRelationship: "Inner", NodeType: "Result", ActualLoops: loops} + } + candidateMetrics := PostgresPlanMetrics{Provenance: map[string]string{}, PlanNodes: []PostgresPlanNodeMetric{ + planNode("sp_i2_distance_bounded", 32), planNode("sp_i2_target", rowCount), + planNode("sp_i2_candidate_marker", 1), planNode("sp_i2_fallback_marker", 0), + planNode("sp_i2_candidate_rows", rowCount), planNode("sp_i2_fallback_rows", 0), + markerGate("candidate", 1), markerGate("fallback", 0), executor("candidate", 1), executor("fallback", 0), + }} + candidate.PostgresMetrics = &candidateMetrics + candidateTelemetry, err := buildPostgresCaseTraversalTelemetry(*candidate.Optimization, candidateMetrics, "101", TraversalTelemetryLevelDiagnostic) + require.NoError(t, err) + enrichInlineDistanceTraversalTelemetry(candidateTelemetry, rowCount) + require.NoError(t, candidateTelemetry.Validate()) + candidate.TraversalTelemetry = candidateTelemetry + return baseline, candidate +} + +// cloneSPI2TestEnvironment returns an independent copy of spi2 test environment. +func cloneSPI2TestEnvironment(environment RunEnvironment, arm string, order int) *RunEnvironment { + copy := environment + copy.Arm = arm + copy.ArmOrder = order + return © +} + +// spI2QualificationTestFreeze prepares or inspects test evidence for sp i2 qualification test freeze. +func spI2QualificationTestFreeze(t *testing.T, discovery SPI2QualificationReport) SPI2QualificationFreezeManifest { + t.Helper() + cohort, err := canonicalSPI2Cohort() + require.NoError(t, err) + return SPI2QualificationFreezeManifest{ + Version: spI2FreezeVersion, + Baseline: discovery.Baseline, + Candidate: discovery.Candidate, + Policy: discovery.Policy, + QuerySHA256: discovery.QuerySHA256, + Caps: discovery.Caps, + Seed: discovery.Seed, + Confidence: discovery.Confidence, + BootstrapCount: discovery.BootstrapCount, + SourceCommit: discovery.SourceCommit, + SourceArchiveSHA256: discovery.SourceArchiveSHA256, + DirtyDiffSHA256: discovery.DirtyDiffSHA256, + BinarySHA256: discovery.BinarySHA256, + TrainingDeclarationSHA256: cohort.trainingDeclarationSHA256, + HoldoutDeclarationSHA256: cohort.holdoutDeclarationSHA256, + FullDeclarationSHA256: cohort.declarationSHA256, + TrainingCorpusSHA256: cohort.trainingCorpusSHA256, + FullCorpusSHA256: cohort.fullCorpusSHA256, + TrainingResolvedSHA256: cohort.trainingResolvedSHA256, + FullResolvedSHA256: cohort.fullResolvedSHA256, + BaselineArtifactSHA256: discovery.BaselineArtifactSHA256, + CandidateArtifactSHA256: discovery.CandidateArtifactSHA256, + ResourceReportSHA256: discovery.ResourceReportSHA256, + DiscoveryReportSHA256: strings.Repeat("4", 64), + TrainingPassed: discovery.TrainingPassed, + } +} diff --git a/cmd/graphbench/sp_i2_simulation_calibration_v2.go b/cmd/graphbench/sp_i2_simulation_calibration_v2.go new file mode 100644 index 00000000..4ab8ed26 --- /dev/null +++ b/cmd/graphbench/sp_i2_simulation_calibration_v2.go @@ -0,0 +1,219 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "math" + "os" + "sort" + "strings" + "time" +) + +const spI2V1CycleControl = "GSP-I2-V1-TRAIN-cycle-control" + +func createSPI2PowerSimulationReportV2(corpusRoot, baselineTrace, candidateTrace, output string) (SPI2PowerSimulationReportV2, error) { + protocol, protocolSHA256, err := loadSPI2ProtocolV2(corpusRoot + "/protocols/sp_i2_distance_v2.json") + if err != nil { + return SPI2PowerSimulationReportV2{}, err + } + if err := verifySPI2SimulationCalibrationV2(protocol, baselineTrace, candidateTrace); err != nil { + return SPI2PowerSimulationReportV2{}, err + } + report, err := buildSPI2PowerSimulationReportV2(protocol, protocolSHA256) + if err != nil { + return SPI2PowerSimulationReportV2{}, err + } + if err := writeSPI2PowerSimulationReportV2(output, report); err != nil { + return SPI2PowerSimulationReportV2{}, err + } + return report, nil +} + +func verifySPI2SimulationCalibrationV2(protocol spI2ProtocolV2, baselinePath, candidatePath string) error { + if baselinePath == "" || candidatePath == "" { + return fmt.Errorf("SP-I2 V2 simulation requires both archived V1 trace artifacts") + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil { + return fmt.Errorf("hash SP-I2 simulation baseline trace: %w", err) + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil { + return fmt.Errorf("hash SP-I2 simulation candidate trace: %w", err) + } + if baselineSHA256 != protocol.Simulation.BaselineTraceSHA256 || candidateSHA256 != protocol.Simulation.CandidateTraceSHA256 { + return fmt.Errorf("SP-I2 simulation trace digest differs from the frozen protocol") + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return fmt.Errorf("read SP-I2 simulation baseline trace: %w", err) + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return fmt.Errorf("read SP-I2 simulation candidate trace: %w", err) + } + records := append(append([]CaseResult(nil), baseline...), candidate...) + if err := verifySPI2SimulationTraceIdentityV2(records, protocol.Simulation.SourceCommit); err != nil { + return err + } + p50Drift, err := spI2RoundDriftV2(records, false) + if err != nil { + return err + } + p95Drift, err := spI2RoundDriftV2(records, true) + if err != nil { + return err + } + if !equalSPI2FloatVectorsV2(p50Drift, protocol.Simulation.P50RoundDrift) || !equalSPI2FloatVectorsV2(p95Drift, protocol.Simulation.P95RoundDrift) { + return fmt.Errorf("SP-I2 simulation round-drift vectors differ from the frozen protocol") + } + calibration, err := deriveSPI2SimulationErrorsV2(baseline) + if err != nil { + return err + } + if calibration.log != protocol.Simulation.LogStandardErrors || calibration.absolute != protocol.Simulation.AbsoluteStandardErrorsUS { + return fmt.Errorf("SP-I2 simulation uncertainty calibration differs from the frozen protocol: log=%+v absolute=%+v", calibration.log, calibration.absolute) + } + return nil +} + +func verifySPI2SimulationTraceIdentityV2(records []CaseResult, sourceCommit string) error { + if len(records) != 240 { + return fmt.Errorf("SP-I2 simulation traces require exactly 240 case records") + } + for _, record := range records { + if !strings.Contains(record.Metadata.DAWGSVersion, sourceCommit) || record.Environment.Round < 1 || record.Environment.Round > 20 || + record.Stats.Iterations != 10 || len(record.Stats.Samples) < 10 { + return fmt.Errorf("SP-I2 simulation trace identity or fixed V1 design is invalid") + } + } + return nil +} + +func spI2RoundDriftV2(records []CaseResult, p95 bool) ([]float64, error) { + logs := make([][]float64, 20) + for _, record := range records { + value := record.Stats.Median + if p95 { + value = record.Stats.P95 + } + if value <= 0 { + return nil, fmt.Errorf("SP-I2 simulation trace contains a non-positive quantile") + } + logs[record.Environment.Round-1] = append(logs[record.Environment.Round-1], math.Log(float64(value))) + } + roundMeans := make([]float64, len(logs)) + grand := 0.0 + for index, values := range logs { + if len(values) != 12 { + return nil, fmt.Errorf("SP-I2 simulation round %d requires exactly 12 trace records", index+1) + } + for _, value := range values { + roundMeans[index] += value + } + roundMeans[index] /= float64(len(values)) + grand += roundMeans[index] + } + grand /= float64(len(roundMeans)) + for index := range roundMeans { + roundMeans[index] = math.Exp(roundMeans[index] - grand) + } + return roundMeans, nil +} + +type spI2DerivedSimulationErrorsV2 struct { + log spI2SimulationErrorsV2 + absolute spI2SimulationErrorsV2 +} + +func deriveSPI2SimulationErrorsV2(records []CaseResult) (spI2DerivedSimulationErrorsV2, error) { + var pooled []time.Duration + cycleRecords := make([]CaseResult, 0, 20) + for _, record := range records { + if record.Name != spI2V1CycleControl { + continue + } + cycleRecords = append(cycleRecords, record) + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" { + pooled = append(pooled, sample.Duration) + } + } + } + if len(cycleRecords) != 20 || len(pooled) != 200 { + return spI2DerivedSimulationErrorsV2{}, fmt.Errorf("SP-I2 simulation calibration requires the complete 20-round V1 cycle control") + } + sort.Slice(pooled, func(left, right int) bool { return pooled[left] < pooled[right] }) + p50 := float64(pooled[99]) + p95 := float64(pooled[189]) + exponent := math.Log(2) / math.Log(p95/p50) + calibrated := roundSamples{} + for _, record := range cycleRecords { + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" { + continue + } + microseconds := time.Duration(1000 * math.Pow(float64(sample.Duration)/p50, exponent)) + for range 10 { + calibrated[record.Environment.Round] = append(calibrated[record.Environment.Round], microseconds*time.Microsecond) + } + } + calibrated[record.Environment.Round+20] = append([]time.Duration(nil), calibrated[record.Environment.Round]...) + } + interval, err := bootstrapSPI2HierarchicalTailV2(calibrated, calibrated, "simulation-calibration", "v1-cycle-control", "p95", 0.95, 0.975, 100_000) + if err != nil { + return spI2DerivedSimulationErrorsV2{}, err + } + z := 2.241402727604947 + pooledLog := roundSPI2CalibrationUpV2((math.Log(interval.Ratio.Upper)-math.Log(interval.Ratio.Lower))/(2*z), 1_000_000) + pooledAbsolute := roundSPI2CalibrationUpV2(math.Max(math.Abs(float64(interval.Change.Lower/time.Microsecond)), math.Abs(float64(interval.Change.Upper/time.Microsecond)))/z, 1_000) + stratumLog := roundSPI2CalibrationUpV2(pooledLog*math.Sqrt2, 1_000_000) + stratumAbsolute := roundSPI2CalibrationUpV2(pooledAbsolute*math.Sqrt2, 1_000) + return spI2DerivedSimulationErrorsV2{ + log: spI2SimulationErrorsV2{Pooled: pooledLog, OrderStratum: stratumLog, FirstPosition: stratumLog}, + absolute: spI2SimulationErrorsV2{Pooled: pooledAbsolute, OrderStratum: stratumAbsolute, FirstPosition: stratumAbsolute}, + }, nil +} + +func roundSPI2CalibrationUpV2(value, precision float64) float64 { + return math.Ceil(value*precision) / precision +} + +func equalSPI2FloatVectorsV2(left, right []float64) bool { + if len(left) != len(right) { + return false + } + for index := range left { + if math.Abs(left[index]-right[index]) > 1e-12 { + return false + } + } + return true +} + +func writeSPI2PowerSimulationReportV2(path string, report SPI2PowerSimulationReportV2) (err error) { + var output *os.File + if path == "" { + output = os.Stdout + } else { + if err := ensureOutputDir(path); err != nil { + return err + } + output, err = os.Create(path) + if err != nil { + return err + } + defer func() { + if closeErr := output.Close(); err == nil && closeErr != nil { + err = closeErr + } + }() + } + encoder := json.NewEncoder(output) + encoder.SetIndent("", " ") + return encoder.Encode(report) +} diff --git a/cmd/graphbench/sp_i2_successor_power_study_v3.go b/cmd/graphbench/sp_i2_successor_power_study_v3.go new file mode 100644 index 00000000..047b0f54 --- /dev/null +++ b/cmd/graphbench/sp_i2_successor_power_study_v3.go @@ -0,0 +1,186 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" +) + +const spI2SuccessorPowerStudyV3Implementation = "sp-i2-power-simulation-v3/chacha8-sha256-normal-pivot" + +type spI2SuccessorPowerStudyV3 struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + Implementation string `json:"implementation"` + Status string `json:"status"` + ArchivedTrace spI2SuccessorArchivedTraceV3 `json:"archived_trace"` + Design spI2SuccessorPowerDesignV3 `json:"design"` + Statistics spI2SuccessorPowerStatisticsV3 `json:"statistics"` + Gates spI2ProtocolGatesV2 `json:"gates"` + Scenarios []spI2SimulationScenarioV2 `json:"scenarios"` +} + +type spI2SuccessorArchivedTraceV3 struct { + SourceCommit string `json:"source_commit"` + BaselineTraceSHA256 string `json:"baseline_trace_sha256"` + CandidateTraceSHA256 string `json:"candidate_trace_sha256"` + Rounds int `json:"rounds"` + CaseRecordsPerRound int `json:"case_records_per_round"` + TimedSamplesPerRecord int `json:"timed_samples_per_record"` +} + +type spI2SuccessorPowerDesignV3 struct { + Blocks int `json:"blocks"` + OrdinaryWarmups int `json:"ordinary_warmups"` + TimedSamplesPerArmCaseBlock int `json:"timed_samples_per_arm_case_block"` + PoolSize int `json:"pool_size"` + Isolation string `json:"isolation"` + ArmOrder string `json:"arm_order"` +} + +type spI2SuccessorPowerStatisticsV3 struct { + BootstrapConfidence float64 `json:"bootstrap_confidence"` + BootstrapReplicates int `json:"bootstrap_replicates"` + Quantile string `json:"quantile"` + WilsonConfidence float64 `json:"wilson_confidence"` + SimulationRunsPerScenario int `json:"simulation_runs_per_scenario"` + RequiredPowerLower float64 `json:"required_power_lower"` + RequiredCoverage float64 `json:"required_coverage"` + P95BoundaryFalsePassUpper float64 `json:"p95_boundary_false_pass_upper"` + DecisionFalsePassUpper float64 `json:"decision_false_pass_upper"` + TraceRescalingTransform string `json:"trace_rescaling_transform"` +} + +type SPI2SuccessorPowerStudyReportV3 struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + Implementation string `json:"implementation"` + ProtocolSHA256 string `json:"protocol_sha256"` + CalibrationScale float64 `json:"calibration_scale"` + LogStandardErrors spI2SimulationErrorsV2 `json:"log_standard_errors"` + AbsoluteStandardErrors spI2SimulationErrorsV2 `json:"absolute_standard_errors_us"` + Passed bool `json:"passed"` + Scenarios []SPI2PowerSimulationScenarioReportV2 `json:"scenarios"` +} + +func loadSPI2SuccessorPowerStudyV3(path string) (spI2SuccessorPowerStudyV3, string, error) { + raw, err := os.ReadFile(path) + if err != nil { + return spI2SuccessorPowerStudyV3{}, "", fmt.Errorf("read SP-I2 successor power study: %w", err) + } + var study spI2SuccessorPowerStudyV3 + if err := json.Unmarshal(raw, &study); err != nil { + return spI2SuccessorPowerStudyV3{}, "", fmt.Errorf("decode SP-I2 successor power study: %w", err) + } + if err := validateSPI2SuccessorPowerStudyV3(study); err != nil { + return spI2SuccessorPowerStudyV3{}, "", err + } + digest := sha256.Sum256(raw) + return study, hex.EncodeToString(digest[:]), nil +} + +func validateSPI2SuccessorPowerStudyV3(study spI2SuccessorPowerStudyV3) error { + if study.Schema != "sp-i2-successor-power-study-v3" || study.Generation != "sp-i2-distance-v3-power-study" || + study.Implementation != spI2SuccessorPowerStudyV3Implementation || study.Status != "prospective" { + return fmt.Errorf("SP-I2 successor power study identity is invalid") + } + trace := study.ArchivedTrace + if trace.SourceCommit != "3865cbc57758b7b20b7ffe431f27235873422eed" || + trace.BaselineTraceSHA256 != "ac3ceb27ee92e3f4e21e3994ff9ee82d483b8081e9d44ddcef8e695ffdb1b6d0" || + trace.CandidateTraceSHA256 != "f6d79e81bdaafedaa95568d57140c14e0808fbb6fc261387abc916081137785a" || + trace.Rounds != 20 || trace.CaseRecordsPerRound != 12 || trace.TimedSamplesPerRecord != 10 { + return fmt.Errorf("SP-I2 successor power study archive contract is invalid") + } + design := study.Design + if design.Blocks != 800 || design.OrdinaryWarmups != 25 || design.TimedSamplesPerArmCaseBlock != 100 || + design.PoolSize != 1 || design.Isolation != "repeatable_read" || + design.ArmOrder != "odd_incumbent_then_candidate_even_candidate_then_incumbent" { + return fmt.Errorf("SP-I2 successor power study design is invalid") + } + stats := study.Statistics + if stats.BootstrapConfidence != 0.975 || stats.BootstrapReplicates != 100_000 || stats.Quantile != "nearest_rank" || + stats.WilsonConfidence != 0.95 || stats.SimulationRunsPerScenario != 20_000 || stats.RequiredPowerLower != 0.90 || + stats.RequiredCoverage != 0.975 || stats.P95BoundaryFalsePassUpper != 0.015 || stats.DecisionFalsePassUpper != 0.0275 || + stats.TraceRescalingTransform != "scaled_v2_calibration_then_paired_empirical_round_drift" { + return fmt.Errorf("SP-I2 successor power study statistics are invalid") + } + if study.Gates != (spI2ProtocolGatesV2{TargetMedianRatioUpper: 0.95, TargetMedianSavingLowerUS: 100, ControlMedianRatioUpper: 1.10, ControlMedianOverheadUpperUS: 100, P95RatioUpper: 1.05, ControlP95OverheadUpperUS: 100, AAEquivalenceRatio: 1.05, AAFirstPositionRatioUpper: 1.10, AAFirstPositionOverheadUpperUS: 100}) { + return fmt.Errorf("SP-I2 successor power study gates are invalid") + } + if len(study.Scenarios) != 11 { + return fmt.Errorf("SP-I2 successor power study scenario count is invalid") + } + for _, scenario := range study.Scenarios { + seed := sha256.Sum256([]byte("sp-i2-power-study-v3\x00" + scenario.Name)) + if scenario.Name == "" || scenario.BaselineP50US <= 0 || scenario.BaselineP95US <= scenario.BaselineP50US || + scenario.CandidateP50US <= 0 || scenario.CandidateP95US <= scenario.CandidateP50US || scenario.Seed != hex.EncodeToString(seed[:]) { + return fmt.Errorf("SP-I2 successor power study scenario %q is invalid", scenario.Name) + } + } + return nil +} + +func buildSPI2SuccessorPowerStudyReportV3(study spI2SuccessorPowerStudyV3, protocolSHA256, baselinePath, candidatePath string) (SPI2SuccessorPowerStudyReportV3, error) { + if err := validateSPI2SuccessorPowerStudyV3(study); err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + baselineSHA256, err := fileSHA256(baselinePath) + if err != nil || baselineSHA256 != study.ArchivedTrace.BaselineTraceSHA256 { + return SPI2SuccessorPowerStudyReportV3{}, fmt.Errorf("SP-I2 successor power study baseline trace digest is invalid") + } + candidateSHA256, err := fileSHA256(candidatePath) + if err != nil || candidateSHA256 != study.ArchivedTrace.CandidateTraceSHA256 { + return SPI2SuccessorPowerStudyReportV3{}, fmt.Errorf("SP-I2 successor power study candidate trace digest is invalid") + } + baseline, err := readJSONLFile(baselinePath) + if err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + candidate, err := readJSONLFile(candidatePath) + if err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + records := append(append([]CaseResult(nil), baseline...), candidate...) + if err := verifySPI2SimulationTraceIdentityV2(records, study.ArchivedTrace.SourceCommit); err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + calibration, err := deriveSPI2SimulationErrorsV2(baseline) + if err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + scale := math.Sqrt(float64(40*100) / float64(study.Design.Blocks*study.Design.TimedSamplesPerArmCaseBlock)) + model := spI2ProtocolV2{Design: spI2ProtocolDesignV2{Rounds: study.Design.Blocks}, Gates: study.Gates, Simulation: spI2ProtocolSimulationV2{ + RunsPerScenario: study.Statistics.SimulationRunsPerScenario, RequiredPowerLower: study.Statistics.RequiredPowerLower, + RequiredCoverage: study.Statistics.RequiredCoverage, P95BoundaryFalsePassUpper: study.Statistics.P95BoundaryFalsePassUpper, + DecisionFalsePassUpper: study.Statistics.DecisionFalsePassUpper, LogStandardErrors: scaleSPI2SimulationErrorsV3(calibration.log, scale), + AbsoluteStandardErrorsUS: scaleSPI2SimulationErrorsV3(calibration.absolute, scale), Scenarios: study.Scenarios, + }} + model.Simulation.P50RoundDrift, err = spI2RoundDriftV2(records, false) + if err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + model.Simulation.P95RoundDrift, err = spI2RoundDriftV2(records, true) + if err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + report := SPI2SuccessorPowerStudyReportV3{Schema: "sp-i2-successor-power-study-report-v3", Generation: study.Generation, Implementation: study.Implementation, ProtocolSHA256: protocolSHA256, CalibrationScale: scale, LogStandardErrors: model.Simulation.LogStandardErrors, AbsoluteStandardErrors: model.Simulation.AbsoluteStandardErrorsUS, Passed: true} + for _, scenario := range study.Scenarios { + result, err := simulateSPI2ScenarioV2(model, scenario) + if err != nil { + return SPI2SuccessorPowerStudyReportV3{}, err + } + report.Scenarios = append(report.Scenarios, result) + report.Passed = report.Passed && result.Passed + } + return report, nil +} + +func scaleSPI2SimulationErrorsV3(errors spI2SimulationErrorsV2, scale float64) spI2SimulationErrorsV2 { + return spI2SimulationErrorsV2{Pooled: errors.Pooled * scale, OrderStratum: errors.OrderStratum * scale, FirstPosition: errors.FirstPosition * scale} +} diff --git a/cmd/graphbench/sp_i2_successor_power_study_v3_test.go b/cmd/graphbench/sp_i2_successor_power_study_v3_test.go new file mode 100644 index 00000000..2ff79974 --- /dev/null +++ b/cmd/graphbench/sp_i2_successor_power_study_v3_test.go @@ -0,0 +1,46 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSPI2SuccessorPowerStudyV3TerminalTombstone(t *testing.T) { + path := filepath.Join("..", "..", "benchmark", "testdata", "scale", "protocols", "sp_i2_successor_power_study_v3_rejection.json") + raw, err := os.ReadFile(path) + require.NoError(t, err) + var tombstone struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + ProtocolSHA string `json:"protocol_sha256"` + Terminal bool `json:"terminal"` + Implemented bool `json:"candidate_implemented"` + Corpus bool `json:"corpus_created"` + Timed bool `json:"database_timing_started"` + Holdout bool `json:"holdout_opened"` + FailedGates []struct { + Scenario string `json:"scenario"` + Observed float64 `json:"observed"` + Required float64 `json:"required"` + } `json:"failed_gates"` + } + require.NoError(t, json.Unmarshal(raw, &tombstone)) + require.Equal(t, "sp-i2-successor-power-study-rejection-v3", tombstone.Schema) + require.Equal(t, "sp-i2-distance-v3-power-study", tombstone.Generation) + require.Equal(t, "e11090bbbe73cc36dfae2af97e26b6e1fc4d42590fc6fd331b2204c7a9e04f31", tombstone.ProtocolSHA) + require.True(t, tombstone.Terminal) + require.False(t, tombstone.Implemented || tombstone.Corpus || tombstone.Timed || tombstone.Holdout) + require.Len(t, tombstone.FailedGates, 2) + require.Equal(t, []string{"aa_order_odd_high", "aa_order_even_high"}, []string{tombstone.FailedGates[0].Scenario, tombstone.FailedGates[1].Scenario}) + require.InDelta(t, 0.14201232557116983, tombstone.FailedGates[0].Observed, 1e-15) + require.InDelta(t, 0.14723913101703448, tombstone.FailedGates[1].Observed, 1e-15) + require.Equal(t, 0.90, tombstone.FailedGates[0].Required) + require.Equal(t, 0.90, tombstone.FailedGates[1].Required) +} diff --git a/cmd/graphbench/sp_i2_tail_qualification_v2.go b/cmd/graphbench/sp_i2_tail_qualification_v2.go new file mode 100644 index 00000000..f852a2db --- /dev/null +++ b/cmd/graphbench/sp_i2_tail_qualification_v2.go @@ -0,0 +1,178 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "sort" + "time" +) + +const ( + spI2TailQualificationSchemaV2 = "sp-i2-tail-qualification-v2" + spI2TailFreezeSchemaV2 = "sp-i2-tail-freeze-v2" + spI2TailAASchemaV2 = "sp-i2-tail-aa-v2" +) + +// SPI2TailQualificationV2 is the generation-specific report schema. It is +// intentionally separate from the archived V1 reporter. +type SPI2TailQualificationV2 struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + ProtocolDeclarationSHA256 string `json:"protocol_declaration_sha256"` + Executor string `json:"executor"` + Policy string `json:"policy"` + Selector string `json:"selector"` + Baseline string `json:"baseline"` + StatisticalImplementation string `json:"statistical_implementation"` + Confidence float64 `json:"confidence"` + BootstrapReplicates int `json:"bootstrap_replicates"` + Rounds int `json:"rounds"` + SamplesPerRound int `json:"samples_per_round"` + MultiplicityRule string `json:"multiplicity_rule"` + Cases []SPI2TailQualificationCaseV2 `json:"cases"` + Passed bool `json:"passed"` +} + +// SPI2TailQualificationCaseV2 contains signs explicitly: P95Change is +// candidate minus baseline, while MedianSaving is baseline minus candidate. +type SPI2TailQualificationCaseV2 struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + Role string `json:"role"` + MedianRatio RatioInterval `json:"median_ratio"` + MedianSaving DurationInterval `json:"median_saving"` + P95Ratio RatioInterval `json:"p95_ratio"` + P95Change DurationInterval `json:"p95_change"` + WorstMedianOverhead time.Duration `json:"worst_median_overhead"` + WorstP95Overhead time.Duration `json:"worst_p95_overhead"` + SemanticPassed bool `json:"semantic_passed"` + ReceiptPassed bool `json:"receipt_passed"` + ResourcePassed bool `json:"resource_passed"` + SchedulePassed bool `json:"schedule_passed"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +// SPI2TailCaseInputV2 supplies already validated native samples and the +// conjunctive non-timing gates. Artifact readers must establish chronology and +// identity before constructing this value. +type SPI2TailCaseInputV2 struct { + Dataset string + Name string + Role string + Baseline roundSamples + Candidate roundSamples + SemanticPassed bool + ReceiptPassed bool + ResourcePassed bool + SchedulePassed bool +} + +func buildSPI2TailQualificationV2(protocol spI2ProtocolV2, protocolSHA256 string, inputs []SPI2TailCaseInputV2) (SPI2TailQualificationV2, error) { + if err := validateSPI2ProtocolV2(protocol); err != nil { + return SPI2TailQualificationV2{}, err + } + if !lowercaseSHA256(protocolSHA256) { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 report requires the exact protocol declaration SHA-256") + } + if len(inputs) == 0 { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 report requires at least one declared case") + } + sort.Slice(inputs, func(left, right int) bool { + if inputs[left].Dataset == inputs[right].Dataset { + return inputs[left].Name < inputs[right].Name + } + return inputs[left].Dataset < inputs[right].Dataset + }) + report := SPI2TailQualificationV2{ + Schema: spI2TailQualificationSchemaV2, + Generation: protocol.Generation, + ProtocolDeclarationSHA256: protocolSHA256, + Executor: protocol.Identities.Executor, + Policy: protocol.Identities.Policy, + Selector: protocol.Identities.Selector, + Baseline: protocol.Identities.FallbackExecutor, + StatisticalImplementation: protocol.Identities.StatisticalImplementation, + Confidence: protocol.Design.ConfidenceLevel, + BootstrapReplicates: protocol.Design.BootstrapReplicates, + Rounds: protocol.Design.Rounds, + SamplesPerRound: protocol.Design.TimedSamplesPerRound, + MultiplicityRule: protocol.MultiplicityRule, + Passed: true, + } + seen := map[string]struct{}{} + for _, input := range inputs { + key := input.Dataset + "\x00" + input.Name + if input.Dataset == "" || input.Name == "" { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 case identity is incomplete") + } + if _, duplicate := seen[key]; duplicate { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 case %s/%s is duplicated", input.Dataset, input.Name) + } + seen[key] = struct{}{} + if input.Role != "adverse_control" && input.Role != "efficacy_target" { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 case %s/%s has invalid preregistered role %q", input.Dataset, input.Name, input.Role) + } + rounds, err := validateSPI2HierarchicalInputs(input.Baseline, input.Candidate, 0.95, protocol.Design.ConfidenceLevel, protocol.Design.BootstrapReplicates) + if err != nil { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 case %s/%s: %w", input.Dataset, input.Name, err) + } + if len(rounds) != protocol.Design.Rounds { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 case %s/%s requires exactly %d rounds", input.Dataset, input.Name, protocol.Design.Rounds) + } + for _, round := range rounds { + if len(input.Baseline[round]) != protocol.Design.TimedSamplesPerRound { + return SPI2TailQualificationV2{}, fmt.Errorf("SP-I2 V2 case %s/%s round %d requires exactly %d samples per arm", input.Dataset, input.Name, round, protocol.Design.TimedSamplesPerRound) + } + } + medianRatio, medianSaving, err := bootstrapSPI2RoundMedianV2(input.Baseline, input.Candidate, input.Dataset, input.Name, "median", protocol.Design.ConfidenceLevel, protocol.Design.BootstrapReplicates) + if err != nil { + return SPI2TailQualificationV2{}, err + } + p95, err := bootstrapSPI2HierarchicalTailV2(input.Baseline, input.Candidate, input.Dataset, input.Name, "p95", 0.95, protocol.Design.ConfidenceLevel, protocol.Design.BootstrapReplicates) + if err != nil { + return SPI2TailQualificationV2{}, err + } + entry := SPI2TailQualificationCaseV2{ + Dataset: input.Dataset, Name: input.Name, Role: input.Role, + MedianRatio: medianRatio, MedianSaving: medianSaving, P95Ratio: p95.Ratio, P95Change: p95.Change, + WorstMedianOverhead: -medianSaving.Lower, WorstP95Overhead: p95.Change.Upper, + SemanticPassed: input.SemanticPassed, ReceiptPassed: input.ReceiptPassed, + ResourcePassed: input.ResourcePassed, SchedulePassed: input.SchedulePassed, Passed: true, + } + if input.Role == "efficacy_target" { + if medianRatio.Upper > protocol.Gates.TargetMedianRatioUpper && medianSaving.Lower < time.Duration(protocol.Gates.TargetMedianSavingLowerUS)*time.Microsecond { + entry.Reasons = append(entry.Reasons, "median materiality gate failed") + } + } else { + if medianRatio.Upper > protocol.Gates.ControlMedianRatioUpper && entry.WorstMedianOverhead > time.Duration(protocol.Gates.ControlMedianOverheadUpperUS)*time.Microsecond { + entry.Reasons = append(entry.Reasons, "adverse-control median containment gate failed") + } + if entry.WorstP95Overhead > time.Duration(protocol.Gates.ControlP95OverheadUpperUS)*time.Microsecond { + entry.Reasons = append(entry.Reasons, "adverse-control absolute p95 containment gate failed") + } + } + if p95.Ratio.Upper > protocol.Gates.P95RatioUpper { + entry.Reasons = append(entry.Reasons, "relative p95 containment gate failed") + } + if !entry.SemanticPassed || !entry.ReceiptPassed || !entry.ResourcePassed || !entry.SchedulePassed { + entry.Reasons = append(entry.Reasons, "one or more non-timing gates failed") + } + entry.Passed = len(entry.Reasons) == 0 + report.Passed = report.Passed && entry.Passed + report.Cases = append(report.Cases, entry) + } + return report, nil +} + +// SPI2TailFreezeV2 binds discovery to the exact protocol and raw native +// artifacts. Creation is permitted only from a passing V2 report. +type SPI2TailFreezeV2 struct { + Schema string `json:"schema"` + Generation string `json:"generation"` + ProtocolDeclarationSHA256 string `json:"protocol_declaration_sha256"` + QualificationReportSHA256 string `json:"qualification_report_sha256"` + RawArtifactSHA256 map[string]string `json:"raw_artifact_sha256"` +} diff --git a/cmd/graphbench/statistical_evidence.go b/cmd/graphbench/statistical_evidence.go new file mode 100644 index 00000000..c267a3cb --- /dev/null +++ b/cmd/graphbench/statistical_evidence.go @@ -0,0 +1,798 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "sort" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +const ( + // defaultConfidenceLevel is the default confidence used by qualification reports. + defaultConfidenceLevel = 0.975 + // minimumTimingNoiseRatio is the smallest relative timing floor accepted for promotion decisions. + minimumTimingNoiseRatio = 0.05 + // minimumTimingNoiseAbsolute is the smallest absolute timing floor accepted for promotion decisions. + minimumTimingNoiseAbsolute = 100 * time.Microsecond +) + +// benchmarkHostIdentity contains stable host properties that must match an A/A calibration. +type benchmarkHostIdentity struct { + // GOOS supplies the goos input to the benchmarkHostIdentity contract. + GOOS string `json:"goos"` + // GOARCH supplies the goarch input to the benchmarkHostIdentity contract. + GOARCH string `json:"goarch"` + // CPUCount records the number of cpu count. + CPUCount int `json:"cpu_count"` + // CPUModel supplies the cpu model input to the benchmarkHostIdentity contract. + CPUModel string `json:"cpu_model"` + // Kernel supplies the kernel input to the benchmarkHostIdentity contract. + Kernel string `json:"kernel"` + // CgroupCPU supplies the cgroup cpu input to the benchmarkHostIdentity contract. + CgroupCPU string `json:"cgroup_cpu,omitempty"` + // CgroupMemory supplies the cgroup memory input to the benchmarkHostIdentity contract. + CgroupMemory string `json:"cgroup_memory,omitempty"` + // CPUGovernor supplies the cpu governor input to the benchmarkHostIdentity contract. + CPUGovernor string `json:"cpu_governor,omitempty"` +} + +// artifactHostFingerprint returns one stable host fingerprint for all PostgreSQL timing records. +func artifactHostFingerprint(records []CaseResult) (string, error) { + fingerprint := "" + found := false + for _, record := range records { + if record.ExecutionMode != ModePostgresSQL || !hasWarmLatencySample(record) { + continue + } + if record.Environment == nil { + return "", fmt.Errorf("%s/%s has no run environment for host calibration", record.Dataset, record.Name) + } + identity := benchmarkHostIdentity{ + GOOS: strings.TrimSpace(record.Environment.GOOS), + GOARCH: strings.TrimSpace(record.Environment.GOARCH), + CPUCount: record.Environment.CPUCount, + CPUModel: strings.TrimSpace(record.Environment.CPUModel), + Kernel: strings.TrimSpace(record.Environment.Kernel), + CgroupCPU: strings.TrimSpace(record.Environment.CgroupCPU), + CgroupMemory: strings.TrimSpace(record.Environment.CgroupMemory), + CPUGovernor: strings.TrimSpace(record.Environment.CPUGovernor), + } + if identity.GOOS == "" || identity.GOARCH == "" || identity.CPUCount < 1 || identity.CPUModel == "" || identity.Kernel == "" { + return "", fmt.Errorf("%s/%s has incomplete host identity", record.Dataset, record.Name) + } + raw, err := json.Marshal(identity) + if err != nil { + return "", err + } + digest := sha256.Sum256(raw) + current := hex.EncodeToString(digest[:]) + if fingerprint != "" && current != fingerprint { + return "", fmt.Errorf("PostgreSQL timing artifact mixes host identities") + } + fingerprint = current + found = true + } + if !found { + return "", fmt.Errorf("artifact has no PostgreSQL warm timing records for host calibration") + } + + return fingerprint, nil +} + +// hasWarmLatencySample reports whether has warm latency sample. +func hasWarmLatencySample(record CaseResult) bool { + for _, sample := range record.Stats.Samples { + if sample.Classification == "warm" && sample.Duration > 0 { + return true + } + } + return false +} + +// validateAAResolutionEvidence verifies schema, checksum, confidence, host, and per-case metric integrity. +func validateAAResolutionEvidence(report *AAResolutionReport, records []CaseResult, confidence float64) error { + if report == nil { + return fmt.Errorf("host A/A resolution report is required") + } + if report.Version != aaReportVersion { + return fmt.Errorf("A/A report version must be %d", aaReportVersion) + } + if report.Confidence <= 0 || report.Confidence >= 1 || math.IsNaN(report.Confidence) || report.Confidence < confidence { + return fmt.Errorf("A/A confidence %.4f is below requested confidence %.4f", report.Confidence, confidence) + } + if !validSHA256(report.ArtifactSHA256) { + return fmt.Errorf("A/A artifact SHA-256 is missing or malformed") + } + chronology := report.PhysicalChronology + if chronology == nil || chronology.Version != aaPhysicalChronologyVersion || !chronology.Validated || + chronology.ArtifactSHA256 != report.ArtifactSHA256 || !validSHA256(chronology.ArtifactSHA256) || + chronology.Rounds < report.MinimumRounds || len(chronology.Arms) != 2 || + strings.TrimSpace(chronology.Arms[0]) == "" || strings.TrimSpace(chronology.Arms[1]) == "" || chronology.Arms[0] == chronology.Arms[1] { + return fmt.Errorf("A/A report lacks artifact-bound physical chronology provenance") + } + hostFingerprint, err := artifactHostFingerprint(records) + if err != nil { + return err + } + if !validSHA256(report.HostFingerprint) || report.HostFingerprint != hostFingerprint { + return fmt.Errorf("A/A host fingerprint does not match timing artifact host") + } + if report.MinimumRounds < minimumGateRounds || report.MinimumSamplesPerArmPerRound < 10 || !report.OrderBalanced { + return fmt.Errorf("A/A report lacks the balanced discovery evidence protocol") + } + if len(report.Cases) == 0 { + return fmt.Errorf("A/A report contains no case resolution evidence") + } + + seen := map[performanceKey]struct{}{} + for _, entry := range report.Cases { + key := performanceKey{ + dataset: entry.Dataset, + name: entry.Name, + backend: entry.Backend, + } + if entry.Dataset == "" || entry.Name == "" || entry.Backend != ModePostgresSQL { + return fmt.Errorf("A/A report contains malformed case identity") + } + if strings.TrimSpace(entry.WorkloadSHA256) == "" { + return fmt.Errorf("A/A case %s/%s has no workload identity", key.dataset, key.name) + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("A/A report contains duplicate case %s/%s/%s", key.dataset, key.name, key.backend) + } + seen[key] = struct{}{} + if entry.Rounds < minimumGateRounds || entry.SamplesPerArm < entry.Rounds*report.MinimumSamplesPerArmPerRound { + return fmt.Errorf("A/A case %s/%s lacks discovery-grade rounds or samples", key.dataset, key.name) + } + if err := validateAAMetric(entry.P50); err != nil { + return fmt.Errorf("A/A case %s/%s p50: %w", key.dataset, key.name, err) + } + if err := validateAAMetric(entry.P95); err != nil { + return fmt.Errorf("A/A case %s/%s p95: %w", key.dataset, key.name, err) + } + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend && record.WorkloadSHA256 != entry.WorkloadSHA256 { + return fmt.Errorf("A/A workload identity does not match %s/%s/%s", key.dataset, key.name, key.backend) + } + } + } + + return nil +} + +// workloadSHA256ForKey derives the lookup key used for workload sha256 for. +func workloadSHA256ForKey(records []CaseResult, key performanceKey) (string, error) { + identity := "" + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if record.WorkloadSHA256 == "" { + return "", fmt.Errorf("%s/%s/%s has no workload identity", key.dataset, key.name, key.backend) + } + if identity != "" && identity != record.WorkloadSHA256 { + return "", fmt.Errorf("%s/%s/%s mixes workload identities", key.dataset, key.name, key.backend) + } + identity = record.WorkloadSHA256 + } + if identity == "" { + return "", fmt.Errorf("%s/%s/%s has no workload record", key.dataset, key.name, key.backend) + } + return identity, nil +} + +// postgresTimingEnvironmentSHA256ForKey derives the lookup key used for postgres timing environment sha256 for. +func postgresTimingEnvironmentSHA256ForKey(records []CaseResult, key performanceKey) (string, error) { + identity := "" + found, missing := false, false + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + found = true + if record.PostgresEnvironment == nil { + missing = true + continue + } + value := *record.PostgresEnvironment + value.AnalyzeState = normalizedAnalyzeState(value.AnalyzeState) + raw, err := json.Marshal(value) + if err != nil { + return "", fmt.Errorf("encode %s/%s/%s PostgreSQL timing environment: %w", key.dataset, key.name, key.backend, err) + } + digest := sha256.Sum256(raw) + current := hex.EncodeToString(digest[:]) + if identity != "" && identity != current { + return "", fmt.Errorf("%s/%s/%s mixes PostgreSQL timing environments", key.dataset, key.name, key.backend) + } + identity = current + } + if !found { + return "", fmt.Errorf("%s/%s/%s has no workload record", key.dataset, key.name, key.backend) + } + if missing && identity != "" { + return "", fmt.Errorf("%s/%s/%s has partially missing PostgreSQL timing environment", key.dataset, key.name, key.backend) + } + return identity, nil +} + +// fixtureSHA256ForKey derives the lookup key used for fixture sha256 for. +func fixtureSHA256ForKey(records []CaseResult, key performanceKey) (string, error) { + identity := "" + found, missing := false, false + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + found = true + if record.Fixture == nil { + missing = true + continue + } + raw, err := json.Marshal(record.Fixture) + if err != nil { + return "", fmt.Errorf("encode %s/%s/%s fixture: %w", key.dataset, key.name, key.backend, err) + } + digest := sha256.Sum256(raw) + current := hex.EncodeToString(digest[:]) + if identity != "" && identity != current { + return "", fmt.Errorf("%s/%s/%s mixes fixture identities", key.dataset, key.name, key.backend) + } + identity = current + } + if !found { + return "", fmt.Errorf("%s/%s/%s has no workload record", key.dataset, key.name, key.backend) + } + if missing && identity != "" { + return "", fmt.Errorf("%s/%s/%s has partially missing fixture identity", key.dataset, key.name, key.backend) + } + return identity, nil +} + +// normalizedAnalyzeState normalizes d analyze state. +func normalizedAnalyzeState(value string) string { + if strings.TrimSpace(value) == "" { + return "" + } + entries := strings.Split(value, ",") + for index, entry := range entries { + relation, state, found := strings.Cut(strings.TrimSpace(entry), ":") + if !found { + entries[index] = relation + continue + } + state = strings.TrimSpace(state) + if state != "" && state != "never" { + state = "analyzed" + } + entries[index] = relation + ":" + state + } + sort.Strings(entries) + return strings.Join(entries, ",") +} + +// validateAAMetric validates aa metric. +func validateAAMetric(metric AAMetricResolution) error { + if metric.Ratio.Estimate <= 0 || metric.Ratio.Lower <= 0 || metric.Ratio.Upper <= 0 || + metric.Ratio.Lower > metric.Ratio.Estimate || metric.Ratio.Estimate > metric.Ratio.Upper || + math.IsNaN(metric.Ratio.Estimate) || math.IsNaN(metric.Ratio.Lower) || math.IsNaN(metric.Ratio.Upper) || + math.IsInf(metric.Ratio.Estimate, 0) || math.IsInf(metric.Ratio.Lower, 0) || math.IsInf(metric.Ratio.Upper, 0) { + return fmt.Errorf("ratio interval is malformed") + } + if metric.RatioResolution < 0 || math.IsNaN(metric.RatioResolution) || math.IsInf(metric.RatioResolution, 0) || metric.AbsoluteResolution < 0 { + return fmt.Errorf("resolution is malformed") + } + if metric.AbsoluteChange.Lower > metric.AbsoluteChange.Estimate || metric.AbsoluteChange.Estimate > metric.AbsoluteChange.Upper || + metric.AbsoluteResolution < max(absDuration(metric.AbsoluteChange.Lower), absDuration(metric.AbsoluteChange.Upper)) { + return fmt.Errorf("absolute-change interval is malformed") + } + return nil +} + +// aaTimingFloor returns host-derived per-case noise with the mandatory relative and absolute minimums. +func aaTimingFloor(report *AAResolutionReport, key performanceKey, p95 bool, configuredRatio float64) (float64, time.Duration, error) { + for _, entry := range report.Cases { + if entry.Dataset != key.dataset || entry.Name != key.name || entry.Backend != key.backend { + continue + } + metric := entry.P50 + if p95 { + metric = entry.P95 + } + return max(minimumTimingNoiseRatio, configuredRatio, metric.RatioResolution), + max(minimumTimingNoiseAbsolute, metric.AbsoluteResolution), nil + } + + return 0, 0, fmt.Errorf("A/A report has no resolution evidence for %s/%s/%s", key.dataset, key.name, key.backend) +} + +// validSHA256 reports whether a value is a canonical lowercase SHA-256 digest. +func validSHA256(value string) bool { + if len(value) != sha256.Size*2 { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} + +// timingTier requires a stable, explicit normal, envelope, or stress classification across artifacts. +func timingTier(key performanceKey, artifacts ...[]CaseResult) (string, error) { + tier := "" + found := false + for _, records := range artifacts { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + current := record.Shape.FixtureTier + if current != "normal" && current != "envelope" && current != "stress" { + return "", fmt.Errorf("%s/%s/%s has missing or unsupported fixture tier %q", key.dataset, key.name, key.backend, current) + } + if tier != "" && tier != current { + return "", fmt.Errorf("%s/%s/%s changes fixture tier across artifacts", key.dataset, key.name, key.backend) + } + tier = current + found = true + } + } + if !found { + return "unknown", nil + } + return tier, nil +} + +// qualificationSplit requires one stable training, holdout, or diagnostic +// partition for prioritized traversal records. The split is part of the +// workload declaration and may not drift between benchmark arms or rounds. +// Legacy non-traversal records may omit it. +func qualificationSplit(key performanceKey, artifacts ...[]CaseResult) (string, error) { + split := "" + found := false + for _, records := range artifacts { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + current := record.Shape.QualificationSplit + if current == "" { + if prioritizedTraversalRecord(record) { + return "", fmt.Errorf("%s/%s/%s has no frozen qualification split", key.dataset, key.name, key.backend) + } + continue + } + if current != "training" && current != "holdout" && current != "diagnostic" { + return "", fmt.Errorf("%s/%s/%s has unsupported qualification split %q", key.dataset, key.name, key.backend, current) + } + if split != "" && split != current { + return "", fmt.Errorf("%s/%s/%s changes qualification split across artifacts", key.dataset, key.name, key.backend) + } + split = current + found = true + } + } + if !found { + return "legacy", nil + } + return split, nil +} + +// prioritizedTraversalCategory identifies result families introduced by the +// traversal-priority qualification program. Their split remains mandatory +// even when an artifact was assembled outside the scale-corpus loader. +func prioritizedTraversalCategory(category string) bool { + switch category { + case "generated_shortest_path_v2", "generated_all_shortest_path_v2", "expand_into_one_hop", "generated_endpoint_seeded_expansion", "generated_fixed_suffix_expansion_v2", "orientation_shadow": + return true + default: + return false + } +} + +// prioritizedTraversalRecord also recognizes the fixed-suffix v2 and +// boundary datasets whose category intentionally remains compatible with the +// original corpus. Artifact consumers must not mistake that shared category +// for permission to omit the frozen qualification split. +func prioritizedTraversalRecord(record CaseResult) bool { + if prioritizedTraversalCategory(record.Category) { + return true + } + + return record.Category == "generated_fixed_suffix_expansion" && + (strings.HasPrefix(record.Dataset, "generated_fixed_suffix_expansion_v2_") || + strings.HasPrefix(record.Dataset, "generated_fixed_suffix_expansion_v3_") || + strings.HasPrefix(record.Name, "GFSE-V2-") || + strings.HasPrefix(record.Name, "GFSE-V3-") || + strings.HasPrefix(record.Name, "GFSE-BOUNDARY-")) +} + +// prioritizedTraversalKey reports whether either artifact identifies a +// matched performance key as part of the traversal qualification program. +// Looking at both artifacts makes the gate fail closed if one side drops or +// changes the category while preserving the logical case identity. +func prioritizedTraversalKey(key performanceKey, artifacts ...[]CaseResult) bool { + for _, records := range artifacts { + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend && prioritizedTraversalRecord(record) { + return true + } + } + } + + return false +} + +// TraversalQualificationStatus reports independent selector-training and +// frozen-holdout coverage for one concrete traversal candidate family. +type TraversalQualificationStatus struct { + // Family supplies the family input to the TraversalQualificationStatus contract. + Family string `json:"family"` + // TrainingCases supplies the training cases input to the TraversalQualificationStatus contract. + TrainingCases int `json:"training_cases"` + // HoldoutCases supplies the holdout cases input to the TraversalQualificationStatus contract. + HoldoutCases int `json:"holdout_cases"` + // TrainingPassed indicates whether training passed applies. + TrainingPassed bool `json:"training_passed"` + // HoldoutPassed indicates whether holdout passed applies. + HoldoutPassed bool `json:"holdout_passed"` + // Passed indicates whether passed applies. + Passed bool `json:"passed"` +} + +// traversalQualificationFamily returns the most specific stable candidate +// identity available for a matched key. Candidate/right artifacts take +// precedence over incumbent/left artifacts. A conservative semantic family +// remains available for externally assembled artifacts without optimizer or +// runtime telemetry. +func traversalQualificationFamily(key performanceKey, artifacts ...[]CaseResult) string { + for artifactIdx := len(artifacts) - 1; artifactIdx >= 0; artifactIdx-- { + for _, record := range artifacts[artifactIdx] { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if record.TraversalTelemetry != nil { + summary := record.TraversalTelemetry.Summary + for _, identity := range []string{summary.EmittedIdentity, summary.SelectorVersion} { + if isOrientationProbePolicy(identity) { + return identity + } + } + if identity := summary.RequestedIdentity; prioritizedTraversalIdentity(identity) { + branch := summary.RuntimeBranch + if branch != "" && branch != "runtime_outcome_unavailable" && branch != "mixed" { + return identity + "@" + branch + } + return identity + } + } + if record.Optimization != nil { + for outcomeIdx := len(record.Optimization.TargetOutcomes) - 1; outcomeIdx >= 0; outcomeIdx-- { + outcome := record.Optimization.TargetOutcomes[outcomeIdx] + for _, identity := range []string{outcome.Candidate, outcome.EmittedPolicy, outcome.PlannedPolicy, outcome.Applied, outcome.Selected} { + if prioritizedTraversalIdentity(identity) { + return identity + } + } + } + } + } + } + for artifactIdx := len(artifacts) - 1; artifactIdx >= 0; artifactIdx-- { + for _, record := range artifacts[artifactIdx] { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend || record.Optimization == nil { + continue + } + for _, outcome := range record.Optimization.TargetOutcomes { + if outcome.TargetKind != "" && outcome.TargetKind != "traversal" { + continue + } + if outcome.Family == "SP" || outcome.Family == "ASP" || strings.Contains(outcome.Family, "expansion") { + return outcome.Family + } + } + } + } + + for _, records := range artifacts { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + if strings.HasPrefix(record.Dataset, "generated_fixed_suffix_expansion_v3_") || strings.HasPrefix(record.Name, "GFSE-V3-") { + return string(optimize.ExpansionSearchPolicyOrientationProbeV2) + } + switch record.Category { + case "generated_shortest_path_v2", "generated_all_shortest_path_v2": + if strings.Contains(strings.ToLower(record.Cypher), "allshortestpaths") || strings.Contains(strings.ToLower(record.Name), "all-shortest") { + return "ASP" + } + return "SP" + case "generated_endpoint_seeded_expansion": + return "fixed_prefix_terminal_expansion" + case "generated_fixed_suffix_expansion", "generated_fixed_suffix_expansion_v2", "orientation_shadow": + return "orientation-probe-v1" + case "expand_into_one_hop": + return "expand-into-study-v1" + } + } + } + + return "prioritized_traversal" +} + +// validateCandidateRuntimeEvidence rejects performance attribution to an +// experimental traversal arm unless every warm sample is bound to one +// singular, non-fallback runtime outcome for that measured invocation. A +// same-case diagnostic replay is useful resource evidence but is not allowed +// to attest latency samples because concurrent graph changes or cap outcomes +// could select a different branch. +func validateCandidateRuntimeEvidence(records []CaseResult, key performanceKey) error { + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend || !requiresCandidateRuntimeEvidence(record) { + continue + } + if record.TraversalTelemetry == nil { + return fmt.Errorf("candidate traversal has no runtime telemetry") + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.RuntimeIdentity == "" || summary.RuntimeBranch == "" || summary.RuntimeBranch == "mixed" || summary.RuntimeBranch == "runtime_outcome_unavailable" { + return fmt.Errorf("candidate traversal runtime outcome is unavailable or mixed") + } + if summary.FallbackExecuted == nil { + return fmt.Errorf("candidate traversal fallback outcome is unavailable") + } + if *summary.FallbackExecuted { + return fmt.Errorf("candidate traversal executed exact fallback %q", summary.FallbackIdentity) + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || sample.RuntimeBranch != summary.RuntimeBranch || sample.FallbackExecuted == nil || *sample.FallbackExecuted || sample.RuntimeAttestation != "timed_invocation" { + return fmt.Errorf("warm sample lacks matching singular runtime attribution") + } + if err := validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted); err != nil { + return fmt.Errorf("warm sample runtime receipt chain: %w", err) + } + } + } + return nil +} + +// validateRuntimeReceiptEvents validates runtime receipt events. +func validateRuntimeReceiptEvents(events []RuntimeReceiptEvent, runtimeIdentity, runtimeBranch string, fallbackExecuted *bool) error { + if len(events) == 0 { + return fmt.Errorf("event chain is missing") + } + for idx, event := range events { + if event.Ordinal != idx+1 || event.RuntimeIdentity == "" || event.RuntimeBranch == "" { + return fmt.Errorf("event chain is not contiguous") + } + } + terminal := events[len(events)-1] + if terminal.RuntimeIdentity != runtimeIdentity || terminal.RuntimeBranch != runtimeBranch { + return fmt.Errorf("terminal event does not match runtime outcome") + } + if fallbackExecuted == nil || terminal.FallbackExecuted != *fallbackExecuted { + return fmt.Errorf("terminal event does not match fallback outcome") + } + return nil +} + +// runtimeReceiptChains supports benchmark evidence processing for runtime receipt chains. +func runtimeReceiptChains(samples []LatencySample) [][]RuntimeReceiptEvent { + chains := make([][]RuntimeReceiptEvent, 0) + for _, sample := range samples { + if len(sample.RuntimeReceiptEvents) == 0 { + continue + } + chains = append(chains, append([]RuntimeReceiptEvent(nil), sample.RuntimeReceiptEvents...)) + } + return chains +} + +// caseRuntimeReceiptChains supports benchmark evidence processing for case runtime receipt chains. +func caseRuntimeReceiptChains(records []CaseResult, key performanceKey) [][]RuntimeReceiptEvent { + chains := make([][]RuntimeReceiptEvent, 0) + for _, record := range records { + if record.Dataset == key.dataset && record.Name == key.name && record.ExecutionMode == key.backend { + chains = append(chains, runtimeReceiptChains(record.Stats.Samples)...) + } + } + return chains +} + +// requiresCandidateRuntimeEvidence reports whether requires candidate runtime evidence. +func requiresCandidateRuntimeEvidence(record CaseResult) bool { + if record.TraversalTelemetry != nil { + summary := record.TraversalTelemetry.Summary + if isOrientationProbePolicy(summary.EmittedIdentity) || isOrientationProbePolicy(summary.SelectorVersion) { + return true + } + requested := summary.RequestedIdentity + if strings.HasPrefix(requested, "SP-B") || strings.HasPrefix(requested, "ASP-B") || isOrientationProbePolicy(requested) { + return true + } + } + if record.Optimization == nil { + return false + } + for _, outcome := range record.Optimization.TargetOutcomes { + for _, identity := range []string{outcome.Candidate, outcome.EmittedPolicy, outcome.Selected} { + if strings.HasPrefix(identity, "SP-B") || strings.HasPrefix(identity, "ASP-B") || isOrientationProbePolicy(identity) { + return true + } + } + } + return false +} + +// prioritizedTraversalIdentity derives the stable identity used to compare prioritized traversal. +func prioritizedTraversalIdentity(identity string) bool { + return strings.HasPrefix(identity, "SP-") || + strings.HasPrefix(identity, "ASP-") || + strings.HasPrefix(identity, "EXPANSION-") || + isOrientationProbePolicy(identity) +} + +// promotionTimingSplit reports whether a frozen qualification partition may +// contribute timing evidence to a promotion decision. Diagnostic records are +// still checked for correctness and resource behavior, but never tune or +// qualify a production selector. +func promotionTimingSplit(split string) bool { + return split != "diagnostic" +} + +// pairedRoundEvidence records independently verifiable observations for paired round. +type pairedRoundEvidence struct { + // Block supplies the block input to the pairedRoundEvidence contract. + Block int + // ArmOrder supplies the arm order input to the pairedRoundEvidence contract. + ArmOrder int + // RunUUID identifies the run uuid. + RunUUID string + // Arm supplies the arm input to the pairedRoundEvidence contract. + Arm string + // Warmups supplies the warmups input to the pairedRoundEvidence contract. + Warmups int +} + +// validatePairedOrderEvidence verifies matched block identity and balanced two-arm ordering for the requested rounds. +func validatePairedOrderEvidence(left, right []CaseResult, key performanceKey, rounds []int, minimumWarmups int) error { + leftEvidence, err := collectPairedRoundEvidence(left, key) + if err != nil { + return err + } + rightEvidence, err := collectPairedRoundEvidence(right, key) + if err != nil { + return err + } + leftFirst := 0 + for _, round := range rounds { + leftRound, leftOK := leftEvidence[round] + rightRound, rightOK := rightEvidence[round] + if !leftOK || !rightOK { + return fmt.Errorf("%s/%s round %d lacks paired order evidence", key.dataset, key.name, round) + } + if leftRound.Warmups < minimumWarmups || rightRound.Warmups < minimumWarmups { + return fmt.Errorf("%s/%s round %d requires at least %d warmups per arm, got %d/%d", key.dataset, key.name, round, minimumWarmups, leftRound.Warmups, rightRound.Warmups) + } + if leftRound.Block < 1 || leftRound.Block != rightRound.Block { + return fmt.Errorf("%s/%s round %d has missing or mismatched paired block", key.dataset, key.name, round) + } + if leftRound.RunUUID == "" || leftRound.RunUUID != rightRound.RunUUID { + return fmt.Errorf("%s/%s round %d has missing or mismatched paired run UUID", key.dataset, key.name, round) + } + if leftRound.Arm == "" || rightRound.Arm == "" || leftRound.Arm == "unlabeled" || rightRound.Arm == "unlabeled" || leftRound.Arm == rightRound.Arm { + return fmt.Errorf("%s/%s round %d has missing or indistinct arm identity", key.dataset, key.name, round) + } + if !((leftRound.ArmOrder == 1 && rightRound.ArmOrder == 2) || (leftRound.ArmOrder == 2 && rightRound.ArmOrder == 1)) { + return fmt.Errorf("%s/%s round %d lacks a complete two-arm order", key.dataset, key.name, round) + } + if leftRound.ArmOrder == 1 { + leftFirst++ + } + } + rightFirst := len(rounds) - leftFirst + if leftFirst-rightFirst > 1 || rightFirst-leftFirst > 1 { + return fmt.Errorf("%s/%s paired arm order is not balanced: %d/%d", key.dataset, key.name, leftFirst, rightFirst) + } + + return nil +} + +// collectPairedRoundEvidence collects paired round evidence. +func collectPairedRoundEvidence(records []CaseResult, key performanceKey) (map[int]pairedRoundEvidence, error) { + evidence := map[int]pairedRoundEvidence{} + for _, record := range records { + if record.Dataset != key.dataset || record.Name != key.name || record.ExecutionMode != key.backend { + continue + } + warmups := record.Stats.WarmupIterations + if record.Environment != nil { + if warmups != 0 && record.Environment.WarmupIterations != 0 && warmups != record.Environment.WarmupIterations { + return nil, fmt.Errorf("%s/%s has inconsistent warmup evidence", key.dataset, key.name) + } + if warmups == 0 { + warmups = record.Environment.WarmupIterations + } + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + round := sample.Round + current := pairedRoundEvidence{ + Block: sample.Block, + ArmOrder: sample.ArmOrder, + RunUUID: sample.RunUUID, + Arm: sample.Arm, + Warmups: warmups, + } + if record.Environment != nil { + if round == 0 { + round = record.Environment.Round + } + if current.Block == 0 { + current.Block = record.Environment.Block + } + if current.ArmOrder == 0 { + current.ArmOrder = record.Environment.ArmOrder + } + if current.RunUUID == "" { + current.RunUUID = record.Environment.RunUUID + } + if current.Arm == "" { + current.Arm = record.Environment.Arm + } + } + if round < 1 { + return nil, fmt.Errorf("%s/%s has warm sample without a round", key.dataset, key.name) + } + if prior, found := evidence[round]; found && prior != current { + return nil, fmt.Errorf("%s/%s round %d has inconsistent paired order metadata", key.dataset, key.name, round) + } + evidence[round] = current + } + } + return evidence, nil +} + +// sortedPerformanceKeys returns stable keys from a set. +func sortedPerformanceKeys(values map[performanceKey]struct{}) []performanceKey { + keys := make([]performanceKey, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].dataset != keys[j].dataset { + return keys[i].dataset < keys[j].dataset + } + if keys[i].name != keys[j].name { + return keys[i].name < keys[j].name + } + return keys[i].backend < keys[j].backend + }) + return keys +} diff --git a/cmd/graphbench/suffix_reverse_guard_integration_test.go b/cmd/graphbench/suffix_reverse_guard_integration_test.go new file mode 100644 index 00000000..24a53724 --- /dev/null +++ b/cmd/graphbench/suffix_reverse_guard_integration_test.go @@ -0,0 +1,477 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package main + +import ( + "context" + "net/url" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestPostgreSQLSuffixReverseGuardPlanAttribution exercises one already-open +// full-path training case against a real PostgreSQL JSON EXPLAIN. It proves +// that the admitted reverse executor is marker-gated and that the exact +// forward fallback remains uninitialized. Protected holdout cases are never +// selected by this test. +func TestPostgreSQLSuffixReverseGuardPlanAttribution(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + const caseName = "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path" + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{caseName}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + require.Equal(t, "training", selected.Cases[0].Shape.QualificationSplit) + require.True(t, selected.Cases[0].Shape.PathMaterializationRequired) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions.EnableExpansionSuffixReverseGuard = true + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotNil(t, record.PostgresMetrics) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + + summary := record.TraversalTelemetry.Summary + require.Equal(t, string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), summary.EmittedIdentity) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), summary.RuntimeIdentity) + require.Equal(t, "suffix_seeded_reverse", summary.RuntimeBranch) + require.False(t, *summary.Overflow) + require.False(t, *summary.FallbackExecuted) + require.Equal(t, optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, summary.Caps["suffix_rows"]) + require.Equal(t, optimize.ExpansionSearchSuffixReverseGuardStateLimit, summary.Caps["state_rows"]) + + diagnostic := record.TraversalTelemetry.Diagnostic + require.Equal(t, TraversalTelemetryCounterStatusComplete, diagnostic.CounterStatus) + require.NotNil(t, diagnostic.PlanReplay) + counters := diagnostic.PlanReplay.Counters + t.Logf("suffix rows=%d state rows=%d output rows=%d", counters["suffix_guard_suffix_rows"], counters["suffix_guard_state_rows"], counters["suffix_guard_output_rows"]) + require.Equal(t, int64(1), counters["suffix_guard_candidate_marker_rows"]) + require.Zero(t, counters["suffix_guard_fallback_marker_rows"]) + require.Equal(t, int64(1), counters["suffix_guard_candidate_executor_loops"]) + require.Zero(t, counters["suffix_guard_fallback_executor_loops"]) + require.Zero(t, counters["suffix_guard_fallback_branch_rows"]) + + // The counter derivation is accepted only when each materialized branch + // body has exactly one marker outer child and one executor inner child. + for _, branch := range []string{"candidate", "fallback"} { + bodySuffix := "suffix_guard_" + branch + "_body" + markerSuffix := "suffix_guard_" + branch + "_marker" + var bodies []PostgresPlanNodeMetric + for _, node := range record.PostgresMetrics.PlanNodes { + if namedCTEBody(node, bodySuffix) { + bodies = append(bodies, node) + } + } + require.Len(t, bodies, 1, branch) + var outer, inner int + for _, node := range record.PostgresMetrics.PlanNodes { + if node.ParentPlanNodeID != bodies[0].PlanNodeID { + continue + } + if strings.EqualFold(node.ParentRelationship, "Outer") && strings.EqualFold(node.NodeType, "CTE Scan") && + strings.HasSuffix(strings.ToLower(node.CTEName), markerSuffix) { + outer++ + } + if strings.EqualFold(node.ParentRelationship, "Inner") { + inner++ + } + } + require.Equal(t, 1, outer, branch) + require.Equal(t, 1, inner, branch) + } +} + +// TestPostgreSQLSuffixReverseRetryPreservesExactRowsAndReceipts exercises the +// reverse-complete, state-overflow, no-path, and output-byte retry paths on +// open P1 training cases. It never selects a protected holdout. +func TestPostgreSQLSuffixReverseRetryPreservesExactRowsAndReceipts(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + for _, test := range []struct { + name string + caseName string + stateLimit int64 + expectedFirstEvent string + expectedFallback bool + expectedFinalEvent string + }{ + { + name: "reverse complete", caseName: "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", + expectedFirstEvent: "reverse_complete", expectedFinalEvent: "reverse_complete", + }, + { + name: "forced state retry", caseName: "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", stateLimit: 1, + expectedFirstEvent: "forward_retry_state_overflow", expectedFallback: true, expectedFinalEvent: "exact_forward_retry_complete", + }, + { + name: "natural high reverse fan-in retry", caseName: "GFSE-P1-TRAIN-D09-F017-R0-X2-I1024-M1-Q1-high_reverse_fanin_path", + expectedFirstEvent: "forward_retry_state_overflow", expectedFallback: true, expectedFinalEvent: "exact_forward_retry_complete", + }, + { + name: "no path exhaustion", caseName: "GFSE-P1-TRAIN-D09-F513-R0-X512-no_path_exhaustion", + expectedFirstEvent: "reverse_complete", expectedFinalEvent: "reverse_complete", + }, + { + name: "output byte retry", caseName: "GFSE-P1-TRAIN-D00-F001-R0-X0-M4-P2100000-output_byte_retry_path", + expectedFirstEvent: "forward_retry_output_bytes", expectedFallback: true, expectedFinalEvent: "exact_forward_retry_complete", + }, + } { + t.Run(test.name, func(t *testing.T) { + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{test.caseName}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + require.Equal(t, "training", selected.Cases[0].Shape.QualificationSplit) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions.EnableExpansionSuffixReverseRetry = true + runner.toolOptions.SuffixReverseGuardStateLimit = test.stateLimit + + records, err := runner.Run(ctx, 0, 2, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.Contains(t, record.SQL, "dawgs.suffix_reverse_retry_status") + require.NotContains(t, record.SQL, "_suffix_guard_fallback_body") + for _, sample := range record.Stats.Samples { + if sample.RuntimeAttestation != "timed_invocation" { + continue + } + require.NotEmpty(t, sample.RuntimeReceiptEvents) + firstEvent := sample.RuntimeReceiptEvents[0] + require.Equal(t, test.expectedFirstEvent, firstEvent.RuntimeBranch) + finalEvent := sample.RuntimeReceiptEvents[len(sample.RuntimeReceiptEvents)-1] + require.Equal(t, test.expectedFinalEvent, finalEvent.RuntimeBranch) + require.NotNil(t, sample.FallbackExecuted) + require.Equal(t, test.expectedFallback, *sample.FallbackExecuted) + } + }) + } +} + +// TestPostgreSQLSuffixRouteComponentPreservesExactRowsAndReceipt exercises the +// default-off direct component arm on an open training fixture. It verifies +// the component is one reverse statement with no guard, retry, or incumbent +// arm; it is not a performance qualification. +func TestPostgreSQLSuffixRouteComponentPreservesExactRowsAndReceipt(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + const caseName = "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path" + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{caseName}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions.EnableExpansionSuffixRouteComponent = true + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + require.NotNil(t, record.TraversalTelemetry.Diagnostic) + require.Equal(t, TraversalTelemetryCounterStatusComplete, record.TraversalTelemetry.Diagnostic.CounterStatus) + require.NotNil(t, record.TraversalTelemetry.Diagnostic.Counters.SuffixComponent) + require.Equal(t, int64(1), *record.TraversalTelemetry.Diagnostic.Counters.SuffixComponent.ReceiptRows) + require.Contains(t, record.SQL, "_suffix_seeded_component_receipt") + require.NotContains(t, record.SQL, "_suffix_guard_") + require.NotContains(t, record.SQL, "EXPANSION-STEPWISE-FORWARD") + + summary := record.TraversalTelemetry.Summary + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), summary.EmittedIdentity) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), summary.RuntimeIdentity) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), summary.AppliedIdentity) + require.Equal(t, optimize.ExpansionSearchSelectorSuffixRouteComponentV1, summary.SelectorVersion) + require.Equal(t, "selected", summary.RuntimeBranch) + require.False(t, *summary.FallbackExecuted) + + for _, sample := range record.Stats.Samples { + if sample.RuntimeAttestation != "timed_invocation" { + continue + } + require.Len(t, sample.RuntimeReceiptEvents, 1) + require.Equal(t, "suffix_route_component", sample.RuntimeReceiptEvents[0].RuntimeBranch) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), sample.RuntimeReceiptEvents[0].RuntimeIdentity) + } +} + +// TestPostgreSQLSuffixRouteComponentClosureRecordsPreparedStateAndWorkspace +// verifies the closure records the first fresh miss, reusable prepared hits, +// same-backend pool reacquisition, and complete component workspace evidence +// without enabling a selector or retry. +func TestPostgreSQLSuffixRouteComponentClosureRecordsPreparedStateAndWorkspace(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{ + Cases: []string{ + "GFSE-SRC-V1-TARGET-D16-F1024-sparse_endpoint_ids", + "GFSE-SRC-V1-TARGET-D17-F1025-sparse_path", + }, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 2) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions.EnableExpansionSuffixRouteComponent = true + runner.suffixRouteComponentClosure = true + runner.sessionMemoryCeilingBytes = 1 << 20 + runner.poolMemoryCeilingBytes = 1 << 20 + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 2) + for _, record := range records { + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotNil(t, record.ClientWaterfall) + require.Len(t, record.ClientWaterfall.Samples, 1) + require.NotNil(t, record.PostgresBoundaryClosure) + closure := record.PostgresBoundaryClosure + require.NotEmpty(t, closure.SQLFingerprint) + expectedObservation, err := stableObservationSHA256(record.ObservedRows) + require.NoError(t, err) + require.Len(t, closure.SameSessionPreparedHits, 1) + require.Len(t, closure.PoolReacquiredPreparedHits, 1) + require.Equal(t, closure.PoolPreparedMiss.ConnectionID, closure.PoolReacquiredPreparedHits[0].ConnectionID) + for _, sample := range postgresBoundaryClosureSamples(*closure) { + require.Equal(t, record.RowCount, sample.Rows) + require.NotNil(t, sample.WorkspaceBytes) + require.NotEmpty(t, sample.ConnectionID) + require.Equal(t, expectedObservation, sample.ObservationSHA256) + } + require.LessOrEqual(t, closure.Workspace.SessionPeakBytes, runner.sessionMemoryCeilingBytes) + require.Equal(t, closure.Workspace.SessionPeakBytes, closure.Workspace.PoolPeakBytes) + require.LessOrEqual(t, closure.Workspace.PerQueryPeakBytes, runner.poolMemoryCeilingBytes) + require.Zero(t, closure.Workspace.PerQueryPeakBytes) + require.Zero(t, closure.Workspace.FreshSessionPeakBytes) + require.Zero(t, closure.Workspace.SessionPeakBytes) + require.Zero(t, closure.Workspace.PoolPeakBytes) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + require.Contains(t, record.TraversalTelemetry.Diagnostic.RequiredFamilies, TraversalTelemetryFamilyWorkspace) + require.NotNil(t, record.TraversalTelemetry.Diagnostic.Counters.Workspace) + require.Equal(t, closure.Workspace.SessionPeakBytes, *record.TraversalTelemetry.Diagnostic.Counters.Workspace.SessionPeakBytes) + } +} + +// TestPostgreSQLSuffixRouteComponentRecordsNoPathReceipt verifies the direct +// component records execution even when its exact reverse query returns no +// public rows. This keeps no-path component measurements fail-closed. +func TestPostgreSQLSuffixRouteComponentRecordsNoPathReceipt(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Cases: []string{"GFSE-P1-TRAIN-D09-F513-R0-X512-no_path_exhaustion"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions.EnableExpansionSuffixRouteComponent = true + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + for _, sample := range record.Stats.Samples { + if sample.RuntimeAttestation != "timed_invocation" { + continue + } + require.Len(t, sample.RuntimeReceiptEvents, 1) + require.Equal(t, "suffix_route_component", sample.RuntimeReceiptEvents[0].RuntimeBranch) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), sample.RuntimeReceiptEvents[0].RuntimeIdentity) + } +} + +// TestPostgreSQLSuffixRouteComponentCancellationReusesPoolSession proves the +// direct component handles PostgreSQL cancellation, rolls the failed +// transaction back, returns its single connection to the pool, and remains +// usable from the reacquired physical backend. It is operational evidence, +// not a timing qualification. +func TestPostgreSQLSuffixRouteComponentCancellationReusesPoolSession(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + connectionURL, err := url.Parse(connection) + require.NoError(t, err) + if connectionURL.Scheme != "postgres" && connectionURL.Scheme != "postgresql" { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{ + Cases: []string{"GFSE-SRC-V1-TARGET-D17-F1025-sparse_path"}, + }) + require.NoError(t, err) + require.Len(t, selected.Cases, 1) + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + runner, err := newPostgresSQLRunner(ctx, "../../integration/testdata", connection, selected, 1, 1, nil, false, nil, "", "") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, runner.Close(context.Background())) }) + runner.repeatableRead = true + runner.traversalTelemetry = postgresTraversalTelemetryDiagnostic + runner.toolOptions.EnableExpansionSuffixRouteComponent = true + + records, err := runner.Run(ctx, 0, 1, selected) + require.NoError(t, err) + require.Len(t, records, 1) + record := records[0] + require.Equal(t, StatusOK, record.Status, record.Error) + require.NotNil(t, record.TraversalTelemetry) + require.NoError(t, record.TraversalTelemetry.Validate()) + require.NotNil(t, record.TraversalTelemetry.Diagnostic.Counters.SuffixComponent) + require.Equal(t, TraversalTelemetryCounterStatusComplete, record.TraversalTelemetry.Diagnostic.CounterStatus) + + translation, sqlQuery, err := runner.translateCypher(ctx, selected.Cases[0].Cypher, record.Params) + require.NoError(t, err) + require.Contains(t, sqlQuery, "_suffix_seeded_component_receipt") + require.NotContains(t, sqlQuery, "EXPANSION-STEPWISE-FORWARD") + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}, pgx.NamedArgs(translation.Parameters)} + + connectionHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + backendPID := connectionHandle.Conn().PgConn().PID() + tx, err := connectionHandle.BeginTx(ctx, postgresConcurrencyTxOptions()) + require.NoError(t, err) + _, err = tx.Exec(ctx, "set local statement_timeout = '1ms'") + require.NoError(t, err) + started := time.Now() + rows, queryErr := tx.Query(ctx, sqlQuery, queryArgs...) + if queryErr == nil { + for rows.Next() { + _, queryErr = rows.Values() + if queryErr != nil { + break + } + } + rows.Close() + if queryErr == nil { + queryErr = rows.Err() + } + } + cancellationLatency := time.Since(started) + var postgresError *pgconn.PgError + require.ErrorAs(t, queryErr, &postgresError) + require.Equal(t, "57014", postgresError.Code) + require.Less(t, cancellationLatency, 250*time.Millisecond) + require.NoError(t, tx.Rollback(ctx)) + connectionHandle.Release() + + reusedHandle, err := runner.pool.Acquire(ctx) + require.NoError(t, err) + defer reusedHandle.Release() + var reusedPID uint32 + require.NoError(t, reusedHandle.QueryRow(ctx, "select pg_backend_pid()").Scan(&reusedPID)) + require.Equal(t, backendPID, reusedPID) + + rows, err = reusedHandle.Query(ctx, sqlQuery, queryArgs...) + require.NoError(t, err) + rowCount := int64(0) + for rows.Next() { + _, err = rows.Values() + require.NoError(t, err) + rowCount++ + } + rows.Close() + require.NoError(t, rows.Err()) + require.Equal(t, record.RowCount, rowCount) + t.Logf("cancelled direct suffix-route component in %s; pool reused backend PID %d", cancellationLatency, backendPID) +} diff --git a/cmd/graphbench/suffix_reverse_guard_report_v1.go b/cmd/graphbench/suffix_reverse_guard_report_v1.go new file mode 100644 index 00000000..9407cfdb --- /dev/null +++ b/cmd/graphbench/suffix_reverse_guard_report_v1.go @@ -0,0 +1,807 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "fmt" + "os" + "slices" + "strings" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + pgdriver "github.com/specterops/dawgs/drivers/pg" +) + +const suffixReverseGuardFeasibilityVersion = 1 + +// SuffixReverseGuardFeasibilityOptions configures the deliberately bounded, +// training-only stop gate. It cannot freeze or authorize production evidence. +type SuffixReverseGuardFeasibilityOptions struct { + Seed int64 + Confidence float64 + BootstrapCount int +} + +// SuffixReverseGuardImprovementGate records material forward improvement and +// p95 containment for one reverse-favorable full-path workload. +type SuffixReverseGuardImprovementGate struct { + MedianRatio RatioInterval `json:"median_ratio_to_forward"` + MedianSaving DurationInterval `json:"median_saving_from_forward"` + P95Ratio RatioInterval `json:"p95_ratio_to_forward"` + RatioUpperLimit float64 `json:"median_ratio_upper_limit"` + SavingLowerLimit time.Duration `json:"median_saving_lower_limit"` + P95UpperLimit float64 `json:"p95_ratio_upper_limit"` + Passed bool `json:"passed"` +} + +// SuffixReverseGuardFeasibilityCase records the three-arm go/no-go decision +// for one already-open, training-only workload. +type SuffixReverseGuardFeasibilityCase struct { + Dataset string `json:"dataset"` + Name string `json:"name"` + QualificationSplit string `json:"qualification_split"` + QuerySHA256 string `json:"query_sha256"` + Rounds int `json:"matched_rounds"` + ExactObservationsMatched bool `json:"exact_observations_matched"` + RuntimeIdentity string `json:"runtime_identity"` + RuntimeBranch string `json:"runtime_branch"` + Overflow bool `json:"overflow"` + FallbackExecuted bool `json:"fallback_executed"` + GuardOverhead OrientationLatencyGate `json:"guard_overhead_to_exact_reverse"` + FastestExactRegret OrientationLatencyGate `json:"regret_to_fastest_exact"` + ForwardImprovement SuffixReverseGuardImprovementGate `json:"improvement_over_forward"` + Passed bool `json:"passed"` + Reasons []string `json:"reasons,omitempty"` +} + +// SuffixReverseGuardFeasibilityReport is negative- or positive-decision +// evidence for whether fresh qualification tooling is warranted. Passed does +// not mean qualified or production-authorized. +type SuffixReverseGuardFeasibilityReport struct { + Version int `json:"version"` + Policy string `json:"policy"` + SelectorVersion string `json:"selector_version"` + Protocol string `json:"protocol"` + Seed int64 `json:"seed"` + Confidence float64 `json:"confidence_level"` + SourceCommit string `json:"source_commit"` + DirtyDiffSHA256 string `json:"dirty_diff_sha256"` + BinarySHA256 string `json:"binary_sha256"` + CorpusSHA256 string `json:"corpus_sha256"` + IncumbentArtifactSHA256 string `json:"incumbent_artifact_sha256,omitempty"` + ReverseArtifactSHA256 string `json:"reverse_artifact_sha256,omitempty"` + GuardedArtifactSHA256 string `json:"guarded_artifact_sha256,omitempty"` + AAReportSHA256 string `json:"aa_report_sha256,omitempty"` + Caps map[string]int64 `json:"caps"` + GuardRatioUpperLimit float64 `json:"guard_ratio_upper_limit"` + GuardAbsoluteUpperLimit time.Duration `json:"guard_absolute_upper_limit"` + ForwardRatioUpperLimit float64 `json:"forward_ratio_upper_limit"` + ForwardSavingLowerLimit time.Duration `json:"forward_saving_lower_limit"` + ForwardP95UpperLimit float64 `json:"forward_p95_upper_limit"` + EvidencePassed bool `json:"evidence_passed"` + Passed bool `json:"passed"` + Cases []SuffixReverseGuardFeasibilityCase `json:"cases"` +} + +type suffixReverseGuardSeries struct { + incumbent roundSamples + reverse roundSamples + guarded roundSamples + runtime string + branch string + overflow bool + fallback bool + observed bool + querySHA string + split string +} + +// suffixReverseGuardInvocationIdentity binds every case in one arm capture to +// the physical GraphBench process interval that executed that scheduled arm. +// The feasibility protocol runs the exact two-case cohort once per arm, so all +// records for an arm/round must carry this same identity and interval. +type suffixReverseGuardInvocationIdentity struct { + round int + block int + order int + arm string + runUUID string + startedAt time.Time + endedAt time.Time +} + +var suffixReverseGuardFeasibilityCases = []struct { + dataset string + name string +}{ + { + dataset: "generated_fixed_suffix_expansion_v3_d2_f4_r0_x2_i0_m1_q4_z1_c1_s1_p0", + name: "GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path", + }, + { + dataset: "generated_fixed_suffix_expansion_v3_d5_f8_r4_x3_i0_m1_q3_z0_c0_s0_p0", + name: "GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path", + }, +} + +// buildSuffixReverseGuardFeasibilityReport evaluates the actual production- +// shaped guard, not a stripped SQL proxy. Protected holdout records are +// rejected before their timing can influence this decision. +func buildSuffixReverseGuardFeasibilityReport( + incumbentRecords, reverseRecords, guardedRecords []CaseResult, + aa *AAResolutionReport, + options SuffixReverseGuardFeasibilityOptions, +) (SuffixReverseGuardFeasibilityReport, error) { + if options.Confidence <= 0 || options.Confidence >= 1 { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("confidence level must be between 0 and 1") + } + if options.BootstrapCount == 0 { + options.BootstrapCount = defaultBootstrapCount + } + if options.BootstrapCount < 1 { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("bootstrap count must be positive") + } + if err := validateAAResolutionEvidence(aa, incumbentRecords, options.Confidence); err != nil { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("incumbent A/A evidence: %w", err) + } + if err := validateOrientationV2AAEvidence(aa, incumbentRecords); err != nil { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("incumbent A/A environment: %w", err) + } + identity, err := validateOrientationV2EvidenceIdentity(incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return SuffixReverseGuardFeasibilityReport{}, err + } + series, keys, err := collectSuffixReverseGuardSeries(incumbentRecords, reverseRecords, guardedRecords) + if err != nil { + return SuffixReverseGuardFeasibilityReport{}, err + } + if err := validateSuffixReverseGuardFeasibilityCohort(keys, incumbentRecords, reverseRecords, guardedRecords); err != nil { + return SuffixReverseGuardFeasibilityReport{}, err + } + if err := validateSuffixReverseGuardRunSchedule(incumbentRecords, reverseRecords, guardedRecords, len(keys)); err != nil { + return SuffixReverseGuardFeasibilityReport{}, err + } + report := SuffixReverseGuardFeasibilityReport{ + Version: suffixReverseGuardFeasibilityVersion, + Policy: string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), + SelectorVersion: optimize.ExpansionSearchSelectorFixedSuffixPathV1, + Protocol: "training_feasibility", + Seed: options.Seed, + Confidence: options.Confidence, + SourceCommit: identity.sourceCommit, + DirtyDiffSHA256: identity.dirtyDiffSHA256, + BinarySHA256: identity.binarySHA256, + CorpusSHA256: identity.corpusSHA256, + Caps: map[string]int64{ + "suffix_rows": optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, + "state_rows": optimize.ExpansionSearchSuffixReverseGuardStateLimit, + }, + GuardRatioUpperLimit: 1.10, GuardAbsoluteUpperLimit: 100 * time.Microsecond, + ForwardRatioUpperLimit: .95, ForwardSavingLowerLimit: 100 * time.Microsecond, ForwardP95UpperLimit: 1.05, + EvidencePassed: true, + } + gateOptions := PerfGateOptions{Seed: options.Seed, Confidence: options.Confidence, BootstrapCount: options.BootstrapCount} + for index, key := range keys { + current := series[key] + if current.split == "holdout" { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("%s/%s feasibility input opens a protected holdout", key.dataset, key.name) + } + if current.split != "training" { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("%s/%s feasibility input must use the predeclared training split, got %q", key.dataset, key.name, current.split) + } + rounds := sortedRounds(current.incumbent) + if len(rounds) != 6 || !slices.Equal(rounds, sortedRounds(current.reverse)) || !slices.Equal(rounds, sortedRounds(current.guarded)) { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("%s/%s requires exactly six matched three-arm rounds", key.dataset, key.name) + } + for _, round := range rounds { + if len(current.incumbent[round]) != 10 || len(current.reverse[round]) != 10 || len(current.guarded[round]) != 10 { + return SuffixReverseGuardFeasibilityReport{}, fmt.Errorf("%s/%s round %d requires exactly ten samples per arm", key.dataset, key.name, round) + } + } + if err := validateSuffixReverseGuardArmOrder(incumbentRecords, reverseRecords, guardedRecords, key, rounds); err != nil { + return SuffixReverseGuardFeasibilityReport{}, err + } + seed := options.Seed + int64(index)*7919 + _, aaFloor, err := aaTimingFloor(aa, key, false, 0) + if err != nil { + return SuffixReverseGuardFeasibilityReport{}, err + } + absoluteFloor := max(report.GuardAbsoluteUpperLimit, aaFloor) + guardOverhead := orientationLatencyGate( + string(optimize.ExpansionSearchSuffixSeededReverse), report.Policy, + current.reverse, current.guarded, report.GuardRatioUpperLimit, report.GuardAbsoluteUpperLimit, seed, gateOptions, + ) + fastestIdentity, fastest := fastestOrientationExactArm(current.incumbent, current.reverse) + fastestRegret := orientationLatencyGate(fastestIdentity, report.Policy, fastest, current.guarded, report.GuardRatioUpperLimit, absoluteFloor, seed+3, gateOptions) + improvement := SuffixReverseGuardImprovementGate{ + MedianRatio: bootstrapRoundMedianRatio(current.incumbent, current.guarded, seed+6, gateOptions), + MedianSaving: bootstrapRoundMedianSaving(current.incumbent, current.guarded, seed+7, gateOptions), + P95Ratio: bootstrapStratifiedP95Ratio(current.incumbent, current.guarded, seed+8, gateOptions), + RatioUpperLimit: report.ForwardRatioUpperLimit, SavingLowerLimit: report.ForwardSavingLowerLimit, P95UpperLimit: report.ForwardP95UpperLimit, + } + improvement.Passed = (improvement.MedianRatio.Upper <= improvement.RatioUpperLimit || improvement.MedianSaving.Lower >= improvement.SavingLowerLimit) && + improvement.P95Ratio.Upper <= improvement.P95UpperLimit + entry := SuffixReverseGuardFeasibilityCase{ + Dataset: key.dataset, Name: key.name, QualificationSplit: current.split, QuerySHA256: current.querySHA, Rounds: len(rounds), + ExactObservationsMatched: true, RuntimeIdentity: current.runtime, RuntimeBranch: current.branch, + Overflow: current.overflow, FallbackExecuted: current.fallback, GuardOverhead: guardOverhead, + FastestExactRegret: fastestRegret, ForwardImprovement: improvement, + } + if current.runtime != string(optimize.ExpansionSearchSuffixSeededReverse) || current.branch != "suffix_seeded_reverse" || current.overflow || current.fallback { + entry.Reasons = append(entry.Reasons, "normal feasibility case did not execute the admitted reverse candidate without fallback") + } + if !guardOverhead.Passed { + entry.Reasons = append(entry.Reasons, "guard overhead exceeds the 1.10/100us stop gate") + } + if !fastestRegret.Passed { + entry.Reasons = append(entry.Reasons, "guard regret exceeds the 1.10/A/A-calibrated stop gate") + } + if !improvement.Passed { + entry.Reasons = append(entry.Reasons, "guard does not materially improve forward p50 with contained p95") + } + entry.Passed = len(entry.Reasons) == 0 + if !entry.Passed { + report.EvidencePassed = false + } + report.Cases = append(report.Cases, entry) + } + report.Passed = len(report.Cases) > 0 && report.EvidencePassed + return report, nil +} + +// validateSuffixReverseGuardFeasibilityCohort binds this early decision to the +// two already-open complete-path V3 training declarations. All three arms +// must carry the same schema-v2 selection declaration and resolved manifest; +// filtered timing from any other training workload is rejected. +func validateSuffixReverseGuardFeasibilityCohort(keys []performanceKey, artifacts ...[]CaseResult) error { + expected := map[performanceKey]struct{}{} + declared := make([]DeclaredCaseBackend, 0, 2*len(suffixReverseGuardFeasibilityCases)) + for _, testCase := range suffixReverseGuardFeasibilityCases { + expected[performanceKey{dataset: testCase.dataset, name: testCase.name, backend: ModePostgresSQL}] = struct{}{} + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + declared = append(declared, DeclaredCaseBackend{Dataset: testCase.dataset, Name: testCase.name, Backend: backend}) + } + } + if !orientationV2KeySetsEqual(expected, performanceKeySet(keys)) { + return fmt.Errorf("suffix-reverse feasibility does not contain the exact two-case training cohort") + } + expectedDeclaration := declarationSHA256(declared) + expectedResolved := "" + for index, records := range artifacts { + selection, err := selectionIdentity(records) + if err != nil { + return fmt.Errorf("suffix-reverse feasibility arm %d selection: %w", index+1, err) + } + if err := validateSelectionManifestAccounting(selection); err != nil { + return fmt.Errorf("suffix-reverse feasibility arm %d selection accounting: %w", index+1, err) + } + if selection.Version != selectionManifestVersion || !selection.DiagnosticOnly || + selection.SelectedDeclarationCount != 2*len(expected) || len(selection.Resolved) != len(expected) || + selection.FullDeclarationCount != selection.SelectedDeclarationCount+selection.OmittedDeclarationCount || + selection.DeclarationSHA256 != expectedDeclaration { + return fmt.Errorf("suffix-reverse feasibility selection does not bind the exact two-case declaration") + } + resolved := map[performanceKey]struct{}{} + for _, item := range selection.Resolved { + if item.Category != "generated_fixed_suffix_expansion" { + return fmt.Errorf("suffix-reverse feasibility selection contains category %q", item.Category) + } + resolved[performanceKey{dataset: item.Dataset, name: item.Name, backend: ModePostgresSQL}] = struct{}{} + } + if !orientationV2KeySetsEqual(expected, resolved) { + return fmt.Errorf("suffix-reverse feasibility selection does not resolve the exact two-case cohort") + } + resolvedSHA := resolvedSelectionSHA256(selection.Resolved) + if expectedResolved == "" { + expectedResolved = resolvedSHA + } else if expectedResolved != resolvedSHA { + return fmt.Errorf("suffix-reverse feasibility arms mix resolved selection identities") + } + } + return nil +} + +func collectSuffixReverseGuardSeries( + incumbentRecords, reverseRecords, guardedRecords []CaseResult, +) (map[performanceKey]*suffixReverseGuardSeries, []performanceKey, error) { + artifacts := []struct { + name string + records []CaseResult + }{{"incumbent", incumbentRecords}, {"reverse", reverseRecords}, {"guarded", guardedRecords}} + keySets := make([]map[performanceKey]struct{}, len(artifacts)) + for index, artifact := range artifacts { + keys, err := orientationV2ArtifactKeys(artifact.name, artifact.records) + if err != nil { + return nil, nil, err + } + keySets[index] = keys + } + if !orientationV2KeySetsEqual(keySets[0], keySets[1]) || !orientationV2KeySetsEqual(keySets[0], keySets[2]) { + return nil, nil, fmt.Errorf("suffix-reverse guard three-arm case sets do not match") + } + series := map[performanceKey]*suffixReverseGuardSeries{} + for key := range keySets[0] { + series[key] = &suffixReverseGuardSeries{incumbent: roundSamples{}, reverse: roundSamples{}, guarded: roundSamples{}} + } + for _, artifact := range artifacts { + seen := map[performanceKey]map[int]struct{}{} + for _, record := range artifact.records { + key := performanceKey{dataset: record.Dataset, name: record.Name, backend: record.ExecutionMode} + current := series[key] + if current == nil { + return nil, nil, fmt.Errorf("%s artifact contains unexpected case %s/%s", artifact.name, key.dataset, key.name) + } + if err := validateSuffixReverseGuardRecord(record, artifact.name); err != nil { + return nil, nil, err + } + round, err := orientationV2RecordRound(record) + if err != nil { + return nil, nil, err + } + if seen[key] == nil { + seen[key] = map[int]struct{}{} + } + if _, duplicate := seen[key][round]; duplicate { + return nil, nil, fmt.Errorf("%s/%s %s artifact duplicates round %d", key.dataset, key.name, artifact.name, round) + } + seen[key][round] = struct{}{} + if strings.TrimSpace(record.Cypher) == "" { + return nil, nil, fmt.Errorf("%s/%s lacks exact Cypher for query-SHA binding", key.dataset, key.name) + } + querySHA := pgdriver.TraversalPolicyQuerySHA256(record.Cypher) + if current.querySHA == "" { + current.querySHA, current.split = querySHA, record.Shape.QualificationSplit + } else if current.querySHA != querySHA || current.split != record.Shape.QualificationSplit { + return nil, nil, fmt.Errorf("%s/%s changes workload digest or split across arms", key.dataset, key.name) + } + switch artifact.name { + case "incumbent": + appendOrientationWarmSamples(current.incumbent, record) + case "reverse": + appendOrientationWarmSamples(current.reverse, record) + case "guarded": + summary := record.TraversalTelemetry.Summary + if current.observed && (current.runtime != summary.RuntimeIdentity || current.branch != summary.RuntimeBranch || current.overflow != *summary.Overflow || current.fallback != *summary.FallbackExecuted) { + return nil, nil, fmt.Errorf("%s/%s changes guarded runtime outcome across rounds", key.dataset, key.name) + } + current.runtime, current.branch, current.overflow, current.fallback, current.observed = + summary.RuntimeIdentity, summary.RuntimeBranch, *summary.Overflow, *summary.FallbackExecuted, true + appendOrientationWarmSamples(current.guarded, record) + } + } + } + keys := sortedPerformanceKeys(keySets[0]) + for _, key := range keys { + if !series[key].observed { + return nil, nil, fmt.Errorf("%s/%s lacks guarded runtime evidence", key.dataset, key.name) + } + if err := validateOrientationExactObservations(key, incumbentRecords, reverseRecords, guardedRecords); err != nil { + return nil, nil, err + } + } + return series, keys, nil +} + +func validateSuffixReverseGuardRecord(record CaseResult, arm string) error { + if record.Status != StatusOK || record.Environment == nil || record.PostgresEnvironment == nil || record.TraversalTelemetry == nil { + return fmt.Errorf("%s/%s %s arm lacks a successful telemetry-bearing PostgreSQL record", record.Dataset, record.Name, arm) + } + if record.Environment.ArtifactSchemaVersion != 2 || record.Environment.PoolSize != 1 || len(record.Environment.Concurrency) != 0 || + record.Environment.ExistingGraph || record.Fixture == nil || record.Fixture.Dataset != record.Dataset || + !lowercaseSHA256(record.Fixture.Checksum) || !record.Fixture.PhysicalValidated { + return fmt.Errorf("%s/%s %s arm lacks the schema-v2 single-session physical-fixture contract", record.Dataset, record.Name, arm) + } + if !lowercaseSHA256(record.WorkloadSHA256) || !lowercaseSHA256(record.SQLFingerprint) || + len(record.Concurrency) != 0 || len(record.PostgresReferences) != 0 || record.ClientWaterfall != nil || + record.RawPGXWaterfall != nil || record.RawPGXRoundTrip != nil { + return fmt.Errorf("%s/%s %s arm mixes incomplete identity with supplemental measurements", record.Dataset, record.Name, arm) + } + if !strings.EqualFold(strings.TrimSpace(record.PostgresEnvironment.TransactionIsolation), "repeatable read") || + !record.Shape.PathMaterializationRequired || record.Shape.QualificationSplit == "holdout" { + return fmt.Errorf("%s/%s %s arm is outside the training-only full-path Repeatable Read envelope", record.Dataset, record.Name, arm) + } + environment := record.Environment + if environment.Round < 1 || environment.Block != environment.Round { + return fmt.Errorf("%s/%s %s arm requires block equal to round", record.Dataset, record.Name, arm) + } + if environment.Arm != arm || environment.ArmOrder < 1 || environment.ArmOrder > 3 || strings.TrimSpace(environment.RunUUID) == "" { + return fmt.Errorf("%s/%s %s arm has malformed three-arm run metadata", record.Dataset, record.Name, arm) + } + if environment.StartedAt.IsZero() || environment.EndedAt.IsZero() || environment.EndedAt.Before(environment.StartedAt) { + return fmt.Errorf("%s/%s %s arm has malformed invocation timestamps", record.Dataset, record.Name, arm) + } + if arm == "incumbent" || arm == "reverse" { + if err := validateOrientationV2Record(record, arm); err != nil { + return fmt.Errorf("%s/%s exact %s arm: %w", record.Dataset, record.Name, arm, err) + } + } + if err := record.TraversalTelemetry.Validate(); err != nil { + return fmt.Errorf("%s/%s %s arm telemetry: %w", record.Dataset, record.Name, arm, err) + } + summary := record.TraversalTelemetry.Summary + if summary.RuntimeOutcomeAvailable == nil || !*summary.RuntimeOutcomeAvailable || summary.Overflow == nil || summary.FallbackExecuted == nil { + return fmt.Errorf("%s/%s %s arm lacks a complete runtime outcome", record.Dataset, record.Name, arm) + } + forward, reverse := string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchSuffixSeededReverse) + switch arm { + case "incumbent": + // validateOrientationV2Record already restricts this to either a direct + // selected-forward tuple or the exact compile-time fallback tuple. Both + // execute the same incumbent SQL boundary and are valid exact controls. + if summary.RuntimeIdentity != forward || summary.AppliedIdentity != forward || *summary.Overflow { + return fmt.Errorf("%s/%s incumbent arm did not execute exact forward", record.Dataset, record.Name) + } + case "reverse": + if summary.RuntimeIdentity != reverse || summary.AppliedIdentity != reverse || *summary.Overflow || *summary.FallbackExecuted { + return fmt.Errorf("%s/%s reverse arm did not execute exact reverse", record.Dataset, record.Name) + } + case "guarded": + if summary.EmittedIdentity != string(optimize.ExpansionSearchPolicySuffixReverseGuardV1) || + summary.SelectorVersion != optimize.ExpansionSearchSelectorFixedSuffixPathV1 || + summary.ExecutionBoundary != optimize.ExpansionSearchExecutionBoundaryGuardedDualArm || + summary.ObservationMode != string(optimize.ExpansionSearchObservationFullPath) || summary.WouldSelectIdentity != "" { + return fmt.Errorf("%s/%s guarded arm does not prove suffix-reverse-guard-v1", record.Dataset, record.Name) + } + if err := validateSuffixReverseGuardDiagnostic(record); err != nil { + return err + } + default: + return fmt.Errorf("unknown suffix-reverse guard arm %q", arm) + } + for _, sample := range record.Stats.Samples { + if sample.Classification != "warm" || sample.Duration <= 0 { + continue + } + if sample.Round != environment.Round || sample.Block != environment.Block || sample.Arm != environment.Arm || + sample.ArmOrder != environment.ArmOrder || sample.RunUUID != environment.RunUUID { + return fmt.Errorf("%s/%s %s warm sample is outside its physical arm invocation", record.Dataset, record.Name, arm) + } + if sample.RequestedIdentity != summary.RequestedIdentity || sample.RuntimeIdentity != summary.RuntimeIdentity || + sample.RuntimeBranch != summary.RuntimeBranch || sample.FallbackExecuted == nil || + *sample.FallbackExecuted != *summary.FallbackExecuted { + return fmt.Errorf("%s/%s %s warm sample contradicts runtime summary", record.Dataset, record.Name, arm) + } + if arm == "guarded" { + if sample.RequestedIdentity != reverse || sample.RuntimeAttestation != "timed_invocation" || strings.TrimSpace(sample.RuntimeInvocationID) == "" || + strings.TrimSpace(sample.ConnectionID) == "" || validateRuntimeReceiptEvents(sample.RuntimeReceiptEvents, sample.RuntimeIdentity, sample.RuntimeBranch, sample.FallbackExecuted) != nil { + return fmt.Errorf("%s/%s guarded warm sample lacks a valid timed receipt", record.Dataset, record.Name) + } + for _, event := range sample.RuntimeReceiptEvents { + if event.InvocationID != sample.RuntimeInvocationID { + return fmt.Errorf("%s/%s guarded warm sample receipt is not bound to its timed invocation", record.Dataset, record.Name) + } + } + } + } + return nil +} + +// validateSuffixReverseGuardRunSchedule proves that the doubled-Williams +// labels describe the order in which the three arm processes actually ran. +// It mirrors the SP-I2 chronology contract: blocks equal rounds, all selected +// cases share one invocation interval per arm/round, arms do not overlap, and +// later rounds neither overlap nor predate earlier rounds. +func validateSuffixReverseGuardRunSchedule( + incumbentRecords, reverseRecords, guardedRecords []CaseResult, + expectedCaseCount int, +) error { + arms := []struct { + name string + records []CaseResult + }{{"incumbent", incumbentRecords}, {"reverse", reverseRecords}, {"guarded", guardedRecords}} + + invocations := make([]map[int]suffixReverseGuardInvocationIdentity, len(arms)) + for index, arm := range arms { + invocations[index] = map[int]suffixReverseGuardInvocationIdentity{} + caseCounts := map[int]int{} + for _, record := range arm.records { + if record.Environment == nil { + return fmt.Errorf("%s/%s %s arm lacks invocation chronology", record.Dataset, record.Name, arm.name) + } + environment := record.Environment + identity := suffixReverseGuardInvocationIdentity{ + round: environment.Round, + block: environment.Block, + order: environment.ArmOrder, + arm: environment.Arm, + runUUID: environment.RunUUID, + startedAt: environment.StartedAt, + endedAt: environment.EndedAt, + } + if identity.startedAt.IsZero() || identity.endedAt.IsZero() || identity.endedAt.Before(identity.startedAt) { + return fmt.Errorf("suffix-reverse guard %s round %d has malformed invocation timestamps", arm.name, identity.round) + } + if prior, found := invocations[index][identity.round]; found && prior != identity { + return fmt.Errorf("suffix-reverse guard %s round %d mixes invocation identities across the exact cohort", arm.name, identity.round) + } + invocations[index][identity.round] = identity + caseCounts[identity.round]++ + } + if len(invocations[index]) != 6 { + return fmt.Errorf("suffix-reverse guard %s artifact does not contain exactly six physical round invocations", arm.name) + } + for round, count := range caseCounts { + if count != expectedCaseCount { + return fmt.Errorf("suffix-reverse guard %s round %d contains %d cases, expected %d", arm.name, round, count, expectedCaseCount) + } + } + } + + positionCounts := make([][4]int, len(arms)) + schedule := map[string]int{} + runUUID := "" + var priorEnded time.Time + for round := 1; round <= 6; round++ { + orderedArms := [4]string{} + orderedInvocations := [4]suffixReverseGuardInvocationIdentity{} + seenPositions := map[int]struct{}{} + seenNames := map[string]struct{}{} + for index, arm := range arms { + current, found := invocations[index][round] + if !found { + return fmt.Errorf("suffix-reverse guard invocation schedule must use contiguous rounds 1 through 6") + } + if current.block != round { + return fmt.Errorf("suffix-reverse guard round %d requires block equal to round", round) + } + if current.arm != arm.name || current.order < 1 || current.order > 3 || strings.TrimSpace(current.runUUID) == "" { + return fmt.Errorf("suffix-reverse guard round %d has malformed %s invocation identity", round, arm.name) + } + if _, duplicate := seenPositions[current.order]; duplicate { + return fmt.Errorf("suffix-reverse guard round %d duplicates a physical three-arm position", round) + } + if _, duplicate := seenNames[current.arm]; duplicate { + return fmt.Errorf("suffix-reverse guard round %d duplicates a physical arm identity", round) + } + seenPositions[current.order], seenNames[current.arm] = struct{}{}, struct{}{} + positionCounts[index][current.order]++ + orderedArms[current.order], orderedInvocations[current.order] = current.arm, current + if runUUID == "" { + runUUID = current.runUUID + } else if runUUID != current.runUUID { + return fmt.Errorf("suffix-reverse guard artifacts mix run UUIDs across arms or rounds") + } + } + if len(seenPositions) != 3 || len(seenNames) != 3 { + return fmt.Errorf("suffix-reverse guard round %d lacks a complete physical three-arm block", round) + } + for position := 2; position <= 3; position++ { + if orderedInvocations[position-1].endedAt.After(orderedInvocations[position].startedAt) { + return fmt.Errorf("suffix-reverse guard round %d arm timestamps contradict the declared execution order", round) + } + } + if !priorEnded.IsZero() && priorEnded.After(orderedInvocations[1].startedAt) { + return fmt.Errorf("suffix-reverse guard round %d overlaps or predates the prior round", round) + } + priorEnded = orderedInvocations[3].endedAt + schedule[strings.Join(orderedArms[1:], "/")]++ + } + for index, counts := range positionCounts { + if counts[1] != 2 || counts[2] != 2 || counts[3] != 2 { + return fmt.Errorf("suffix-reverse guard %s arm does not physically follow the six-round doubled-Williams schedule", arms[index].name) + } + } + if len(schedule) != 6 { + return fmt.Errorf("suffix-reverse guard physical schedule does not contain all six doubled-Williams arm orders exactly once") + } + for _, count := range schedule { + if count != 1 { + return fmt.Errorf("suffix-reverse guard physical schedule repeats a doubled-Williams arm order") + } + } + return nil +} + +// validateSuffixReverseGuardDiagnostic binds feasibility timing to the exact +// immutable caps, the independent suffix-guard counter family, and a +// marker-first plan shape that proves the inactive arm did not initialize. +func validateSuffixReverseGuardDiagnostic(record CaseResult) error { + telemetry := record.TraversalTelemetry + if telemetry.Level != TraversalTelemetryLevelDiagnostic || telemetry.Diagnostic == nil || + telemetry.Diagnostic.CounterStatus != TraversalTelemetryCounterStatusComplete || telemetry.Diagnostic.PlanReplay == nil { + return fmt.Errorf("%s/%s guarded arm lacks a complete untimed diagnostic replay", record.Dataset, record.Name) + } + if record.PostgresMetrics == nil { + return fmt.Errorf("%s/%s guarded arm lacks the measured PostgreSQL plan used by its replay", record.Dataset, record.Name) + } + for _, family := range []TraversalTelemetryFamily{ + TraversalTelemetryFamilySuffixGuard, TraversalTelemetryFamilyOrdinary, TraversalTelemetryFamilyHydration, + } { + if !slices.Contains(telemetry.Diagnostic.RequiredFamilies, family) { + return fmt.Errorf("%s/%s guarded arm lacks required %s telemetry", record.Dataset, record.Name, family) + } + } + if telemetry.Diagnostic.Counters.Orientation != nil { + return fmt.Errorf("%s/%s guarded arm incorrectly carries orientation topology counters", record.Dataset, record.Name) + } + summary := telemetry.Summary + if len(summary.Caps) != 2 || + summary.Caps["suffix_rows"] != optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit || + summary.Caps["state_rows"] != optimize.ExpansionSearchSuffixReverseGuardStateLimit { + return fmt.Errorf("%s/%s guarded arm does not bind the immutable suffix/state caps", record.Dataset, record.Name) + } + gate := ResourceGateCase{} + appendSuffixGuardAttributionReasons(&gate, telemetry.Diagnostic) + if len(gate.Reasons) > 0 { + return fmt.Errorf("%s/%s guarded arm plan attribution: %s", record.Dataset, record.Name, strings.Join(gate.Reasons, "; ")) + } + + counters := telemetry.Diagnostic.Counters.SuffixGuard + if counters == nil { + return fmt.Errorf("%s/%s guarded arm lacks typed suffix-guard counters", record.Dataset, record.Name) + } + planCounters := telemetry.Diagnostic.PlanReplay.Counters + derivedCounters := postgresTraversalPlanReplay(*record.PostgresMetrics).Counters + for name, typed := range map[string]*int64{ + "suffix_guard_root_presence_rows": counters.RootPresenceRows, + "suffix_guard_suffix_rows": counters.SuffixRows, + "suffix_guard_boundary_rows": counters.DistinctBoundaryRows, + "suffix_guard_state_rows": counters.StateRows, + "suffix_guard_output_rows": counters.OutputRows, + "suffix_guard_candidate_marker_rows": counters.CandidateMarkerRows, + "suffix_guard_fallback_marker_rows": counters.FallbackMarkerRows, + "suffix_guard_candidate_branch_rows": counters.CandidateBranchRows, + "suffix_guard_fallback_branch_rows": counters.FallbackBranchRows, + "suffix_guard_candidate_executor_loops": counters.CandidateExecutorLoops, + "suffix_guard_fallback_executor_loops": counters.FallbackExecutorLoops, + } { + planValue, planPresent := planCounters[name] + derivedValue, derivedPresent := derivedCounters[name] + if typed == nil || !planPresent || !derivedPresent || *typed != planValue || planValue != derivedValue { + return fmt.Errorf("%s/%s guarded arm counter %s is not bound to its measured plan", record.Dataset, record.Name, name) + } + } + rootRows, suffixRows := *counters.RootPresenceRows, *counters.SuffixRows + boundaryRows, stateRows, outputRows := *counters.DistinctBoundaryRows, *counters.StateRows, *counters.OutputRows + suffixOverflow := suffixRows > optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit + stateOverflow := stateRows > optimize.ExpansionSearchSuffixReverseGuardStateLimit + overflow := suffixOverflow || stateOverflow + if rootRows < 0 || rootRows > 1 || suffixRows < 0 || + suffixRows > optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit+1 || + boundaryRows < 0 || boundaryRows > suffixRows || stateRows < 0 || + stateRows > optimize.ExpansionSearchSuffixReverseGuardStateLimit+1 || outputRows != record.RowCount { + return fmt.Errorf("%s/%s guarded arm has impossible bounded-relation counters", record.Dataset, record.Name) + } + if rootRows == 0 && (suffixRows != 0 || boundaryRows != 0 || stateRows != 0) { + return fmt.Errorf("%s/%s guarded arm performed suffix/reverse work without a bound root", record.Dataset, record.Name) + } + if suffixOverflow && (boundaryRows != 0 || stateRows != 0) { + return fmt.Errorf("%s/%s guarded arm performed reverse work after suffix overflow", record.Dataset, record.Name) + } + if *counters.SuffixOverflow != suffixOverflow || *counters.StateOverflow != stateOverflow || + *summary.Overflow != overflow || *summary.FallbackExecuted != overflow { + return fmt.Errorf("%s/%s guarded arm overflow flags contradict its cap+1 counters", record.Dataset, record.Name) + } + if !overflow { + if summary.RuntimeIdentity != string(optimize.ExpansionSearchSuffixSeededReverse) || summary.RuntimeBranch != "suffix_seeded_reverse" || + *counters.CandidateMarkerRows != 1 || *counters.FallbackMarkerRows != 0 { + return fmt.Errorf("%s/%s guarded arm did not execute the admitted reverse candidate", record.Dataset, record.Name) + } + return nil + } + expectedBranch := "exact_forward_state_overflow" + if suffixOverflow { + expectedBranch = "exact_forward_suffix_overflow" + } + if summary.RuntimeIdentity != string(optimize.ExpansionSearchStepwiseForward) || summary.RuntimeBranch != expectedBranch || + summary.FallbackIdentity != string(optimize.ExpansionSearchStepwiseForward) || + *counters.CandidateMarkerRows != 0 || *counters.FallbackMarkerRows != 1 { + return fmt.Errorf("%s/%s guarded arm did not execute the exact overflow fallback", record.Dataset, record.Name) + } + return nil +} + +func validateSuffixReverseGuardArmOrder( + incumbentRecords, reverseRecords, guardedRecords []CaseResult, + key performanceKey, + rounds []int, +) error { + arms := []struct { + name string + records []CaseResult + }{{"incumbent", incumbentRecords}, {"reverse", reverseRecords}, {"guarded", guardedRecords}} + evidence := make([]map[int]pairedRoundEvidence, len(arms)) + positions := make([][4]int, len(arms)) + schedule := map[string]int{} + for index, arm := range arms { + current, err := collectPairedRoundEvidence(arm.records, key) + if err != nil { + return err + } + evidence[index] = current + } + for _, round := range rounds { + seenPositions, seenNames := map[int]struct{}{}, map[string]struct{}{} + orderedArms := [4]string{} + block, runUUID := 0, "" + for index, arm := range arms { + current, found := evidence[index][round] + if !found || current.Warmups != 5 || current.Arm != arm.name || current.ArmOrder < 1 || current.ArmOrder > 3 { + return fmt.Errorf("%s/%s round %d lacks %s arm identity, warmups, or order", key.dataset, key.name, round, arm.name) + } + if _, duplicate := seenPositions[current.ArmOrder]; duplicate { + return fmt.Errorf("%s/%s round %d duplicates a three-arm position", key.dataset, key.name, round) + } + if _, duplicate := seenNames[current.Arm]; duplicate { + return fmt.Errorf("%s/%s round %d duplicates an arm label", key.dataset, key.name, round) + } + seenPositions[current.ArmOrder], seenNames[current.Arm], positions[index][current.ArmOrder] = struct{}{}, struct{}{}, positions[index][current.ArmOrder]+1 + orderedArms[current.ArmOrder] = current.Arm + if block == 0 { + block, runUUID = current.Block, current.RunUUID + } else if current.Block != block || current.RunUUID != runUUID { + return fmt.Errorf("%s/%s round %d has mismatched block or run UUID", key.dataset, key.name, round) + } + } + if block < 1 || runUUID == "" || len(seenPositions) != 3 || len(seenNames) != 3 { + return fmt.Errorf("%s/%s round %d lacks a complete three-arm block", key.dataset, key.name, round) + } + schedule[strings.Join(orderedArms[1:], "/")]++ + } + for index, counts := range positions { + if counts[1] != 2 || counts[2] != 2 || counts[3] != 2 { + return fmt.Errorf("%s/%s %s arm does not follow the six-round doubled-Williams schedule", key.dataset, key.name, arms[index].name) + } + } + if len(schedule) != 6 { + return fmt.Errorf("%s/%s does not contain all six doubled-Williams arm orders exactly once", key.dataset, key.name) + } + for _, count := range schedule { + if count != 1 { + return fmt.Errorf("%s/%s repeats a doubled-Williams arm order", key.dataset, key.name) + } + } + return nil +} + +// createSuffixReverseGuardFeasibilityReport loads, evaluates, and writes the +// bounded stop-gate artifact. +func createSuffixReverseGuardFeasibilityReport( + incumbentPath, reversePath, guardedPath, aaPath, outputPath string, + options SuffixReverseGuardFeasibilityOptions, +) (bool, error) { + incumbent, err := readJSONLFile(incumbentPath) + if err != nil { + return false, fmt.Errorf("read suffix-guard incumbent artifact: %w", err) + } + reverse, err := readJSONLFile(reversePath) + if err != nil { + return false, fmt.Errorf("read suffix-guard reverse artifact: %w", err) + } + guarded, err := readJSONLFile(guardedPath) + if err != nil { + return false, fmt.Errorf("read suffix-guard guarded artifact: %w", err) + } + aa, aaSHA, err := loadAAResolutionReport(aaPath) + if err != nil { + return false, fmt.Errorf("read suffix-guard A/A report: %w", err) + } + report, err := buildSuffixReverseGuardFeasibilityReport(incumbent, reverse, guarded, aa, options) + if err != nil { + return false, err + } + for destination, source := range map[*string]string{ + &report.IncumbentArtifactSHA256: incumbentPath, &report.ReverseArtifactSHA256: reversePath, + &report.GuardedArtifactSHA256: guardedPath, &report.AAReportSHA256: aaPath, + } { + digest, err := fileSHA256(source) + if err != nil { + return false, err + } + *destination = digest + } + report.AAReportSHA256 = aaSHA + raw, err := json.MarshalIndent(report, "", " ") + if err != nil { + return false, err + } + if outputPath == "" { + _, err = os.Stdout.Write(append(raw, '\n')) + } else { + err = os.WriteFile(outputPath, append(raw, '\n'), 0o644) + } + return report.Passed, err +} diff --git a/cmd/graphbench/suffix_reverse_guard_report_v1_test.go b/cmd/graphbench/suffix_reverse_guard_report_v1_test.go new file mode 100644 index 00000000..68557c14 --- /dev/null +++ b/cmd/graphbench/suffix_reverse_guard_report_v1_test.go @@ -0,0 +1,465 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +// TestSuffixReverseGuardFeasibilityPassesSixRoundStopGate verifies the report +// is an early three-arm decision, not a qualification or holdout report. +func TestSuffixReverseGuardFeasibilityPassesSixRoundStopGate(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + report, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 7, Confidence: defaultConfidenceLevel, BootstrapCount: 100}, + ) + require.NoError(t, err) + require.True(t, report.Passed) + require.Equal(t, "training_feasibility", report.Protocol) + require.Equal(t, string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), report.Policy) + require.Len(t, report.Cases, 2) + for _, reportCase := range report.Cases { + require.True(t, reportCase.GuardOverhead.Passed) + require.True(t, reportCase.FastestExactRegret.Passed) + require.True(t, reportCase.ForwardImprovement.Passed) + } +} + +// TestSuffixReverseGuardFeasibilityRequiresExactSelectionCohort verifies both +// key membership and each arm's schema-v2 selection declaration are sealed. +func TestSuffixReverseGuardFeasibilityRequiresExactSelectionCohort(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + keepFirstCase := func(records []CaseResult) []CaseResult { + result := records[:0] + for _, record := range records { + if record.Name == suffixReverseGuardFeasibilityCases[0].name { + result = append(result, record) + } + } + return result + } + _, err := buildSuffixReverseGuardFeasibilityReport( + keepFirstCase(incumbent), keepFirstCase(reverse), keepFirstCase(guarded), testAAReportForRecords(t, keepFirstCase(incumbent)), + SuffixReverseGuardFeasibilityOptions{Seed: 29, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "exact two-case training cohort") + + incumbent, reverse, guarded = suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + for index := range reverse { + reverse[index].Environment.Selection.DeclarationSHA256 = testSHA("9") + } + _, err = buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 29, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "exact two-case declaration") +} + +// TestSuffixReverseGuardFeasibilityRequiresPhysicalAAChronology ensures a +// label-balanced legacy A/A report cannot calibrate the stop gate without +// artifact-bound validation of its source process intervals. +func TestSuffixReverseGuardFeasibilityRequiresPhysicalAAChronology(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + aa := testAAReportForRecords(t, incumbent) + aa.PhysicalChronology = nil + _, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, aa, + SuffixReverseGuardFeasibilityOptions{Seed: 41, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "physical chronology provenance") +} + +// TestSuffixReverseGuardFeasibilityAcceptsExactCompileTimeForwardFallback +// preserves the incumbent tuple emitted when static lowering requests reverse +// but safely falls back to exact stepwise forward at compile time. +func TestSuffixReverseGuardFeasibilityAcceptsExactCompileTimeForwardFallback(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + forward := string(optimize.ExpansionSearchStepwiseForward) + requested := string(optimize.ExpansionSearchSuffixSeededReverse) + for recordIndex := range incumbent { + summary := &incumbent[recordIndex].TraversalTelemetry.Summary + summary.RequestedIdentity = requested + summary.RuntimeBranch = "compile_time_fallback" + summary.FallbackExecuted = boolPointer(true) + summary.FallbackIdentity = forward + summary.Provenance["fallback_identity"] = "test" + for sampleIndex := range incumbent[recordIndex].Stats.Samples { + sample := &incumbent[recordIndex].Stats.Samples[sampleIndex] + if sample.Classification == "warm" && sample.Duration > 0 { + sample.RequestedIdentity = requested + sample.RuntimeBranch = "compile_time_fallback" + sample.FallbackExecuted = boolPointer(true) + } + } + } + report, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 31, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.NoError(t, err) + require.True(t, report.Passed) +} + +// TestSuffixReverseGuardFeasibilityRejectsOverheadAndProtectedHoldout verifies +// the predeclared stop gate cannot be relaxed or fed v2/future holdout timing. +func TestSuffixReverseGuardFeasibilityRejectsOverheadAndProtectedHoldout(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 750*time.Microsecond) + report, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 11, Confidence: defaultConfidenceLevel, BootstrapCount: 50}, + ) + require.NoError(t, err) + require.False(t, report.Passed) + require.Contains(t, report.Cases[0].Reasons, "guard overhead exceeds the 1.10/100us stop gate") + + for _, records := range [][]CaseResult{incumbent, reverse, guarded} { + for index := range records { + records[index].Shape.QualificationSplit = "holdout" + } + } + _, err = buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 11, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "training-only full-path") + + incumbent, reverse, guarded = suffixReverseGuardTestArtifacts("diagnostic", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + _, err = buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 11, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "predeclared training split") +} + +// TestSuffixReverseGuardFeasibilityFailsClosedOnDiagnosticSubstitution verifies +// feasibility cannot be manufactured from summary-only, cap-substituted, or +// typed-counter evidence that differs from the measured PostgreSQL plan. +func TestSuffixReverseGuardFeasibilityFailsClosedOnDiagnosticSubstitution(t *testing.T) { + tests := []struct { + name string + mutate func(*CaseResult) + problem string + }{ + { + name: "summary only", + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Level = TraversalTelemetryLevelSummary + record.TraversalTelemetry.Diagnostic = nil + }, + problem: "complete untimed diagnostic replay", + }, + { + name: "cap substitution", + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Summary.Caps["suffix_rows"]-- + }, + problem: "immutable suffix/state caps", + }, + { + name: "typed counter substitution", + mutate: func(record *CaseResult) { + *record.TraversalTelemetry.Diagnostic.Counters.SuffixGuard.StateRows++ + }, + problem: "not bound to its measured plan", + }, + { + name: "inactive arm work", + mutate: func(record *CaseResult) { + record.TraversalTelemetry.Diagnostic.PlanReplay.Counters["suffix_guard_fallback_executor_loops"] = 1 + }, + problem: "did not suppress the fallback executor", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + test.mutate(&guarded[0]) + _, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 19, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, test.problem) + }) + } +} + +// TestSuffixReverseGuardFeasibilityRequiresAllWilliamsOrders prevents a +// position-balanced but repeated three-arm schedule from passing as the +// predeclared six-order design. +func TestSuffixReverseGuardFeasibilityRequiresAllWilliamsOrders(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + artifacts := [][]CaseResult{incumbent, reverse, guarded} + setRoundOrder := func(records []CaseResult, round, order int) { + for recordIndex := range records { + record := &records[recordIndex] + if record.Environment.Round != round { + continue + } + record.Environment.ArmOrder = order + roundStart := time.Unix(1_700_000_000+int64(round)*10, 0).UTC() + record.Environment.StartedAt = roundStart.Add(time.Duration(order-1) * 2 * time.Second) + record.Environment.EndedAt = record.Environment.StartedAt.Add(time.Second) + for sampleIndex := range record.Stats.Samples { + record.Stats.Samples[sampleIndex].ArmOrder = order + } + } + } + // Repeat the first three cyclic orders. Every arm still occupies every + // position twice, and the physical timestamps follow the substituted + // labels, so only exact schedule validation detects substitution. + for index, orders := range [][3]int{{1, 2, 3}, {2, 3, 1}, {3, 1, 2}} { + round := index + 4 + for artifactIndex, order := range orders { + setRoundOrder(artifacts[artifactIndex], round, order) + } + } + _, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 23, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "all six doubled-Williams arm orders exactly once") +} + +// TestSuffixReverseGuardFeasibilityRejectsPhysicalScheduleTampering verifies +// labels cannot manufacture a doubled-Williams study whose arm processes did +// not execute sequentially in the declared order and round chronology. +func TestSuffixReverseGuardFeasibilityRejectsPhysicalScheduleTampering(t *testing.T) { + tests := []struct { + name string + mutate func([]CaseResult, []CaseResult, []CaseResult) + problem string + }{ + { + name: "block differs from round", + mutate: func(incumbent, _, _ []CaseResult) { + for index := range incumbent { + if incumbent[index].Environment.Round == 2 { + incumbent[index].Environment.Block = 1 + for sampleIndex := range incumbent[index].Stats.Samples { + incumbent[index].Stats.Samples[sampleIndex].Block = 1 + } + } + } + }, + problem: "requires block equal to round", + }, + { + name: "missing invocation timestamp", + mutate: func(_, reverse, _ []CaseResult) { + for index := range reverse { + if reverse[index].Environment.Round == 1 { + reverse[index].Environment.StartedAt = time.Time{} + } + } + }, + problem: "malformed invocation timestamps", + }, + { + name: "mixed cohort invocation", + mutate: func(incumbent, _, _ []CaseResult) { + incumbent[0].Environment.StartedAt = incumbent[0].Environment.StartedAt.Add(100 * time.Millisecond) + incumbent[0].Environment.EndedAt = incumbent[0].Environment.EndedAt.Add(100 * time.Millisecond) + }, + problem: "mixes invocation identities across the exact cohort", + }, + { + name: "declared arm order contradicts execution", + mutate: func(incumbent, reverse, _ []CaseResult) { + firstStarted := incumbent[0].Environment.StartedAt + for index := range reverse { + if reverse[index].Environment.Round == 1 { + reverse[index].Environment.StartedAt = firstStarted.Add(500 * time.Millisecond) + reverse[index].Environment.EndedAt = firstStarted.Add(1500 * time.Millisecond) + } + } + }, + problem: "arm timestamps contradict the declared execution order", + }, + { + name: "round overlaps prior round", + mutate: func(incumbent, reverse, guarded []CaseResult) { + artifacts := [][]CaseResult{incumbent, reverse, guarded} + priorEnded := time.Time{} + for _, records := range artifacts { + for index := range records { + environment := records[index].Environment + if environment.Round == 1 && environment.ArmOrder == 3 { + priorEnded = environment.EndedAt + } + } + } + firstStarted := priorEnded.Add(-500 * time.Millisecond) + for _, records := range artifacts { + for index := range records { + environment := records[index].Environment + if environment.Round != 2 { + continue + } + environment.StartedAt = firstStarted.Add(time.Duration(environment.ArmOrder-1) * 2 * time.Second) + environment.EndedAt = environment.StartedAt.Add(time.Second) + } + } + }, + problem: "overlaps or predates the prior round", + }, + { + name: "mixed run UUID across rounds", + mutate: func(incumbent, reverse, guarded []CaseResult) { + for _, records := range [][]CaseResult{incumbent, reverse, guarded} { + for index := range records { + if records[index].Environment.Round != 6 { + continue + } + records[index].Environment.RunUUID = "substituted-run" + for sampleIndex := range records[index].Stats.Samples { + records[index].Stats.Samples[sampleIndex].RunUUID = "substituted-run" + } + } + } + }, + problem: "mix run UUIDs across arms or rounds", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + aa := testAAReportForRecords(t, incumbent) + test.mutate(incumbent, reverse, guarded) + _, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, aa, + SuffixReverseGuardFeasibilityOptions{Seed: 37, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, test.problem) + }) + } +} + +func TestParseConfigAcceptsAndIsolatesSuffixGuardFeasibilityReport(t *testing.T) { + cfg, err := parseConfig([]string{ + "-suffix-guard-incumbent-artifact", "forward.jsonl", + "-suffix-guard-reverse-artifact", "reverse.jsonl", + "-suffix-guard-guarded-artifact", "guarded.jsonl", + "-suffix-guard-aa", "aa.json", + "-suffix-guard-output", "report.json", + }, func(string) string { return "" }) + require.NoError(t, err) + require.Equal(t, "guarded.jsonl", cfg.SuffixGuardGuardedArtifact) + + for _, args := range [][]string{ + {"-suffix-guard-output", "report.json"}, + { + "-suffix-guard-incumbent-artifact", "same.jsonl", + "-suffix-guard-reverse-artifact", "same.jsonl", + "-suffix-guard-guarded-artifact", "guarded.jsonl", + "-suffix-guard-aa", "aa.json", + "-suffix-guard-output", "report.json", + }, + } { + _, err := parseConfig(args, func(string) string { return "" }) + require.Error(t, err, args) + } +} + +func TestSuffixGuardFeasibilityRejectsReceiptFromAnotherInvocation(t *testing.T) { + incumbent, reverse, guarded := suffixReverseGuardTestArtifacts("training", time.Millisecond, 500*time.Microsecond, 550*time.Microsecond) + guarded[0].Stats.Samples[0].RuntimeReceiptEvents[0].InvocationID = "other-invocation" + _, err := buildSuffixReverseGuardFeasibilityReport( + incumbent, reverse, guarded, testAAReportForRecords(t, incumbent), + SuffixReverseGuardFeasibilityOptions{Seed: 7, Confidence: defaultConfidenceLevel, BootstrapCount: 10}, + ) + require.ErrorContains(t, err, "not bound to its timed invocation") +} + +func suffixReverseGuardTestArtifacts(split string, incumbentDuration, reverseDuration, guardedDuration time.Duration) ([]CaseResult, []CaseResult, []CaseResult) { + orders := [][3]int{{1, 2, 3}, {2, 3, 1}, {3, 1, 2}, {3, 2, 1}, {1, 3, 2}, {2, 1, 3}} + incumbent := make([]CaseResult, 0, len(orders)*len(suffixReverseGuardFeasibilityCases)) + reverse := make([]CaseResult, 0, len(orders)*len(suffixReverseGuardFeasibilityCases)) + guarded := make([]CaseResult, 0, len(orders)*len(suffixReverseGuardFeasibilityCases)) + for caseIndex, testCase := range suffixReverseGuardFeasibilityCases { + for index, order := range orders { + round := index + 1 + incumbentRecord := orientationSelectorV2Record(round, order[0], "incumbent", split, "", incumbentDuration, false) + reverseRecord := orientationSelectorV2Record(round, order[1], "reverse", split, "", reverseDuration, false) + guardedRecord := orientationSelectorV2Record(round, order[2], "guarded", split, string(optimize.ExpansionSearchSuffixSeededReverse), guardedDuration, false) + for _, record := range []*CaseResult{&incumbentRecord, &reverseRecord, &guardedRecord} { + record.Dataset = testCase.dataset + record.Name = testCase.name + record.Fixture.Dataset = testCase.dataset + record.WorkloadSHA256 = sqlFingerprint("suffix-guard-workload-" + testCase.name) + record.Fixture.Checksum = sqlFingerprint("suffix-guard-fixture-" + testCase.name) + record.PostgresEnvironment.NodeRelationBytes += int64(caseIndex + 1) + record.PostgresEnvironment.EdgeRelationBytes += int64(caseIndex + 1) + } + metrics := suffixGuardTestMetrics(1, 0, 1, 0, 2, 9) + telemetry, err := buildPostgresCaseTraversalTelemetry( + translate.OptimizationSummary{TargetOutcomes: []translate.TargetLoweringOutcome{suffixGuardTestOutcome()}}, + metrics, "9123", TraversalTelemetryLevelDiagnostic, + ) + if err != nil { + panic(err) + } + enrichSuffixGuardTraversalTelemetry(telemetry, metrics, guardedRecord.RowCount, guardedRecord.ObservedRows) + guardedRecord.TraversalTelemetry = telemetry + guardedRecord.PostgresMetrics = &metrics + incumbent = append(incumbent, incumbentRecord) + reverse = append(reverse, reverseRecord) + guarded = append(guarded, guardedRecord) + } + } + for _, records := range [][]CaseResult{incumbent, reverse, guarded} { + for index := range records { + records[index].Cypher = "MATCH p=(root)-[:Edge*1..5]->()-[:Edge]->()-[:Edge]->(terminal) WHERE root.key = $root_key RETURN p" + records[index].Shape.PathMaterializationRequired = true + records[index].Environment.RunUUID = "suffix-guard-run" + roundStart := time.Unix(1_700_000_000+int64(records[index].Environment.Round)*10, 0).UTC() + records[index].Environment.StartedAt = roundStart.Add(time.Duration(records[index].Environment.ArmOrder-1) * 2 * time.Second) + records[index].Environment.EndedAt = records[index].Environment.StartedAt.Add(time.Second) + records[index].Stats.WarmupIterations = 5 + records[index].Environment.WarmupIterations = 5 + records[index].Stats.Samples = records[index].Stats.Samples[:10] + for sampleIndex := range records[index].Stats.Samples { + records[index].Stats.Samples[sampleIndex].RunUUID = "suffix-guard-run" + if records[index].Stats.Samples[sampleIndex].RuntimeAttestation == "timed_invocation" { + invocation := fmt.Sprintf("suffix-guard-%d-%d", records[index].Stats.Samples[sampleIndex].Round, sampleIndex+1) + records[index].Stats.Samples[sampleIndex].RuntimeInvocationID = invocation + records[index].Stats.Samples[sampleIndex].ConnectionID = "901" + for eventIndex := range records[index].Stats.Samples[sampleIndex].RuntimeReceiptEvents { + records[index].Stats.Samples[sampleIndex].RuntimeReceiptEvents[eventIndex].InvocationID = invocation + } + } + } + } + } + stampSuffixReverseGuardSelections(incumbent, reverse, guarded) + return incumbent, reverse, guarded +} + +func stampSuffixReverseGuardSelections(artifacts ...[]CaseResult) { + declared := make([]DeclaredCaseBackend, 0, 2*len(suffixReverseGuardFeasibilityCases)) + resolved := make([]ResolvedCaseSelector, 0, len(suffixReverseGuardFeasibilityCases)) + for _, testCase := range suffixReverseGuardFeasibilityCases { + for _, backend := range []ExecutionMode{ModePostgresSQL, ModeNeo4j} { + declared = append(declared, DeclaredCaseBackend{Dataset: testCase.dataset, Name: testCase.name, Backend: backend}) + } + resolved = append(resolved, ResolvedCaseSelector{Dataset: testCase.dataset, Name: testCase.name, Category: "generated_fixed_suffix_expansion"}) + } + selection := SelectionManifest{ + Version: selectionManifestVersion, Resolved: resolved, DiagnosticOnly: true, + FullDeclarationCount: len(declared), SelectedDeclarationCount: len(declared), DeclarationSHA256: declarationSHA256(declared), + } + for _, records := range artifacts { + for index := range records { + copy := selection + copy.Resolved = append([]ResolvedCaseSelector(nil), selection.Resolved...) + records[index].Environment.Selection = © + } + } +} diff --git a/cmd/graphbench/suffix_reverse_retry.go b/cmd/graphbench/suffix_reverse_retry.go new file mode 100644 index 00000000..270c960e --- /dev/null +++ b/cmd/graphbench/suffix_reverse_retry.go @@ -0,0 +1,52 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +// suffixReverseRetryDatabase limits the tool-only retry behavior to one exact +// translated candidate statement. Every other database operation delegates +// unchanged to the underlying PostgreSQL database. +type suffixReverseRetryDatabase struct { + graph.Database + candidateSQL string + fallbackSQL string + candidateParameters map[string]any + fallbackParameters map[string]any + limits pg.SuffixReverseRetryLimits +} + +func (s *suffixReverseRetryDatabase) ReadTransaction(ctx context.Context, delegate graph.TransactionDelegate, options ...graph.TransactionOption) error { + return s.Database.ReadTransaction(ctx, func(tx graph.Transaction) error { + return delegate(&suffixReverseRetryTransaction{Transaction: tx, owner: s}) + }, options...) +} + +type suffixReverseRetryTransaction struct { + graph.Transaction + owner *suffixReverseRetryDatabase +} + +func (s *suffixReverseRetryTransaction) Raw(query string, parameters map[string]any) graph.Result { + if query != s.owner.candidateSQL { + return s.Transaction.Raw(query, parameters) + } + retry, ok := s.Transaction.(pg.SuffixReverseRetryTransaction) + if !ok { + return graph.NewErrorResult(fmt.Errorf("PostgreSQL transaction does not expose suffix reverse retry tooling")) + } + return retry.RawSuffixReverseRetry( + s.owner.candidateSQL, + s.owner.fallbackSQL, + s.owner.candidateParameters, + s.owner.fallbackParameters, + s.owner.limits, + ) +} diff --git a/cmd/graphbench/suffix_route_cache_feasibility_protocol_test.go b/cmd/graphbench/suffix_route_cache_feasibility_protocol_test.go new file mode 100644 index 00000000..88acd306 --- /dev/null +++ b/cmd/graphbench/suffix_route_cache_feasibility_protocol_test.go @@ -0,0 +1,152 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "os" + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestSuffixRouteCacheFeasibilityV1FreezesTransactionOnlySafety verifies the +// pre-implementation contract cannot silently widen cache ownership or permit +// writes before a dedicated feasibility implementation is reviewed. +func TestSuffixRouteCacheFeasibilityV1FreezesTransactionOnlySafety(t *testing.T) { + contents, err := os.ReadFile("../../benchmark/testdata/scale/protocols/suffix_route_cache_feasibility_v1.json") + require.NoError(t, err) + + var protocol struct { + Schema string `json:"schema"` + Status string `json:"status"` + ProductionDefault string `json:"production_default"` + Scope struct { + Backend string `json:"backend"` + Transaction string `json:"transaction"` + Location string `json:"cache_location"` + } `json:"scope"` + CacheKey struct { + Required []string `json:"required_components"` + Snapshot string `json:"snapshot_rule"` + OnMissing string `json:"missing_or_unverifiable_component"` + } `json:"cache_key"` + Ownership struct { + Allowed string `json:"allowed_transactions"` + Write string `json:"write_or_savepoint_boundary"` + Commit string `json:"commit"` + Rollback string `json:"rollback_or_cancellation"` + Retry string `json:"retry"` + PoolRelease string `json:"pool_reacquisition"` + } `json:"ownership_and_invalidation"` + MissHit struct { + Miss string `json:"miss"` + Hit string `json:"hit"` + } `json:"miss_and_hit_contract"` + Resources struct { + Entries int64 `json:"maximum_entries_per_transaction"` + TotalBytes int64 `json:"maximum_total_bytes_per_transaction"` + EntryBytes int64 `json:"maximum_entry_bytes"` + WAL string `json:"read_path_wal"` + State string `json:"persistent_state"` + } `json:"resource_and_write_boundary"` + Acceptance map[string]bool `json:"acceptance"` + } + require.NoError(t, json.Unmarshal(contents, &protocol)) + + require.Equal(t, "suffix-route-cache-feasibility-v1", protocol.Schema) + require.Equal(t, "frozen_preimplementation_feasibility", protocol.Status) + require.Equal(t, "off", protocol.ProductionDefault) + require.Equal(t, "postgres_sql", protocol.Scope.Backend) + require.Contains(t, protocol.Scope.Transaction, "read-only") + require.Contains(t, protocol.Scope.Transaction, "repeatable_read") + require.Contains(t, protocol.Scope.Location, "exactly one active PostgreSQL transaction") + require.ElementsMatch(t, []string{ + "opaque transaction-owner token minted after BEGIN", + "graph_id", + "normalized Cypher shape fingerprint", + "canonical parameter names, types, and values fingerprint", + "frozen routing-policy identity and threshold version", + "transaction-local invalidation generation", + }, protocol.CacheKey.Required) + require.Contains(t, protocol.CacheKey.Snapshot, "no process, pool, connection") + require.Contains(t, protocol.CacheKey.OnMissing, "ordinary incumbent") + require.Contains(t, protocol.Ownership.Allowed, "no savepoint lifecycle") + require.Contains(t, protocol.Ownership.Write, "invalidate every entry") + require.Contains(t, protocol.Ownership.Commit, "no entry survives commit") + require.Contains(t, protocol.Ownership.Rollback, "must not publish") + require.Equal(t, "forbidden; a replacement transaction receives a new owner token and an empty cache", protocol.Ownership.Retry) + require.Contains(t, protocol.Ownership.PoolRelease, "cannot carry an entry") + require.Contains(t, protocol.MissHit.Miss, "exact ordinary EXPANSION-STEPWISE-FORWARD incumbent") + require.Contains(t, protocol.MissHit.Hit, "same active owner transaction") + require.Equal(t, int64(64), protocol.Resources.Entries) + require.Equal(t, int64(65536), protocol.Resources.TotalBytes) + require.Equal(t, int64(4096), protocol.Resources.EntryBytes) + require.Contains(t, protocol.Resources.WAL, "zero cache-attributable WAL") + require.Contains(t, protocol.Resources.State, "forbidden") + for _, requirement := range []string{ + "all_misses_incumbent_only", + "all_hits_owner_snapshot_key_and_generation_bound", + "all_transaction_end_and_invalidation_boundaries_empty_cache", + "all_memory_limits_observed", + "zero_cache_attributable_wal_and_persistent_state", + "cancellation_and_rollback_replay_exact", + "no_selector_or_translation_cache_change", + } { + require.True(t, protocol.Acceptance[requirement], requirement) + } + require.False(t, slices.Contains(protocol.CacheKey.Required, "backend_pid")) +} + +func TestTopologySelectedRoutingV1FreezesDefaultOffSnapshotContract(t *testing.T) { + contents, err := os.ReadFile("../../benchmark/testdata/scale/protocols/topology_selected_routing_v1.json") + require.NoError(t, err) + + var protocol struct { + Schema string `json:"schema"` + Status string `json:"status"` + ProductionDefault string `json:"production_default"` + Transaction struct { + Isolation []string `json:"required_isolation"` + ReadOnly bool `json:"read_only"` + SameSnap bool `json:"same_snapshot_synopsis_read"` + } `json:"transaction"` + Selector struct { + EstimatorVersion string `json:"estimator_version"` + MaximumDensity int64 `json:"maximum_edge_to_node_ratio_per_mille"` + Comparison string `json:"comparison"` + } `json:"selector"` + RouteCache struct { + Scope string `json:"scope"` + Entries int64 `json:"maximum_entries"` + TotalBytes int64 `json:"maximum_total_bytes"` + EntryBytes int64 `json:"maximum_entry_bytes"` + Miss string `json:"miss"` + Invalidates []string `json:"invalidation"` + } `json:"route_cache"` + Execution struct { + SingleArm bool `json:"single_arm"` + Fallback string `json:"fallback"` + } `json:"execution"` + } + require.NoError(t, json.Unmarshal(contents, &protocol)) + require.Equal(t, "topology-selected-routing-v1", protocol.Schema) + require.Equal(t, "frozen_implementation_protocol", protocol.Status) + require.Equal(t, "off", protocol.ProductionDefault) + require.ElementsMatch(t, []string{"repeatable_read", "serializable"}, protocol.Transaction.Isolation) + require.True(t, protocol.Transaction.ReadOnly) + require.True(t, protocol.Transaction.SameSnap) + require.Equal(t, "topology-fixed-suffix-counts-v1", protocol.Selector.EstimatorVersion) + require.Equal(t, int64(1000), protocol.Selector.MaximumDensity) + require.Contains(t, protocol.Selector.Comparison, "edge_count * 1000") + require.Equal(t, "one_active_transaction", protocol.RouteCache.Scope) + require.Equal(t, int64(64), protocol.RouteCache.Entries) + require.Equal(t, int64(65536), protocol.RouteCache.TotalBytes) + require.Equal(t, int64(4096), protocol.RouteCache.EntryBytes) + require.Equal(t, "incumbent_only", protocol.RouteCache.Miss) + require.Contains(t, protocol.RouteCache.Invalidates, "cancellation") + require.True(t, protocol.Execution.SingleArm) + require.Equal(t, "exact_forward_same_snapshot", protocol.Execution.Fallback) +} diff --git a/cmd/graphbench/suffix_route_component_roster_test.go b/cmd/graphbench/suffix_route_component_roster_test.go new file mode 100644 index 00000000..1fd70dc9 --- /dev/null +++ b/cmd/graphbench/suffix_route_component_roster_test.go @@ -0,0 +1,62 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestSuffixRouteComponentV1RosterFreezesFreshOpenTargetAndControls keeps the +// direct-component preflight isolated from every terminal fixed-suffix cohort. +func TestSuffixRouteComponentV1RosterFreezesFreshOpenTargetAndControls(t *testing.T) { + corpus, err := loadScaleCorpus("../../benchmark/testdata/scale") + require.NoError(t, err) + selected, _, err := selectScaleCorpus(corpus, CorpusSelectors{Tags: []string{"suffix-route-component-v1"}}) + require.NoError(t, err) + require.Len(t, selected.Cases, 11) + + seenDatasets := map[string]bool{} + targets, controls := 0, 0 + classes := map[string]bool{} + for _, testCase := range selected.Cases { + require.Equal(t, "training", testCase.Shape.QualificationSplit, testCase.Name) + require.Equal(t, "forbidden", testCase.Shape.FallbackExpectation, testCase.Name) + require.True(t, testCase.Supports(ModePostgresSQL), testCase.Name) + require.False(t, testCase.Supports(ModeNeo4j), testCase.Name) + require.False(t, seenDatasets[testCase.Dataset], "dataset identity reused: %s", testCase.Dataset) + seenDatasets[testCase.Dataset] = true + if testCase.Expected.ResultKind == "id_rows" { + require.Len(t, testCase.Expected.IDRows, int(*testCase.Expected.RowCount), testCase.Name) + } else { + require.Len(t, testCase.Expected.PathRows, int(*testCase.Expected.RowCount), testCase.Name) + } + metadata, err := fixtureMetadata("unused", testCase.Dataset) + require.NoError(t, err, testCase.Name) + require.NotNil(t, metadata.FixedSuffixExpansion, testCase.Name) + require.Equal(t, metadata.FixedSuffixExpansion.CompleteOutputTrails, *testCase.Expected.RowCount, testCase.Name) + require.False(t, slices.Contains(testCase.Tags, "orientation-v2-training"), testCase.Name) + require.False(t, slices.Contains(testCase.Tags, "orientation-v2-holdout"), testCase.Name) + require.False(t, slices.Contains(testCase.Tags, "suffix-reverse-retry-v1-training"), testCase.Name) + + switch testCase.Shape.QualificationRole { + case "efficacy_target": + targets++ + case "adverse_control": + controls++ + default: + t.Fatalf("%s has unexpected roster role %q", testCase.Name, testCase.Shape.QualificationRole) + } + for _, tag := range testCase.Tags { + classes[tag] = true + } + } + require.Equal(t, 2, targets) + require.Equal(t, 9, controls) + for _, class := range []string{"sparse-suffix", "high-reverse-fanin", "dense-suffix", "no-path", "suffix-cap-511", "suffix-cap-512", "suffix-cap-513", "productive-cycle", "productive-self-loop", "multiple-path"} { + require.True(t, classes[class], class) + } +} diff --git a/cmd/graphbench/summary.go b/cmd/graphbench/summary.go index ba21fd9a..3b95fbbf 100644 --- a/cmd/graphbench/summary.go +++ b/cmd/graphbench/summary.go @@ -24,51 +24,128 @@ import ( "sort" "strings" "time" + + "github.com/specterops/dawgs/testutil" ) +// Summary aggregates benchmark records into cases, modes, improvements, and cost models. type Summary struct { - GeneratedAt time.Time `json:"generated_at"` - Modes []ModeSummary `json:"modes"` - Cases []CaseSummary `json:"cases"` - Regressions []BaselineEntry `json:"regressions,omitempty"` + // GeneratedAt records when the summary was assembled. + GeneratedAt time.Time `json:"generated_at"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Modes lists aggregate mode summaries in deterministic report order. + Modes []ModeSummary `json:"modes"` + // Cases contains per-workload aggregates in deterministic report order. + Cases []CaseSummary `json:"cases"` + // Regressions lists baseline comparisons classified as regressions. + Regressions []BaselineEntry `json:"regressions,omitempty"` + // Improvements lists baseline comparisons classified as improvements. Improvements []BaselineEntry `json:"improvements,omitempty"` + // CostModels lists per-case client/backend latency attribution models. + CostModels []CostModelCase `json:"cost_models,omitempty"` +} + +// CostModelCase attributes one case's end-to-end latency across compile and backend boundary components. +type CostModelCase struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Boundary identifies the measured execution boundary. + Boundary string `json:"boundary"` + // E2EMedian records median end-to-end latency attributed by the cost model. + E2EMedian time.Duration `json:"e2e_median"` + // Attribution reports the fraction of median end-to-end latency explained by measured components. + Attribution float64 `json:"attribution"` + // Components lists cost-model components in display order. + Components []CostModelComponent `json:"components"` } +// CostModelComponent attributes a duration and share to one benchmark boundary component. +type CostModelComponent struct { + // Name labels the measured latency component shown in the cost model. + Name string `json:"name"` + // Interval states whether the component is exclusive, derived, or inclusive and overlapping. + Interval string `json:"interval"` + // Median supplies the median input to the CostModelComponent contract. + Median time.Duration `json:"median"` + // P95 supplies the p95 input to the CostModelComponent contract. + P95 time.Duration `json:"p95"` + // Rows records the number of rows. + Rows int64 `json:"rows,omitempty"` + // ShareOfE2E reports this component's fraction of end-to-end latency. + ShareOfE2E float64 `json:"share_of_e2e,omitempty"` + // Confidence describes whether the component is directly observed, derived, or diagnostic. + Confidence string `json:"confidence"` +} + +// ModeSummary aggregates sample and latency statistics for one execution mode. type ModeSummary struct { - Mode ExecutionMode `json:"mode"` - Total int `json:"total"` - OK int `json:"ok"` - RowMismatch int `json:"row_mismatch"` - Error int `json:"error"` - NotImplemented int `json:"not_implemented"` + // Mode identifies the backend whose result statuses are aggregated. + Mode ExecutionMode `json:"mode"` + // Total counts all results emitted for the execution mode. + Total int `json:"total"` + // OK counts successful results for an execution mode. + OK int `json:"ok"` + // RowMismatch counts results whose row cardinality differed from expectation. + RowMismatch int `json:"row_mismatch"` + // Error counts results that failed during backend execution. + Error int `json:"error"` + // NotImplemented counts cases unsupported by the execution mode. + NotImplemented int `json:"not_implemented"` } +// CaseSummary aggregates all backend results for one dataset case. type CaseSummary struct { - Source string `json:"source"` - Dataset string `json:"dataset"` - Name string `json:"name"` - Category string `json:"category"` - Modes map[ExecutionMode]ModeCaseCell `json:"modes"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Category groups cases by workload category. + Category string `json:"category"` + // Modes maps execution mode to its status, statistics, and baseline comparison. + Modes map[ExecutionMode]ModeCaseCell `json:"modes"` } +// ModeCaseCell contains the status, statistics, and baseline comparison rendered in one summary cell. type ModeCaseCell struct { - Status string `json:"status"` - Rows int64 `json:"rows,omitempty"` - Median time.Duration `json:"median,omitempty"` - Baseline *BaselineComparison `json:"baseline,omitempty"` - FallbackReason string `json:"fallback_reason,omitempty"` - Error string `json:"error,omitempty"` + // Status supplies the status input to the ModeCaseCell contract. + Status string `json:"status"` + // Rows records the number of rows. + Rows int64 `json:"rows,omitempty"` + // Median supplies the median input to the ModeCaseCell contract. + Median time.Duration `json:"median,omitempty"` + // Baseline contains the latency comparison with a matching baseline record. + Baseline *BaselineComparison `json:"baseline,omitempty"` + // FallbackReason explains why execution used a fallback architecture. + FallbackReason string `json:"fallback_reason,omitempty"` + // Error supplies the error input to the ModeCaseCell contract. + Error string `json:"error,omitempty"` + // RuntimeReceiptChains preserves every measured invocation's complete + // ordered traversal branch chain. + RuntimeReceiptChains [][]RuntimeReceiptEvent `json:"runtime_receipt_chains,omitempty"` } +// BaselineEntry stores one case/backend baseline median used for future comparison. type BaselineEntry struct { - Dataset string `json:"dataset"` - Name string `json:"name"` - Mode ExecutionMode `json:"mode"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Mode identifies the backend to which the baseline comparison applies. + Mode ExecutionMode `json:"mode"` + // BaselineMedian supplies the baseline median input to the BaselineEntry contract. BaselineMedian time.Duration `json:"baseline_median"` - CurrentMedian time.Duration `json:"current_median"` - Ratio float64 `json:"ratio"` + // CurrentMedian supplies the current median input to the BaselineEntry contract. + CurrentMedian time.Duration `json:"current_median"` + // Ratio reports the candidate-to-baseline latency ratio. + Ratio float64 `json:"ratio"` } +// buildSummary aggregates benchmark records by case and mode and derives boundary cost models. func buildSummary(records []CaseResult) Summary { var ( summary = Summary{ @@ -79,6 +156,9 @@ func buildSummary(records []CaseResult) Summary { ) for _, record := range records { + if summary.Metadata == (testutil.BaselineMetadata{}) { + summary.Metadata = record.Metadata + } modeSummary := modeSummaries[record.ExecutionMode] if modeSummary == nil { modeSummary = &ModeSummary{Mode: record.ExecutionMode} @@ -114,12 +194,13 @@ func buildSummary(records []CaseResult) Summary { } caseSummary.Modes[record.ExecutionMode] = ModeCaseCell{ - Status: record.Status, - Rows: record.RowCount, - Median: record.Stats.Median, - Baseline: record.Baseline, - FallbackReason: record.FallbackReason, - Error: record.Error, + Status: record.Status, + Rows: record.RowCount, + Median: record.Stats.Median, + Baseline: record.Baseline, + FallbackReason: record.FallbackReason, + Error: record.Error, + RuntimeReceiptChains: runtimeReceiptChains(record.Stats.Samples), } if record.Baseline != nil { @@ -137,6 +218,9 @@ func buildSummary(records []CaseResult) Summary { summary.Improvements = append(summary.Improvements, entry) } } + if record.RawPGXWaterfall != nil && len(record.RawPGXWaterfall.Samples) > 0 { + summary.CostModels = append(summary.CostModels, buildBoundaryCostModel(record)) + } } for _, modeSummary := range modeSummaries { @@ -163,9 +247,119 @@ func buildSummary(records []CaseResult) Summary { sortBaselineEntries(summary.Regressions, true) sortBaselineEntries(summary.Improvements, false) + sort.Slice(summary.CostModels, func(i, j int) bool { + if summary.CostModels[i].Dataset != summary.CostModels[j].Dataset { + return summary.CostModels[i].Dataset < summary.CostModels[j].Dataset + } + return summary.CostModels[i].Name < summary.CostModels[j].Name + }) return summary } +// buildBoundaryCostModel attributes end-to-end latency among compile, driver, planning, execution, and decode stages. +func buildBoundaryCostModel(record CaseResult) CostModelCase { + samples := record.RawPGXWaterfall.Samples + total := boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.Total }) + e2e := durationFromQuantile(total, 0.50) + components := []struct { + // name labels the latency component in the rendered cost model. + name string + // values contains the observed durations attributed to the component. + values []time.Duration + }{ + { + name: "Pool acquisition", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.PoolWait }), + }, + { + name: "Transaction setup", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.Transaction }), + }, + { + name: "Bind/prepare", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.BindPrepare }), + }, + { + name: "First-row transfer/decode", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.FirstRow }), + }, + { + name: "Remaining transfer/decode", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.AllRowsDecode }), + }, + { + name: "Drain/close", + values: boundaryDurations(samples, func(sample BoundarySample) time.Duration { return sample.DrainClose }), + }, + } + model := CostModelCase{ + Dataset: record.Dataset, + Name: record.Name, + Boundary: record.RawPGXWaterfall.Boundary, + E2EMedian: e2e, + } + var attributed time.Duration + for _, component := range components { + median := durationFromQuantile(component.values, 0.50) + attributed += median + model.Components = append(model.Components, CostModelComponent{ + Name: component.name, + Interval: "exclusive", + Median: median, + P95: durationFromQuantile(component.values, 0.95), + Rows: samples[0].Rows, + ShareOfE2E: durationShare(median, e2e), + Confidence: "raw-pgx observed boundary", + }) + } + residual := e2e - attributed + if residual < 0 { + residual = 0 + } + model.Components = append(model.Components, CostModelComponent{ + Name: "Unexplained residual", + Interval: "derived", + Median: residual, + ShareOfE2E: durationShare(residual, e2e), + Confidence: "derived", + }) + model.Attribution = durationShare(e2e-residual, e2e) + if record.PostgresMetrics != nil && record.PostgresMetrics.ExecutionMS != nil { + server := time.Duration(*record.PostgresMetrics.ExecutionMS * float64(time.Millisecond)) + model.Components = append(model.Components, CostModelComponent{ + Name: "Server execution", + Interval: "inclusive/overlapping", + Median: server, + ShareOfE2E: durationShare(server, e2e), + Confidence: "single EXPLAIN diagnostic", + }) + } + return model +} + +// boundaryDurations extracts positive boundary-stage durations from benchmark samples. +func boundaryDurations(samples []BoundarySample, selectDuration func(BoundarySample) time.Duration) []time.Duration { + values := make([]time.Duration, len(samples)) + for idx, sample := range samples { + values[idx] = selectDuration(sample) + } + return values +} + +// durationFromQuantile converts a floating-point duration quantile to time.Duration. +func durationFromQuantile(values []time.Duration, probability float64) time.Duration { + return time.Duration(durationQuantile(values, probability)) +} + +// durationShare returns a component's fraction of total latency. +func durationShare(component, total time.Duration) float64 { + if total <= 0 { + return 0 + } + return float64(component) / float64(total) +} + +// sortBaselineEntries orders baseline entries by dataset, case, and execution mode. func sortBaselineEntries(entries []BaselineEntry, descending bool) { sort.Slice(entries, func(i, j int) bool { if descending { @@ -176,6 +370,7 @@ func sortBaselineEntries(entries []BaselineEntry, descending bool) { }) } +// writeMarkdownSummaryFile creates a Markdown summary file and propagates write or close failures. func writeMarkdownSummaryFile(path string, summary Summary) error { if err := ensureOutputDir(path); err != nil { return err @@ -190,6 +385,7 @@ func writeMarkdownSummaryFile(path string, summary Summary) error { return writeMarkdownSummary(output, summary) } +// writeJSONSummaryFile creates a JSON summary file and propagates encode or close failures. func writeJSONSummaryFile(path string, summary Summary) error { if err := ensureOutputDir(path); err != nil { return err @@ -206,9 +402,11 @@ func writeJSONSummaryFile(path string, summary Summary) error { return encoder.Encode(summary) } +// writeMarkdownSummary renders benchmark overview, case matrix, improvements, and cost models as Markdown. func writeMarkdownSummary(w io.Writer, summary Summary) error { fmt.Fprintf(w, "# GraphBench Summary\n\n") fmt.Fprintf(w, "Generated: %s\n\n", summary.GeneratedAt.Format(time.RFC3339)) + fmt.Fprintf(w, "DAWGS version: `%s`\n\n", summary.Metadata.DAWGSVersion) fmt.Fprintf(w, "## Modes\n\n") fmt.Fprintf(w, "| Mode | Total | OK | Row Mismatch | Error | Not Implemented |\n") @@ -246,10 +444,23 @@ func writeMarkdownSummary(w io.Writer, summary Summary) error { fmt.Fprintf(w, "\n## Baseline Improvements\n\n") writeBaselineTable(w, summary.Improvements) } + if len(summary.CostModels) > 0 { + fmt.Fprintf(w, "\n## Raw PostgreSQL Cost Models\n\n") + for _, model := range summary.CostModels { + fmt.Fprintf(w, "### %s / %s\n\n", escapeMarkdown(model.Dataset), escapeMarkdown(model.Name)) + fmt.Fprintf(w, "Boundary attribution: %.1f%% of %s.\n\n", model.Attribution*100, formatDuration(model.E2EMedian)) + fmt.Fprintf(w, "| Component | Interval | Median | p95 | Share of E2E | Confidence |\n") + fmt.Fprintf(w, "| --- | --- | ---: | ---: | ---: | --- |\n") + for _, component := range model.Components { + fmt.Fprintf(w, "| %s | %s | %s | %s | %.1f%% | %s |\n", escapeMarkdown(component.Name), component.Interval, formatDuration(component.Median), formatDuration(component.P95), component.ShareOfE2E*100, escapeMarkdown(component.Confidence)) + } + } + } return nil } +// writeBaselineTable renders baseline comparisons for one summary section. func writeBaselineTable(w io.Writer, entries []BaselineEntry) { fmt.Fprintf(w, "| Case | Dataset | Mode | Baseline | Current | Ratio |\n") fmt.Fprintf(w, "| --- | --- | --- | ---: | ---: | ---: |\n") @@ -265,6 +476,7 @@ func writeBaselineTable(w io.Writer, entries []BaselineEntry) { } } +// formatModeCell formats one backend result and its baseline comparison for Markdown. func formatModeCell(cell ModeCaseCell) string { if cell.Status == "" { return "-" @@ -296,6 +508,7 @@ func formatModeCell(cell ModeCaseCell) string { return escapeMarkdown(strings.Join(parts, "; ")) } +// formatDuration formats a duration for compact benchmark tables. func formatDuration(duration time.Duration) string { ms := float64(duration.Microseconds()) / 1000.0 if ms < 1 { @@ -308,6 +521,7 @@ func formatDuration(duration time.Duration) string { return fmt.Sprintf("%.0fms", ms) } +// escapeMarkdown escapes table delimiters and normalizes line breaks for Markdown cells. func escapeMarkdown(value string) string { return strings.ReplaceAll(value, "|", "\\|") } diff --git a/cmd/graphbench/summary_test.go b/cmd/graphbench/summary_test.go index e7a080bd..acb5b7a4 100644 --- a/cmd/graphbench/summary_test.go +++ b/cmd/graphbench/summary_test.go @@ -25,6 +25,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestApplyBaseline verifies that matching dataset/name/backend records receive the expected 1.5 ratio and five-millisecond absolute change. func TestApplyBaseline(t *testing.T) { var ( dir = t.TempDir() @@ -57,6 +58,7 @@ func TestApplyBaseline(t *testing.T) { require.Equal(t, 5*time.Millisecond, records[0].Baseline.Change) } +// TestBuildSummarySortsCaseSourceTieBreaker verifies deterministic source-path ordering when dataset, case name, and backend keys are otherwise identical. func TestBuildSummarySortsCaseSourceTieBreaker(t *testing.T) { summary := buildSummary([]CaseResult{ { @@ -80,6 +82,7 @@ func TestBuildSummarySortsCaseSourceTieBreaker(t *testing.T) { require.Equal(t, "cases/b.json", summary.Cases[1].Source) } +// TestWriteMarkdownSummary verifies that one row combines PostgreSQL timing/cardinality with an unavailable local-executor status and leaves absent backends blank. func TestWriteMarkdownSummary(t *testing.T) { var ( summary = buildSummary([]CaseResult{ @@ -110,3 +113,32 @@ func TestWriteMarkdownSummary(t *testing.T) { require.NoError(t, writeMarkdownSummary(&output, summary)) require.Contains(t, output.String(), "| case | base | counts | 2.0ms; rows=1 | not_implemented; local traversal executor unavailable | - |") } + +// TestBuildSummaryIncludesExclusiveRawPGXCostModel verifies that mutually exclusive boundary components reconcile to total latency and retain an explicit residual component. +func TestBuildSummaryIncludesExclusiveRawPGXCostModel(t *testing.T) { + record := CaseResult{ + Dataset: "base", + Name: "large", + ExecutionMode: ModePostgresSQL, + Status: StatusOK, + RawPGXWaterfall: &PostgresBoundaryWaterfall{ + Boundary: "raw", + Samples: []BoundarySample{{ + PoolWait: time.Millisecond, + Transaction: time.Millisecond, + BindPrepare: 2 * time.Millisecond, + FirstRow: 2 * time.Millisecond, + AllRowsDecode: 3 * time.Millisecond, + DrainClose: time.Millisecond, + Total: 10 * time.Millisecond, + Rows: 1000, + }}, + }, + } + + summary := buildSummary([]CaseResult{record}) + require.Len(t, summary.CostModels, 1) + require.Equal(t, 10*time.Millisecond, summary.CostModels[0].E2EMedian) + require.InDelta(t, 1.0, summary.CostModels[0].Attribution, 0.0001) + require.Equal(t, "Unexplained residual", summary.CostModels[0].Components[6].Name) +} diff --git a/cmd/graphbench/traversal_telemetry.go b/cmd/graphbench/traversal_telemetry.go new file mode 100644 index 00000000..e1c30979 --- /dev/null +++ b/cmd/graphbench/traversal_telemetry.go @@ -0,0 +1,931 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "slices" + "strings" +) + +const ( + // TraversalExecutionTelemetrySchemaVersion is the current serialized telemetry schema revision. + TraversalExecutionTelemetrySchemaVersion = 2 + + // TraversalTelemetryLevelSummary records only the production execution identity and outcome. + TraversalTelemetryLevelSummary TraversalTelemetryLevel = "summary" + // TraversalTelemetryLevelDiagnostic adds counters from a separate untimed replay. + TraversalTelemetryLevelDiagnostic TraversalTelemetryLevel = "diagnostic" + + // TraversalTelemetryCounterStatusComplete records a replay with every declared family populated by invocation-local counters. + TraversalTelemetryCounterStatusComplete TraversalTelemetryCounterStatus = "complete" + // TraversalTelemetryCounterStatusPlanPartial records honest SQL-visible EXPLAIN evidence that is insufficient for qualification. + TraversalTelemetryCounterStatusPlanPartial TraversalTelemetryCounterStatus = "plan_derived_partial" + // TraversalTelemetryCounterStatusHiddenUnavailable records a function-backed executor whose internal work counters were unavailable. + TraversalTelemetryCounterStatusHiddenUnavailable TraversalTelemetryCounterStatus = "hidden_counters_unavailable" + + // TraversalTelemetryFamilyOrdinary identifies ordinary DFS or recursive-CTE traversal work. + TraversalTelemetryFamilyOrdinary TraversalTelemetryFamily = "ordinary" + // TraversalTelemetryFamilyOrientation identifies runtime orientation-policy work. + TraversalTelemetryFamilyOrientation TraversalTelemetryFamily = "orientation" + // TraversalTelemetryFamilySuffixGuard identifies bounded reverse-first + // fixed-suffix admission and its exact fallback boundary. + TraversalTelemetryFamilySuffixGuard TraversalTelemetryFamily = "suffix_guard" + // TraversalTelemetryFamilySuffixComponent identifies the direct, one-arm + // fixed-suffix reverse component used by the routing preflight. + TraversalTelemetryFamilySuffixComponent TraversalTelemetryFamily = "suffix_component" + // TraversalTelemetryFamilySP identifies singleton shortest-path work. + TraversalTelemetryFamilySP TraversalTelemetryFamily = "shortest_path" + // TraversalTelemetryFamilyASP identifies all-shortest-path work. + TraversalTelemetryFamilyASP TraversalTelemetryFamily = "all_shortest_paths" + // TraversalTelemetryFamilyHydration identifies post-discovery path hydration work. + TraversalTelemetryFamilyHydration TraversalTelemetryFamily = "hydration" + // TraversalTelemetryFamilyWorkspace identifies measured session and pool workspace high-water marks. + TraversalTelemetryFamilyWorkspace TraversalTelemetryFamily = "workspace" +) + +// TraversalTelemetryLevel identifies whether a record contains only lightweight summary data or an untimed diagnostic replay. +type TraversalTelemetryLevel string + +// TraversalTelemetryFamily identifies a counter family required for an invocation. +type TraversalTelemetryFamily string + +// TraversalTelemetryCounterStatus identifies whether an untimed replay exposes every required invocation-local counter. +type TraversalTelemetryCounterStatus string + +// TraversalExecutionTelemetry records versioned execution identity and optional diagnostic replay counters. +type TraversalExecutionTelemetry struct { + // SchemaVersion identifies the serialized telemetry schema revision. + SchemaVersion int `json:"schema_version"` + // Level identifies the instrumentation boundary represented by this record. + Level TraversalTelemetryLevel `json:"level"` + // Summary contains lightweight data captured for the production invocation. + Summary TraversalExecutionSummary `json:"summary"` + // Diagnostic contains counters from a separate untimed replay when Level is diagnostic. + Diagnostic *TraversalExecutionDiagnostic `json:"diagnostic,omitempty"` +} + +// TraversalExecutionSummary identifies the planned and executed traversal policy without detailed work counters. +type TraversalExecutionSummary struct { + // RequestedIdentity identifies the requested identity. + RequestedIdentity string `json:"requested_identity"` + // PlannedIdentities supplies the planned identities input to the TraversalExecutionSummary contract. + PlannedIdentities []string `json:"planned_identities"` + // EmittedIdentity identifies the emitted identity. + EmittedIdentity string `json:"emitted_identity"` + // RuntimeIdentity identifies the runtime identity. + RuntimeIdentity string `json:"runtime_identity"` + // AppliedIdentity identifies the applied identity. + AppliedIdentity string `json:"applied_identity"` + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string `json:"selector_version"` + // SchedulerVersion identifies the schema version for scheduler version. + SchedulerVersion string `json:"scheduler_version"` + // ExecutionBoundary supplies the execution boundary input to the TraversalExecutionSummary contract. + ExecutionBoundary string `json:"execution_boundary,omitempty"` + // ObservationMode identifies whether the public boundary consumes scalar, + // ordered-ID, or hydrated path values. + ObservationMode string `json:"observation_mode,omitempty"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps map[string]int64 `json:"caps"` + // RuntimeOutcomeAvailable distinguishes executor evidence from a + // translator prediction. When false, runtime-dependent facts stay unset. + RuntimeOutcomeAvailable *bool `json:"runtime_outcome_available,omitempty"` + // RuntimeBranch supplies the runtime branch input to the TraversalExecutionSummary contract. + RuntimeBranch string `json:"runtime_branch"` + // Overflow supplies the overflow input to the TraversalExecutionSummary contract. + Overflow *bool `json:"overflow"` + // FallbackExecuted supplies the fallback executed input to the TraversalExecutionSummary contract. + FallbackExecuted *bool `json:"fallback_executed"` + // FallbackIdentity identifies the fallback identity. + FallbackIdentity string `json:"fallback_identity,omitempty"` + // WouldSelectIdentity records a shadow policy choice while RuntimeIdentity + // and AppliedIdentity remain bound to the only executed incumbent arm. + WouldSelectIdentity string `json:"would_select_identity,omitempty"` + // Provenance maps summary field paths to the optimizer, SQL branch, function, or executor fact that produced them. + Provenance map[string]string `json:"provenance"` +} + +// TraversalExecutionDiagnostic contains counters from one tool-only replay, separate from all timed samples. +type TraversalExecutionDiagnostic struct { + // InvocationID uniquely identifies the diagnostic invocation and its session-local workspace. + InvocationID string `json:"invocation_id"` + // ConnectionID identifies the same backend connection used by the production invocation. + ConnectionID string `json:"connection_id"` + // TimedSample is required and must be false so replay resources cannot be attributed to latency samples. + TimedSample *bool `json:"timed_sample"` + // RequiredFamilies declares exactly which counter groups must be complete for this invocation. + RequiredFamilies []TraversalTelemetryFamily `json:"required_families"` + // Counters supplies the counters input to the TraversalExecutionDiagnostic contract. + Counters TraversalDiagnosticCounters `json:"counters"` + // CounterStatus distinguishes qualification-complete invocation metrics from partial plan evidence or opaque function work. + CounterStatus TraversalTelemetryCounterStatus `json:"counter_status"` + // IncompleteReasons explains why a diagnostic replay cannot qualify when CounterStatus is not complete. + IncompleteReasons []string `json:"incomplete_reasons,omitempty"` + // PlanReplay records only counters PostgreSQL exposes through the separate TIMING OFF JSON EXPLAIN replay. + PlanReplay *TraversalPlanReplayEvidence `json:"plan_replay,omitempty"` + // Provenance maps diagnostic counter paths to the function, CTE, or executor metric that produced them. + Provenance map[string]string `json:"provenance"` +} + +// TraversalPlanReplayEvidence contains honest SQL-visible counters without pretending an outer Function Scan exposes hidden executor work. +type TraversalPlanReplayEvidence struct { + // Source identifies the exact untimed diagnostic boundary. + Source string `json:"source"` + // Counters contains only values with explicit PostgreSQL plan provenance. + Counters map[string]int64 `json:"counters,omitempty"` + // Flags contains only boolean outcomes observable from named plan branches or guards. + Flags map[string]bool `json:"flags,omitempty"` + // Provenance maps every counter and flag to its JSON EXPLAIN derivation. + Provenance map[string]string `json:"provenance"` +} + +// TraversalDiagnosticCounters groups independent runtime counter families. +type TraversalDiagnosticCounters struct { + // Ordinary supplies the ordinary input to the TraversalDiagnosticCounters contract. + Ordinary *OrdinaryTraversalCounters `json:"ordinary,omitempty"` + // Orientation supplies the orientation input to the TraversalDiagnosticCounters contract. + Orientation *OrientationTraversalCounters `json:"orientation,omitempty"` + // SuffixGuard records reverse-first fixed-suffix admission independently of + // topology-scored orientation probes. + SuffixGuard *SuffixGuardTraversalCounters `json:"suffix_guard,omitempty"` + // SuffixComponent records the direct fixed-suffix reverse component. It is + // deliberately distinct from SuffixGuard: the component has one exact arm, + // no selector probe, no cap admission, and no fallback branch. + SuffixComponent *SuffixComponentTraversalCounters `json:"suffix_component,omitempty"` + // ShortestPath identifies the filesystem shortest path. + ShortestPath *ShortestPathTraversalCounters `json:"shortest_path,omitempty"` + // AllShortestPaths identifies the filesystem all shortest paths. + AllShortestPaths *AllShortestPathsTraversalCounters `json:"all_shortest_paths,omitempty"` + // InlineASP supplies the inline asp input to the TraversalDiagnosticCounters contract. + InlineASP *InlinePredecessorTraversalCounters `json:"inline_asp,omitempty"` + // InlineShortestPath identifies the filesystem inline shortest path. + InlineShortestPath *InlinePredecessorTraversalCounters `json:"inline_shortest_path,omitempty"` + // InlineShortestDistance records guarded SP-I2 distance-only work. + InlineShortestDistance *InlineDistanceTraversalCounters `json:"inline_shortest_distance,omitempty"` + // Hydration supplies the hydration input to the TraversalDiagnosticCounters contract. + Hydration *TraversalHydrationCounters `json:"hydration,omitempty"` + // Workspace supplies the workspace input to the TraversalDiagnosticCounters contract. + Workspace *TraversalWorkspaceCounters `json:"workspace,omitempty"` +} + +// InlineDistanceTraversalCounters records the bounded reverse-physical +// distance relation and complementary candidate/fallback branch receipts. +type InlineDistanceTraversalCounters struct { + StateRows *int64 `json:"state_rows"` + FrontierRows *int64 `json:"frontier_rows"` + AdmissionProbeRows *int64 `json:"admission_probe_rows,omitempty"` + AdmissionProbeLoops *int64 `json:"admission_probe_loops,omitempty"` + DirectProbeRows *int64 `json:"direct_probe_rows,omitempty"` + DirectProbeLoops *int64 `json:"direct_probe_loops,omitempty"` + TargetRows *int64 `json:"target_rows,omitempty"` + OutputRows *int64 `json:"output_rows"` + CandidateMarkerRows *int64 `json:"candidate_marker_rows"` + FallbackMarkerRows *int64 `json:"fallback_marker_rows"` + CandidateBranchRows *int64 `json:"candidate_branch_rows"` + FallbackBranchRows *int64 `json:"fallback_branch_rows"` + CandidateExecutorLoops *int64 `json:"candidate_executor_loops"` + FallbackExecutorLoops *int64 `json:"fallback_executor_loops"` + FrontierGuardDominated *bool `json:"frontier_guard_dominated,omitempty"` + CapRelationship string `json:"cap_relationship,omitempty"` + ObservedOverflowReason string `json:"observed_overflow_reason,omitempty"` +} + +// SuffixGuardTraversalCounters records the complete bounded relations and +// complementary branch markers exposed by suffix-reverse-guard-v1. Boolean +// overflow fields remain pointers so a measured false cannot be confused with +// missing evidence. +type SuffixGuardTraversalCounters struct { + RootPresenceRows *int64 `json:"root_presence_rows"` + SuffixRows *int64 `json:"suffix_rows"` + DistinctBoundaryRows *int64 `json:"distinct_boundary_rows"` + StateRows *int64 `json:"state_rows"` + OutputRows *int64 `json:"output_rows"` + CandidateMarkerRows *int64 `json:"candidate_marker_rows"` + FallbackMarkerRows *int64 `json:"fallback_marker_rows"` + CandidateBranchRows *int64 `json:"candidate_branch_rows"` + FallbackBranchRows *int64 `json:"fallback_branch_rows"` + CandidateExecutorLoops *int64 `json:"candidate_executor_loops"` + FallbackExecutorLoops *int64 `json:"fallback_executor_loops"` + SuffixOverflow *bool `json:"suffix_overflow"` + StateOverflow *bool `json:"state_overflow"` +} + +// SuffixComponentTraversalCounters records the complete plan-visible work of +// the direct suffix-route component. Planning and execution timings come from +// the separate untimed EXPLAIN replay; the public output count is bound to the +// exact observed query result rather than inferred from a consumer CTE scan. +type SuffixComponentTraversalCounters struct { + SuffixRows *int64 `json:"suffix_rows"` + BoundaryRows *int64 `json:"boundary_rows"` + ReverseStateRows *int64 `json:"reverse_state_rows"` + OrderedNodeHydrationLoops *int64 `json:"ordered_node_hydration_loops"` + OrderedNodeHydrationRows *int64 `json:"ordered_node_hydration_rows"` + OrderedEdgeHydrationLoops *int64 `json:"ordered_edge_hydration_loops"` + OrderedEdgeHydrationRows *int64 `json:"ordered_edge_hydration_rows"` + OutputRows *int64 `json:"output_rows"` + ReceiptRows *int64 `json:"receipt_rows"` + PlanningTimeNS *int64 `json:"planning_time_ns"` + ExecutionTimeNS *int64 `json:"execution_time_ns"` +} + +// InlinePredecessorTraversalCounters records the complete set of bounded +// relations and complementary branch markers exposed by an inline I1 +// predecessor statement. ASP and canonical one-witness policies serialize +// into separate fields so their resource evidence cannot be interchanged. +type InlinePredecessorTraversalCounters struct { + // DistanceRows records the number of distance rows. + DistanceRows *int64 `json:"distance_rows"` + // PredecessorRows records the number of predecessor rows. + PredecessorRows *int64 `json:"predecessor_rows"` + // EnumerationRows records the number of enumeration rows. + EnumerationRows *int64 `json:"enumeration_rows"` + // OutputPaths identifies the filesystem output paths. + OutputPaths *int64 `json:"output_paths"` + // OutputBytes supplies the output bytes input to the InlinePredecessorTraversalCounters contract. + OutputBytes *int64 `json:"output_bytes"` + // CandidateMarkerRows records the number of candidate marker rows. + CandidateMarkerRows *int64 `json:"candidate_marker_rows"` + // FallbackMarkerRows records the number of fallback marker rows. + FallbackMarkerRows *int64 `json:"fallback_marker_rows"` + // CandidateBranchRows records the number of candidate branch rows. + CandidateBranchRows *int64 `json:"candidate_branch_rows"` + // FallbackBranchRows records the number of fallback branch rows. + FallbackBranchRows *int64 `json:"fallback_branch_rows"` + // CandidateExecutorLoops supplies the candidate executor loops input to the InlinePredecessorTraversalCounters contract. + CandidateExecutorLoops *int64 `json:"candidate_executor_loops"` + // FallbackExecutorLoops supplies the fallback executor loops input to the InlinePredecessorTraversalCounters contract. + FallbackExecutorLoops *int64 `json:"fallback_executor_loops"` +} + +// InlineASPTraversalCounters preserves the source-level name used by existing +// ASP telemetry producers while sharing the exact bounded-relation schema. +type InlineASPTraversalCounters = InlinePredecessorTraversalCounters + +// OrdinaryTraversalCounters records DFS or recursive-CTE discovery work. +type OrdinaryTraversalCounters struct { + // Roots supplies the roots input to the OrdinaryTraversalCounters contract. + Roots *int64 `json:"roots"` + // EdgeCandidates supplies the edge candidates input to the OrdinaryTraversalCounters contract. + EdgeCandidates *int64 `json:"edge_candidates"` + // AdmittedStates supplies the admitted states input to the OrdinaryTraversalCounters contract. + AdmittedStates *int64 `json:"admitted_states"` + // RelationshipRepeatRejects supplies the relationship repeat rejects input to the OrdinaryTraversalCounters contract. + RelationshipRepeatRejects *int64 `json:"relationship_repeat_rejects"` + // RecursiveRows records the number of recursive rows. + RecursiveRows *int64 `json:"recursive_rows"` + // PeakState supplies the peak state input to the OrdinaryTraversalCounters contract. + PeakState *int64 `json:"peak_state"` + // EmittedTrails supplies the emitted trails input to the OrdinaryTraversalCounters contract. + EmittedTrails *int64 `json:"emitted_trails"` + // HydrationRows records the number of hydration rows. + HydrationRows *int64 `json:"hydration_rows"` +} + +// OrientationTraversalCounters records bounded policy probes and selected-branch work. +type OrientationTraversalCounters struct { + // ForwardSeeds supplies the forward seeds input to the OrientationTraversalCounters contract. + ForwardSeeds *int64 `json:"forward_seeds"` + // ReverseSeeds supplies the reverse seeds input to the OrientationTraversalCounters contract. + ReverseSeeds *int64 `json:"reverse_seeds"` + // DuplicateSeeds supplies the duplicate seeds input to the OrientationTraversalCounters contract. + DuplicateSeeds *int64 `json:"duplicate_seeds"` + // SuffixRows records the number of suffix rows. + SuffixRows *int64 `json:"suffix_rows"` + // DistinctBoundaries supplies the distinct boundaries input to the OrientationTraversalCounters contract. + DistinctBoundaries *int64 `json:"distinct_boundaries"` + // TypedDirectionalDegreeSamples supplies the typed directional degree samples input to the OrientationTraversalCounters contract. + TypedDirectionalDegreeSamples *int64 `json:"typed_directional_degree_samples"` + // ForwardDegreeSamples supplies the forward degree samples input to the OrientationTraversalCounters contract. + ForwardDegreeSamples *int64 `json:"forward_degree_samples"` + // ReverseDegreeSamples supplies the reverse degree samples input to the OrientationTraversalCounters contract. + ReverseDegreeSamples *int64 `json:"reverse_degree_samples"` + // ShallowSurvivalRows records the number of shallow survival rows. + ShallowSurvivalRows *int64 `json:"shallow_survival_rows"` + // ShallowSurvival supplies the shallow survival input to the OrientationTraversalCounters contract. + ShallowSurvival *float64 `json:"shallow_survival"` + // ProbeRows records the number of probe rows. + ProbeRows *int64 `json:"probe_rows"` + // ProbeTimeNS supplies the probe time ns input to the OrientationTraversalCounters contract. + ProbeTimeNS *int64 `json:"probe_time_ns"` + // ProbeBufferHits supplies the probe buffer hits input to the OrientationTraversalCounters contract. + ProbeBufferHits *int64 `json:"probe_buffer_hits"` + // ProbeBufferReads supplies the probe buffer reads input to the OrientationTraversalCounters contract. + ProbeBufferReads *int64 `json:"probe_buffer_reads"` + // ForwardScore supplies the forward score input to the OrientationTraversalCounters contract. + ForwardScore *float64 `json:"forward_score"` + // ReverseScore supplies the reverse score input to the OrientationTraversalCounters contract. + ReverseScore *float64 `json:"reverse_score"` + // SelectedSide supplies the selected side input to the OrientationTraversalCounters contract. + SelectedSide string `json:"selected_side"` + // SentinelOverflow supplies the sentinel overflow input to the OrientationTraversalCounters contract. + SentinelOverflow *bool `json:"sentinel_overflow"` + // BranchLoops supplies the branch loops input to the OrientationTraversalCounters contract. + BranchLoops *int64 `json:"branch_loops"` +} + +// ShortestPathLevelCounters records one scheduler action and the two-sided frontier state it observed. +type ShortestPathLevelCounters struct { + // SearchID identifies the search id. + SearchID int64 `json:"search_id"` + // ActionIndex supplies the action index input to the ShortestPathLevelCounters contract. + ActionIndex int64 `json:"action_index"` + // Side supplies the side input to the ShortestPathLevelCounters contract. + Side string `json:"side"` + // Action supplies the action input to the ShortestPathLevelCounters contract. + Action string `json:"action"` + // Depth supplies the depth input to the ShortestPathLevelCounters contract. + Depth *int64 `json:"depth"` + // FrontierRows records the number of frontier rows. + FrontierRows *int64 `json:"frontier_rows"` + // CandidateEdges supplies the candidate edges input to the ShortestPathLevelCounters contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the ShortestPathLevelCounters contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenRows records the number of seen rows. + SeenRows *int64 `json:"seen_rows"` + // QueueRows records the number of queue rows. + QueueRows *int64 `json:"queue_rows"` + // PredecessorRows records the number of predecessor rows. + PredecessorRows *int64 `json:"predecessor_rows"` + // MeetingCandidates supplies the meeting candidates input to the ShortestPathLevelCounters contract. + MeetingCandidates *int64 `json:"meeting_candidates"` + // Provenance names the invocation-local stage or executor metric that produced this level row. + Provenance string `json:"provenance"` +} + +// ShortestPathTraversalCounters records bidirectional scheduler, frontier, and witness work. +type ShortestPathTraversalCounters struct { + // SchedulerActions supplies the scheduler actions input to the ShortestPathTraversalCounters contract. + SchedulerActions *int64 `json:"scheduler_actions"` + // Levels supplies the levels input to the ShortestPathTraversalCounters contract. + Levels []ShortestPathLevelCounters `json:"levels"` + // CandidateEdges supplies the candidate edges input to the ShortestPathTraversalCounters contract. + CandidateEdges *int64 `json:"candidate_edges"` + // DistinctNewNodes supplies the distinct new nodes input to the ShortestPathTraversalCounters contract. + DistinctNewNodes *int64 `json:"distinct_new_nodes"` + // SeenPeak supplies the seen peak input to the ShortestPathTraversalCounters contract. + SeenPeak *int64 `json:"seen_peak"` + // FrontierPeak supplies the frontier peak input to the ShortestPathTraversalCounters contract. + FrontierPeak *int64 `json:"frontier_peak"` + // QueuePeak supplies the queue peak input to the ShortestPathTraversalCounters contract. + QueuePeak *int64 `json:"queue_peak"` + // PredecessorPeak supplies the predecessor peak input to the ShortestPathTraversalCounters contract. + PredecessorPeak *int64 `json:"predecessor_peak"` + // MeetingCandidates supplies the meeting candidates input to the ShortestPathTraversalCounters contract. + MeetingCandidates *int64 `json:"meeting_candidates"` + // FrozenDistance supplies the frozen distance input to the ShortestPathTraversalCounters contract. + FrozenDistance *int64 `json:"frozen_distance"` + // WitnessRows records the number of witness rows. + WitnessRows *int64 `json:"witness_rows"` + // FallbackExecuted supplies the fallback executed input to the ShortestPathTraversalCounters contract. + FallbackExecuted *bool `json:"fallback_executed"` +} + +// AllShortestPathsTraversalCounters records SP search work plus predecessor and output enumeration work. +type AllShortestPathsTraversalCounters struct { + // Search supplies the search input to the AllShortestPathsTraversalCounters contract. + Search ShortestPathTraversalCounters `json:"search"` + // SameDepthPredecessorAdditions supplies the same depth predecessor additions input to the AllShortestPathsTraversalCounters contract. + SameDepthPredecessorAdditions *int64 `json:"same_depth_predecessor_additions"` + // PredecessorPeak supplies the predecessor peak input to the AllShortestPathsTraversalCounters contract. + PredecessorPeak *int64 `json:"predecessor_peak"` + // MeetingNodes supplies the meeting nodes input to the AllShortestPathsTraversalCounters contract. + MeetingNodes *int64 `json:"meeting_nodes"` + // CutDepth supplies the cut depth input to the AllShortestPathsTraversalCounters contract. + CutDepth *int64 `json:"cut_depth"` + // PathCountEstimate supplies the path count estimate input to the AllShortestPathsTraversalCounters contract. + PathCountEstimate *int64 `json:"path_count_estimate"` + // PathCountSaturated supplies the path count saturated input to the AllShortestPathsTraversalCounters contract. + PathCountSaturated *bool `json:"path_count_saturated"` + // EnumeratedCandidates supplies the enumerated candidates input to the AllShortestPathsTraversalCounters contract. + EnumeratedCandidates *int64 `json:"enumerated_candidates"` + // DuplicateRejects supplies the duplicate rejects input to the AllShortestPathsTraversalCounters contract. + DuplicateRejects *int64 `json:"duplicate_rejects"` + // OutputPaths identifies the filesystem output paths. + OutputPaths *int64 `json:"output_paths"` + // OutputEdgeCells supplies the output edge cells input to the AllShortestPathsTraversalCounters contract. + OutputEdgeCells *int64 `json:"output_edge_cells"` + // OutputBytes supplies the output bytes input to the AllShortestPathsTraversalCounters contract. + OutputBytes *int64 `json:"output_bytes"` +} + +// TraversalHydrationCounters records post-discovery materialization separately from traversal work. +type TraversalHydrationCounters struct { + // PathCount records the number of path count. + PathCount *int64 `json:"path_count"` + // NodeLookups supplies the node lookups input to the TraversalHydrationCounters contract. + NodeLookups *int64 `json:"node_lookups"` + // EdgeLookups supplies the edge lookups input to the TraversalHydrationCounters contract. + EdgeLookups *int64 `json:"edge_lookups"` + // Loops supplies the loops input to the TraversalHydrationCounters contract. + Loops *int64 `json:"loops"` + // Rows records the number of rows. + Rows *int64 `json:"rows"` + // TimeNS supplies the time ns input to the TraversalHydrationCounters contract. + TimeNS *int64 `json:"time_ns"` + // Bytes supplies the bytes input to the TraversalHydrationCounters contract. + Bytes *int64 `json:"bytes"` +} + +// TraversalWorkspaceCounters records measured high-water memory attributed to +// one diagnostic invocation and to all simultaneously active pool sessions. +type TraversalWorkspaceCounters struct { + // SessionPeakBytes supplies the session peak bytes input to the TraversalWorkspaceCounters contract. + SessionPeakBytes *int64 `json:"session_peak_bytes"` + // PoolPeakBytes supplies the pool peak bytes input to the TraversalWorkspaceCounters contract. + PoolPeakBytes *int64 `json:"pool_peak_bytes"` +} + +// ValidateTraversalExecutionTelemetry rejects incomplete or contradictory telemetry. +func ValidateTraversalExecutionTelemetry(telemetry *TraversalExecutionTelemetry) error { + if telemetry == nil { + return fmt.Errorf("traversal execution telemetry is missing") + } + + return telemetry.Validate() +} + +// Validate rejects unsupported schema versions, incomplete summaries, timed diagnostic replays, and missing counters or provenance. +func (s TraversalExecutionTelemetry) Validate() error { + var problems []string + + if s.SchemaVersion != TraversalExecutionTelemetrySchemaVersion { + problems = append(problems, fmt.Sprintf("schema_version must be %d", TraversalExecutionTelemetrySchemaVersion)) + } + if s.Level != TraversalTelemetryLevelSummary && s.Level != TraversalTelemetryLevelDiagnostic { + problems = append(problems, "level must be summary or diagnostic") + } + + validateTraversalSummary(s.Summary, &problems) + + switch s.Level { + case TraversalTelemetryLevelSummary: + if s.Diagnostic != nil { + problems = append(problems, "summary telemetry must not contain a diagnostic replay") + } + case TraversalTelemetryLevelDiagnostic: + validateTraversalDiagnostic(s.Diagnostic, &problems) + } + + if len(problems) > 0 { + return fmt.Errorf("invalid traversal execution telemetry: %s", strings.Join(problems, "; ")) + } + + return nil +} + +// validateTraversalSummary validates traversal summary. +func validateTraversalSummary(summary TraversalExecutionSummary, problems *[]string) { + requireText("summary.requested_identity", summary.RequestedIdentity, problems) + if len(summary.PlannedIdentities) == 0 { + *problems = append(*problems, "summary.planned_identities is missing") + } + planned := map[string]struct{}{} + for idx, identity := range summary.PlannedIdentities { + requireText(fmt.Sprintf("summary.planned_identities[%d]", idx), identity, problems) + if _, duplicate := planned[identity]; duplicate { + *problems = append(*problems, fmt.Sprintf("summary.planned_identities contains duplicate %q", identity)) + } + planned[identity] = struct{}{} + } + requireText("summary.emitted_identity", summary.EmittedIdentity, problems) + runtimeOutcomeAvailable := summary.RuntimeOutcomeAvailable == nil || *summary.RuntimeOutcomeAvailable + if runtimeOutcomeAvailable { + requireText("summary.runtime_identity", summary.RuntimeIdentity, problems) + requireText("summary.applied_identity", summary.AppliedIdentity, problems) + } else { + if summary.RuntimeIdentity != "" || summary.AppliedIdentity != "" { + *problems = append(*problems, "summary unavailable runtime outcome must not assert runtime or applied identity") + } + if summary.RuntimeBranch != "runtime_outcome_unavailable" { + *problems = append(*problems, "summary unavailable runtime outcome must use runtime_outcome_unavailable branch") + } + if summary.Overflow != nil || summary.FallbackExecuted != nil || summary.FallbackIdentity != "" { + *problems = append(*problems, "summary unavailable runtime outcome must not assert overflow or fallback facts") + } + } + requireText("summary.selector_version", summary.SelectorVersion, problems) + requireText("summary.scheduler_version", summary.SchedulerVersion, problems) + requireText("summary.runtime_branch", summary.RuntimeBranch, problems) + if runtimeOutcomeAvailable { + requirePointer("summary.overflow", summary.Overflow, problems) + requirePointer("summary.fallback_executed", summary.FallbackExecuted, problems) + } + if runtimeOutcomeAvailable && summary.RuntimeIdentity != "" { + if _, ok := planned[summary.RuntimeIdentity]; !ok { + *problems = append(*problems, "summary.runtime_identity is not a planned identity") + } + } + if runtimeOutcomeAvailable && summary.FallbackExecuted != nil && *summary.FallbackExecuted { + requireText("summary.fallback_identity", summary.FallbackIdentity, problems) + if summary.FallbackIdentity != "" { + if _, ok := planned[summary.FallbackIdentity]; !ok { + *problems = append(*problems, "summary.fallback_identity is not a planned identity") + } + if summary.AppliedIdentity != summary.FallbackIdentity { + *problems = append(*problems, "summary.applied_identity must equal fallback_identity when fallback executes") + } + } + } else if runtimeOutcomeAvailable && summary.FallbackExecuted != nil && summary.AppliedIdentity != "" && summary.RuntimeIdentity != "" && summary.AppliedIdentity != summary.RuntimeIdentity { + *problems = append(*problems, "summary.applied_identity must equal runtime_identity when fallback does not execute") + } + if summary.WouldSelectIdentity != "" { + if _, ok := planned[summary.WouldSelectIdentity]; !ok { + *problems = append(*problems, "summary.would_select_identity is not a planned identity") + } + requireProvenance("summary.would_select_identity", summary.Provenance["would_select_identity"], problems) + } + + for _, path := range []string{ + "requested_identity", "planned_identities", "emitted_identity", "runtime_identity", "applied_identity", + "selector_version", "scheduler_version", "runtime_branch", + } { + requireProvenance("summary."+path, summary.Provenance[path], problems) + } + if summary.RuntimeOutcomeAvailable != nil { + requireProvenance("summary.runtime_outcome_available", summary.Provenance["runtime_outcome_available"], problems) + } + if summary.ObservationMode != "" { + requireProvenance("summary.observation_mode", summary.Provenance["observation_mode"], problems) + } + if runtimeOutcomeAvailable { + for _, path := range []string{"overflow", "fallback_executed"} { + requireProvenance("summary."+path, summary.Provenance[path], problems) + } + } + for capName := range summary.Caps { + requireProvenance("summary.caps."+capName, summary.Provenance["caps."+capName], problems) + } + if runtimeOutcomeAvailable && summary.FallbackExecuted != nil && *summary.FallbackExecuted { + requireProvenance("summary.fallback_identity", summary.Provenance["fallback_identity"], problems) + } +} + +// validateTraversalDiagnostic validates traversal diagnostic. +func validateTraversalDiagnostic(diagnostic *TraversalExecutionDiagnostic, problems *[]string) { + if diagnostic == nil { + *problems = append(*problems, "diagnostic replay is missing") + return + } + + requireText("diagnostic.invocation_id", diagnostic.InvocationID, problems) + requireText("diagnostic.connection_id", diagnostic.ConnectionID, problems) + requirePointer("diagnostic.timed_sample", diagnostic.TimedSample, problems) + if diagnostic.TimedSample != nil && *diagnostic.TimedSample { + *problems = append(*problems, "diagnostic.timed_sample must be false") + } + if len(diagnostic.RequiredFamilies) == 0 { + *problems = append(*problems, "diagnostic.required_families is missing") + } + counterStatus := diagnostic.CounterStatus + if counterStatus == "" { + // Version-one in-memory callers predate the explicit completeness field; + // their fully populated typed counters retain complete semantics. + counterStatus = TraversalTelemetryCounterStatusComplete + } + if counterStatus != TraversalTelemetryCounterStatusComplete && + counterStatus != TraversalTelemetryCounterStatusPlanPartial && + counterStatus != TraversalTelemetryCounterStatusHiddenUnavailable { + *problems = append(*problems, "diagnostic.counter_status is unsupported") + } + if counterStatus != TraversalTelemetryCounterStatusComplete && len(diagnostic.IncompleteReasons) == 0 { + *problems = append(*problems, "diagnostic.incomplete_reasons is missing for incomplete counters") + } + if counterStatus == TraversalTelemetryCounterStatusPlanPartial && diagnostic.PlanReplay == nil { + *problems = append(*problems, "diagnostic.plan_replay is missing for plan-derived counters") + } + if diagnostic.PlanReplay != nil { + validateTraversalPlanReplay(diagnostic.PlanReplay, problems) + } + + seen := map[TraversalTelemetryFamily]struct{}{} + for _, family := range diagnostic.RequiredFamilies { + if _, duplicate := seen[family]; duplicate { + *problems = append(*problems, fmt.Sprintf("diagnostic.required_families contains duplicate %q", family)) + continue + } + seen[family] = struct{}{} + + if counterStatus != TraversalTelemetryCounterStatusComplete { + continue + } + + switch family { + case TraversalTelemetryFamilyOrdinary: + validateOrdinaryCounters(diagnostic.Counters.Ordinary, diagnostic.Provenance, problems) + case TraversalTelemetryFamilyOrientation: + validateOrientationCounters(diagnostic.Counters.Orientation, diagnostic.Provenance, problems) + case TraversalTelemetryFamilySuffixGuard: + validateSuffixGuardCounters(diagnostic.Counters.SuffixGuard, diagnostic.Provenance, problems) + case TraversalTelemetryFamilySuffixComponent: + validateSuffixComponentCounters(diagnostic.Counters.SuffixComponent, diagnostic.Provenance, problems) + case TraversalTelemetryFamilySP: + if diagnostic.Counters.InlineShortestDistance != nil { + validateInlineDistanceCounters(diagnostic.Counters.InlineShortestDistance, diagnostic.Provenance, problems) + } else if diagnostic.Counters.InlineShortestPath != nil { + validateInlinePredecessorCounters("inline_shortest_path", diagnostic.Counters.InlineShortestPath, diagnostic.Provenance, problems) + } else { + validateShortestPathCounters("shortest_path", diagnostic.Counters.ShortestPath, diagnostic.Provenance, problems) + } + case TraversalTelemetryFamilyASP: + if diagnostic.Counters.InlineASP != nil { + validateInlinePredecessorCounters("inline_asp", diagnostic.Counters.InlineASP, diagnostic.Provenance, problems) + } else { + validateAllShortestPathsCounters(diagnostic.Counters.AllShortestPaths, diagnostic.Provenance, problems) + } + case TraversalTelemetryFamilyHydration: + validateHydrationCounters(diagnostic.Counters.Hydration, diagnostic.Provenance, problems) + case TraversalTelemetryFamilyWorkspace: + validateWorkspaceCounters(diagnostic.Counters.Workspace, diagnostic.Provenance, problems) + default: + *problems = append(*problems, fmt.Sprintf("diagnostic.required_families contains unsupported family %q", family)) + } + } + + if counterStatus != TraversalTelemetryCounterStatusComplete { + return + } + + for family, present := range map[TraversalTelemetryFamily]bool{ + TraversalTelemetryFamilyOrdinary: diagnostic.Counters.Ordinary != nil, + TraversalTelemetryFamilyOrientation: diagnostic.Counters.Orientation != nil, + TraversalTelemetryFamilySuffixGuard: diagnostic.Counters.SuffixGuard != nil, + TraversalTelemetryFamilySuffixComponent: diagnostic.Counters.SuffixComponent != nil, + TraversalTelemetryFamilySP: diagnostic.Counters.ShortestPath != nil || diagnostic.Counters.InlineShortestPath != nil || diagnostic.Counters.InlineShortestDistance != nil, + TraversalTelemetryFamilyASP: diagnostic.Counters.AllShortestPaths != nil || diagnostic.Counters.InlineASP != nil, + TraversalTelemetryFamilyHydration: diagnostic.Counters.Hydration != nil, + TraversalTelemetryFamilyWorkspace: diagnostic.Counters.Workspace != nil, + } { + if present && !slices.Contains(diagnostic.RequiredFamilies, family) { + *problems = append(*problems, fmt.Sprintf("diagnostic counter family %q is present but not declared", family)) + } + } +} + +// validateSuffixGuardCounters validates the reverse-first guard's complete +// admission, branch, and sentinel evidence without requiring orientation-only +// degree samples or scores. +func validateSuffixGuardCounters(counters *SuffixGuardTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.suffix_guard is missing") + return + } + requireCounters("suffix_guard", provenance, problems, map[string]*int64{ + "root_presence_rows": counters.RootPresenceRows, + "suffix_rows": counters.SuffixRows, + "distinct_boundary_rows": counters.DistinctBoundaryRows, + "state_rows": counters.StateRows, + "output_rows": counters.OutputRows, + "candidate_marker_rows": counters.CandidateMarkerRows, + "fallback_marker_rows": counters.FallbackMarkerRows, + "candidate_branch_rows": counters.CandidateBranchRows, + "fallback_branch_rows": counters.FallbackBranchRows, + "candidate_executor_loops": counters.CandidateExecutorLoops, + "fallback_executor_loops": counters.FallbackExecutorLoops, + }) + requirePointerAndProvenance("suffix_guard.suffix_overflow", counters.SuffixOverflow, provenance, problems) + requirePointerAndProvenance("suffix_guard.state_overflow", counters.StateOverflow, provenance, problems) +} + +// validateSuffixComponentCounters validates the one-arm component's exact +// CTE materializations, runtime receipt, observed output, and replay timings. +func validateSuffixComponentCounters(counters *SuffixComponentTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.suffix_component is missing") + return + } + requireCounters("suffix_component", provenance, problems, map[string]*int64{ + "suffix_rows": counters.SuffixRows, + "boundary_rows": counters.BoundaryRows, + "reverse_state_rows": counters.ReverseStateRows, + "ordered_node_hydration_loops": counters.OrderedNodeHydrationLoops, + "ordered_node_hydration_rows": counters.OrderedNodeHydrationRows, + "ordered_edge_hydration_loops": counters.OrderedEdgeHydrationLoops, + "ordered_edge_hydration_rows": counters.OrderedEdgeHydrationRows, + "output_rows": counters.OutputRows, + "receipt_rows": counters.ReceiptRows, + "planning_time_ns": counters.PlanningTimeNS, + "execution_time_ns": counters.ExecutionTimeNS, + }) +} + +func validateInlineDistanceCounters(counters *InlineDistanceTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.inline_shortest_distance is missing") + return + } + requireCounters("inline_shortest_distance", provenance, problems, map[string]*int64{ + "state_rows": counters.StateRows, "frontier_rows": counters.FrontierRows, "output_rows": counters.OutputRows, + "candidate_marker_rows": counters.CandidateMarkerRows, "fallback_marker_rows": counters.FallbackMarkerRows, + "candidate_branch_rows": counters.CandidateBranchRows, "fallback_branch_rows": counters.FallbackBranchRows, + "candidate_executor_loops": counters.CandidateExecutorLoops, "fallback_executor_loops": counters.FallbackExecutorLoops, + }) + if counters.FrontierGuardDominated != nil { + requireCounters("inline_shortest_distance", provenance, problems, map[string]*int64{ + "admission_probe_rows": counters.AdmissionProbeRows, + "admission_probe_loops": counters.AdmissionProbeLoops, + "target_rows": counters.TargetRows, + }) + requireText("diagnostic.counters.inline_shortest_distance.cap_relationship", counters.CapRelationship, problems) + requireText("diagnostic.counters.inline_shortest_distance.observed_overflow_reason", counters.ObservedOverflowReason, problems) + requireProvenance("diagnostic.counters.inline_shortest_distance.frontier_guard_dominated", provenance["inline_shortest_distance.frontier_guard_dominated"], problems) + requireProvenance("diagnostic.counters.inline_shortest_distance.cap_relationship", provenance["inline_shortest_distance.cap_relationship"], problems) + requireProvenance("diagnostic.counters.inline_shortest_distance.observed_overflow_reason", provenance["inline_shortest_distance.observed_overflow_reason"], problems) + } +} + +// validateInlinePredecessorCounters validates inline predecessor counters. +func validateInlinePredecessorCounters(prefix string, counters *InlinePredecessorTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters."+prefix+" is missing") + return + } + requireCounters(prefix, provenance, problems, map[string]*int64{ + "distance_rows": counters.DistanceRows, "predecessor_rows": counters.PredecessorRows, + "enumeration_rows": counters.EnumerationRows, "output_paths": counters.OutputPaths, + "output_bytes": counters.OutputBytes, "candidate_marker_rows": counters.CandidateMarkerRows, + "fallback_marker_rows": counters.FallbackMarkerRows, "candidate_branch_rows": counters.CandidateBranchRows, + "fallback_branch_rows": counters.FallbackBranchRows, "candidate_executor_loops": counters.CandidateExecutorLoops, + "fallback_executor_loops": counters.FallbackExecutorLoops, + }) +} + +// validateTraversalPlanReplay validates traversal plan replay. +func validateTraversalPlanReplay(replay *TraversalPlanReplayEvidence, problems *[]string) { + if replay == nil { + return + } + requireText("diagnostic.plan_replay.source", replay.Source, problems) + if len(replay.Counters) == 0 && len(replay.Flags) == 0 { + *problems = append(*problems, "diagnostic.plan_replay contains no observable counters or flags") + } + for name := range replay.Counters { + requireProvenance("diagnostic.plan_replay.counters."+name, replay.Provenance["counters."+name], problems) + } + for name := range replay.Flags { + requireProvenance("diagnostic.plan_replay.flags."+name, replay.Provenance["flags."+name], problems) + } +} + +// validateOrdinaryCounters validates ordinary counters. +func validateOrdinaryCounters(counters *OrdinaryTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.ordinary is missing") + return + } + + requireCounters("ordinary", provenance, problems, map[string]*int64{ + "roots": counters.Roots, "edge_candidates": counters.EdgeCandidates, "admitted_states": counters.AdmittedStates, + "relationship_repeat_rejects": counters.RelationshipRepeatRejects, "recursive_rows": counters.RecursiveRows, + "peak_state": counters.PeakState, "emitted_trails": counters.EmittedTrails, "hydration_rows": counters.HydrationRows, + }) +} + +// validateOrientationCounters validates orientation counters. +func validateOrientationCounters(counters *OrientationTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.orientation is missing") + return + } + + requireCounters("orientation", provenance, problems, map[string]*int64{ + "forward_seeds": counters.ForwardSeeds, "reverse_seeds": counters.ReverseSeeds, "duplicate_seeds": counters.DuplicateSeeds, + "suffix_rows": counters.SuffixRows, "distinct_boundaries": counters.DistinctBoundaries, + "typed_directional_degree_samples": counters.TypedDirectionalDegreeSamples, "probe_rows": counters.ProbeRows, + "forward_degree_samples": counters.ForwardDegreeSamples, "reverse_degree_samples": counters.ReverseDegreeSamples, + "shallow_survival_rows": counters.ShallowSurvivalRows, + "probe_time_ns": counters.ProbeTimeNS, "probe_buffer_hits": counters.ProbeBufferHits, + "probe_buffer_reads": counters.ProbeBufferReads, "branch_loops": counters.BranchLoops, + }) + requirePointerAndProvenance("orientation.shallow_survival", counters.ShallowSurvival, provenance, problems) + requirePointerAndProvenance("orientation.forward_score", counters.ForwardScore, provenance, problems) + requirePointerAndProvenance("orientation.reverse_score", counters.ReverseScore, provenance, problems) + requireText("diagnostic.counters.orientation.selected_side", counters.SelectedSide, problems) + requireProvenance("diagnostic.counters.orientation.selected_side", provenance["orientation.selected_side"], problems) + requirePointerAndProvenance("orientation.sentinel_overflow", counters.SentinelOverflow, provenance, problems) +} + +// validateShortestPathCounters validates shortest path counters. +func validateShortestPathCounters(prefix string, counters *ShortestPathTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters."+prefix+" is missing") + return + } + + requireCounters(prefix, provenance, problems, map[string]*int64{ + "scheduler_actions": counters.SchedulerActions, "candidate_edges": counters.CandidateEdges, + "distinct_new_nodes": counters.DistinctNewNodes, "seen_peak": counters.SeenPeak, "frontier_peak": counters.FrontierPeak, + "queue_peak": counters.QueuePeak, "predecessor_peak": counters.PredecessorPeak, "meeting_candidates": counters.MeetingCandidates, + "frozen_distance": counters.FrozenDistance, "witness_rows": counters.WitnessRows, + }) + requirePointerAndProvenance(prefix+".fallback_executed", counters.FallbackExecuted, provenance, problems) + if len(counters.Levels) == 0 { + *problems = append(*problems, "diagnostic.counters."+prefix+".levels is missing") + } + for idx, level := range counters.Levels { + levelPath := fmt.Sprintf("diagnostic.counters.%s.levels[%d]", prefix, idx) + requireText(levelPath+".side", level.Side, problems) + requireText(levelPath+".action", level.Action, problems) + requirePointer(levelPath+".depth", level.Depth, problems) + requirePointer(levelPath+".frontier_rows", level.FrontierRows, problems) + requirePointer(levelPath+".candidate_edges", level.CandidateEdges, problems) + requirePointer(levelPath+".distinct_new_nodes", level.DistinctNewNodes, problems) + requirePointer(levelPath+".seen_rows", level.SeenRows, problems) + requirePointer(levelPath+".queue_rows", level.QueueRows, problems) + requirePointer(levelPath+".predecessor_rows", level.PredecessorRows, problems) + requirePointer(levelPath+".meeting_candidates", level.MeetingCandidates, problems) + requireProvenance(levelPath, level.Provenance, problems) + } +} + +// validateAllShortestPathsCounters validates all shortest paths counters. +func validateAllShortestPathsCounters(counters *AllShortestPathsTraversalCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.all_shortest_paths is missing") + return + } + + validateShortestPathCounters("all_shortest_paths.search", &counters.Search, provenance, problems) + requireCounters("all_shortest_paths", provenance, problems, map[string]*int64{ + "same_depth_predecessor_additions": counters.SameDepthPredecessorAdditions, "predecessor_peak": counters.PredecessorPeak, + "meeting_nodes": counters.MeetingNodes, "cut_depth": counters.CutDepth, "path_count_estimate": counters.PathCountEstimate, + "enumerated_candidates": counters.EnumeratedCandidates, "duplicate_rejects": counters.DuplicateRejects, + "output_paths": counters.OutputPaths, "output_edge_cells": counters.OutputEdgeCells, "output_bytes": counters.OutputBytes, + }) + requirePointerAndProvenance("all_shortest_paths.path_count_saturated", counters.PathCountSaturated, provenance, problems) +} + +// validateHydrationCounters validates hydration counters. +func validateHydrationCounters(counters *TraversalHydrationCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.hydration is missing") + return + } + + requireCounters("hydration", provenance, problems, map[string]*int64{ + "path_count": counters.PathCount, "node_lookups": counters.NodeLookups, "edge_lookups": counters.EdgeLookups, + "loops": counters.Loops, "rows": counters.Rows, "time_ns": counters.TimeNS, "bytes": counters.Bytes, + }) +} + +// validateWorkspaceCounters validates workspace counters. +func validateWorkspaceCounters(counters *TraversalWorkspaceCounters, provenance map[string]string, problems *[]string) { + if counters == nil { + *problems = append(*problems, "diagnostic.counters.workspace is missing") + return + } + + requireCounters("workspace", provenance, problems, map[string]*int64{ + "session_peak_bytes": counters.SessionPeakBytes, + "pool_peak_bytes": counters.PoolPeakBytes, + }) +} + +// requireCounters supports benchmark evidence processing for require counters. +func requireCounters(prefix string, provenance map[string]string, problems *[]string, counters map[string]*int64) { + for name, value := range counters { + requirePointerAndProvenance(prefix+"."+name, value, provenance, problems) + } +} + +// requirePointerAndProvenance supports benchmark evidence processing for require pointer and provenance. +func requirePointerAndProvenance[T any](path string, value *T, provenance map[string]string, problems *[]string) { + requirePointer("diagnostic.counters."+path, value, problems) + requireProvenance("diagnostic.counters."+path, provenance[path], problems) +} + +// requirePointer returns an addressable representation of require. +func requirePointer[T any](path string, value *T, problems *[]string) { + if value == nil { + *problems = append(*problems, path+" is missing") + } +} + +// requireText supports benchmark evidence processing for require text. +func requireText(path, value string, problems *[]string) { + if strings.TrimSpace(value) == "" { + *problems = append(*problems, path+" is missing") + } +} + +// requireProvenance supports benchmark evidence processing for require provenance. +func requireProvenance(path, value string, problems *[]string) { + if strings.TrimSpace(value) == "" { + *problems = append(*problems, path+" provenance is missing") + } +} diff --git a/cmd/graphbench/traversal_telemetry_test.go b/cmd/graphbench/traversal_telemetry_test.go new file mode 100644 index 00000000..ce1ddd10 --- /dev/null +++ b/cmd/graphbench/traversal_telemetry_test.go @@ -0,0 +1,205 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestTraversalExecutionTelemetrySummaryValidation verifies traversal execution telemetry summary validation behavior. +func TestTraversalExecutionTelemetrySummaryValidation(t *testing.T) { + telemetry := validTraversalTelemetry() + + require.NoError(t, telemetry.Validate()) + + telemetry.Summary.Overflow = nil + err := telemetry.Validate() + require.ErrorContains(t, err, "summary.overflow is missing") + + telemetry = validTraversalTelemetry() + delete(telemetry.Summary.Provenance, "runtime_identity") + err = telemetry.Validate() + require.ErrorContains(t, err, "summary.runtime_identity provenance is missing") +} + +// TestTraversalExecutionTelemetrySummaryRejectsContradictoryIdentityChain verifies traversal execution telemetry summary rejects contradictory identity chain behavior. +func TestTraversalExecutionTelemetrySummaryRejectsContradictoryIdentityChain(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Summary.RuntimeIdentity = "unplanned-v1" + + require.ErrorContains(t, telemetry.Validate(), "summary.runtime_identity is not a planned identity") + + telemetry = validTraversalTelemetry() + telemetry.Summary.FallbackExecuted = telemetryBool(true) + telemetry.Summary.FallbackIdentity = "incumbent-v1" + telemetry.Summary.Provenance["fallback_identity"] = "executor.fallback_identity" + + require.ErrorContains(t, telemetry.Validate(), "summary.applied_identity must equal fallback_identity") +} + +// TestTraversalExecutionTelemetryDiagnosticRequiresPointerCountersAndProvenance verifies traversal execution telemetry diagnostic requires pointer counters and provenance behavior. +func TestTraversalExecutionTelemetryDiagnosticRequiresPointerCountersAndProvenance(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Diagnostic = ordinaryDiagnostic() + + require.NoError(t, telemetry.Validate()) + + telemetry.Diagnostic.Counters.Ordinary.RecursiveRows = nil + err := telemetry.Validate() + require.ErrorContains(t, err, "diagnostic.counters.ordinary.recursive_rows is missing") + + telemetry.Diagnostic.Counters.Ordinary.RecursiveRows = telemetryInt64(0) + delete(telemetry.Diagnostic.Provenance, "ordinary.recursive_rows") + err = telemetry.Validate() + require.ErrorContains(t, err, "diagnostic.counters.ordinary.recursive_rows provenance is missing") +} + +// TestTraversalExecutionTelemetryDiagnosticCannotBeTimed verifies traversal execution telemetry diagnostic cannot be timed behavior. +func TestTraversalExecutionTelemetryDiagnosticCannotBeTimed(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + telemetry.Diagnostic = ordinaryDiagnostic() + telemetry.Diagnostic.TimedSample = telemetryBool(true) + + require.ErrorContains(t, telemetry.Validate(), "diagnostic.timed_sample must be false") +} + +// TestTraversalExecutionTelemetryValidatesSuffixGuardIndependently verifies +// suffix-guard evidence does not require fabricated orientation scores while +// still failing closed on a missing sentinel outcome. +func TestTraversalExecutionTelemetryValidatesSuffixGuardIndependently(t *testing.T) { + telemetry := validTraversalTelemetry() + telemetry.Level = TraversalTelemetryLevelDiagnostic + provenance := map[string]string{} + for _, name := range []string{ + "root_presence_rows", "suffix_rows", "distinct_boundary_rows", "state_rows", "output_rows", "candidate_marker_rows", + "fallback_marker_rows", "candidate_branch_rows", "fallback_branch_rows", "candidate_executor_loops", "fallback_executor_loops", + "suffix_overflow", "state_overflow", + } { + provenance["suffix_guard."+name] = "plan." + name + } + telemetry.Diagnostic = &TraversalExecutionDiagnostic{ + InvocationID: "invocation-1", ConnectionID: "backend-123", TimedSample: telemetryBool(false), + RequiredFamilies: []TraversalTelemetryFamily{TraversalTelemetryFamilySuffixGuard}, CounterStatus: TraversalTelemetryCounterStatusComplete, + Counters: TraversalDiagnosticCounters{SuffixGuard: &SuffixGuardTraversalCounters{ + RootPresenceRows: telemetryInt64(1), SuffixRows: telemetryInt64(2), DistinctBoundaryRows: telemetryInt64(1), StateRows: telemetryInt64(9), + OutputRows: telemetryInt64(1), CandidateMarkerRows: telemetryInt64(1), FallbackMarkerRows: telemetryInt64(0), + CandidateBranchRows: telemetryInt64(1), FallbackBranchRows: telemetryInt64(0), CandidateExecutorLoops: telemetryInt64(1), + FallbackExecutorLoops: telemetryInt64(0), SuffixOverflow: telemetryBool(false), StateOverflow: telemetryBool(false), + }}, Provenance: provenance, + } + require.NoError(t, telemetry.Validate()) + telemetry.Diagnostic.Counters.SuffixGuard.StateOverflow = nil + require.ErrorContains(t, telemetry.Validate(), "suffix_guard.state_overflow is missing") +} + +// TestTraversalExecutionTelemetryAttachmentsSerializeVersionedSchema verifies traversal execution telemetry attachments serialize versioned schema behavior. +func TestTraversalExecutionTelemetryAttachmentsSerializeVersionedSchema(t *testing.T) { + telemetry := validTraversalTelemetry() + encoded, err := json.Marshal(struct { + // Case supplies the case input to the anonymous record contract. + Case CaseResult `json:"case"` + // Reference supplies the reference input to the anonymous record contract. + Reference PostgresReferenceResult `json:"reference"` + }{ + Case: CaseResult{TraversalTelemetry: &telemetry}, + Reference: PostgresReferenceResult{TraversalTelemetry: &telemetry}, + }) + + require.NoError(t, err) + require.Contains(t, string(encoded), `"traversal_execution_telemetry":{"schema_version":2`) +} + +// validTraversalTelemetry returns a self-consistent telemetry fixture for the requested architecture. +func validTraversalTelemetry() TraversalExecutionTelemetry { + return TraversalExecutionTelemetry{ + SchemaVersion: TraversalExecutionTelemetrySchemaVersion, + Level: TraversalTelemetryLevelSummary, + Summary: TraversalExecutionSummary{ + RequestedIdentity: "requested-v1", + PlannedIdentities: []string{"candidate-v1", "incumbent-v1"}, + EmittedIdentity: "policy-v1", + RuntimeIdentity: "candidate-v1", + AppliedIdentity: "candidate-v1", + SelectorVersion: "selector-v1", + SchedulerVersion: "scheduler-v1", + Caps: map[string]int64{"state": 32}, + RuntimeBranch: "candidate", + Overflow: telemetryBool(false), + FallbackExecuted: telemetryBool(false), + Provenance: map[string]string{ + "requested_identity": "optimizer.request", + "planned_identities": "optimizer.candidates", + "emitted_identity": "translator.policy", + "runtime_identity": "executor.branch", + "applied_identity": "executor.applied", + "selector_version": "optimizer.selector", + "scheduler_version": "executor.scheduler", + "caps.state": "policy.state_cap", + "runtime_branch": "executor.branch", + "overflow": "executor.guard", + "fallback_executed": "executor.fallback", + }, + }, + } +} + +// ordinaryDiagnostic prepares or inspects test evidence for ordinary diagnostic. +func ordinaryDiagnostic() *TraversalExecutionDiagnostic { + provenance := map[string]string{} + for _, name := range []string{ + "roots", "edge_candidates", "admitted_states", "relationship_repeat_rejects", "recursive_rows", + "peak_state", "emitted_trails", "hydration_rows", + } { + provenance["ordinary."+name] = "traversal_recursive_cte." + name + } + + return &TraversalExecutionDiagnostic{ + InvocationID: "invocation-1", + ConnectionID: "backend-123", + TimedSample: telemetryBool(false), + RequiredFamilies: []TraversalTelemetryFamily{TraversalTelemetryFamilyOrdinary}, + CounterStatus: TraversalTelemetryCounterStatusComplete, + Counters: TraversalDiagnosticCounters{ + Ordinary: &OrdinaryTraversalCounters{ + Roots: telemetryInt64(0), + EdgeCandidates: telemetryInt64(0), + AdmittedStates: telemetryInt64(0), + RelationshipRepeatRejects: telemetryInt64(0), + RecursiveRows: telemetryInt64(0), + PeakState: telemetryInt64(0), + EmittedTrails: telemetryInt64(0), + HydrationRows: telemetryInt64(0), + }, + }, + Provenance: provenance, + } +} + +// telemetryInt64 prepares or inspects test evidence for telemetry int64. +func telemetryInt64(value int64) *int64 { + return &value +} + +// telemetryBool prepares or inspects test evidence for telemetry bool. +func telemetryBool(value bool) *bool { + return &value +} diff --git a/cmd/graphbench/types.go b/cmd/graphbench/types.go index c941a01a..697931c3 100644 --- a/cmd/graphbench/types.go +++ b/cmd/graphbench/types.go @@ -20,14 +20,22 @@ import ( "fmt" "slices" "strings" + + "github.com/specterops/dawgs/testutil" ) const ( - ModePostgresSQL ExecutionMode = "postgres_sql" + // ModePostgresSQL selects translated PostgreSQL execution. + ModePostgresSQL ExecutionMode = "postgres_sql" + + // ModeLocalTraversal selects in-process traversal execution. ModeLocalTraversal ExecutionMode = "local_traversal" - ModeNeo4j ExecutionMode = "neo4j" + + // ModeNeo4j selects Neo4j execution. + ModeNeo4j ExecutionMode = "neo4j" ) +// validExecutionModes lists every execution mode accepted by graphbench. var validExecutionModes = []ExecutionMode{ ModePostgresSQL, ModeLocalTraversal, @@ -36,10 +44,12 @@ var validExecutionModes = []ExecutionMode{ type ExecutionMode string +// Valid reports whether the execution mode is one of the supported backend modes. func (s ExecutionMode) Valid() bool { return slices.Contains(validExecutionModes, s) } +// parseExecutionMode returns the execution mode named by text or an error for unsupported values. func parseExecutionMode(raw string) (ExecutionMode, error) { mode := ExecutionMode(strings.TrimSpace(raw)) if mode.Valid() { @@ -49,56 +59,219 @@ func parseExecutionMode(raw string) (ExecutionMode, error) { return "", fmt.Errorf("unsupported execution mode %q", raw) } +// ScaleCorpus contains the ordered benchmark cases loaded from the scale corpus. type ScaleCorpus struct { + // Cases contains loaded workloads in deterministic corpus order. Cases []ScaleCase } +// DeclaredCaseBackend identifies one case/backend combination and any declared unsupported reason. +type DeclaredCaseBackend struct { + // Dataset identifies the fixture dataset. + Dataset string + // Name identifies the case or record within its dataset. + Name string + // Backend identifies the execution backend. + Backend ExecutionMode + // UnsupportedReason explains why a declared case cannot run on the selected backend. + UnsupportedReason string +} + +// DeclaredBackends expands a scale case into the backend declarations consumed during gate validation. +func (s ScaleCorpus) DeclaredBackends() []DeclaredCaseBackend { + declared := make([]DeclaredCaseBackend, 0, len(s.Cases)*2) + for _, testCase := range s.Cases { + for _, backend := range testCase.CandidateModes { + declared = append(declared, DeclaredCaseBackend{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Backend: backend, + }) + } + for backend, reason := range testCase.UnsupportedModes { + declared = append(declared, DeclaredCaseBackend{ + Dataset: testCase.Dataset, + Name: testCase.Name, + Backend: backend, + UnsupportedReason: reason, + }) + } + } + return declared +} + +// ScaleCaseFile models the JSON envelope containing a group of scale cases. type ScaleCaseFile struct { + // Cases contains the workload declarations decoded from one corpus file. Cases []ScaleCase `json:"cases"` } +// ScaleCase declares one executable workload, its parameters, backend support, and exact expectations. type ScaleCase struct { - Source string `json:"-"` - Name string `json:"name"` - Dataset string `json:"dataset"` - Category string `json:"category"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - NodeParams map[string]string `json:"node_params,omitempty"` - Expected ExpectedResult `json:"expected"` - Observes ObservedValues `json:"observes"` - Shape WorkloadShape `json:"shape"` - CandidateModes []ExecutionMode `json:"candidate_modes"` - Tags []string `json:"tags,omitempty"` - ReferenceDesign *ReferenceDesign `json:"reference_design,omitempty"` + // Source identifies the source corpus file. + Source string `json:"-"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Category groups cases by workload category. + Category string `json:"category"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // GeneratedNodeListParams maps query parameters to generated fixture node sets. + GeneratedNodeListParams map[string]testutil.GeneratedNodeListParam `json:"generated_node_list_params,omitempty"` + // Expected supplies the expected input to the ScaleCase contract. + Expected ExpectedResult `json:"expected"` + // Observes identifies the normalized observation contract declared by the scale case. + Observes ObservedValues `json:"observes"` + // Shape describes the workload shape used for selection and comparison. + Shape WorkloadShape `json:"shape"` + // CandidateModes lists backends expected to participate in cross-backend comparison. + CandidateModes []ExecutionMode `json:"candidate_modes"` + // UnsupportedModes maps unsupported execution modes to their declared reasons. + UnsupportedModes map[ExecutionMode]string `json:"unsupported_modes,omitempty"` + // Tags lists selectors attached to the case. + Tags []string `json:"tags,omitempty"` + // ReferenceDesign documents reference arms and validation boundaries applicable to the scale case. + ReferenceDesign *ReferenceDesign `json:"reference_design,omitempty"` + // WriteScenario supplies the write scenario input to the ScaleCase contract. + WriteScenario *WriteScenario `json:"write_scenario,omitempty"` } +// ExpectedResult groups state that must remain consistent while processing expected result. type ExpectedResult struct { - RowCount *int64 `json:"row_count,omitempty"` + // RowCount records the number of row count. + RowCount *int64 `json:"row_count,omitempty"` + // ScalarInt sets the required scalar result when ResultKind is scalar_int. + ScalarInt *int64 `json:"scalar_int,omitempty"` + // ResultKind identifies how returned values must be normalized. ResultKind string `json:"result_kind,omitempty"` + // IDRows contains the expected ordered identifier rows. + IDRows [][]string `json:"id_rows,omitempty"` + // PathRows contains the expected stable paths. + PathRows []ExpectedPath `json:"path_rows,omitempty"` } +// ExpectedPath defines one expected stable node and relationship sequence. +type ExpectedPath struct { + // Nodes contains the stable node sequence. + Nodes []string `json:"nodes"` + // RelationshipKinds contains the expected relationship-kind sequence. + RelationshipKinds []string `json:"relationship_kinds"` + // RelationshipKeys contains the expected fixture relationship-key sequence. + RelationshipKeys []string `json:"relationship_keys,omitempty"` +} + +// WriteScenario defines a measured mutation and the state checks that validate it. +type WriteScenario struct { + // SelectionCypher contains the write-selection Cypher statement. + SelectionCypher string `json:"selection_cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // GeneratedNodeListParams maps query parameters to generated fixture node sets. + GeneratedNodeListParams map[string]testutil.GeneratedNodeListParam `json:"generated_node_list_params,omitempty"` + // AffectedEntity identifies the entity class counted after a write. + AffectedEntity string `json:"affected_entity"` + // ExpectedMatched sets the required number of matched entities. + ExpectedMatched *int64 `json:"expected_matched"` + // ExpectedAffected sets the required number of affected entities. + ExpectedAffected *int64 `json:"expected_affected"` + // PostState supplies the post state input to the WriteScenario contract. + PostState []ScaleStateQuery `json:"post_state"` +} + +// ScaleStateQuery defines a post-mutation query and its scalar or row-count expectation. +type ScaleStateQuery struct { + // Name labels the post-write state assertion in diagnostics and results. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // GeneratedNodeListParams maps query parameters to generated fixture node sets. + GeneratedNodeListParams map[string]testutil.GeneratedNodeListParam `json:"generated_node_list_params,omitempty"` + // Expected supplies the expected input to the ScaleStateQuery contract. + Expected ExpectedResult `json:"expected"` +} + +// ObservedValues declares which entity and path features a case exposes for normalized comparison. type ObservedValues struct { - Paths bool `json:"paths"` - Nodes bool `json:"nodes"` + // Paths reports whether the normalized result includes materialized paths. + Paths bool `json:"paths"` + // Nodes reports whether the normalized result includes node values. + Nodes bool `json:"nodes"` + // Relationships reports whether the normalized result includes relationship values. Relationships bool `json:"relationships"` - Properties bool `json:"properties"` + // Properties reports whether normalized entity observations include properties. + Properties bool `json:"properties"` } +// WorkloadShape describes traversal depth, direction, projection, and expected complexity. type WorkloadShape struct { - RootPredicate string `json:"root_predicate,omitempty"` - TerminalPredicate string `json:"terminal_predicate,omitempty"` - EdgeKinds []string `json:"edge_kinds,omitempty"` - MinDepth *int `json:"min_depth,omitempty"` - MaxDepth *int `json:"max_depth,omitempty"` - PathMaterializationRequired bool `json:"path_materialization_required"` + // QualificationSplit identifies whether a topology bucket is training, + // holdout, or a diagnostic boundary. Selector tuning must not consume + // holdout records. + QualificationSplit string `json:"qualification_split,omitempty"` + // QualificationRole freezes the statistical role independently of observed + // performance. Formal SP-I2 V2 cases use adverse_control or efficacy_target. + QualificationRole string `json:"qualification_role,omitempty"` + // FallbackExpectation is the typed runtime contract for candidate execution: + // forbidden, required, or allowed. Prioritized corpus declarations receive a + // deterministic value during loading when older files omit it. + FallbackExpectation string `json:"fallback_expectation,omitempty"` + // RootPredicate describes how the traversal root is constrained. + RootPredicate string `json:"root_predicate,omitempty"` + // TerminalPredicate describes how the traversal terminal is constrained. + TerminalPredicate string `json:"terminal_predicate,omitempty"` + // EdgeKinds lists the relationship kinds traversed by the workload. + EdgeKinds []string `json:"edge_kinds,omitempty"` + // Direction sets the traversal direction. + Direction string `json:"direction,omitempty"` + // RelationshipKindCount records the number of relationship kind count. + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + // FixtureTier identifies the fixture scale tier. + FixtureTier string `json:"fixture_tier,omitempty"` + // ExpectedStateClass identifies the expected recursive-state complexity class. + ExpectedStateClass string `json:"expected_state_class,omitempty"` + // ResultCardinalityClass identifies the expected result-cardinality class. + ResultCardinalityClass string `json:"result_cardinality_class,omitempty"` + // MinDepth is the shallowest traversal depth permitted by the workload. + MinDepth *int `json:"min_depth,omitempty"` + // MaxDepth sets the maximum traversal depth. + MaxDepth *int `json:"max_depth,omitempty"` + // PathMaterializationRequired reports whether the workload must materialize complete paths. + PathMaterializationRequired bool `json:"path_materialization_required"` } +// ReferenceDesign documents the independent reference implementations applicable to a case. type ReferenceDesign struct { + // AGERelevance documents how the reference design relates to Apache AGE execution. AGERelevance []string `json:"age_relevance,omitempty"` - Notes string `json:"notes,omitempty"` + // Notes contains human-readable caveats attached to the artifact or case. + Notes string `json:"notes,omitempty"` } +// Supports reports whether the case declares the requested execution mode as a candidate backend. func (s ScaleCase) Supports(mode ExecutionMode) bool { return slices.Contains(s.CandidateModes, mode) } + +// UnsupportedReason returns the declared reason that a scale case cannot run in the requested mode. +func (s ScaleCase) UnsupportedReason(mode ExecutionMode) (string, bool) { + reason, unsupported := s.UnsupportedModes[mode] + return reason, unsupported +} diff --git a/cmd/graphbench/waterfall.go b/cmd/graphbench/waterfall.go new file mode 100644 index 00000000..04539bbb --- /dev/null +++ b/cmd/graphbench/waterfall.go @@ -0,0 +1,556 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "fmt" + "runtime" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/graph" +) + +// postgresBoundarySession is the small shared execution surface of a dedicated +// pgx connection and a pooled pgx connection. +type postgresBoundarySession interface { + BeginTx(context.Context, pgx.TxOptions) (pgx.Tx, error) +} + +// postgresBoundaryPIDReader provides a physical PostgreSQL backend identity. +type postgresBoundaryPIDReader interface { + QueryRow(context.Context, string, ...any) pgx.Row +} + +// postgresBoundaryObservationNormalizer converts raw pgx values into the +// same fixture-stable public observations used by the primary measurement. +// It is intentionally part of the decode stage: a closure cannot claim exact +// raw-PGX output evidence without decoding the returned values. +type postgresBoundaryObservationNormalizer struct { + mapper graph.ValueMapper + reversedIDs map[graph.ID]string + scalarNodeIDs bool + pathValues bool +} + +func (s postgresBoundaryObservationNormalizer) normalize(values []any, fields []pgconn.FieldDescription) (string, error) { + decodePostgresBoundaryJSONValues(values, fields) + stableValues, err := stableRowValues(values, s.mapper, s.reversedIDs, s.scalarNodeIDs, s.pathValues) + if err != nil { + return "", err + } + if s.pathValues { + // The path-set corpus contract has exactly one returned path column per + // row. Do not let an unregistered composite OID become an opaque binary + // value and receive a misleading, but stable, observation digest. + if len(stableValues) != 1 { + return "", fmt.Errorf("expected one decoded path column, got %d", len(stableValues)) + } + if _, ok := stableValues[0].(stablePathObservation); !ok { + return "", fmt.Errorf("expected decoded path value, got %T", values[0]) + } + } + encoded, err := json.Marshal(stableValues) + if err != nil { + return "", err + } + return string(encoded), nil +} + +// decodePostgresBoundaryJSONValues mirrors the PostgreSQL driver's raw-result +// normalization so a direct pgx boundary records the same public value shape. +func decodePostgresBoundaryJSONValues(values []any, fields []pgconn.FieldDescription) { + for idx, field := range fields { + if field.DataTypeOID != pgtype.JSONOID && field.DataTypeOID != pgtype.JSONBOID { + continue + } + if decoded, ok := decodePostgresBoundaryJSONValue(values[idx]); ok { + values[idx] = decoded + } + } +} + +func decodePostgresBoundaryJSONValue(value any) (any, bool) { + switch typedValue := value.(type) { + case []byte: + var decoded any + if err := json.Unmarshal(typedValue, &decoded); err == nil { + return decoded, true + } + case string: + trimmedValue := strings.TrimSpace(typedValue) + if len(trimmedValue) == 0 { + return nil, false + } + switch trimmedValue[0] { + case '{', '[', '"': + default: + return nil, false + } + var decoded any + if err := json.Unmarshal([]byte(trimmedValue), &decoded); err == nil { + return decoded, true + } + } + return nil, false +} + +// measureCompileWaterfall times Cypher parse, translate, and SQL rendering separately. +func measureCompileWaterfall( + ctx context.Context, + cypherQuery string, + params map[string]any, + kindMapper pgsql.KindMapper, + graphID int32, + iterations int, + toolOptions translate.ToolOptions, +) (ClientWaterfall, error) { + waterfall := ClientWaterfall{ + IntervalsOverlap: true, + Notes: "translate_including_optimize repeats optimization internally; parse, optimize, translate, and render must not be summed as an additive client attribution", + Samples: make([]CompileSample, 0, iterations), + } + for iteration := 1; iteration <= iterations; iteration++ { + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + totalStart := time.Now() + + parseStart := time.Now() + query, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + return ClientWaterfall{}, fmt.Errorf("parse: %w", err) + } + parseDuration := time.Since(parseStart) + + optimizeStart := time.Now() + if _, err := optimize.Optimize(query); err != nil { + return ClientWaterfall{}, fmt.Errorf("optimize: %w", err) + } + optimizeDuration := time.Since(optimizeStart) + + translateStart := time.Now() + var translation translate.Result + if !hasForcedToolOptions(toolOptions) { + translation, err = translate.Translate(ctx, query, kindMapper, params, graphID) + } else { + translation, err = translate.TranslateForTool(ctx, query, kindMapper, params, graphID, toolOptions) + } + if err != nil { + return ClientWaterfall{}, fmt.Errorf("translate: %w", err) + } + translateDuration := time.Since(translateStart) + + renderStart := time.Now() + if _, err := translate.Translated(translation); err != nil { + return ClientWaterfall{}, fmt.Errorf("render: %w", err) + } + renderDuration := time.Since(renderStart) + totalDuration := time.Since(totalStart) + runtime.ReadMemStats(&after) + + waterfall.Samples = append(waterfall.Samples, CompileSample{ + Iteration: iteration, + Parse: parseDuration, + Optimize: optimizeDuration, + TranslateIncludingOptimize: translateDuration, + Render: renderDuration, + Total: totalDuration, + Allocations: after.Mallocs - before.Mallocs, + AllocatedBytes: after.TotalAlloc - before.TotalAlloc, + }) + } + return waterfall, nil +} + +// measureRawPGXWaterfall times PostgreSQL bind, first row, drain, and close stages separately. +func measureRawPGXWaterfall(ctx context.Context, pool *pgxpool.Pool, sqlQuery string, params map[string]any, warmupIterations, iterations int, isolation ...pgx.TxIsoLevel) (PostgresBoundaryWaterfall, error) { + if warmupIterations < 0 || iterations < 1 { + return PostgresBoundaryWaterfall{}, fmt.Errorf("invalid raw pgx warmup/iteration counts") + } + run := func(iteration int, retain bool) (BoundarySample, error) { + totalStart := time.Now() + acquireStart := time.Now() + connection, err := pool.Acquire(ctx) + if err != nil { + return BoundarySample{}, err + } + defer connection.Release() + return measureRawPGXOnSession(ctx, connection, sqlQuery, params, nil, iteration, totalStart, time.Since(acquireStart), retain, false, isolation...) + } + for idx := 0; idx < warmupIterations; idx++ { + if _, err := run(-(idx + 1), false); err != nil { + return PostgresBoundaryWaterfall{}, err + } + } + result := PostgresBoundaryWaterfall{ + Boundary: "identical translated SQL through raw pgx pool/transaction/decode/drain", + SQLFingerprint: sqlFingerprint(sqlQuery), + WarmupIterations: warmupIterations, + Samples: make([]BoundarySample, 0, iterations), + } + var expectedRows int64 = -1 + for iteration := 1; iteration <= iterations; iteration++ { + sample, err := run(iteration, true) + if err != nil { + return PostgresBoundaryWaterfall{}, err + } + if expectedRows < 0 { + expectedRows = sample.Rows + } + if sample.Rows != expectedRows { + return PostgresBoundaryWaterfall{}, fmt.Errorf("raw pgx row count changed from %d to %d", expectedRows, sample.Rows) + } + result.Samples = append(result.Samples, sample) + } + return result, nil +} + +// measureRawPGXOnSession executes one exact translated statement through an +// already-selected PostgreSQL session. Workspace collection happens after the +// result is drained and is excluded from all timing intervals. +func measureRawPGXOnSession( + ctx context.Context, + session postgresBoundarySession, + sqlQuery string, + params map[string]any, + normalizer *postgresBoundaryObservationNormalizer, + iteration int, + totalStart time.Time, + poolWait time.Duration, + retainAllocations bool, + captureWorkspace bool, + isolation ...pgx.TxIsoLevel, +) (BoundarySample, error) { + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + transactionStart := time.Now() + // DAWGS read queries may invoke session-local workspace DDL/DML. The raw + // boundary therefore uses a rollback-only read-write transaction, preserving + // the exact translated SQL without committing benchmark side effects. + txOptions := pgx.TxOptions{AccessMode: pgx.ReadWrite} + if len(isolation) > 0 { + txOptions.IsoLevel = isolation[0] + } + tx, err := session.BeginTx(ctx, txOptions) + if err != nil { + return BoundarySample{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + transactionDuration := time.Since(transactionStart) + + bindStart := time.Now() + queryArgs := []any{pgx.QueryExecModeCacheStatement, pgx.QueryResultFormats{pgx.BinaryFormatCode}} + if len(params) > 0 { + queryArgs = append(queryArgs, pgx.NamedArgs(params)) + } + rows, err := tx.Query(ctx, sqlQuery, queryArgs...) + if err != nil { + return BoundarySample{}, err + } + bindDuration := time.Since(bindStart) + firstRowStart := time.Now() + var rowCount int64 + var observedRows []string + observeRow := func() error { + values, err := rows.Values() + if err != nil { + return err + } + if normalizer != nil { + observed, err := normalizer.normalize(values, rows.FieldDescriptions()) + if err != nil { + return err + } + observedRows = append(observedRows, observed) + } + return nil + } + if rows.Next() { + rowCount++ + if err := observeRow(); err != nil { + rows.Close() + return BoundarySample{}, err + } + } + firstRowDuration := time.Since(firstRowStart) + allRowsStart := time.Now() + for rows.Next() { + rowCount++ + if err := observeRow(); err != nil { + rows.Close() + return BoundarySample{}, err + } + } + allRowsDuration := time.Since(allRowsStart) + + drainStart := time.Now() + rows.Close() + if err := rows.Err(); err != nil { + return BoundarySample{}, err + } + var observationSHA256 string + if normalizer != nil { + sort.Strings(observedRows) + observationSHA256, err = stableObservationSHA256(observedRows) + if err != nil { + return BoundarySample{}, err + } + } + drainDuration := time.Since(drainStart) + var workspaceBytes *int64 + if captureWorkspace { + workspaceStart := time.Now() + workspace, err := measurePostgresTemporaryWorkspace(ctx, tx) + if err != nil { + return BoundarySample{}, err + } + workspaceBytes = &workspace + // The observation query is intentionally outside the raw execution + // intervals; only cleanup remains part of the direct request boundary. + workspaceDuration := time.Since(workspaceStart) + totalStart = totalStart.Add(workspaceDuration) + } + rollbackStart := time.Now() + if err := tx.Rollback(ctx); err != nil && err != pgx.ErrTxClosed { + return BoundarySample{}, err + } + drainDuration += time.Since(rollbackStart) + runtime.ReadMemStats(&after) + sample := BoundarySample{ + Iteration: iteration, + PoolWait: poolWait, + Transaction: transactionDuration, + BindPrepare: bindDuration, + FirstRow: firstRowDuration, + AllRowsDecode: allRowsDuration, + DrainClose: drainDuration, + Total: time.Since(totalStart), + Rows: rowCount, + WorkspaceBytes: workspaceBytes, + ObservationSHA256: observationSHA256, + } + if retainAllocations { + sample.Allocations = after.Mallocs - before.Mallocs + sample.AllocatedBytes = after.TotalAlloc - before.TotalAlloc + } + return sample, nil +} + +// measurePostgresTemporaryWorkspace reports all non-diagnostic temporary +// relations visible to the exact raw query transaction. Runtime attestation +// and telemetry tables are measurement scaffolding, so they are excluded from +// the component's performance-workspace budget. +func measurePostgresTemporaryWorkspace(ctx context.Context, tx pgx.Tx) (int64, error) { + var workspace int64 + if err := tx.QueryRow(ctx, ` + select coalesce(sum(pg_total_relation_size(c.oid)), 0)::int8 + from pg_class c + where c.relnamespace = pg_my_temp_schema() + and c.relname <> 'traversal_runtime_attestation_v1' + and c.relname not like '%telemetry%' + and not exists ( + select 1 + from pg_index i + join pg_class indexed on indexed.oid = i.indrelid + where i.indexrelid = c.oid + and indexed.relnamespace = pg_my_temp_schema() + and (indexed.relname = 'traversal_runtime_attestation_v1' or indexed.relname like '%telemetry%') + ) + `).Scan(&workspace); err != nil { + return 0, fmt.Errorf("measure temporary workspace high-water: %w", err) + } + return workspace, nil +} + +// postgresBackendPID reads one physical PostgreSQL connection identity outside +// the timing boundary so closure records can prove pool release/reacquisition. +func postgresBackendPID(ctx context.Context, connection postgresBoundaryPIDReader) (string, error) { + var backendPID int64 + if err := connection.QueryRow(ctx, "select pg_backend_pid()").Scan(&backendPID); err != nil { + return "", fmt.Errorf("read PostgreSQL backend PID: %w", err) + } + return fmt.Sprintf("%d", backendPID), nil +} + +// measurePostgresBoundaryClosure captures explicit fresh-session, prepared-hit, +// and release/reacquisition strata without changing the SQL under measurement. +func measurePostgresBoundaryClosure(ctx context.Context, pool *pgxpool.Pool, sqlQuery string, params map[string]any, normalizer postgresBoundaryObservationNormalizer, iterations int, sessionCeilingBytes, poolCeilingBytes int64, isolation ...pgx.TxIsoLevel) (PostgresBoundaryClosure, error) { + if iterations < 1 { + return PostgresBoundaryClosure{}, fmt.Errorf("closure iterations must be positive") + } + if sessionCeilingBytes <= 0 || poolCeilingBytes <= 0 { + return PostgresBoundaryClosure{}, fmt.Errorf("closure workspace ceilings must be positive") + } + poolConfig := pool.Config().Copy() + freshConfig := poolConfig.ConnConfig.Copy() + fresh, err := pgx.ConnectConfig(ctx, freshConfig) + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("open fresh PostgreSQL closure session: %w", err) + } + defer fresh.Close(ctx) + // pgx.ConnectConfig does not invoke pgxpool.AfterConnect. Run the same + // hook used by the benchmark pool before collecting the backend ID or + // timing the first statement so direct and pooled sessions decode driver + // composites identically while retaining an honest statement-cache miss. + if poolConfig.AfterConnect != nil { + if err := poolConfig.AfterConnect(ctx, fresh); err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("initialize fresh PostgreSQL closure session: %w", err) + } + } + // The runner pool has already executed the primary benchmark statement, so + // it cannot establish an honest prepared-statement miss. Build a separate + // size-one pool from the same connection configuration for the raw closure. + // This pool is used only below: its first query is the recorded miss and its + // later release/reacquisition samples prove reusable prepared-hit behavior. + closurePoolConfig := poolConfig.Copy() + closurePoolConfig.MaxConns = 1 + closurePoolConfig.MinConns = 0 + closurePool, err := pgxpool.NewWithConfig(ctx, closurePoolConfig) + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("open dedicated PostgreSQL closure pool: %w", err) + } + defer closurePool.Close() + + closure := PostgresBoundaryClosure{ + Boundary: "identical translated SQL through fresh and size-one raw pgx sessions/transactions/decode/drain", + SQLFingerprint: sqlFingerprint(sqlQuery), + SameSessionPreparedHits: make([]BoundarySample, 0, iterations), + PoolReacquiredPreparedHits: make([]BoundarySample, 0, iterations), + } + freshPID, err := postgresBackendPID(ctx, fresh) + if err != nil { + return PostgresBoundaryClosure{}, err + } + closure.FreshSessionPreparedMiss, err = measureRawPGXOnSession(ctx, fresh, sqlQuery, params, &normalizer, 1, time.Now(), 0, true, true, isolation...) + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("fresh-session prepared miss: %w", err) + } + closure.FreshSessionPreparedMiss.ConnectionID = freshPID + for iteration := 1; iteration <= iterations; iteration++ { + sample, err := measureRawPGXOnSession(ctx, fresh, sqlQuery, params, &normalizer, iteration, time.Now(), 0, true, true, isolation...) + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("same-session prepared hit %d: %w", iteration, err) + } + sample.ConnectionID = freshPID + closure.SameSessionPreparedHits = append(closure.SameSessionPreparedHits, sample) + } + + acquireStart := time.Now() + pooled, err := closurePool.Acquire(ctx) + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("acquire pooled prepared miss session: %w", err) + } + poolWait := time.Since(acquireStart) + pooledPID, err := postgresBackendPID(ctx, pooled) + if err == nil { + closure.PoolPreparedMiss, err = measureRawPGXOnSession(ctx, pooled, sqlQuery, params, &normalizer, 1, acquireStart, poolWait, true, true, isolation...) + } + pooled.Release() + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("pooled prepared miss: %w", err) + } + closure.PoolPreparedMiss.ConnectionID = pooledPID + + for iteration := 1; iteration <= iterations; iteration++ { + acquireStart := time.Now() + pooled, err := closurePool.Acquire(ctx) + if err != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("reacquire pooled prepared-hit session %d: %w", iteration, err) + } + poolWait := time.Since(acquireStart) + currentPID, pidErr := postgresBackendPID(ctx, pooled) + if pidErr == nil && currentPID != pooledPID { + pidErr = fmt.Errorf("pooled backend changed after release: %s -> %s", pooledPID, currentPID) + } + var sample BoundarySample + if pidErr == nil { + sample, pidErr = measureRawPGXOnSession(ctx, pooled, sqlQuery, params, &normalizer, iteration, acquireStart, poolWait, true, true, isolation...) + } + pooled.Release() + if pidErr != nil { + return PostgresBoundaryClosure{}, fmt.Errorf("pooled prepared hit %d: %w", iteration, pidErr) + } + sample.ConnectionID = currentPID + closure.PoolReacquiredPreparedHits = append(closure.PoolReacquiredPreparedHits, sample) + } + + closure, err = finalizePostgresBoundaryClosure(closure, sessionCeilingBytes, poolCeilingBytes) + if err != nil { + return PostgresBoundaryClosure{}, err + } + return closure, nil +} + +// finalizePostgresBoundaryClosure fails closed on an incomplete stratum, +// changing pooled connection, absent workspace observation, or ceiling breach. +func finalizePostgresBoundaryClosure(closure PostgresBoundaryClosure, sessionCeilingBytes, poolCeilingBytes int64) (PostgresBoundaryClosure, error) { + if closure.SQLFingerprint == "" || closure.FreshSessionPreparedMiss.WorkspaceBytes == nil || closure.FreshSessionPreparedMiss.ObservationSHA256 == "" || + len(closure.SameSessionPreparedHits) == 0 || closure.PoolPreparedMiss.WorkspaceBytes == nil || + len(closure.PoolReacquiredPreparedHits) == 0 { + return PostgresBoundaryClosure{}, fmt.Errorf("closure lacks a complete fresh, same-session, or pooled prepared-state stratum") + } + if closure.PoolPreparedMiss.ConnectionID == "" { + return PostgresBoundaryClosure{}, fmt.Errorf("closure pooled prepared miss lacks a backend identity") + } + all := []BoundarySample{closure.FreshSessionPreparedMiss, closure.PoolPreparedMiss} + all = append(all, closure.SameSessionPreparedHits...) + all = append(all, closure.PoolReacquiredPreparedHits...) + for _, sample := range all { + if sample.WorkspaceBytes == nil || sample.Rows < 0 || sample.Total <= 0 || sample.ConnectionID == "" || sample.ObservationSHA256 == "" { + return PostgresBoundaryClosure{}, fmt.Errorf("closure contains an incomplete boundary sample") + } + if sample.ObservationSHA256 != closure.FreshSessionPreparedMiss.ObservationSHA256 { + return PostgresBoundaryClosure{}, fmt.Errorf("closure raw observations differ between prepared-state strata") + } + if *sample.WorkspaceBytes > closure.Workspace.PerQueryPeakBytes { + closure.Workspace.PerQueryPeakBytes = *sample.WorkspaceBytes + } + } + for _, sample := range append([]BoundarySample{closure.FreshSessionPreparedMiss}, closure.SameSessionPreparedHits...) { + if *sample.WorkspaceBytes > closure.Workspace.FreshSessionPeakBytes { + closure.Workspace.FreshSessionPeakBytes = *sample.WorkspaceBytes + } + } + for _, sample := range append([]BoundarySample{closure.PoolPreparedMiss}, closure.PoolReacquiredPreparedHits...) { + if sample.ConnectionID != closure.PoolPreparedMiss.ConnectionID { + return PostgresBoundaryClosure{}, fmt.Errorf("closure pooled prepared-hit backend identity differs from prepared miss") + } + if *sample.WorkspaceBytes > closure.Workspace.SessionPeakBytes { + closure.Workspace.SessionPeakBytes = *sample.WorkspaceBytes + } + } + closure.Workspace.PoolPeakBytes = closure.Workspace.SessionPeakBytes + if closure.Workspace.SessionPeakBytes > sessionCeilingBytes { + return PostgresBoundaryClosure{}, fmt.Errorf("closure session workspace high-water %d exceeds ceiling %d", closure.Workspace.SessionPeakBytes, sessionCeilingBytes) + } + if closure.Workspace.PoolPeakBytes > poolCeilingBytes { + return PostgresBoundaryClosure{}, fmt.Errorf("closure pool workspace high-water %d exceeds ceiling %d", closure.Workspace.PoolPeakBytes, poolCeilingBytes) + } + return closure, nil +} + +// postgresBoundaryClosureSamples returns every raw SQL observation contained in +// a closure, preserving its explicit prepared-state strata in their declared +// order for exact-result validation. +func postgresBoundaryClosureSamples(closure PostgresBoundaryClosure) []BoundarySample { + samples := make([]BoundarySample, 0, 2+len(closure.SameSessionPreparedHits)+len(closure.PoolReacquiredPreparedHits)) + samples = append(samples, closure.FreshSessionPreparedMiss) + samples = append(samples, closure.SameSessionPreparedHits...) + samples = append(samples, closure.PoolPreparedMiss) + samples = append(samples, closure.PoolReacquiredPreparedHits...) + return samples +} diff --git a/cmd/graphbench/waterfall_test.go b/cmd/graphbench/waterfall_test.go new file mode 100644 index 00000000..c639e925 --- /dev/null +++ b/cmd/graphbench/waterfall_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg/pgutil" + "github.com/stretchr/testify/require" +) + +// TestMeasureCompileWaterfallMarksOverlappingIntervals verifies that compile phase timings are labeled non-additive and each requested sample records elapsed time and allocations. +func TestMeasureCompileWaterfallMarksOverlappingIntervals(t *testing.T) { + waterfall, err := measureCompileWaterfall(context.Background(), "MATCH (n) RETURN id(n)", nil, pgutil.NewInMemoryKindMapper(), 1, 2, translate.ToolOptions{}) + + require.NoError(t, err) + require.True(t, waterfall.IntervalsOverlap) + require.Contains(t, waterfall.Notes, "must not be summed") + require.Len(t, waterfall.Samples, 2) + for _, sample := range waterfall.Samples { + require.Positive(t, sample.Total) + require.Positive(t, sample.Allocations) + } +} + +// TestFinalizePostgresBoundaryClosureCompletesWorkspaceHighWater verifies the +// closure derives per-query, fresh-session, size-one session, and pool maxima +// only from complete prepared-state strata. +func TestFinalizePostgresBoundaryClosureCompletesWorkspaceHighWater(t *testing.T) { + workspace := func(bytes int64) *int64 { return &bytes } + observation, err := stableObservationSHA256([]string{"[1]"}) + require.NoError(t, err) + sample := func(connection string, bytes int64) BoundarySample { + return BoundarySample{ + Total: time.Millisecond, Rows: 1, ConnectionID: connection, WorkspaceBytes: workspace(bytes), ObservationSHA256: observation, + } + } + closure, err := finalizePostgresBoundaryClosure(PostgresBoundaryClosure{ + SQLFingerprint: "sql", + FreshSessionPreparedMiss: sample("fresh", 4), + SameSessionPreparedHits: []BoundarySample{sample("fresh", 7), sample("fresh", 5)}, + PoolPreparedMiss: sample("pool", 3), + PoolReacquiredPreparedHits: []BoundarySample{sample("pool", 9), sample("pool", 8)}, + }, 10, 10) + + require.NoError(t, err) + require.Equal(t, int64(9), closure.Workspace.PerQueryPeakBytes) + require.Equal(t, int64(7), closure.Workspace.FreshSessionPeakBytes) + require.Equal(t, int64(9), closure.Workspace.SessionPeakBytes) + require.Equal(t, int64(9), closure.Workspace.PoolPeakBytes) + require.Len(t, postgresBoundaryClosureSamples(closure), 6) +} + +// TestFinalizePostgresBoundaryClosureFailsClosed verifies absent workspace +// observations, pool identity drift, and budget overage cannot produce closure +// evidence. +func TestFinalizePostgresBoundaryClosureFailsClosed(t *testing.T) { + workspace := int64(1) + observation, err := stableObservationSHA256([]string{"[1]"}) + require.NoError(t, err) + base := PostgresBoundaryClosure{ + SQLFingerprint: "sql", + FreshSessionPreparedMiss: BoundarySample{Total: time.Millisecond, ConnectionID: "fresh", WorkspaceBytes: &workspace, ObservationSHA256: observation}, + SameSessionPreparedHits: []BoundarySample{{Total: time.Millisecond, ConnectionID: "fresh", WorkspaceBytes: &workspace, ObservationSHA256: observation}}, + PoolPreparedMiss: BoundarySample{Total: time.Millisecond, ConnectionID: "pool", WorkspaceBytes: &workspace, ObservationSHA256: observation}, + PoolReacquiredPreparedHits: []BoundarySample{{ + Total: time.Millisecond, ConnectionID: "pool", WorkspaceBytes: &workspace, ObservationSHA256: observation, + }}, + } + + _, err = finalizePostgresBoundaryClosure(base, 0, 1) + require.ErrorContains(t, err, "exceeds ceiling") + + changedConnection := base + changedConnection.PoolReacquiredPreparedHits[0].ConnectionID = "other" + _, err = finalizePostgresBoundaryClosure(changedConnection, 1, 1) + require.ErrorContains(t, err, "backend identity differs") + + missingWorkspace := base + missingWorkspace.SameSessionPreparedHits[0].WorkspaceBytes = nil + _, err = finalizePostgresBoundaryClosure(missingWorkspace, 1, 1) + require.ErrorContains(t, err, "incomplete boundary sample") + base.SameSessionPreparedHits[0].WorkspaceBytes = &workspace + + missingObservation := base + missingObservation.SameSessionPreparedHits[0].ObservationSHA256 = "" + _, err = finalizePostgresBoundaryClosure(missingObservation, 1, 1) + require.ErrorContains(t, err, "incomplete boundary sample") + base.SameSessionPreparedHits[0].ObservationSHA256 = observation + + mismatchedObservation := base + mismatchedObservation.SameSessionPreparedHits[0].ObservationSHA256 = "different" + _, err = finalizePostgresBoundaryClosure(mismatchedObservation, 1, 1) + require.ErrorContains(t, err, "raw observations differ") +} + +func TestPostgresBoundaryObservationNormalizerUsesStablePublicRows(t *testing.T) { + normalizer := postgresBoundaryObservationNormalizer{} + row, err := normalizer.normalize([]any{int64(42)}, nil) + require.NoError(t, err) + require.Equal(t, "[42]", row) + + jsonRow, err := normalizer.normalize([]any{[]byte(`{"value":42}`)}, []pgconn.FieldDescription{{DataTypeOID: pgtype.JSONBOID}}) + require.NoError(t, err) + require.Equal(t, `[{"value":42}]`, jsonRow) +} + +func TestPostgresBoundaryObservationNormalizerRejectsOpaquePathValue(t *testing.T) { + normalizer := postgresBoundaryObservationNormalizer{pathValues: true} + + _, err := normalizer.normalize([]any{[]byte{1, 2, 3}}, nil) + + require.ErrorContains(t, err, "expected decoded path value") +} diff --git a/cmd/plancorpus/README.md b/cmd/plancorpus/README.md index 75d8ee64..4c2749e7 100644 --- a/cmd/plancorpus/README.md +++ b/cmd/plancorpus/README.md @@ -3,20 +3,33 @@ `plancorpus` captures query-plan diagnostics for the shared integration corpus. It reads `integration/testdata/cases` and `integration/testdata/templates`, loads the same datasets and inline fixtures used by the integration tests, and writes backend-specific JSONL plan records plus markdown and JSON summaries. +Fixture-backed `node_params` and `node_list_params` are resolved after each +fixture load, preserving ID-anchored production query shapes in captured plans. Use this command to baseline PostgreSQL translator and optimizer changes. PostgreSQL captures include translated SQL, `EXPLAIN` output, plan operator counts, estimated plan cost, recursive CTE indicators, path materialization indicators, -planned lowerings, applied lowerings, skipped lowerings, and skipped-lowering reasons. Neo4j captures include logical -plan operator trees for cross-backend plan-shape comparison. +planned lowerings, applied lowerings, skipped lowerings, and skipped-lowering reasons. Neo4j read captures use `PROFILE` +after execution and retain ordered operators, estimated and actual rows, DB and page-cache hits, loops, and operator +time when the server exposes them. Writes remain `EXPLAIN`-only. + +Every run also writes a semantic PostgreSQL/Neo4j delta over the union of captured workloads. The delta is keyed by +workload hash and source revision, fingerprints each backend plan, compares access side, physical direction, predicate +placement, endpoint binding, traversal family, estimates, and PostgreSQL planned/emitted/fallback identities, and ranks +the largest disagreements. A missing or failed backend remains an explicit incomplete pair; it is never discarded by an +intersection-only comparison. Runtime-arm attribution remains GraphBench's responsibility. ## Usage ```bash PG_CONNECTION_STRING="postgres://postgres:password@localhost/db" \ -NEO4J_CONNECTION_STRING="neo4j://neo4j:password@localhost:7687" \ -go run ./cmd/plancorpus + NEO4J_CONNECTION_STRING="neo4j://neo4j:password@localhost:7687" \ + go run ./cmd/plancorpus ``` +Plan capture reloads fixtures and refuses to open a selected backend unless the destructive acknowledgement is set and +its exact credential-free target is allowlisted. PostgreSQL aliases and omitted default ports are canonicalized; +multi-host PostgreSQL URLs are accepted only when every fallback resolves to the same target. + Useful flags: | Flag | Default | Description | @@ -28,16 +41,20 @@ Useful flags: | `-neo4j-connection` | `NEO4J_CONNECTION_STRING` | Neo4j backend | | `-summary` | `.coverage/plan-corpus-summary.md` | Markdown summary | | `-summary-json` | `.coverage/plan-corpus-summary.json` | JSON summary | +| `-plan-delta-json` | `.coverage/plan-corpus-delta.json` | Versioned paired semantic delta, including incomplete backend pairs | | `-top` | `25` | Number of expensive PostgreSQL plans to include in summaries | +| `-dawgs-version` | auto-detected | DAWGS source version recorded in output | ## Reviewing Captures The markdown summary is intended for human review. It ranks the highest-cost PostgreSQL plans, reports feature counts such as `Recursive Union`, `SubPlan`, and `Function Scan on unnest`, and summarizes planned/applied/skipped lowerings. -The JSON summary is intended for automation and baseline comparison. For optimizer work, check that intentional SQL +The JSON summary and paired delta are intended for automation and baseline comparison. For optimizer work, check that intentional SQL shape changes are explained and that skipped-lowering accounting remains actionable. A planned lowering without a matching applied lowering should either have a specific skipped reason or indicate a translator consumption bug. +Both per-query JSONL records and summaries include the DAWGS source version +needed to compare captures made from different worktrees. Expected capture errors should be limited to invalid-query cases surfaced by the integration corpus or backend-specific syntax differences. Unexpected capture errors should be treated as validation failures for planner or translator work. diff --git a/cmd/plancorpus/capture.go b/cmd/plancorpus/capture.go index d05a7046..30d2d2a3 100644 --- a/cmd/plancorpus/capture.go +++ b/cmd/plancorpus/capture.go @@ -7,12 +7,14 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "github.com/jackc/pgx/v5/pgxpool" neo4jcore "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/specterops/dawgs" "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" "github.com/specterops/dawgs/drivers/neo4j" @@ -22,22 +24,34 @@ import ( "github.com/specterops/dawgs/util/size" ) +// defaultGraphName names the isolated graph populated while capturing corpus plans. const defaultGraphName = "integration_test" +// captureSpec binds a requested driver name to the connection string used for capture. type captureSpec struct { + // DriverName identifies the database driver selected for this capture. DriverName string + // Connection contains the backend connection string. Connection string } +// backendCapture owns one plan-capture backend and its graph database handle. type backendCapture struct { - spec captureSpec - db graph.Database - pgDriver *pg.Driver - pgGraphID int32 + // spec identifies the backend connection and driver being captured. + spec captureSpec + // db provides graph transactions for fixture preparation and query execution. + db graph.Database + // pgDriver provides PostgreSQL graph access and kind mapping. + pgDriver *pg.Driver + // pgGraphID selects the PostgreSQL graph partition cleared, populated, and queried during capture. + pgGraphID int32 + // neo4jDriver owns the Neo4j connection used for plan capture. neo4jDriver neo4jcore.Driver + // neo4jDBName selects the Neo4j database used for plan capture. neo4jDBName string } +// driverFromConnectionString selects a graph driver from the connection URI scheme. func driverFromConnectionString(connStr string) (string, error) { u, err := url.Parse(connStr) if err != nil { @@ -54,6 +68,7 @@ func driverFromConnectionString(connStr string) (string, error) { } } +// captureCorpus loads each required fixture and captures every corpus query for one backend. func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec captureSpec) ([]PlanRecord, error) { backend, err := openBackend(ctx, suite, spec) if err != nil { @@ -87,23 +102,28 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca for _, file := range group.files { for _, testCase := range file.Cases { + var idMap opengraph.IDMap if testCase.Fixture == nil { if err := ensureDatasetLoaded(); err != nil { return nil, err } } else { - if err := loadCommittedFixture(ctx, backend.db, testCase.Fixture); err != nil { + if idMap, err = loadCommittedFixture(ctx, backend.db, testCase.Fixture); err != nil { return nil, err } datasetLoaded = false } + params, err := resolveFixtureParams(testCase.Params, testCase.NodeParams, testCase.NodeListParams, idMap) + if err != nil { + return nil, fmt.Errorf("%s/%s: %w", file.path, testCase.Name, err) + } record := backend.capture(ctx, CorpusQuery{ Source: file.path, Dataset: datasetName, Name: testCase.Name, Cypher: testCase.Cypher, - Params: testCase.Params, + Params: params, }) records = append(records, record) } @@ -123,15 +143,25 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca if err != nil { return nil, fmt.Errorf("%s/%s/%s: %w", file.path, family.Name, variant.Name, err) } - if err := loadCommittedFixture(ctx, backend.db, family.Fixture); err != nil { + idMap, err := loadCommittedFixture(ctx, backend.db, family.Fixture) + if err != nil { return nil, err } + params, err := resolveFixtureParams( + mergeParams(family.Params, variant.Params), + mergeStringMap(family.NodeParams, variant.NodeParams), + mergeStringListMap(family.NodeListParams, variant.NodeListParams), + idMap, + ) + if err != nil { + return nil, fmt.Errorf("%s/%s/%s: %w", file.path, family.Name, variant.Name, err) + } record := backend.capture(ctx, CorpusQuery{ Source: file.path, Name: fileName + "/" + family.Name + "/" + variant.Name, Cypher: rendered, - Params: mergeParams(family.Params, variant.Params), + Params: params, }) records = append(records, record) } @@ -141,7 +171,7 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca if family.Fixture == nil { return nil, fmt.Errorf("%s/%s has no fixture", file.path, family.Name) } - if err := loadCommittedFixture(ctx, backend.db, family.Fixture); err != nil { + if _, err := loadCommittedFixture(ctx, backend.db, family.Fixture); err != nil { return nil, err } @@ -160,6 +190,7 @@ func captureCorpus(ctx context.Context, datasetDir string, suite corpus, spec ca return records, nil } +// openBackend opens the requested graph backend, asserts the capture schema, and retains driver-specific plan handles. func openBackend(ctx context.Context, suite corpus, spec captureSpec) (*backendCapture, error) { cfg := dawgs.Config{ GraphQueryMemoryLimit: size.Gibibyte, @@ -232,6 +263,7 @@ func openBackend(ctx context.Context, suite corpus, spec captureSpec) (*backendC return backend, nil } +// close closes the backend driver resources owned by a capture. func (s *backendCapture) close(ctx context.Context) { if s.neo4jDriver != nil { _ = s.neo4jDriver.Close() @@ -241,14 +273,17 @@ func (s *backendCapture) close(ctx context.Context) { } } +// capture captures one query plan with driver, workload, and fixture metadata. func (s *backendCapture) capture(ctx context.Context, query CorpusQuery) PlanRecord { record := PlanRecord{ - Driver: s.spec.DriverName, - Source: query.Source, - Dataset: query.Dataset, - Name: query.Name, - Cypher: query.Cypher, - Params: query.Params, + SchemaVersion: planRecordSchemaVersion, + Driver: s.spec.DriverName, + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + WorkloadSHA256: workloadFingerprint(query), + Cypher: query.Cypher, + Params: query.Params, } switch s.spec.DriverName { @@ -257,10 +292,13 @@ func (s *backendCapture) capture(ctx context.Context, query CorpusQuery) PlanRec case neo4j.DriverName: s.captureNeo4j(query.Cypher, query.Params, &record) } + record.PGPlanFingerprint = postgresPlanFingerprint(record.PGPlan) + record.Neo4jPlanFingerprint = neo4jPlanFingerprint(record.Neo4jPlan) return record } +// capturePostgres translates a Cypher query and attaches PostgreSQL EXPLAIN evidence to its record. func (s *backendCapture) capturePostgres(ctx context.Context, cypherQuery string, params map[string]any, record *PlanRecord) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) if err != nil { @@ -307,14 +345,27 @@ func (s *backendCapture) capturePostgres(ctx context.Context, cypherQuery string record.Optimization = &translation.Optimization } +// captureNeo4j runs PROFILE for reads and EXPLAIN for writes, then attaches its normalized operator tree. func (s *backendCapture) captureNeo4j(cypherQuery string, params map[string]any, record *PlanRecord) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + if err != nil { + record.Error = err.Error() + return + } + write := regularQueryHasUpdates(regularQuery) + accessMode := neo4jcore.AccessModeRead + command := "PROFILE " + if write { + accessMode = neo4jcore.AccessModeWrite + command = "EXPLAIN " + } session := s.neo4jDriver.NewSession(neo4jcore.SessionConfig{ - AccessMode: neo4jcore.AccessModeWrite, + AccessMode: accessMode, DatabaseName: s.neo4jDBName, }) defer session.Close() - result, err := session.Run("EXPLAIN "+cypherWithoutTerminator(cypherQuery), params) + result, err := session.Run(command+cypherWithoutTerminator(cypherQuery), params) if err != nil { record.Error = err.Error() return @@ -326,20 +377,50 @@ func (s *backendCapture) captureNeo4j(cypherQuery string, params map[string]any, return } - if plan := summary.Plan(); plan != nil { + if profile := summary.Profile(); profile != nil { + planNode := convertNeo4jProfile(profile) + record.Neo4jPlan = &planNode + record.Neo4jOperators = neo4jOperators(planNode) + } else if plan := summary.Plan(); plan != nil { planNode := convertNeo4jPlan(plan) record.Neo4jPlan = &planNode record.Neo4jOperators = neo4jOperators(planNode) } } +// regularQueryHasUpdates reports whether any query part contains a mutation. +func regularQueryHasUpdates(query *cypher.RegularQuery) bool { + if query == nil || query.SingleQuery == nil { + return false + } + if single := query.SingleQuery.SinglePartQuery; single != nil { + return len(single.UpdatingClauses) > 0 + } + multi := query.SingleQuery.MultiPartQuery + if multi == nil { + return false + } + for _, part := range multi.Parts { + if part != nil && len(part.UpdatingClauses) > 0 { + return true + } + } + return multi.SinglePartQuery != nil && len(multi.SinglePartQuery.UpdatingClauses) > 0 +} + +// neo4jPlanDriverConfig contains a Neo4j server URI and optional target database parsed from a connection string. type neo4jPlanDriverConfig struct { - Target string - Username string - Password string + // Target contains the Neo4j server URI without a database path. + Target string + // Username contains the Neo4j username decoded from the connection URI. + Username string + // Password contains the Neo4j password decoded from the connection URI. + Password string + // DatabaseName selects the Neo4j database targeted by the session. DatabaseName string } +// parseNeo4jPlanDriverConfig parses a Neo4j connection string while preserving its server URI and database path. func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { connectionURL, err := url.Parse(connStr) if err != nil { @@ -376,6 +457,7 @@ func parseNeo4jPlanDriverConfig(connStr string) (neo4jPlanDriverConfig, error) { }, nil } +// neo4jDatabaseName returns the optional single-segment database name encoded in a Neo4j URI path. func neo4jDatabaseName(connectionURL *url.URL) (string, error) { databasePath := strings.Trim(connectionURL.EscapedPath(), "/") if databasePath == "" { @@ -397,6 +479,7 @@ func neo4jDatabaseName(connectionURL *url.URL) (string, error) { return databaseName, nil } +// openNeo4jPlanDriver parses the capture connection settings and returns a driver together with the selected Neo4j database name. func openNeo4jPlanDriver(connStr string) (neo4jcore.Driver, string, error) { cfg, err := parseNeo4jPlanDriverConfig(connStr) if err != nil { @@ -414,12 +497,45 @@ func openNeo4jPlanDriver(connStr string) (neo4jcore.Driver, string, error) { return driver, cfg.DatabaseName, nil } +// clearGraph removes relationships before nodes, using PostgreSQL partition truncation when available. func clearGraph(ctx context.Context, db graph.Database) error { + if pgDriver, isPostgres := db.(*pg.Driver); isPostgres { + graphTarget, hasDefaultGraph := pgDriver.DefaultGraph() + if !hasDefaultGraph { + return fmt.Errorf("PostgreSQL default graph is not set") + } + + return clearPostgresGraph(ctx, db, graphTarget.ID) + } + + return db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if err := tx.Relationships().Delete(); err != nil { + return fmt.Errorf("delete relationships: %w", err) + } + + if err := tx.Nodes().Delete(); err != nil { + return fmt.Errorf("delete nodes: %w", err) + } + + return nil + }) +} + +// clearPostgresGraph truncates one PostgreSQL graph's edge and node partitions in a transaction. +func clearPostgresGraph(ctx context.Context, db graph.Database, graphID int32) error { return db.WriteTransaction(ctx, func(tx graph.Transaction) error { - return tx.Nodes().Delete() + statement := fmt.Sprintf("truncate table edge_%d, node_%d", graphID, graphID) + result := tx.Raw(statement, nil) + result.Close() + if err := result.Error(); err != nil { + return fmt.Errorf("execute PostgreSQL graph reset: %w", err) + } + + return nil }) } +// loadDataset decodes and loads a named fixture dataset into an empty graph. func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string) error { f, err := os.Open(filepath.Join(datasetDir, name+".json")) if err != nil { @@ -433,27 +549,36 @@ func loadDataset(ctx context.Context, db graph.Database, datasetDir, name string return nil } -func loadCommittedFixture(ctx context.Context, db graph.Database, fixture *opengraph.Graph) error { +// loadCommittedFixture loads an inline fixture graph and returns its stable key-to-ID mapping. +func loadCommittedFixture(ctx context.Context, db graph.Database, fixture *opengraph.Graph) (opengraph.IDMap, error) { if fixture == nil { - return fmt.Errorf("fixture is nil") + return nil, fmt.Errorf("fixture is nil") } if err := clearGraph(ctx, db); err != nil { - return err + return nil, err } - return db.WriteTransaction(ctx, func(tx graph.Transaction) error { - _, err := opengraph.WriteGraphTx(tx, fixture) + var idMap opengraph.IDMap + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + var err error + idMap, err = opengraph.WriteGraphTx(tx, fixture) return err - }) + }); err != nil { + return nil, err + } + + return idMap, nil } +// convertNeo4jPlan recursively converts a Neo4j plan into the stable serialized plan-node schema. func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { node := Neo4jPlanNode{ - Operator: plan.Operator(), + Operator: normalizeNeo4jOperator(plan.Operator()), Arguments: stringifyArguments(plan.Arguments()), Identifiers: append([]string(nil), plan.Identifiers()...), } + node.EstimatedRows = neo4jArgumentFloat(node.Arguments, "EstimatedRows") for _, child := range plan.Children() { node.Children = append(node.Children, convertNeo4jPlan(child)) @@ -462,6 +587,60 @@ func convertNeo4jPlan(plan neo4jcore.Plan) Neo4jPlanNode { return node } +// convertNeo4jProfile recursively converts executed read-plan evidence. +func convertNeo4jProfile(plan neo4jcore.ProfiledPlan) Neo4jPlanNode { + rows, dbHits := plan.Records(), plan.DbHits() + node := Neo4jPlanNode{ + Operator: normalizeNeo4jOperator(plan.Operator()), + Arguments: stringifyArguments(plan.Arguments()), + Identifiers: append([]string(nil), plan.Identifiers()...), + ActualRows: &rows, + DBHits: optionalNonnegativeInt64(dbHits), + PageCacheHits: optionalNonnegativeInt64(plan.PageCacheHits()), + PageCacheMisses: optionalNonnegativeInt64(plan.PageCacheMisses()), + TimeNS: optionalNonnegativeInt64(plan.Time()), + } + node.EstimatedRows = neo4jArgumentFloat(node.Arguments, "EstimatedRows") + for _, child := range plan.Children() { + node.Children = append(node.Children, convertNeo4jProfile(child)) + } + return node +} + +// optionalNonnegativeInt64 distinguishes unavailable profiler values from zero. +func optionalNonnegativeInt64(value int64) *int64 { + if value < 0 { + return nil + } + return &value +} + +// neo4jArgumentFloat parses an optional numeric plan argument. +func neo4jArgumentFloat(arguments map[string]string, key string) *float64 { + value, found := arguments[key] + if !found { + return nil + } + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return nil + } + return &parsed +} + +// normalizeNeo4jOperator removes repeated backend suffixes and applies exactly one. +func normalizeNeo4jOperator(operator string) string { + operator = strings.TrimSpace(operator) + for strings.HasSuffix(operator, "@neo4j") { + operator = strings.TrimSuffix(operator, "@neo4j") + } + if operator == "" { + return "" + } + return operator + "@neo4j" +} + +// stringifyArguments converts plan arguments to stable strings in a fresh map. func stringifyArguments(arguments map[string]any) map[string]string { if len(arguments) == 0 { return nil @@ -474,6 +653,7 @@ func stringifyArguments(arguments map[string]any) map[string]string { return values } +// postgresOperators extracts normalized operator names from PostgreSQL text plans. func postgresOperators(plan []string) []string { operators := make([]string, 0, len(plan)) for _, line := range plan { @@ -491,6 +671,7 @@ func postgresOperators(plan []string) []string { return operators } +// neo4jOperators flattens a Neo4j plan tree into sorted unique operator names. func neo4jOperators(root Neo4jPlanNode) []string { var ( operators []string @@ -498,7 +679,7 @@ func neo4jOperators(root Neo4jPlanNode) []string { ) walk = func(node Neo4jPlanNode) { - operators = append(operators, node.Operator) + operators = append(operators, normalizeNeo4jOperator(node.Operator)) for _, child := range node.Children { walk(child) } @@ -507,6 +688,7 @@ func neo4jOperators(root Neo4jPlanNode) []string { return operators } +// loweringNames returns sorted unique names of applied SQL lowering decisions. func loweringNames(decisions []optimize.LoweringDecision) []string { if len(decisions) == 0 { return nil @@ -529,6 +711,7 @@ func loweringNames(decisions []optimize.LoweringDecision) []string { return names } +// cypherWithoutTerminator trims surrounding whitespace and one trailing Cypher semicolon. func cypherWithoutTerminator(cypherQuery string) string { return strings.TrimSuffix(strings.TrimSpace(cypherQuery), ";") } diff --git a/cmd/plancorpus/corpus.go b/cmd/plancorpus/corpus.go index 46fdd4e0..b398f6a1 100644 --- a/cmd/plancorpus/corpus.go +++ b/cmd/plancorpus/corpus.go @@ -10,66 +10,120 @@ import ( "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) +// corpus contains loaded corpus queries and their dataset definitions. type corpus struct { - caseGroups map[string]*caseGroup - datasetNames []string + // caseGroups indexes loaded corpus cases by dataset name. + caseGroups map[string]*caseGroup + // datasetNames lists fixture datasets in deterministic plan-capture order. + datasetNames []string + // templateFiles retains decoded template files for corpus expansion. templateFiles []templateFile - nodeKinds graph.Kinds - edgeKinds graph.Kinds + // nodeKinds contains every node kind declared by loaded fixtures. + nodeKinds graph.Kinds + // edgeKinds contains every relationship kind declared by loaded fixtures. + edgeKinds graph.Kinds } +// caseGroup models a case-group entry in a scale-corpus JSON file. type caseGroup struct { + // dataset names the fixture shared by every case file in the group. dataset string - files []caseFile + // files retains source case files contributing to a dataset group. + files []caseFile } +// caseFile models the top-level groups in a scale-corpus case file. type caseFile struct { - path string - Dataset string `json:"dataset"` - Cases []caseEntry `json:"cases"` + // path retains the source path used in errors and provenance. + path string + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset"` + // Cases contains query cases declared by this source file. + Cases []caseEntry `json:"cases"` } +// caseEntry models one named query case and its parameter declarations. type caseEntry struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` + // Name identifies the query case within its dataset. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // Fixture captures the fixture identity and cardinality contract. Fixture *opengraph.Graph `json:"fixture,omitempty"` } +// templateFile models template and metamorphic query families from a corpus template file. type templateFile struct { - path string - Families []templateFamily `json:"families,omitempty"` + // path retains the source path used in errors and provenance. + path string + // Families lists query-template families decoded from the file. + Families []templateFamily `json:"families,omitempty"` + // Metamorphic lists metamorphic query families decoded from the file. Metamorphic []metamorphicFamily `json:"metamorphic,omitempty"` } +// templateFamily defines a base query and the variants rendered from it. type templateFamily struct { - Name string `json:"name"` - Template string `json:"template"` - Params map[string]any `json:"params,omitempty"` - Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Name identifies the query-template family in expanded case names. + Name string `json:"name"` + // Template contains the Cypher template rendered for each variant. + Template string `json:"template"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + // Fixture captures the fixture identity and cardinality contract. + Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Variants lists substitutions rendered from the base query template. Variants []templateVariant `json:"variants"` } +// templateVariant defines one named substitution set for a query template. type templateVariant struct { - Name string `json:"name"` - Vars map[string]string `json:"vars"` - Params map[string]any `json:"params,omitempty"` + // Name identifies this substitution set in the rendered case name. + Name string `json:"name"` + // Vars maps template placeholders to replacement text. + Vars map[string]string `json:"vars"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` + // NodeParams maps query parameters to fixture node keys. + NodeParams map[string]string `json:"node_params,omitempty"` + // NodeListParams maps query parameters to ordered fixture node-key lists. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` } +// metamorphicFamily groups semantically equivalent queries used for plan comparison. type metamorphicFamily struct { - Name string `json:"name"` - Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Name identifies the family of queries expected to remain semantically equivalent. + Name string `json:"name"` + // Fixture captures the fixture identity and cardinality contract. + Fixture *opengraph.Graph `json:"fixture,omitempty"` + // Queries lists semantically equivalent queries in the metamorphic family. Queries []metamorphicQuery `json:"queries"` } +// metamorphicQuery defines one named query in a metamorphic family. type metamorphicQuery struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` + // Name identifies one query variant within its metamorphic family. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params testutil.Params `json:"params,omitempty"` } +// loadCorpus loads case, template, and dataset-kind declarations from a corpus directory. func loadCorpus(datasetDir string) (corpus, error) { var loaded corpus loaded.caseGroups = map[string]*caseGroup{} @@ -88,6 +142,7 @@ func loadCorpus(datasetDir string) (corpus, error) { return loaded, nil } +// loadCaseFiles decodes case files and indexes them by dataset while retaining source paths. func (s *corpus) loadCaseFiles(datasetDir string) error { paths, err := filepath.Glob(filepath.Join(datasetDir, "cases", "*.json")) if err != nil { @@ -123,6 +178,7 @@ func (s *corpus) loadCaseFiles(datasetDir string) error { return nil } +// loadTemplateFiles renders template variants and metamorphic families into executable corpus cases. func (s *corpus) loadTemplateFiles(datasetDir string) error { paths, err := filepath.Glob(filepath.Join(datasetDir, "templates", "*.json")) if err != nil { @@ -149,6 +205,7 @@ func (s *corpus) loadTemplateFiles(datasetDir string) error { return nil } +// loadDatasetKinds loads fixture graphs and accumulates the node and relationship kinds they declare. func (s *corpus) loadDatasetKinds(datasetDir string) error { for _, datasetName := range s.datasetNames { path := filepath.Join(datasetDir, datasetName+".json") @@ -174,6 +231,7 @@ func (s *corpus) loadDatasetKinds(datasetDir string) error { return nil } +// addFixtureKinds unions a fixture's node and relationship kinds into the corpus kind sets. func (s *corpus) addFixtureKinds(fixture *opengraph.Graph) { if fixture == nil { return @@ -184,6 +242,7 @@ func (s *corpus) addFixtureKinds(fixture *opengraph.Graph) { s.edgeKinds = s.edgeKinds.Add(edgeKinds...) } +// decodeJSONFile reads a JSON file and decodes it into the supplied destination. func decodeJSONFile(path string, target any) error { raw, err := os.ReadFile(path) if err != nil { @@ -195,6 +254,7 @@ func decodeJSONFile(path string, target any) error { return nil } +// renderTemplate substitutes every named placeholder and rejects any unresolved template markers. func renderTemplate(template string, vars map[string]string) (string, error) { rendered := template for name, value := range vars { @@ -206,6 +266,7 @@ func renderTemplate(template string, vars map[string]string) (string, error) { return rendered, nil } +// mergeParams returns a copied parameter map in which override values take precedence. func mergeParams(base, overrides map[string]any) map[string]any { if len(base) == 0 && len(overrides) == 0 { return nil @@ -220,3 +281,73 @@ func mergeParams(base, overrides map[string]any) map[string]any { } return merged } + +// mergeStringMap returns a copied string map in which override values take precedence. +func mergeStringMap(base, overrides map[string]string) map[string]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = value + } + for key, value := range overrides { + merged[key] = value + } + return merged +} + +// mergeStringListMap returns a deep-enough copy of string-list parameters with overrides applied. +func mergeStringListMap(base, overrides map[string][]string) map[string][]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string][]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = append([]string(nil), value...) + } + for key, value := range overrides { + merged[key] = append([]string(nil), value...) + } + return merged +} + +// resolveFixtureParams replaces symbolic node keys and key lists with fixture database identifiers. +func resolveFixtureParams( + params map[string]any, + nodeParams map[string]string, + nodeListParams map[string][]string, + idMap opengraph.IDMap, +) (map[string]any, error) { + resolved := make(map[string]any, len(params)+len(nodeParams)+len(nodeListParams)) + for name, value := range params { + resolved[name] = value + } + + for paramName, fixtureID := range nodeParams { + id, found := idMap[fixtureID] + if !found { + return nil, fmt.Errorf("node parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + resolved[paramName] = id.Int64() + } + + for paramName, fixtureIDs := range nodeListParams { + ids := make([]int64, len(fixtureIDs)) + for idx, fixtureID := range fixtureIDs { + id, found := idMap[fixtureID] + if !found { + return nil, fmt.Errorf("node list parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + ids[idx] = id.Int64() + } + resolved[paramName] = ids + } + + if len(resolved) == 0 { + return nil, nil + } + return resolved, nil +} diff --git a/cmd/plancorpus/corpus_test.go b/cmd/plancorpus/corpus_test.go index 141fa515..5b8996d3 100644 --- a/cmd/plancorpus/corpus_test.go +++ b/cmd/plancorpus/corpus_test.go @@ -4,9 +4,13 @@ import ( "path/filepath" "testing" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" "github.com/stretchr/testify/require" ) +// TestLoadCorpus verifies that integration fixtures populate case groups, datasets, templates, and both node and edge kind catalogs. func TestLoadCorpus(t *testing.T) { suite, err := loadCorpus(filepath.Join("..", "..", "integration", "testdata")) require.NoError(t, err) @@ -18,6 +22,26 @@ func TestLoadCorpus(t *testing.T) { require.NotEmpty(t, suite.edgeKinds) } +// TestCorpusTemplatesParse verifies that every declared template variant renders without placeholders and parses as Cypher. +func TestCorpusTemplatesParse(t *testing.T) { + suite, err := loadCorpus(filepath.Join("..", "..", "integration", "testdata")) + require.NoError(t, err) + + for _, file := range suite.templateFiles { + for _, family := range file.Families { + for _, variant := range family.Variants { + t.Run(family.Name+"/"+variant.Name, func(t *testing.T) { + rendered, err := renderTemplate(family.Template, variant.Vars) + require.NoError(t, err) + _, err = frontend.ParseCypher(frontend.NewContext(), rendered) + require.NoError(t, err) + }) + } + } + } +} + +// TestRenderTemplateRequiresAllPlaceholders verifies successful substitution and rejection when any template marker remains unresolved. func TestRenderTemplateRequiresAllPlaceholders(t *testing.T) { rendered, err := renderTemplate("match ({{name}}) return {{name}}", map[string]string{"name": "n"}) require.NoError(t, err) @@ -27,8 +51,28 @@ func TestRenderTemplateRequiresAllPlaceholders(t *testing.T) { require.ErrorContains(t, err, "unresolved placeholders") } +// TestMergeParams verifies right-hand override precedence, retention of unrelated values, and a nil result for two absent maps. func TestMergeParams(t *testing.T) { merged := mergeParams(map[string]any{"a": 1, "b": 2}, map[string]any{"b": 3}) require.Equal(t, map[string]any{"a": 1, "b": 3}, merged) require.Nil(t, mergeParams(nil, nil)) } + +// TestResolveFixtureParams verifies scalar/list key resolution to ordered int64 IDs and reports an unknown fixture key. +func TestResolveFixtureParams(t *testing.T) { + params, err := resolveFixtureParams( + map[string]any{"literal": "value"}, + map[string]string{"start_id": "start"}, + map[string][]string{"end_ids": {"end", "start"}}, + opengraph.IDMap{"start": graph.ID(11), "end": graph.ID(22)}, + ) + require.NoError(t, err) + require.Equal(t, map[string]any{ + "literal": "value", + "start_id": int64(11), + "end_ids": []int64{22, 11}, + }, params) + + _, err = resolveFixtureParams(nil, map[string]string{"missing": "unknown"}, nil, opengraph.IDMap{}) + require.ErrorContains(t, err, "unknown fixture ID") +} diff --git a/cmd/plancorpus/dormant_forms_guard_test.go b/cmd/plancorpus/dormant_forms_guard_test.go new file mode 100644 index 00000000..45bdb5c1 --- /dev/null +++ b/cmd/plancorpus/dormant_forms_guard_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestDormantFormsStayOutOfPlanCorpus verifies that active cases, variants, and metamorphic query names never expose FUTURE-prefixed query forms. +func TestDormantFormsStayOutOfPlanCorpus(t *testing.T) { + suite, err := loadCorpus("../../integration/testdata") + require.NoError(t, err) + + for _, group := range suite.caseGroups { + for _, file := range group.files { + for _, testCase := range file.Cases { + requireNoDormantPlanQueryFormID(t, file.path+" case", testCase.Name) + } + } + } + + for _, file := range suite.templateFiles { + for _, family := range file.Families { + requireNoDormantPlanQueryFormID(t, file.path+" family", family.Name) + for _, variant := range family.Variants { + requireNoDormantPlanQueryFormID(t, file.path+" variant", variant.Name) + } + } + for _, family := range file.Metamorphic { + requireNoDormantPlanQueryFormID(t, file.path+" metamorphic family", family.Name) + for _, query := range family.Queries { + requireNoDormantPlanQueryFormID(t, file.path+" metamorphic query", query.Name) + } + } + } +} + +// requireNoDormantPlanQueryFormID rejects a corpus field containing the reserved FUTURE marker, independent of letter case. +func requireNoDormantPlanQueryFormID(t *testing.T, field, value string) { + t.Helper() + require.False(t, strings.Contains(strings.ToUpper(value), "FUTURE-"), + "%s %q places a dormant query form in the active plan corpus", field, value) +} diff --git a/cmd/plancorpus/main.go b/cmd/plancorpus/main.go index 152a0beb..1430a251 100644 --- a/cmd/plancorpus/main.go +++ b/cmd/plancorpus/main.go @@ -8,29 +8,47 @@ import ( "io" "os" "path/filepath" + + "github.com/specterops/dawgs/testutil" ) +// commandConfig contains plancorpus command-line inputs and output selections. type commandConfig struct { - DatasetDir string - OutputDir string + // DatasetDir locates fixture datasets loaded before plan capture. + DatasetDir string + // OutputDir selects the directory that receives captured plans and summaries. + OutputDir string + // SummaryMarkdown selects the Markdown plan-summary destination. SummaryMarkdown string - SummaryJSON string - Connection string - PGConnection string + // SummaryJSON selects the JSON summary destination. + SummaryJSON string + // PlanDeltaJSON selects the versioned paired plan-delta destination. + PlanDeltaJSON string + // Connection contains the backend connection string. + Connection string + // PGConnection contains the PostgreSQL connection string. + PGConnection string + // Neo4jConnection contains the Neo4j connection string. Neo4jConnection string - TopPlans int + // TopPlans limits expensive PostgreSQL plans included in the summary. + TopPlans int + // DAWGSVersion identifies the schema version for dawgs version. + DAWGSVersion string } +// main runs the plancorpus command. func main() { cfg := commandConfig{} flag.StringVar(&cfg.DatasetDir, "dataset-dir", "integration/testdata", "integration testdata directory") flag.StringVar(&cfg.OutputDir, "output-dir", ".coverage", "directory for JSONL plan captures") flag.StringVar(&cfg.SummaryMarkdown, "summary", "", "markdown summary path (default: output-dir/plan-corpus-summary.md)") flag.StringVar(&cfg.SummaryJSON, "summary-json", "", "JSON summary path (default: output-dir/plan-corpus-summary.json)") + flag.StringVar(&cfg.PlanDeltaJSON, "plan-delta-json", "", "paired semantic plan-delta path (default: output-dir/plan-corpus-delta.json)") flag.StringVar(&cfg.Connection, "connection", os.Getenv("CONNECTION_STRING"), "single backend connection string") flag.StringVar(&cfg.PGConnection, "pg-connection", os.Getenv("PG_CONNECTION_STRING"), "PostgreSQL connection string") flag.StringVar(&cfg.Neo4jConnection, "neo4j-connection", os.Getenv("NEO4J_CONNECTION_STRING"), "Neo4j connection string") flag.IntVar(&cfg.TopPlans, "top", defaultTopPlans, "number of expensive PostgreSQL plans to include in summaries") + flag.StringVar(&cfg.DAWGSVersion, "dawgs-version", "", "DAWGS source version (auto-detected when empty)") flag.Parse() if err := run(context.Background(), cfg); err != nil { @@ -39,6 +57,7 @@ func main() { } } +// run captures plans for each configured backend and writes aggregate summaries. func run(ctx context.Context, cfg commandConfig) error { specs, err := captureSpecs(cfg) if err != nil { @@ -55,12 +74,17 @@ func run(ctx context.Context, cfg commandConfig) error { } var allRecords []PlanRecord + metadata := testutil.ResolveBaselineMetadata(cfg.DAWGSVersion) for _, spec := range specs { records, err := captureCorpus(ctx, cfg.DatasetDir, suite, spec) if err != nil { return err } + for idx := range records { + records[idx].Metadata = metadata + } + outputPath := filepath.Join(cfg.OutputDir, "plan-corpus-"+spec.DriverName+".jsonl") if err := writePlanRecords(outputPath, records); err != nil { return err @@ -81,10 +105,22 @@ func run(ctx context.Context, cfg commandConfig) error { if err := writeSummaryFiles(cfg.SummaryMarkdown, cfg.SummaryJSON, summary); err != nil { return err } + planDelta, err := buildPlanDeltaReport(allRecords) + if err != nil { + return err + } + if cfg.PlanDeltaJSON == "" { + cfg.PlanDeltaJSON = filepath.Join(cfg.OutputDir, "plan-corpus-delta.json") + } + if err := writePlanDeltaReport(cfg.PlanDeltaJSON, planDelta); err != nil { + return err + } fmt.Fprintf(os.Stderr, "wrote summaries to %s and %s\n", cfg.SummaryMarkdown, cfg.SummaryJSON) + fmt.Fprintf(os.Stderr, "wrote paired plan delta to %s\n", cfg.PlanDeltaJSON) return nil } +// captureSpecs validates connection inputs and returns one deterministic capture specification per driver. func captureSpecs(cfg commandConfig) ([]captureSpec, error) { specsByDriver := map[string]captureSpec{} @@ -129,14 +165,17 @@ func captureSpecs(cfg commandConfig) ([]captureSpec, error) { return specs, nil } +// pgDriverName returns the registered driver name for PostgreSQL connections. func pgDriverName() string { return "pg" } +// neo4jDriverName returns the registered driver name for Neo4j connections. func neo4jDriverName() string { return "neo4j" } +// writePlanRecords creates a JSON Lines artifact and writes every captured plan record to it. func writePlanRecords(path string, records []PlanRecord) error { out, err := os.Create(path) if err != nil { @@ -146,6 +185,7 @@ func writePlanRecords(path string, records []PlanRecord) error { return writePlanRecordsTo(out, path, records) } +// writePlanRecordsTo encodes plan records as JSON Lines and reports both encode and close failures. func writePlanRecordsTo(out io.WriteCloser, path string, records []PlanRecord) error { encoder := json.NewEncoder(out) for _, record := range records { @@ -162,6 +202,7 @@ func writePlanRecordsTo(out io.WriteCloser, path string, records []PlanRecord) e return nil } +// writeSummaryFiles writes the requested Markdown and JSON plan summaries and closes each output. func writeSummaryFiles(markdownPath, jsonPath string, summary PlanSummary) error { if markdownPath != "" { out, err := os.Create(markdownPath) diff --git a/cmd/plancorpus/main_test.go b/cmd/plancorpus/main_test.go index 17aca49a..e01b1c3a 100644 --- a/cmd/plancorpus/main_test.go +++ b/cmd/plancorpus/main_test.go @@ -7,18 +7,25 @@ import ( "path/filepath" "testing" + "github.com/specterops/dawgs/cypher/frontend" "github.com/stretchr/testify/require" ) +// closeErrorWriter wraps an in-memory buffer and injects a Close error for output tests. type closeErrorWriter struct { + // Buffer captures bytes written before the injected Close failure. bytes.Buffer + + // err is returned after serialization attempts to close the destination. err error } +// Close returns the injected failure used to verify output finalization errors. func (s *closeErrorWriter) Close() error { return s.err } +// TestCaptureSpecs verifies that backend-specific connection flags override the generic URI and produce PostgreSQL then Neo4j capture specs. func TestCaptureSpecs(t *testing.T) { specs, err := captureSpecs(commandConfig{ Connection: "neo4j://neo4j:password@localhost:7687", @@ -35,32 +42,42 @@ func TestCaptureSpecs(t *testing.T) { }}, specs) } +// TestCaptureSpecsRequiresConnection verifies that capture cannot proceed when no generic or backend-specific connection URI is supplied. func TestCaptureSpecsRequiresConnection(t *testing.T) { _, err := captureSpecs(commandConfig{}) require.ErrorContains(t, err, "no connection string supplied") } +// TestWritePlanRecordsWritesJSONLines verifies the stable JSON Lines schema, including source query identity and default metadata. func TestWritePlanRecordsWritesJSONLines(t *testing.T) { path := filepath.Join(t.TempDir(), "records.jsonl") err := writePlanRecords(path, []PlanRecord{{ - Driver: "pg", - Source: "cases/example.json", - Name: "example", - Cypher: "MATCH (n) RETURN n", + SchemaVersion: planRecordSchemaVersion, + Driver: "pg", + Source: "cases/example.json", + Name: "example", + WorkloadSHA256: "workload", + Cypher: "MATCH (n) RETURN n", }}) require.NoError(t, err) contents, err := os.ReadFile(path) require.NoError(t, err) require.JSONEq(t, `{ + "schema_version": 2, "driver": "pg", "source": "cases/example.json", "name": "example", - "cypher": "MATCH (n) RETURN n" + "workload_sha256": "workload", + "cypher": "MATCH (n) RETURN n", + "metadata": { + "dawgs_version": "" + } }`, string(bytes.TrimSpace(contents))) } +// TestWritePlanRecordsToReturnsCloseError verifies that destination close failures retain the output path in their diagnostic. func TestWritePlanRecordsToReturnsCloseError(t *testing.T) { writer := &closeErrorWriter{err: errors.New("close failed")} @@ -70,6 +87,7 @@ func TestWritePlanRecordsToReturnsCloseError(t *testing.T) { require.ErrorContains(t, err, "close failed") } +// TestWritePlanRecordsToClosesAfterEncodeError verifies that encoding and close failures are joined so cleanup is attempted without losing the primary serialization error. func TestWritePlanRecordsToClosesAfterEncodeError(t *testing.T) { writer := &closeErrorWriter{err: errors.New("close failed")} @@ -85,6 +103,7 @@ func TestWritePlanRecordsToClosesAfterEncodeError(t *testing.T) { require.ErrorContains(t, err, "close failed") } +// TestDriverFromConnectionString verifies PostgreSQL and all supported Neo4j routing schemes and rejects an unrelated database protocol. func TestDriverFromConnectionString(t *testing.T) { driverName, err := driverFromConnectionString("postgresql://postgres:password@localhost/db") require.NoError(t, err) @@ -104,11 +123,19 @@ func TestDriverFromConnectionString(t *testing.T) { require.ErrorContains(t, err, "unknown connection string scheme") } +// TestParseNeo4jPlanDriverConfigPreservesURI verifies credentials extraction while preserving routing security, host, query, and an optional single database name. func TestParseNeo4jPlanDriverConfigPreservesURI(t *testing.T) { testCases := []struct { - name string - connStr string - expectedTarget string + // name identifies the routing form in subtest diagnostics. + name string + + // connStr is the credential-bearing URI accepted by the parser. + connStr string + + // expectedTarget is the credential-free driver URI after database-path extraction. + expectedTarget string + + // expectedDatabase is the optional database parsed from the sole path segment. expectedDatabase string }{{ name: "plain routing", @@ -139,6 +166,7 @@ func TestParseNeo4jPlanDriverConfigPreservesURI(t *testing.T) { } } +// TestParseNeo4jPlanDriverConfigRejectsNestedDatabasePath verifies that literal and percent-encoded nested paths cannot masquerade as one Neo4j database name. func TestParseNeo4jPlanDriverConfigRejectsNestedDatabasePath(t *testing.T) { for _, connStr := range []string{ "neo4j://neo4j:password@localhost:7687/db/extra", @@ -148,3 +176,33 @@ func TestParseNeo4jPlanDriverConfigRejectsNestedDatabasePath(t *testing.T) { require.ErrorContains(t, err, "single database name") } } + +// TestRegularQueryHasUpdatesDistinguishesReadProfilesFromWriteExplains verifies +// the PlanCorpus Neo4j command boundary cannot execute mutations during capture. +func TestRegularQueryHasUpdatesDistinguishesReadProfilesFromWriteExplains(t *testing.T) { + for _, testCase := range []struct { + // query retains the query while anonymous record is assembled or evaluated. + query string + // write indicates whether write applies. + write bool + }{{ + query: "MATCH (n) RETURN n", + write: false, + }, { + query: "CREATE (n) RETURN n", + write: true, + }, { + query: "MATCH (n) WITH n SET n.x = 1 RETURN n", + write: true, + }} { + parsed, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.NoError(t, err) + require.Equal(t, testCase.write, regularQueryHasUpdates(parsed), testCase.query) + } +} + +// TestNormalizeNeo4jOperatorAppliesOneSuffix verifies historical doubled backend suffixes are canonicalized. +func TestNormalizeNeo4jOperatorAppliesOneSuffix(t *testing.T) { + require.Equal(t, "ShortestPath@neo4j", normalizeNeo4jOperator("ShortestPath@neo4j@neo4j")) + require.Equal(t, "ShortestPath@neo4j", normalizeNeo4jOperator("ShortestPath")) +} diff --git a/cmd/plancorpus/plan_delta.go b/cmd/plancorpus/plan_delta.go new file mode 100644 index 00000000..d6019abf --- /dev/null +++ b/cmd/plancorpus/plan_delta.go @@ -0,0 +1,800 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "math" + "os" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// planDeltaSchemaVersion reserves the stable protocol value used to recognize plan delta schema version across artifacts and executions. +const planDeltaSchemaVersion = 2 + +// planRowsPattern contains the frozen plan rows pattern declaration consulted by package validation. +var planRowsPattern = regexp.MustCompile(`\brows=([0-9]+)\b`) + +// workloadFingerprint hashes backend-independent query identity and parameter +// type shape. Physical fixture IDs are deliberately excluded so captures from +// independently loaded backends still pair. +func workloadFingerprint(query CorpusQuery) string { + parameterTypes := make(map[string]string, len(query.Params)) + for name, value := range query.Params { + if value == nil { + parameterTypes[name] = "nil" + } else { + parameterTypes[name] = reflect.TypeOf(value).String() + } + } + + return jsonFingerprint(struct { + // Source supplies the source input to the anonymous record contract. + Source string `json:"source"` + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset,omitempty"` + // Name identifies the name. + Name string `json:"name"` + // Cypher supplies the cypher input to the anonymous record contract. + Cypher string `json:"cypher"` + // ParameterTypes supplies the parameter types input to the anonymous record contract. + ParameterTypes map[string]string `json:"parameter_types,omitempty"` + }{ + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + Cypher: strings.TrimSpace(query.Cypher), + ParameterTypes: parameterTypes, + }) +} + +// postgresPlanFingerprint hashes one normalized PostgreSQL text plan. +func postgresPlanFingerprint(plan []string) string { + if len(plan) == 0 { + return "" + } + return jsonFingerprint(plan) +} + +// neo4jPlanFingerprint hashes one normalized Neo4j plan tree. +func neo4jPlanFingerprint(plan *Neo4jPlanNode) string { + if plan == nil { + return "" + } + + // fingerprintNode contains only the normalized plan attributes bound into the digest. + type fingerprintNode struct { + // Operator supplies the operator input to the fingerprintNode contract. + Operator string `json:"operator"` + // Arguments supplies the arguments input to the fingerprintNode contract. + Arguments map[string]string `json:"arguments,omitempty"` + // Identifiers supplies the identifiers input to the fingerprintNode contract. + Identifiers []string `json:"identifiers,omitempty"` + // Children supplies the children input to the fingerprintNode contract. + Children []fingerprintNode `json:"children,omitempty"` + } + var project func(Neo4jPlanNode) fingerprintNode + project = func(node Neo4jPlanNode) fingerprintNode { + projected := fingerprintNode{ + Operator: normalizeNeo4jOperator(node.Operator), + Arguments: structuralNeo4jArguments(node.Arguments), + Identifiers: append([]string(nil), node.Identifiers...), + } + for _, child := range node.Children { + projected.Children = append(projected.Children, project(child)) + } + return projected + } + return jsonFingerprint(project(*plan)) +} + +// structuralNeo4jArguments removes execution-only counters from a PROFILE so +// the plan fingerprint remains stable when the same operator tree is replayed. +func structuralNeo4jArguments(arguments map[string]string) map[string]string { + if len(arguments) == 0 { + return nil + } + filtered := map[string]string{} + for name, value := range arguments { + canonical := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(name, "_", ""), " ", "")) + switch canonical { + case "rows", "dbhits", "pagecachehits", "pagecachemisses", "time", "timens", "actualrows", "actualloops": + continue + default: + filtered[name] = value + } + } + if len(filtered) == 0 { + return nil + } + return filtered +} + +// jsonFingerprint returns a stable SHA-256 digest for a JSON-serializable value. +func jsonFingerprint(value any) string { + raw, err := json.Marshal(value) + if err != nil { + return "" + } + digest := sha256.Sum256(raw) + return hex.EncodeToString(digest[:]) +} + +// buildPlanDeltaReport pairs records by workload union so a missing backend is +// preserved as evidence instead of disappearing through intersection-only reporting. +func buildPlanDeltaReport(records []PlanRecord) (PlanDeltaReport, error) { + // pair holds the PostgreSQL and Neo4j plans available for one workload revision. + type pair struct { + // postgres retains the postgres while pair is assembled or evaluated. + postgres *PlanRecord + // neo4j retains the neo4j while pair is assembled or evaluated. + neo4j *PlanRecord + } + + // pairKey identifies one workload at one DAWGS revision. + type pairKey struct { + // workload retains the workload while pairKey is assembled or evaluated. + workload string + // revision retains the revision while pairKey is assembled or evaluated. + revision string + } + pairs := map[pairKey]pair{} + for idx := range records { + record := &records[idx] + if record.WorkloadSHA256 == "" { + record.WorkloadSHA256 = workloadFingerprint(CorpusQuery{ + Source: record.Source, + Dataset: record.Dataset, + Name: record.Name, + Cypher: record.Cypher, + Params: record.Params, + }) + } + key := pairKey{ + workload: record.WorkloadSHA256, + revision: record.Metadata.DAWGSVersion, + } + next := pairs[key] + switch record.Driver { + case pgDriverName(): + if next.postgres != nil { + return PlanDeltaReport{}, fmt.Errorf("duplicate PostgreSQL plan for workload %s at source revision %q", record.WorkloadSHA256, key.revision) + } + next.postgres = record + case neo4jDriverName(): + if next.neo4j != nil { + return PlanDeltaReport{}, fmt.Errorf("duplicate Neo4j plan for workload %s at source revision %q", record.WorkloadSHA256, key.revision) + } + next.neo4j = record + default: + return PlanDeltaReport{}, fmt.Errorf("unsupported plan-delta driver %q", record.Driver) + } + pairs[key] = next + } + + report := PlanDeltaReport{Version: planDeltaSchemaVersion} + for key, next := range pairs { + identity := next.postgres + if identity == nil { + identity = next.neo4j + } + delta := PlanDeltaRecord{ + Dataset: identity.Dataset, + Source: identity.Source, + Name: identity.Name, + WorkloadSHA256: key.workload, + SourceRevision: key.revision, + } + if next.postgres != nil { + plan := semanticPostgresPlan(*next.postgres) + delta.Postgres = &plan + } + if next.neo4j != nil { + plan := semanticNeo4jPlan(*next.neo4j) + delta.Neo4j = &plan + } + delta.Complete, delta.IncompleteReason = planDeltaCompleteness(delta) + if delta.Postgres != nil && delta.Neo4j != nil { + delta.OppositeStartingSides = comparableDifferent(accessSide(delta.Postgres.StartingAccess), accessSide(delta.Neo4j.StartingAccess)) + delta.OppositePhysicalDirections = comparableDifferent(delta.Postgres.PhysicalDirection, delta.Neo4j.PhysicalDirection) + delta.Neo4jReorderedPattern = neo4jReorderedPattern(identity.Cypher, delta.Neo4j.StartingAccess) + delta.ChosenSideDidLessObservedWork = lessObservedSeedWork(delta.Neo4j) + delta.SeedEstimateQError = estimateQError(delta.Postgres.EstimatedSeeds, delta.Neo4j.EstimatedSeeds) + delta.TraversalEstimateQError = estimateQError(delta.Postgres.EstimatedTraversal, delta.Neo4j.EstimatedTraversal) + delta.OutputEstimateQError = estimateQError(delta.Postgres.EstimatedOutput, delta.Neo4j.EstimatedOutput) + delta.PredicatePlacementMoved = predicatePlacementMoved(delta.Postgres.PredicatePlacement, delta.Neo4j.PredicatePlacement) + delta.HydrationEstimateQError = estimateQError(delta.Postgres.EstimatedHydration, delta.Neo4j.EstimatedHydration) + } + delta.PairSHA256 = planDeltaPairFingerprint(delta) + report.Records = append(report.Records, delta) + } + + sort.Slice(report.Records, func(i, j int) bool { + left, right := report.Records[i], report.Records[j] + if left.Dataset != right.Dataset { + return left.Dataset < right.Dataset + } + if left.Source != right.Source { + return left.Source < right.Source + } + return left.Name < right.Name + }) + report.RankedFindings = rankPlanDeltaFindings(report.Records) + return report, nil +} + +// planDeltaPairFingerprint binds source and both backend plan identities without embedding raw plans. +func planDeltaPairFingerprint(delta PlanDeltaRecord) string { + postgresFingerprint, neo4jFingerprint := "", "" + if delta.Postgres != nil { + postgresFingerprint = delta.Postgres.PlanFingerprint + } + if delta.Neo4j != nil { + neo4jFingerprint = delta.Neo4j.PlanFingerprint + } + return jsonFingerprint(struct { + // Dataset identifies the fixture dataset that supplies the workload graph. + Dataset string `json:"dataset,omitempty"` + // Source supplies the source input to the anonymous record contract. + Source string `json:"source"` + // Name identifies the name. + Name string `json:"name"` + // WorkloadSHA256 binds the referenced workload content by SHA-256 digest. + WorkloadSHA256 string `json:"workload_sha256"` + // SourceRevision supplies the source revision input to the anonymous record contract. + SourceRevision string `json:"source_revision,omitempty"` + // PostgresFingerprint supplies the postgres fingerprint input to the anonymous record contract. + PostgresFingerprint string `json:"postgres_plan_fingerprint,omitempty"` + // Neo4jFingerprint supplies the neo4j fingerprint input to the anonymous record contract. + Neo4jFingerprint string `json:"neo4j_plan_fingerprint,omitempty"` + }{ + Dataset: delta.Dataset, + Source: delta.Source, + Name: delta.Name, + WorkloadSHA256: delta.WorkloadSHA256, + SourceRevision: delta.SourceRevision, + PostgresFingerprint: postgresFingerprint, + Neo4jFingerprint: neo4jFingerprint, + }) +} + +// accessSide maps backend-specific access labels onto a root/terminal side when possible. +func accessSide(access string) string { + lower := strings.ToLower(access) + switch { + case strings.Contains(lower, "terminal"), strings.Contains(lower, "target"), strings.Contains(lower, " n1"), strings.Contains(lower, "(n1"): + return "terminal" + case strings.Contains(lower, "root"), strings.Contains(lower, "source"), strings.Contains(lower, " n0"), strings.Contains(lower, "(n0"): + return "root" + default: + return "" + } +} + +// planDeltaCompleteness reports whether both sides contain successful plan evidence. +func planDeltaCompleteness(delta PlanDeltaRecord) (bool, string) { + var reasons []string + if delta.Postgres == nil { + reasons = append(reasons, "missing_postgres") + } else if delta.Postgres.Error != "" || delta.Postgres.PlanFingerprint == "" { + reasons = append(reasons, "failed_postgres") + } + if delta.Neo4j == nil { + reasons = append(reasons, "missing_neo4j") + } else if delta.Neo4j.Error != "" || delta.Neo4j.PlanFingerprint == "" { + reasons = append(reasons, "failed_neo4j") + } + return len(reasons) == 0, strings.Join(reasons, ",") +} + +// comparableDifferent compares nonempty semantic labels. +func comparableDifferent(left, right string) bool { + return left != "" && right != "" && left != right +} + +// neo4jReorderedPattern reports a conservative endpoint reversal relative to the textual first relationship. +func neo4jReorderedPattern(cypherQuery, startingAccess string) bool { + if logicalDirection(cypherQuery) == "" { + return false + } + return accessSide(startingAccess) == "terminal" +} + +// lessObservedSeedWork compares profiled leaf work only when both endpoint leaves expose it. +func lessObservedSeedWork(plan *SemanticPlan) *bool { + if plan == nil || plan.ObservedSeedWork == nil || plan.ObservedAlternativeSeedWork == nil { + return nil + } + value := *plan.ObservedSeedWork <= *plan.ObservedAlternativeSeedWork + return &value +} + +// estimateQError reports symmetric disagreement between two positive backend estimates. +func estimateQError(left, right *float64) *float64 { + if left == nil || right == nil || *left <= 0 || *right <= 0 { + return nil + } + value := math.Max(*left / *right, *right / *left) + return &value +} + +// predicatePlacementMoved compares normalized predicate-bearing stage families rather than raw backend syntax. +func predicatePlacementMoved(postgres, neo4j []string) bool { + if len(postgres) == 0 && len(neo4j) == 0 { + return false + } + postgresStages := normalizedPredicateStages(postgres) + neo4jStages := normalizedPredicateStages(neo4j) + return !reflect.DeepEqual(postgresStages, neo4jStages) +} + +// normalizedPredicateStages reduces backend syntax to access/filter/join stage counts. +func normalizedPredicateStages(stages []string) map[string]int { + normalized := map[string]int{} + for _, stage := range stages { + lower := strings.ToLower(stage) + switch { + case strings.Contains(lower, "join filter"), strings.Contains(lower, "apply"): + normalized["join"]++ + case strings.Contains(lower, "index cond"), strings.Contains(lower, "seek"): + normalized["access"]++ + default: + normalized["filter"]++ + } + } + return normalized +} + +// neo4jNodeObservedWork prefers DB hits and otherwise uses profiled output rows. +func neo4jNodeObservedWork(node Neo4jPlanNode) *int64 { + if node.DBHits != nil { + return node.DBHits + } + return node.ActualRows +} + +// rankPlanDeltaFindings produces category-local scores and a stable global review order. +func rankPlanDeltaFindings(records []PlanDeltaRecord) []PlanDeltaFinding { + var findings []PlanDeltaFinding + appendFinding := func(record PlanDeltaRecord, category string, score float64, summary string) { + findings = append(findings, PlanDeltaFinding{ + Category: category, + Dataset: record.Dataset, + Source: record.Source, + Name: record.Name, + PairSHA256: record.PairSHA256, + Score: score, + Summary: summary, + }) + } + for _, record := range records { + if !record.Complete { + appendFinding(record, "incomplete_pair", math.MaxFloat64, record.IncompleteReason) + continue + } + if record.OppositeStartingSides { + summary := "backends start from opposite endpoint sides" + if record.ChosenSideDidLessObservedWork != nil { + summary += fmt.Sprintf("; Neo4j lower-work choice=%t", *record.ChosenSideDidLessObservedWork) + } + appendFinding(record, "opposite_starting_side", 1, summary) + } + for category, value := range map[string]*float64{ + "seed_estimate_disagreement": record.SeedEstimateQError, + "traversal_estimate_disagreement": record.TraversalEstimateQError, + "output_estimate_disagreement": record.OutputEstimateQError, + "hydration_estimate_disagreement": record.HydrationEstimateQError, + } { + if value != nil && *value > 1 { + appendFinding(record, category, *value, fmt.Sprintf("backend estimate Q-error %.4g", *value)) + } + } + if record.PredicatePlacementMoved { + appendFinding(record, "predicate_placement_move", 1, "predicate-bearing stage families differ") + } + if record.Postgres != nil && (record.Postgres.FallbackReason != "" || len(record.Postgres.ProbeCaps) > 0) { + summary := "bounded candidate or fallback is present" + if record.Postgres.FallbackReason != "" { + summary = "fallback: " + record.Postgres.FallbackReason + } + appendFinding(record, "fallback_or_cap", float64(len(record.Postgres.ProbeCaps)+1), summary) + } + } + categoryPriority := map[string]int{ + "incomplete_pair": 0, "fallback_or_cap": 1, "opposite_starting_side": 2, + "traversal_estimate_disagreement": 3, "seed_estimate_disagreement": 4, + "output_estimate_disagreement": 5, "predicate_placement_move": 6, "hydration_estimate_disagreement": 7, + } + sort.Slice(findings, func(i, j int) bool { + leftPriority, rightPriority := categoryPriority[findings[i].Category], categoryPriority[findings[j].Category] + if leftPriority != rightPriority { + return leftPriority < rightPriority + } + if findings[i].Score != findings[j].Score { + return findings[i].Score > findings[j].Score + } + if findings[i].Dataset != findings[j].Dataset { + return findings[i].Dataset < findings[j].Dataset + } + if findings[i].Source != findings[j].Source { + return findings[i].Source < findings[j].Source + } + return findings[i].Name < findings[j].Name + }) + for idx := range findings { + findings[idx].Rank = idx + 1 + } + return findings +} + +// semanticPostgresPlan projects PostgreSQL operators and translator outcomes +// onto backend-neutral traversal stages. +func semanticPostgresPlan(record PlanRecord) SemanticPlan { + plan := SemanticPlan{ + Driver: record.Driver, + PlanFingerprint: record.PGPlanFingerprint, + LogicalDirection: logicalDirection(record.Cypher), + PhysicalDirection: postgresPhysicalDirection(record.PGPlan), + PredicatePlacement: postgresPredicatePlacement(record.PGPlan), + EndpointBinding: postgresEndpointBinding(record.PGPlan), + OperatorFamily: postgresOperatorFamily(record.PGPlan), + RuntimeIdentityKnown: false, + Error: record.Error, + RawOptimization: record.Optimization, + } + accesses := postgresAccesses(record.PGPlan) + if len(accesses) > 0 { + plan.StartingAccess = accesses[0] + plan.EstimatedSeeds = postgresRowsEstimate(accesses[0]) + } + if len(accesses) > 1 { + plan.TerminalAccess = accesses[1] + } + if len(record.PGPlan) > 0 { + plan.EstimatedOutput = postgresRowsEstimate(record.PGPlan[0]) + } + for _, line := range record.PGPlan { + lower := strings.ToLower(line) + if strings.Contains(line, "Recursive Union") || strings.Contains(lower, "shortest_path") { + plan.EstimatedTraversal = postgresRowsEstimate(line) + } + if plan.EstimatedHydration == nil && (strings.Contains(lower, "hydrat") || strings.Contains(lower, "materializ")) { + plan.EstimatedHydration = postgresRowsEstimate(line) + } + } + plan.PlannedIdentity, plan.EmittedIdentity, plan.PlannedCandidates, plan.EmittedCandidates, + plan.FallbackIdentity, plan.FallbackReason, plan.SelectorVersion, plan.ProbeCaps = postgresPlanIdentities(record.Optimization) + return plan +} + +// semanticNeo4jPlan projects the ordered Neo4j tree onto comparable stages. +func semanticNeo4jPlan(record PlanRecord) SemanticPlan { + plan := SemanticPlan{ + Driver: record.Driver, + PlanFingerprint: record.Neo4jPlanFingerprint, + LogicalDirection: logicalDirection(record.Cypher), + PhysicalDirection: neo4jPhysicalDirection(record.Neo4jPlan), + PredicatePlacement: neo4jPredicatePlacement(record.Neo4jPlan), + EndpointBinding: neo4jEndpointBinding(record.Neo4jPlan), + OperatorFamily: neo4jOperatorFamily(record.Neo4jPlan), + RuntimeIdentityKnown: false, + Error: record.Error, + } + if record.Neo4jPlan == nil { + return plan + } + leaves := neo4jLeaves(*record.Neo4jPlan) + if len(leaves) > 0 { + plan.StartingAccess = neo4jAccessLabel(leaves[0]) + plan.EstimatedSeeds = neo4jEstimatedRows(leaves[0]) + plan.ObservedSeedWork = neo4jNodeObservedWork(leaves[0]) + } + if len(leaves) > 1 { + plan.TerminalAccess = neo4jAccessLabel(leaves[1]) + plan.ObservedAlternativeSeedWork = neo4jNodeObservedWork(leaves[1]) + } + plan.EstimatedOutput = neo4jEstimatedRows(*record.Neo4jPlan) + plan.ActualOutput = record.Neo4jPlan.ActualRows + plan.OutputQError = qError(plan.EstimatedOutput, plan.ActualOutput) + var traversal *Neo4jPlanNode + walkNeo4jPlan(*record.Neo4jPlan, func(node Neo4jPlanNode) { + if traversal == nil && (strings.Contains(node.Operator, "Expand") || strings.Contains(node.Operator, "ShortestPath")) { + copyNode := node + traversal = ©Node + } + lower := strings.ToLower(node.Operator + " " + node.Arguments["Details"]) + if strings.Contains(lower, "project") || strings.Contains(lower, "materializ") || strings.Contains(lower, "path") && !strings.Contains(lower, "shortestpath") { + if plan.EstimatedHydration == nil { + plan.EstimatedHydration = neo4jEstimatedRows(node) + } + if plan.ObservedHydrationRows == nil { + plan.ObservedHydrationRows = node.ActualRows + } + } + }) + if traversal != nil { + plan.EstimatedTraversal = neo4jEstimatedRows(*traversal) + plan.ObservedTraversalWork = traversal.DBHits + if strings.Contains(traversal.Operator, "ShortestPath") { + plan.InternalTraversalWork = "opaque" + } + } + return plan +} + +// postgresPlanIdentities returns selected/emitted identities, complete candidate sets, fallback, selector, and bounded probe caps. +func postgresPlanIdentities(optimization *translate.OptimizationSummary) (string, string, []string, []string, string, string, string, map[string]int64) { + if optimization == nil { + return "", "", nil, nil, "", "", "", nil + } + for _, outcome := range optimization.TargetOutcomes { + if outcome.Family == "SP" || outcome.Family == "ASP" || strings.Contains(outcome.Family, "expansion") { + caps := map[string]int64{} + if outcome.ProbeCaps != nil { + caps["root_rows"] = outcome.ProbeCaps.RootRowLimit + caps["reverse_seed_rows"] = outcome.ProbeCaps.ReverseSeedRowLimit + caps["directional_degree_rows"] = outcome.ProbeCaps.DirectionalDegreeRowLimit + caps["survival_rows"] = outcome.ProbeCaps.SurvivalRowLimit + } + if outcome.StateLimit > 0 { + caps["state_rows"] = outcome.StateLimit + } + if outcome.EndpointLimit > 0 { + caps["endpoint_rows"] = outcome.EndpointLimit + } + for name, value := range caps { + if value <= 0 { + delete(caps, name) + } + } + if len(caps) == 0 { + caps = nil + } + return outcome.Selected, outcome.Applied, + append([]string(nil), outcome.PlannedCandidates...), append([]string(nil), outcome.EmittedCandidates...), + outcome.Fallback, outcome.SkipReason, outcome.SelectorVersion, caps + } + } + return "", "", nil, nil, "", "", "", nil +} + +// postgresAccesses returns leaf access lines in execution order. +func postgresAccesses(plan []string) []string { + var accesses []string + for idx := len(plan) - 1; idx >= 0; idx-- { + line := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(plan[idx]), "->")) + if strings.Contains(line, " Scan") && !strings.Contains(line, "CTE Scan") && !strings.Contains(line, "Subquery Scan") { + accesses = append(accesses, line) + } + } + return accesses +} + +// postgresRowsEstimate extracts a planner row estimate from a text-plan line. +func postgresRowsEstimate(line string) *float64 { + match := planRowsPattern.FindStringSubmatch(line) + if len(match) != 2 { + return nil + } + value, err := strconv.ParseFloat(match[1], 64) + if err != nil { + return nil + } + return &value +} + +// postgresPhysicalDirection identifies the adjacency endpoint used by a plan. +func postgresPhysicalDirection(plan []string) string { + joined := strings.ToLower(strings.Join(plan, "\n")) + start, end := strings.Contains(joined, "start_id"), strings.Contains(joined, "end_id") + switch { + case start && end: + return "mixed" + case start: + return "start_id" + case end: + return "end_id" + default: + return "" + } +} + +// postgresPredicatePlacement lists plan stages containing filters. +func postgresPredicatePlacement(plan []string) []string { + var stages []string + for _, line := range plan { + trimmed := strings.TrimSpace(line) + if strings.Contains(trimmed, "Filter:") || strings.Contains(trimmed, "Index Cond:") || strings.Contains(trimmed, "Join Filter:") { + stages = append(stages, trimmed) + } + } + return stages +} + +// postgresEndpointBinding classifies evidence that a bound endpoint pair is materialized. +func postgresEndpointBinding(plan []string) string { + joined := strings.ToLower(strings.Join(plan, "\n")) + if strings.Contains(joined, "pair_filter") || strings.Contains(joined, "cartesian") { + return "both_before_traversal" + } + if strings.Contains(joined, "terminal_filter") { + return "terminal_before_traversal" + } + return "" +} + +// postgresOperatorFamily classifies PostgreSQL traversal execution. +func postgresOperatorFamily(plan []string) string { + joined := strings.ToLower(strings.Join(plan, "\n")) + switch { + case strings.Contains(joined, "all_shortest_paths"): + return "all_shortest_paths" + case strings.Contains(joined, "shortest_path"): + return "shortest_path" + case strings.Contains(joined, "recursive union"): + return "ordinary_expand" + case strings.Contains(joined, "edge"): + return "fixed_hop" + default: + return "" + } +} + +// logicalDirection extracts the first directed relationship orientation. +func logicalDirection(cypherQuery string) string { + compact := strings.ReplaceAll(cypherQuery, " ", "") + switch { + case strings.Contains(compact, "]->"): + return "outbound" + case strings.Contains(compact, "<-["): + return "inbound" + case strings.Contains(compact, "]-[") || strings.Contains(compact, "]-"): + return "directionless" + default: + return "" + } +} + +// neo4jLeaves returns leaf operators in backend child order. +func neo4jLeaves(root Neo4jPlanNode) []Neo4jPlanNode { + var leaves []Neo4jPlanNode + walkNeo4jPlan(root, func(node Neo4jPlanNode) { + if len(node.Children) == 0 { + leaves = append(leaves, node) + } + }) + return leaves +} + +// walkNeo4jPlan visits a plan in parent-before-child order while retaining backend child order. +func walkNeo4jPlan(root Neo4jPlanNode, visit func(Neo4jPlanNode)) { + visit(root) + for _, child := range root.Children { + walkNeo4jPlan(child, visit) + } +} + +// neo4jAccessLabel renders an access operator with its stable details. +func neo4jAccessLabel(node Neo4jPlanNode) string { + details := node.Arguments["Details"] + if details == "" { + return node.Operator + } + return node.Operator + ": " + details +} + +// neo4jEstimatedRows returns an estimate from a typed field or stable argument. +func neo4jEstimatedRows(node Neo4jPlanNode) *float64 { + if node.EstimatedRows != nil { + return node.EstimatedRows + } + value, err := strconv.ParseFloat(node.Arguments["EstimatedRows"], 64) + if err != nil { + return nil + } + return &value +} + +// neo4jPhysicalDirection classifies expansion direction from operator details. +func neo4jPhysicalDirection(root *Neo4jPlanNode) string { + if root == nil { + return "" + } + var directions []string + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + if !strings.Contains(node.Operator, "Expand") && !strings.Contains(node.Operator, "ShortestPath") { + return + } + details := strings.ToLower(node.Arguments["Details"]) + switch { + case strings.Contains(details, "incoming") || strings.Contains(details, "<-"): + directions = append(directions, "incoming") + case strings.Contains(details, "outgoing") || strings.Contains(details, "->"): + directions = append(directions, "outgoing") + } + }) + if len(directions) == 0 { + return "" + } + for _, direction := range directions[1:] { + if direction != directions[0] { + return "mixed" + } + } + return directions[0] +} + +// neo4jPredicatePlacement lists operators whose details expose predicates. +func neo4jPredicatePlacement(root *Neo4jPlanNode) []string { + if root == nil { + return nil + } + var stages []string + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + if strings.Contains(node.Operator, "Filter") || strings.Contains(node.Operator, "Seek") { + stages = append(stages, neo4jAccessLabel(node)) + } + }) + return stages +} + +// neo4jEndpointBinding recognizes the pair-producing plan boundary. +func neo4jEndpointBinding(root *Neo4jPlanNode) string { + if root == nil { + return "" + } + bound := "" + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + if strings.Contains(node.Operator, "CartesianProduct") || strings.Contains(node.Operator, "Apply") { + bound = "both_before_traversal" + } + }) + return bound +} + +// neo4jOperatorFamily classifies Neo4j traversal operators. +func neo4jOperatorFamily(root *Neo4jPlanNode) string { + if root == nil { + return "" + } + family := "" + walkNeo4jPlan(*root, func(node Neo4jPlanNode) { + switch { + case strings.Contains(node.Operator, "ShortestPath"): + family = "shortest_path" + case family == "" && strings.Contains(node.Operator, "VarLengthExpand"): + family = "ordinary_expand" + case family == "" && strings.Contains(node.Operator, "Expand"): + family = "fixed_hop" + } + }) + return family +} + +// qError returns symmetric estimate error when both values are positive. +func qError(estimated *float64, actual *int64) *float64 { + if estimated == nil || actual == nil || *estimated <= 0 || *actual <= 0 { + return nil + } + value := math.Max(*estimated/float64(*actual), float64(*actual)/(*estimated)) + return &value +} + +// writePlanDeltaReport writes one indented, newline-terminated paired report. +func writePlanDeltaReport(path string, report PlanDeltaReport) error { + raw, err := json.MarshalIndent(report, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(path, append(raw, '\n'), 0o644); err != nil { + return fmt.Errorf("write plan delta %s: %w", path, err) + } + return nil +} diff --git a/cmd/plancorpus/plan_delta_test.go b/cmd/plancorpus/plan_delta_test.go new file mode 100644 index 00000000..2e706008 --- /dev/null +++ b/cmd/plancorpus/plan_delta_test.go @@ -0,0 +1,187 @@ +package main + +import ( + "path/filepath" + "testing" + + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestBuildPlanDeltaReportPairsByWorkloadAndPreservesSemanticDifferences verifies +// stable pairing, plan fingerprints, direction classification, and opaque Neo4j +// shortest-path work. +func TestBuildPlanDeltaReportPairsByWorkloadAndPreservesSemanticDifferences(t *testing.T) { + query := CorpusQuery{ + Source: "cases/shortest.json", + Dataset: "shortest", + Name: "bound", + Cypher: "MATCH p = shortestPath((root)-[*1..4]->(terminal)) RETURN p", + Params: map[string]any{"root_id": int64(1), "terminal_id": int64(2)}, + } + workload := workloadFingerprint(query) + pgPlan := []string{ + "Function Scan on shortest_path_compact (cost=0.25..0.26 rows=1 width=8)", + "Index Scan using node_id_idx on node root (cost=0.10..1.00 rows=1 width=8)", + "Index Cond: (start_id = root.id)", + } + neoPlan := &Neo4jPlanNode{ + Operator: "ProduceResults", + Arguments: map[string]string{"EstimatedRows": "1"}, + Children: []Neo4jPlanNode{{ + Operator: "ShortestPath", + Arguments: map[string]string{"EstimatedRows": "1", "Details": "(terminal)<-[*]-(root)"}, + Children: []Neo4jPlanNode{{ + Operator: "NodeByIdSeek", + Arguments: map[string]string{"Details": "terminal"}, + }}, + }}, + } + records := []PlanRecord{{ + SchemaVersion: planRecordSchemaVersion, + Driver: pgDriverName(), + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + WorkloadSHA256: workload, + Cypher: query.Cypher, + PGPlan: pgPlan, + PGPlanFingerprint: postgresPlanFingerprint(pgPlan), + }, { + SchemaVersion: planRecordSchemaVersion, + Driver: neo4jDriverName(), + Source: query.Source, + Dataset: query.Dataset, + Name: query.Name, + WorkloadSHA256: workload, + Cypher: query.Cypher, + Neo4jPlan: neoPlan, + Neo4jPlanFingerprint: neo4jPlanFingerprint(neoPlan), + }} + + report, err := buildPlanDeltaReport(records) + require.NoError(t, err) + require.Equal(t, planDeltaSchemaVersion, report.Version) + require.Len(t, report.Records, 1) + delta := report.Records[0] + require.True(t, delta.Complete) + require.Empty(t, delta.IncompleteReason) + require.Equal(t, "shortest_path", delta.Postgres.OperatorFamily) + require.Equal(t, "shortest_path", delta.Neo4j.OperatorFamily) + require.Equal(t, "opaque", delta.Neo4j.InternalTraversalWork) + require.True(t, delta.OppositeStartingSides) + require.NotEmpty(t, delta.Postgres.PlanFingerprint) + require.NotEmpty(t, delta.Neo4j.PlanFingerprint) + require.NotEmpty(t, delta.PairSHA256) + require.NotEmpty(t, report.RankedFindings) + require.Equal(t, "opposite_starting_side", report.RankedFindings[0].Category) +} + +// TestBuildPlanDeltaReportKeepsSourceRevisionsSeparate verifies captures from different source trees cannot silently pair. +func TestBuildPlanDeltaReportKeepsSourceRevisionsSeparate(t *testing.T) { + postgres := PlanRecord{ + Driver: pgDriverName(), + Source: "cases/a.json", + Name: "a", + WorkloadSHA256: "workload", + PGPlanFingerprint: "pg-plan", + Metadata: testutil.BaselineMetadata{DAWGSVersion: "revision-a"}, + } + neo4j := PlanRecord{ + Driver: neo4jDriverName(), + Source: "cases/a.json", + Name: "a", + WorkloadSHA256: "workload", + Neo4jPlanFingerprint: "neo-plan", + Metadata: testutil.BaselineMetadata{DAWGSVersion: "revision-b"}, + } + report, err := buildPlanDeltaReport([]PlanRecord{postgres, neo4j}) + require.NoError(t, err) + require.Len(t, report.Records, 2) + require.False(t, report.Records[0].Complete) + require.False(t, report.Records[1].Complete) +} + +// TestBuildPlanDeltaReportRetainsIncompletePairs verifies union-based pairing. +func TestBuildPlanDeltaReportRetainsIncompletePairs(t *testing.T) { + report, err := buildPlanDeltaReport([]PlanRecord{{ + Driver: pgDriverName(), + Source: "cases/a.json", + Name: "a", + WorkloadSHA256: "workload", + PGPlan: []string{"Result (cost=0.00..0.01 rows=1 width=4)"}, + PGPlanFingerprint: "pg-plan", + }}) + + require.NoError(t, err) + require.Len(t, report.Records, 1) + require.False(t, report.Records[0].Complete) + require.Equal(t, "missing_neo4j", report.Records[0].IncompleteReason) + require.NotNil(t, report.Records[0].Postgres) + require.Nil(t, report.Records[0].Neo4j) +} + +// TestBuildPlanDeltaReportRejectsDuplicateBackendSides verifies ambiguous pairing fails closed. +func TestBuildPlanDeltaReportRejectsDuplicateBackendSides(t *testing.T) { + _, err := buildPlanDeltaReport([]PlanRecord{{ + Driver: pgDriverName(), + WorkloadSHA256: "same", + }, { + Driver: pgDriverName(), + WorkloadSHA256: "same", + }}) + require.ErrorContains(t, err, "duplicate PostgreSQL") +} + +// TestWritePlanDeltaReportWritesVersionedJSON verifies portable serialization. +func TestWritePlanDeltaReportWritesVersionedJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "delta.json") + require.NoError(t, writePlanDeltaReport(path, PlanDeltaReport{Version: planDeltaSchemaVersion})) + require.FileExists(t, path) +} + +// TestWorkloadFingerprintIgnoresPhysicalValuesButIncludesTypeShape verifies independently loaded backend IDs pair safely. +func TestWorkloadFingerprintIgnoresPhysicalValuesButIncludesTypeShape(t *testing.T) { + base := CorpusQuery{ + Source: "cases/a.json", + Name: "a", + Cypher: "RETURN $id", + Params: map[string]any{"id": int64(1)}, + } + otherID := base + otherID.Params = map[string]any{"id": int64(999)} + otherType := base + otherType.Params = map[string]any{"id": "1"} + + require.Equal(t, workloadFingerprint(base), workloadFingerprint(otherID)) + require.NotEqual(t, workloadFingerprint(base), workloadFingerprint(otherType)) +} + +// TestNeo4jPlanFingerprintExcludesProfileMeasurements verifies replay counters do not make an identical plan shape look like a different plan. +func TestNeo4jPlanFingerprintExcludesProfileMeasurements(t *testing.T) { + firstRows, secondRows := int64(1), int64(99) + first := &Neo4jPlanNode{ + Operator: "ProduceResults@neo4j", + Arguments: map[string]string{"EstimatedRows": "1", "Rows": "1", "Details": "n"}, + ActualRows: &firstRows, + DBHits: &firstRows, + Children: []Neo4jPlanNode{{ + Operator: "NodeByLabelScan", + Arguments: map[string]string{"Details": "n:Node"}, + }}, + } + second := &Neo4jPlanNode{ + Operator: "ProduceResults@neo4j@neo4j", + Arguments: map[string]string{"EstimatedRows": "1", "Rows": "99", "Details": "n"}, + ActualRows: &secondRows, + DBHits: &secondRows, + Children: []Neo4jPlanNode{{ + Operator: "NodeByLabelScan@neo4j", + Arguments: map[string]string{"Details": "n:Node"}, + }}, + } + + require.Equal(t, neo4jPlanFingerprint(first), neo4jPlanFingerprint(second)) + second.Children[0].Operator = "NodeIndexSeek" + require.NotEqual(t, neo4jPlanFingerprint(first), neo4jPlanFingerprint(second)) +} diff --git a/cmd/plancorpus/report.go b/cmd/plancorpus/report.go index 654067cc..e2104d1f 100644 --- a/cmd/plancorpus/report.go +++ b/cmd/plancorpus/report.go @@ -10,56 +10,96 @@ import ( "strings" "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/testutil" ) +// defaultTopPlans limits an unconfigured report to its 25 most expensive PostgreSQL plans. const defaultTopPlans = 25 +// postgresCostPattern extracts the total-cost upper bound from a PostgreSQL plan's cost range. var postgresCostPattern = regexp.MustCompile(`cost=[0-9.]+\.\.([0-9.]+)`) +// PlanSummary aggregates captured plans by driver, lowering, and cost. type PlanSummary struct { - Drivers []DriverSummary `json:"drivers"` - TopPostgresPlans []CostedPlan `json:"top_postgres_plans,omitempty"` - PostgresOperators []Count `json:"postgres_operators,omitempty"` - Neo4jOperators []Count `json:"neo4j_operators,omitempty"` - PlannedLowerings []Count `json:"planned_lowerings,omitempty"` - AppliedLowerings []Count `json:"applied_lowerings,omitempty"` - SkippedLowerings []Count `json:"skipped_lowerings,omitempty"` - SkippedReasons []Count `json:"skipped_reasons,omitempty"` - FeatureCounts []Count `json:"feature_counts,omitempty"` - Errors []PlanError `json:"errors,omitempty"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Drivers lists driver summaries in deterministic display order. + Drivers []DriverSummary `json:"drivers"` + // TopPostgresPlans lists the highest-cost PostgreSQL plans selected for the summary. + TopPostgresPlans []CostedPlan `json:"top_postgres_plans,omitempty"` + // PostgresOperators counts normalized PostgreSQL plan operators. + PostgresOperators []Count `json:"postgres_operators,omitempty"` + // Neo4jOperators lists normalized Neo4j operators found in the captured plan. + Neo4jOperators []Count `json:"neo4j_operators,omitempty"` + // PlannedLowerings lists SQL lowering opportunities identified before optimization. + PlannedLowerings []Count `json:"planned_lowerings,omitempty"` + // AppliedLowerings lists SQL lowerings actually applied during translation. + AppliedLowerings []Count `json:"applied_lowerings,omitempty"` + // SkippedLowerings lists identified SQL lowerings not applied. + SkippedLowerings []Count `json:"skipped_lowerings,omitempty"` + // SkippedReasons counts reasons identified lowerings were not applied. + SkippedReasons []Count `json:"skipped_reasons,omitempty"` + // FeatureCounts counts captured plans containing each normalized plan feature. + FeatureCounts []Count `json:"feature_counts,omitempty"` + // Errors lists failures observed while processing the record. + Errors []PlanError `json:"errors,omitempty"` } +// DriverSummary aggregates plan counts and operators for one database driver. type DriverSummary struct { - Driver string `json:"driver"` - Records int `json:"records"` - Errors int `json:"errors"` + // Driver identifies the database driver that produced the plan or summary. + Driver string `json:"driver"` + // Records counts captured plan records produced by the driver. + Records int `json:"records"` + // Errors counts plan-capture failures reported by the driver. + Errors int `json:"errors"` } +// Count pairs a label with an aggregate count for serialized summaries. type Count struct { - Name string `json:"name"` - Count int `json:"count"` + // Name labels the operator, lowering, feature, or reason being counted. + Name string `json:"name"` + // Count records how many plan records contributed the named item. + Count int `json:"count"` } +// CostedPlan identifies a captured plan and its parsed PostgreSQL estimated cost. type CostedPlan struct { - Cost float64 `json:"cost"` - Driver string `json:"driver"` - Source string `json:"source"` - Dataset string `json:"dataset,omitempty"` - Name string `json:"name"` - Cypher string `json:"cypher"` - PlanRoot string `json:"plan_root"` + // Cost records the PostgreSQL planner's estimated total cost. + Cost float64 `json:"cost"` + // Driver identifies the database driver that produced the plan or summary. + Driver string `json:"driver"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // PlanRoot identifies the root operator of the captured plan. + PlanRoot string `json:"plan_root"` + // PlannedLowerings lists SQL lowering opportunities identified before optimization. PlannedLowerings []string `json:"planned_lowerings,omitempty"` + // AppliedLowerings lists SQL lowerings actually applied during translation. AppliedLowerings []string `json:"applied_lowerings,omitempty"` + // SkippedLowerings lists identified SQL lowerings not applied. SkippedLowerings []string `json:"skipped_lowerings,omitempty"` } +// PlanError records the driver, query, and failure for a plan that could not be summarized. type PlanError struct { + // Driver identifies the database driver that produced the plan or summary. Driver string `json:"driver"` + // Source identifies the source corpus file. Source string `json:"source"` - Name string `json:"name"` - Error string `json:"error"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // Error records the failure message when the operation did not succeed. + Error string `json:"error"` } +// buildSummary aggregates plan records by driver, operator, lowering, error, and estimated cost. func buildSummary(records []PlanRecord, topN int) PlanSummary { if topN <= 0 { topN = defaultTopPlans @@ -74,11 +114,15 @@ func buildSummary(records []PlanRecord, topN int) PlanSummary { skippedLoweringCounts = map[string]int{} skippedReasonCounts = map[string]int{} featureCounts = map[string]int{} + summaryMetadata testutil.BaselineMetadata errors []PlanError topPG []CostedPlan ) for _, record := range records { + if summaryMetadata == (testutil.BaselineMetadata{}) { + summaryMetadata = record.Metadata + } driver := driverCounts[record.Driver] if driver == nil { driver = &DriverSummary{Driver: record.Driver} @@ -150,6 +194,7 @@ func buildSummary(records []PlanRecord, topN int) PlanSummary { } return PlanSummary{ + Metadata: summaryMetadata, Drivers: sortedDriverSummaries(driverCounts), TopPostgresPlans: topPG, PostgresOperators: sortedCounts(postgresOperatorCounts), @@ -163,6 +208,7 @@ func buildSummary(records []PlanRecord, topN int) PlanSummary { } } +// skippedLoweringLabels renders skipped lowering names and reasons as stable report labels, preserving their plan order. func skippedLoweringLabels(lowerings []translate.SkippedLowering) []string { if len(lowerings) == 0 { return nil @@ -176,6 +222,7 @@ func skippedLoweringLabels(lowerings []translate.SkippedLowering) []string { return labels } +// postgresEstimatedCost extracts the PostgreSQL planner's estimated total cost from plan text. func postgresEstimatedCost(planRoot string) float64 { match := postgresCostPattern.FindStringSubmatch(planRoot) if len(match) != 2 { @@ -189,6 +236,7 @@ func postgresEstimatedCost(planRoot string) float64 { return cost } +// normalizePostgresOperator removes plan decoration so equivalent PostgreSQL operator lines share one name. func normalizePostgresOperator(operator string) string { operator = strings.TrimSpace(operator) if operator == "" { @@ -206,6 +254,7 @@ func normalizePostgresOperator(operator string) string { return operator } +// sortedDriverSummaries returns driver summaries ordered by driver name. func sortedDriverSummaries(drivers map[string]*DriverSummary) []DriverSummary { sorted := make([]DriverSummary, 0, len(drivers)) for _, summary := range drivers { @@ -217,6 +266,7 @@ func sortedDriverSummaries(drivers map[string]*DriverSummary) []DriverSummary { return sorted } +// sortedCounts converts a count map to descending-count, name-tiebroken entries. func sortedCounts(counts map[string]int) []Count { sorted := make([]Count, 0, len(counts)) for name, count := range counts { @@ -234,12 +284,14 @@ func sortedCounts(counts map[string]int) []Count { return sorted } +// writeJSONSummary encodes a plan summary as indented JSON. func writeJSONSummary(w io.Writer, summary PlanSummary) error { encoder := json.NewEncoder(w) encoder.SetIndent("", " ") return encoder.Encode(summary) } +// writeMarkdownSummary renders aggregate counts, expensive plans, and errors as Markdown. func writeMarkdownSummary(w io.Writer, summary PlanSummary) error { writef := func(format string, args ...any) error { _, err := fmt.Fprintf(w, format, args...) @@ -272,6 +324,9 @@ func writeMarkdownSummary(w io.Writer, summary PlanSummary) error { if err := writeln("# Cypher Plan Corpus Summary"); err != nil { return err } + if err := writef("\nDAWGS version: `%s`\n", summary.Metadata.DAWGSVersion); err != nil { + return err + } if err := writeln("\n## Drivers\n\n| Driver | Records | Errors |\n| --- | ---: | ---: |"); err != nil { return err } @@ -341,6 +396,7 @@ func writeMarkdownSummary(w io.Writer, summary PlanSummary) error { return nil } +// markdownCell escapes table delimiters and line breaks for a Markdown cell. func markdownCell(value string) string { value = strings.ReplaceAll(value, "\n", " ") value = strings.ReplaceAll(value, "|", "\\|") diff --git a/cmd/plancorpus/types.go b/cmd/plancorpus/types.go index 9c4fa662..418bb4b4 100644 --- a/cmd/plancorpus/types.go +++ b/cmd/plancorpus/types.go @@ -1,37 +1,236 @@ package main -import "github.com/specterops/dawgs/cypher/models/pgsql/translate" +import ( + "encoding/json" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/testutil" +) + +// planRecordSchemaVersion reserves the stable protocol value used to recognize plan record schema version across artifacts and executions. +const planRecordSchemaVersion = 2 + +// PlanRecord captures a query plan together with workload, fixture, and environment identity. type PlanRecord struct { - Driver string `json:"driver"` - Source string `json:"source"` - Dataset string `json:"dataset,omitempty"` - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - SQL string `json:"sql,omitempty"` - PGPlan []string `json:"pg_plan,omitempty"` - PGOperators []string `json:"pg_operators,omitempty"` - Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` - Neo4jOperators []string `json:"neo4j_operators,omitempty"` - PlannedLowerings []string `json:"planned_lowerings,omitempty"` - AppliedLowerings []string `json:"applied_lowerings,omitempty"` - SkippedLowerings []translate.SkippedLowering `json:"skipped_lowerings,omitempty"` - Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` - Error string `json:"error,omitempty"` + // SchemaVersion identifies the serialized plan-record schema revision. + SchemaVersion int `json:"schema_version"` + // Metadata captures build and baseline metadata. + Metadata testutil.BaselineMetadata `json:"metadata"` + // Driver identifies the database driver that produced the plan or summary. + Driver string `json:"driver"` + // Source identifies the source corpus file. + Source string `json:"source"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Name identifies the case or record within its dataset. + Name string `json:"name"` + // WorkloadSHA256 identifies the backend-independent source workload. + WorkloadSHA256 string `json:"workload_sha256"` + // Cypher contains the Cypher statement under test. + Cypher string `json:"cypher"` + // Params supplies literal query parameters. + Params map[string]any `json:"params,omitempty"` + // SQL contains the rendered SQL statement. + SQL string `json:"sql,omitempty"` + // PGPlan contains the normalized PostgreSQL text plan. + PGPlan []string `json:"pg_plan,omitempty"` + // PGPlanFingerprint identifies the normalized PostgreSQL plan without retaining another copy. + PGPlanFingerprint string `json:"pg_plan_fingerprint,omitempty"` + // PGOperators lists normalized PostgreSQL operators found in the captured plan. + PGOperators []string `json:"pg_operators,omitempty"` + // Neo4jPlan contains the normalized Neo4j operator tree. + Neo4jPlan *Neo4jPlanNode `json:"neo4j_plan,omitempty"` + // Neo4jPlanFingerprint identifies the normalized Neo4j plan tree. + Neo4jPlanFingerprint string `json:"neo4j_plan_fingerprint,omitempty"` + // Neo4jOperators lists normalized Neo4j operators found in the captured plan. + Neo4jOperators []string `json:"neo4j_operators,omitempty"` + // PlannedLowerings lists SQL lowering opportunities identified before optimization. + PlannedLowerings []string `json:"planned_lowerings,omitempty"` + // AppliedLowerings lists SQL lowerings actually applied during translation. + AppliedLowerings []string `json:"applied_lowerings,omitempty"` + // SkippedLowerings lists identified SQL lowerings not applied. + SkippedLowerings []translate.SkippedLowering `json:"skipped_lowerings,omitempty"` + // Optimization captures translation optimization and lowering decisions. + Optimization *translate.OptimizationSummary `json:"optimization,omitempty"` + // Error supplies the error input to the PlanRecord contract. + Error string `json:"error,omitempty"` } +// Neo4jPlanNode models the recursive operator tree returned by Neo4j EXPLAIN. type Neo4jPlanNode struct { - Operator string `json:"operator"` - Arguments map[string]string `json:"arguments,omitempty"` - Identifiers []string `json:"identifiers,omitempty"` - Children []Neo4jPlanNode `json:"children,omitempty"` + // Operator identifies the backend plan operator at this node. + Operator string `json:"operator"` + // Arguments maps backend plan argument names to stable string representations. + Arguments map[string]string `json:"arguments,omitempty"` + // Identifiers lists variables or identifiers referenced by the Neo4j plan node. + Identifiers []string `json:"identifiers,omitempty"` + // Children contains child Neo4j plan operators in backend order. + Children []Neo4jPlanNode `json:"children,omitempty"` + // EstimatedRows records planner cardinality when exposed by the server. + EstimatedRows *float64 `json:"estimated_rows,omitempty"` + // ActualRows records profiled output cardinality when this is an executed read plan. + ActualRows *int64 `json:"actual_rows,omitempty"` + // DBHits records profiled store accesses when exposed by the server. + DBHits *int64 `json:"db_hits,omitempty"` + // PageCacheHits records profiled page-cache hits when exposed by the server. + PageCacheHits *int64 `json:"page_cache_hits,omitempty"` + // PageCacheMisses records profiled page-cache misses when exposed by the server. + PageCacheMisses *int64 `json:"page_cache_misses,omitempty"` + // TimeNS records profiled operator time in nanoseconds when exposed by the server. + TimeNS *int64 `json:"time_ns,omitempty"` +} + +// PlanDeltaReport contains backend-paired semantic plan comparisons without +// treating backend-specific operator counters as interchangeable. +type PlanDeltaReport struct { + // Version identifies the serialized plan-delta schema revision. + Version int `json:"version"` + // Records contains complete and explicitly incomplete backend pairs. + Records []PlanDeltaRecord `json:"records"` + // RankedFindings prioritizes semantic disagreements and qualification cases. + RankedFindings []PlanDeltaFinding `json:"ranked_findings,omitempty"` +} + +// PlanDeltaFinding ranks one cross-backend semantic observation for review. +type PlanDeltaFinding struct { + // Rank is the one-based position after stable severity ordering. + Rank int `json:"rank"` + // Category identifies the semantic disagreement being ranked. + Category string `json:"category"` + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Source identifies the corpus declaration. + Source string `json:"source"` + // Name identifies the workload case. + Name string `json:"name"` + // PairSHA256 identifies the exact paired record. + PairSHA256 string `json:"pair_sha256"` + // Score is a category-local descending severity score. + Score float64 `json:"score"` + // Summary is a compact stable explanation of the finding. + Summary string `json:"summary"` +} + +// PlanDeltaRecord compares one source workload across PostgreSQL and Neo4j. +type PlanDeltaRecord struct { + // Dataset identifies the fixture dataset. + Dataset string `json:"dataset,omitempty"` + // Source identifies the source corpus declaration. + Source string `json:"source"` + // Name identifies the case within its source. + Name string `json:"name"` + // WorkloadSHA256 identifies the backend-independent source workload. + WorkloadSHA256 string `json:"workload_sha256"` + // SourceRevision identifies the DAWGS source used for capture. + SourceRevision string `json:"source_revision,omitempty"` + // PairSHA256 binds workload, source revision, and both backend plan fingerprints. + PairSHA256 string `json:"pair_sha256"` + // Postgres supplies the postgres input to the PlanDeltaRecord contract. + Postgres *SemanticPlan `json:"postgres,omitempty"` + // Neo4j supplies the neo4j input to the PlanDeltaRecord contract. + Neo4j *SemanticPlan `json:"neo4j,omitempty"` + // Complete reports whether both backend plans were captured successfully. + Complete bool `json:"complete"` + // IncompleteReason explains a missing or failed backend side. + IncompleteReason string `json:"incomplete_reason,omitempty"` + // OppositeStartingSides reports a material starting-side disagreement. + OppositeStartingSides bool `json:"opposite_starting_sides,omitempty"` + // OppositePhysicalDirections reports a physical adjacency disagreement. + OppositePhysicalDirections bool `json:"opposite_physical_directions,omitempty"` + // Neo4jReorderedPattern reports that Neo4j started from the opposite logical endpoint. + Neo4jReorderedPattern bool `json:"neo4j_reordered_pattern,omitempty"` + // ChosenSideDidLessObservedWork reports whether Neo4j's first leaf had no more profiled work than the alternative leaf. + ChosenSideDidLessObservedWork *bool `json:"chosen_side_did_less_observed_work,omitempty"` + // SeedEstimateQError reports symmetric disagreement between backend seed estimates. + SeedEstimateQError *float64 `json:"seed_estimate_q_error,omitempty"` + // TraversalEstimateQError reports symmetric disagreement between backend traversal estimates. + TraversalEstimateQError *float64 `json:"traversal_estimate_q_error,omitempty"` + // OutputEstimateQError reports symmetric disagreement between backend output estimates. + OutputEstimateQError *float64 `json:"output_estimate_q_error,omitempty"` + // PredicatePlacementMoved reports a backend disagreement in predicate-bearing stages. + PredicatePlacementMoved bool `json:"predicate_placement_moved,omitempty"` + // HydrationEstimateQError reports symmetric disagreement in identifiable hydration work. + HydrationEstimateQError *float64 `json:"hydration_estimate_q_error,omitempty"` +} + +// SemanticPlan normalizes one backend plan into comparable traversal stages. +type SemanticPlan struct { + // Driver identifies the backend that produced this plan. + Driver string `json:"driver"` + // PlanFingerprint identifies the complete normalized backend plan. + PlanFingerprint string `json:"plan_fingerprint"` + // StartingAccess describes the first observed leaf access. + StartingAccess string `json:"starting_access,omitempty"` + // TerminalAccess describes the opposite endpoint access when identifiable. + TerminalAccess string `json:"terminal_access,omitempty"` + // LogicalDirection describes the query's directed traversal orientation. + LogicalDirection string `json:"logical_direction,omitempty"` + // PhysicalDirection identifies start_id or end_id adjacency use. + PhysicalDirection string `json:"physical_direction,omitempty"` + // PredicatePlacement lists stages carrying predicates or filters. + PredicatePlacement []string `json:"predicate_placement,omitempty"` + // EndpointBinding reports whether both endpoints are available before traversal. + EndpointBinding string `json:"endpoint_binding,omitempty"` + // OperatorFamily classifies ordinary expansion, SP, ASP, or fixed-hop work. + OperatorFamily string `json:"operator_family,omitempty"` + // EstimatedSeeds records a comparable seed estimate when exposed. + EstimatedSeeds *float64 `json:"estimated_seeds,omitempty"` + // EstimatedTraversal records a comparable traversal estimate when exposed. + EstimatedTraversal *float64 `json:"estimated_traversal,omitempty"` + // EstimatedOutput records a comparable output estimate when exposed. + EstimatedOutput *float64 `json:"estimated_output,omitempty"` + // EstimatedHydration records rows at an identifiable hydration/materialization stage. + EstimatedHydration *float64 `json:"estimated_hydration,omitempty"` + // ActualOutput records profiled output rows when exposed. + ActualOutput *int64 `json:"actual_output,omitempty"` + // ObservedSeedWork records actual rows or store hits at the selected seed leaf when exposed. + ObservedSeedWork *int64 `json:"observed_seed_work,omitempty"` + // ObservedAlternativeSeedWork supplies the observed alternative seed work input to the SemanticPlan contract. + ObservedAlternativeSeedWork *int64 `json:"observed_alternative_seed_work,omitempty"` + // ObservedTraversalWork records profiled traversal DB hits when exposed. + ObservedTraversalWork *int64 `json:"observed_traversal_work,omitempty"` + // ObservedHydrationRows records profiled hydration rows when exposed. + ObservedHydrationRows *int64 `json:"observed_hydration_rows,omitempty"` + // OutputQError records estimate error when both estimate and actual output exist. + OutputQError *float64 `json:"output_q_error,omitempty"` + // PlannedIdentity identifies the planned identity. + PlannedIdentity string `json:"planned_identity,omitempty"` + // EmittedIdentity identifies the emitted identity. + EmittedIdentity string `json:"emitted_identity,omitempty"` + // PlannedCandidates lists the complete typed candidate set. + PlannedCandidates []string `json:"planned_candidates,omitempty"` + // EmittedCandidates lists the arms present in translated SQL. + EmittedCandidates []string `json:"emitted_candidates,omitempty"` + // FallbackIdentity identifies the fallback identity. + FallbackIdentity string `json:"fallback_identity,omitempty"` + // FallbackReason records static qualification failure or guarded fallback intent. + FallbackReason string `json:"fallback_reason,omitempty"` + // SelectorVersion identifies the policy that produced the plan. + SelectorVersion string `json:"selector_version,omitempty"` + // ProbeCaps records bounded runtime evidence limits declared by the plan. + ProbeCaps map[string]int64 `json:"probe_caps,omitempty"` + // RuntimeIdentityKnown is false for PlanCorpus because execution telemetry is GraphBench authority. + RuntimeIdentityKnown bool `json:"runtime_identity_known"` + // InternalTraversalWork marks backend work that profiling cannot expose. + InternalTraversalWork string `json:"internal_traversal_work,omitempty"` + // Error retains a capture failure without dropping the pair. + Error string `json:"error,omitempty"` + // RawOptimization retains typed translation diagnostics for PostgreSQL. + RawOptimization *translate.OptimizationSummary `json:"raw_optimization,omitempty"` + // PlanJSON optionally retains a stable semantic projection for downstream tools. + PlanJSON json.RawMessage `json:"plan_json,omitempty"` } +// CorpusQuery defines one corpus query and the fixture parameters needed to execute it. type CorpusQuery struct { - Source string + // Source identifies the source corpus file. + Source string + // Dataset identifies the fixture dataset. Dataset string - Name string - Cypher string - Params map[string]any + // Name identifies the case or record within its dataset. + Name string + // Cypher contains the Cypher statement under test. + Cypher string + // Params supplies literal query parameters. + Params map[string]any } diff --git a/cypher/frontend/expression.go b/cypher/frontend/expression.go index 8385d0db..bdb692b4 100644 --- a/cypher/frontend/expression.go +++ b/cypher/frontend/expression.go @@ -423,6 +423,7 @@ func (s *NonArithmeticOperatorExpressionVisitor) EnterOC_PropertyKeyName(ctx *pa s.ctx.Enter(&SymbolicNameOrReservedWordVisitor{}) } +// ExitOC_PropertyKeyName assigns the parsed key to the property lookup under construction. func (s *NonArithmeticOperatorExpressionVisitor) ExitOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNameContext) { s.PropertyKeyName = extractPropertyKeyName(s.ctx, ctx) } diff --git a/cypher/frontend/literal.go b/cypher/frontend/literal.go index 45503a81..4797d304 100644 --- a/cypher/frontend/literal.go +++ b/cypher/frontend/literal.go @@ -44,6 +44,7 @@ func (s *MapLiteralVisitor) EnterOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNa s.ctx.Enter(&SymbolicNameOrReservedWordVisitor{}) } +// ExitOC_PropertyKeyName decodes and retains the key for the next map-literal entry. func (s *MapLiteralVisitor) ExitOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNameContext) { s.nextPropertyKey = cypher.UnescapePropertyKeyName(s.ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name) } diff --git a/cypher/frontend/property_key.go b/cypher/frontend/property_key.go index f6ef9ca7..c4fd6841 100644 --- a/cypher/frontend/property_key.go +++ b/cypher/frontend/property_key.go @@ -5,6 +5,7 @@ import ( "github.com/specterops/dawgs/cypher/parser" ) +// extractPropertyKeyName decodes a parsed property-key token and records a syntax error when the decoded key is invalid. func extractPropertyKeyName(ctx *Context, cypherCtx *parser.OC_PropertyKeyNameContext) string { name := cypher.UnescapePropertyKeyName(ctx.Exit().(*SymbolicNameOrReservedWordVisitor).Name) if err := cypher.ValidatePropertyKeyName(name); err != nil { diff --git a/cypher/frontend/property_key_test.go b/cypher/frontend/property_key_test.go index 754be892..ea169b68 100644 --- a/cypher/frontend/property_key_test.go +++ b/cypher/frontend/property_key_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestParsePropertyLookupStoresRawPropertyKeyNames verifies that lookup tokens are decoded before storage in the AST. func TestParsePropertyLookupStoresRawPropertyKeyNames(t *testing.T) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN n.match, n.`a-aaa`, n.`has``tick`, n.` `") require.NoError(t, err) @@ -24,6 +25,7 @@ func TestParsePropertyLookupStoresRawPropertyKeyNames(t *testing.T) { require.Equal(t, []string{"match", "a-aaa", "has`tick", " "}, symbols) } +// TestParsePropertyLookupStoresQuotePropertyKeyNames verifies that quote characters survive property-key parsing unchanged. func TestParsePropertyLookupStoresQuotePropertyKeyNames(t *testing.T) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN n.`'`, n.`\"`") require.NoError(t, err) @@ -39,6 +41,7 @@ func TestParsePropertyLookupStoresQuotePropertyKeyNames(t *testing.T) { require.Equal(t, []string{"'", "\""}, symbols) } +// TestParsePropertyLookupStoresUnicodePropertyKeyNames verifies the Unicode classes accepted in raw property keys. func TestParsePropertyLookupStoresUnicodePropertyKeyNames(t *testing.T) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN n.\u2118, n.a\u00b7, n.a\u0301, n.a\u093e, n.a$, n.`a\u20dd`") require.NoError(t, err) @@ -54,6 +57,7 @@ func TestParsePropertyLookupStoresUnicodePropertyKeyNames(t *testing.T) { require.Equal(t, []string{"\u2118", "a\u00b7", "a\u0301", "a\u093e", "a$", "a\u20dd"}, symbols) } +// TestParseMapLiteralStoresRawPropertyKeyNames verifies that map keys are decoded before storage in the AST. func TestParseMapLiteralStoresRawPropertyKeyNames(t *testing.T) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN {match: 1, `a-aaa`: 2, `has``tick`: 3, ``: 4, ` `: 5}") require.NoError(t, err) @@ -69,6 +73,7 @@ func TestParseMapLiteralStoresRawPropertyKeyNames(t *testing.T) { require.ElementsMatch(t, []string{"match", "a-aaa", "has`tick", "", " "}, keys) } +// TestParseMapLiteralStoresQuotePropertyKeyNames verifies that quote characters survive map-key parsing unchanged. func TestParseMapLiteralStoresQuotePropertyKeyNames(t *testing.T) { regularQuery, err := frontend.ParseCypher(frontend.NewContext(), "RETURN {`'`: 1, `\"`: 2}") require.NoError(t, err) @@ -84,9 +89,12 @@ func TestParseMapLiteralStoresQuotePropertyKeyNames(t *testing.T) { require.ElementsMatch(t, []string{"'", "\""}, keys) } +// TestParseRejectsEmptyPropertyKeyNames verifies that empty escaped keys are rejected in every property-key position. func TestParseRejectsEmptyPropertyKeyNames(t *testing.T) { testCases := []struct { - name string + // name labels the property-key syntax under test. + name string + // query contains an empty escaped key in the named syntax position. query string }{ {name: "property lookup", query: "RETURN n.``"}, diff --git a/cypher/frontend/query.go b/cypher/frontend/query.go index 4207d5bb..36a338ed 100644 --- a/cypher/frontend/query.go +++ b/cypher/frontend/query.go @@ -706,6 +706,7 @@ func (s *PropertyExpressionVisitor) EnterOC_PropertyKeyName(ctx *parser.OC_Prope s.ctx.Enter(&SymbolicNameOrReservedWordVisitor{}) } +// ExitOC_PropertyKeyName assigns the parsed key to the property expression under construction. func (s *PropertyExpressionVisitor) ExitOC_PropertyKeyName(ctx *parser.OC_PropertyKeyNameContext) { s.PropertyLookup.SetSymbol(extractPropertyKeyName(s.ctx, ctx)) } diff --git a/cypher/models/cypher/format/format.go b/cypher/models/cypher/format/format.go index 67f566c3..bcd7b934 100644 --- a/cypher/models/cypher/format/format.go +++ b/cypher/models/cypher/format/format.go @@ -13,6 +13,7 @@ import ( "github.com/specterops/dawgs/graph" ) +// strippedLiteral replaces literal values when emitting a privacy-preserving Cypher query. const strippedLiteral = "$STRIPPED" func writeJoinedKinds(output io.Writer, delimiter string, kinds graph.Kinds) error { @@ -318,6 +319,7 @@ func (s Emitter) formatWhere(output io.Writer, whereClause *cypher.Where) error return nil } +// formatMapLiteral renders a Cypher map literal with each property key escaped as needed. func (s Emitter) formatMapLiteral(output io.Writer, mapLiteral cypher.MapLiteral) error { if _, err := io.WriteString(output, "{"); err != nil { return err @@ -447,6 +449,7 @@ func (s Emitter) formatLiteral(output io.Writer, literal *cypher.Literal) error return nil } +// WriteExpression renders an expression and its nested operands as Cypher syntax. func (s Emitter) WriteExpression(output io.Writer, expression cypher.Expression) error { switch typedExpression := expression.(type) { case *cypher.ProjectionItem: diff --git a/cypher/models/cypher/format/format_test.go b/cypher/models/cypher/format/format_test.go index 52eab612..c4d3620e 100644 --- a/cypher/models/cypher/format/format_test.go +++ b/cypher/models/cypher/format/format_test.go @@ -68,6 +68,7 @@ func TestRegularQueryWithParameterSequenceRejectsMismatchedSymbols(t *testing.T) require.ErrorContains(t, err, "more symbols") } +// TestCypherEmitter_FormatsMapLiteralPropertyKeys verifies that map keys are emitted bare or escaped according to property-key grammar. func TestCypherEmitter_FormatsMapLiteralPropertyKeys(t *testing.T) { var ( buffer = &bytes.Buffer{} @@ -87,10 +88,14 @@ func TestCypherEmitter_FormatsMapLiteralPropertyKeys(t *testing.T) { require.Equal(t, "{``: 4, ` `: 5, `'`: 6, `a-aaa`: 2, `has``tick`: 3, match: 1}", buffer.String()) } +// TestCypherEmitter_FormatsPropertyLookupKeys verifies canonical rendering of bare and escaped lookup keys. func TestCypherEmitter_FormatsPropertyLookupKeys(t *testing.T) { testCases := []struct { - name string - symbol string + // name labels the property-key form under test. + name string + // symbol is the raw property key stored in the AST. + symbol string + // expected is the canonical rendered property lookup. expected string }{ { @@ -141,6 +146,7 @@ func TestCypherEmitter_FormatsPropertyLookupKeys(t *testing.T) { } } +// TestCypherEmitter_RejectsEmptyPropertyLookupKey verifies that an empty raw lookup key cannot be rendered. func TestCypherEmitter_RejectsEmptyPropertyLookupKey(t *testing.T) { buffer := &bytes.Buffer{} emitter := format.NewCypherEmitter(false) diff --git a/cypher/models/cypher/functions.go b/cypher/models/cypher/functions.go index 21dcac2b..96e42117 100644 --- a/cypher/models/cypher/functions.go +++ b/cypher/models/cypher/functions.go @@ -1,46 +1,126 @@ package cypher const ( - CountFunction = "count" - DateFunction = "date" - TimeFunction = "time" - LocalTimeFunction = "localtime" - DateTimeFunction = "datetime" - LocalDateTimeFunction = "localdatetime" - DurationFunction = "duration" - IdentityFunction = "id" - ToLowerFunction = "tolower" - ToUpperFunction = "toupper" - NodeLabelsFunction = "labels" - EdgeTypeFunction = "type" - StartNodeFunction = "startnode" - EndNodeFunction = "endnode" + // CountFunction identifies the Cypher aggregate that counts non-null values or rows. + CountFunction = "count" + + // DateFunction identifies the Cypher constructor for date values. + DateFunction = "date" + + // TimeFunction identifies the Cypher constructor for zoned time values. + TimeFunction = "time" + + // LocalTimeFunction identifies the Cypher constructor for local time values. + LocalTimeFunction = "localtime" + + // DateTimeFunction identifies the Cypher constructor for zoned date-time values. + DateTimeFunction = "datetime" + + // LocalDateTimeFunction identifies the Cypher constructor for local date-time values. + LocalDateTimeFunction = "localdatetime" + + // DurationFunction identifies the Cypher constructor for duration values. + DurationFunction = "duration" + + // IdentityFunction identifies the Cypher function that returns an entity ID. + IdentityFunction = "id" + + // ToLowerFunction identifies the Cypher function that lowercases text. + ToLowerFunction = "tolower" + + // ToUpperFunction identifies the Cypher function that uppercases text. + ToUpperFunction = "toupper" + + // NodeLabelsFunction identifies the Cypher function that returns a node's labels. + NodeLabelsFunction = "labels" + + // EdgeTypeFunction identifies the Cypher function that returns a relationship's type. + EdgeTypeFunction = "type" + + // StartNodeFunction identifies the Cypher function that returns a relationship's start node. + StartNodeFunction = "startnode" + + // EndNodeFunction identifies the Cypher function that returns a relationship's end node. + EndNodeFunction = "endnode" + + // StringSplitToArrayFunction identifies the Cypher function that splits text into a list. StringSplitToArrayFunction = "split" - ToStringFunction = "tostring" - ToIntegerFunction = "tointeger" - ListSizeFunction = "size" - HeadFunction = "head" - TailFunction = "tail" - NodesFunction = "nodes" - RelationshipsFunction = "relationships" - CoalesceFunction = "coalesce" - CollectFunction = "collect" - SumFunction = "sum" - AvgFunction = "avg" - MinFunction = "min" - MaxFunction = "max" - - // ITTC - Instant Type; Temporal Component (https://neo4j.com/docs/cypher-manual/current/functions/temporal/) - ITTCYear = "year" - ITTCMonth = "month" - ITTCDay = "day" - ITTCHour = "hour" - ITTCMinute = "minute" - ITTCSecond = "second" - ITTCMillisecond = "millisecond" - ITTCMicrosecond = "microsecond" - ITTCNanosecond = "nanosecond" - ITTCTimeZone = "timezone" - ITTCEpochSeconds = "epochseconds" + + // ToStringFunction identifies the Cypher function that converts a value to text. + ToStringFunction = "tostring" + + // ToIntegerFunction identifies the Cypher function that converts a value to an integer. + ToIntegerFunction = "tointeger" + + // ListSizeFunction identifies the Cypher function that returns the size of a list or string. + ListSizeFunction = "size" + + // HeadFunction identifies the Cypher function that returns the first list element. + HeadFunction = "head" + + // TailFunction identifies the Cypher function that returns all but the first list element. + TailFunction = "tail" + + // NodesFunction identifies the Cypher function that returns a path's nodes in order. + NodesFunction = "nodes" + + // RelationshipsFunction identifies the Cypher function that returns a path's relationships in order. + RelationshipsFunction = "relationships" + + // PathLengthFunction identifies the Cypher function that returns the number of relationships in a path. + PathLengthFunction = "length" + + // CoalesceFunction identifies the Cypher function that returns the first non-null argument. + CoalesceFunction = "coalesce" + + // CollectFunction identifies the Cypher aggregate that collects values into a list. + CollectFunction = "collect" + + // SumFunction identifies the Cypher aggregate that sums numeric values. + SumFunction = "sum" + + // AvgFunction identifies the Cypher aggregate that averages numeric values. + AvgFunction = "avg" + + // MinFunction identifies the Cypher aggregate that returns the minimum value. + MinFunction = "min" + + // MaxFunction identifies the Cypher aggregate that returns the maximum value. + MaxFunction = "max" + + // ITTCYear identifies the year component of a Cypher instant value. + ITTCYear = "year" + + // ITTCMonth identifies the month component of a Cypher instant value. + ITTCMonth = "month" + + // ITTCDay identifies the day component of a Cypher instant value. + ITTCDay = "day" + + // ITTCHour identifies the hour component of a Cypher instant value. + ITTCHour = "hour" + + // ITTCMinute identifies the minute component of a Cypher instant value. + ITTCMinute = "minute" + + // ITTCSecond identifies the second component of a Cypher instant value. + ITTCSecond = "second" + + // ITTCMillisecond identifies the millisecond component of a Cypher instant value. + ITTCMillisecond = "millisecond" + + // ITTCMicrosecond identifies the microsecond component of a Cypher instant value. + ITTCMicrosecond = "microsecond" + + // ITTCNanosecond identifies the nanosecond component of a Cypher instant value. + ITTCNanosecond = "nanosecond" + + // ITTCTimeZone identifies the time-zone component of a Cypher instant value. + ITTCTimeZone = "timezone" + + // ITTCEpochSeconds identifies the epoch-seconds component of a Cypher instant value. + ITTCEpochSeconds = "epochseconds" + + // ITTCEpochMilliseconds identifies the epoch-milliseconds component of a Cypher instant value. ITTCEpochMilliseconds = "epochmillis" ) diff --git a/cypher/models/cypher/model.go b/cypher/models/cypher/model.go index b1b22a87..2fc6c842 100644 --- a/cypher/models/cypher/model.go +++ b/cypher/models/cypher/model.go @@ -1306,7 +1306,9 @@ func (s *ProjectionItem) copy() *ProjectionItem { } } +// PropertyLookup represents access to a named property on an expression. type PropertyLookup struct { + // Atom is the expression whose property is accessed. Atom Expression // Symbol is the raw property key, not an already-rendered Cypher token. diff --git a/cypher/models/cypher/property_key.go b/cypher/models/cypher/property_key.go index e39baeab..3cb92f19 100644 --- a/cypher/models/cypher/property_key.go +++ b/cypher/models/cypher/property_key.go @@ -6,20 +6,25 @@ import ( "unicode" ) +// ErrEmptyPropertyKeyName reports that a property-key token decoded to an empty name. var ErrEmptyPropertyKeyName = errors.New("property key name must not be empty") +// isCypherIDStart reports whether char may begin an unescaped Cypher identifier. func isCypherIDStart(char rune) bool { return unicode.IsLetter(char) || unicode.In(char, unicode.Nl, unicode.Other_ID_Start) } +// isCypherIDContinue reports whether char may follow the first rune of an unescaped Cypher identifier. func isCypherIDContinue(char rune) bool { return isCypherIDStart(char) || unicode.In(char, unicode.Mn, unicode.Mc, unicode.Nd, unicode.Pc, unicode.Other_ID_Continue) } +// isCypherSymbolStart reports whether char may begin an unescaped symbolic name, including connector punctuation. func isCypherSymbolStart(char rune) bool { return isCypherIDStart(char) || unicode.In(char, unicode.Pc) } +// isCypherSymbolPart reports whether char may appear after the first rune of an unescaped symbolic name. func isCypherSymbolPart(char rune) bool { return isCypherIDContinue(char) || unicode.In(char, unicode.Sc) } @@ -48,6 +53,7 @@ func CanEmitBarePropertyKeyName(name string) bool { return true } +// ValidatePropertyKeyName rejects empty decoded property-key names. func ValidatePropertyKeyName(name string) error { if name == "" { return ErrEmptyPropertyKeyName diff --git a/cypher/models/cypher/property_key_test.go b/cypher/models/cypher/property_key_test.go index 4ddfd9b2..7d7c2f25 100644 --- a/cypher/models/cypher/property_key_test.go +++ b/cypher/models/cypher/property_key_test.go @@ -7,10 +7,14 @@ import ( "github.com/stretchr/testify/require" ) +// TestCanEmitBarePropertyKeyName verifies the Unicode and punctuation rules for unescaped property keys. func TestCanEmitBarePropertyKeyName(t *testing.T) { testCases := []struct { - name string - input string + // name labels the property-key form under test. + name string + // input is the decoded property-key name. + input string + // expected indicates whether input may be rendered without backticks. expected bool }{ {name: "simple", input: "name", expected: true}, @@ -36,10 +40,14 @@ func TestCanEmitBarePropertyKeyName(t *testing.T) { } } +// TestEscapePropertyKeyName verifies canonical quoting and embedded-backtick escaping for property keys. func TestEscapePropertyKeyName(t *testing.T) { testCases := []struct { - name string - input string + // name labels the property-key form under test. + name string + // input is the decoded property-key name. + input string + // expected is the canonical property-key token. expected string }{ {name: "simple", input: "name", expected: "name"}, @@ -67,15 +75,20 @@ func TestEscapePropertyKeyName(t *testing.T) { } } +// TestValidatePropertyKeyName verifies that only empty decoded property-key names are invalid. func TestValidatePropertyKeyName(t *testing.T) { require.NoError(t, cypher.ValidatePropertyKeyName(" ")) require.ErrorIs(t, cypher.ValidatePropertyKeyName(""), cypher.ErrEmptyPropertyKeyName) } +// TestUnescapePropertyKeyName verifies decoding of quoted keys and doubled backticks. func TestUnescapePropertyKeyName(t *testing.T) { testCases := []struct { - name string - input string + // name labels the property-key token under test. + name string + // input is the rendered property-key token. + input string + // expected is the decoded property-key name. expected string }{ {name: "simple", input: "name", expected: "name"}, diff --git a/cypher/models/pgsql/format/format.go b/cypher/models/pgsql/format/format.go index 9cbed49c..4806b624 100644 --- a/cypher/models/pgsql/format/format.go +++ b/cypher/models/pgsql/format/format.go @@ -8,11 +8,38 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// OutputBuilder accumulates PostgreSQL text with graph targeting and optional parameter materialization. type OutputBuilder struct { + // MaterializeParameters substitutes configured values for parameter references during rendering. MaterializeParameters bool - StripLiterals bool - parameters map[string]any - builder *strings.Builder + // StripLiterals records the requested literal-redaction mode for formatter configuration. + StripLiterals bool + // TargetGraphID selects the concrete graph partitions used to render persistent node and edge references. + TargetGraphID int32 + // parameters contains values substituted when MaterializeParameters is enabled. + parameters map[string]any + // builder accumulates the rendered PostgreSQL text. + builder *strings.Builder +} + +// formatIdentifier preserves the wildcard and quotes names containing characters outside the formatter's unquoted ASCII subset. +func formatIdentifier(identifier pgsql.Identifier) string { + value := identifier.String() + if value == pgsql.WildcardIdentifier.String() { + return value + } + + for idx, character := range value { + valid := character == '_' || character >= 'a' && character <= 'z' || character >= 'A' && character <= 'Z' + if idx > 0 { + valid = valid || character >= '0' && character <= '9' || character == '$' + } + if !valid { + return `"` + strings.ReplaceAll(value, `"`, `""`) + `"` + } + } + + return value } func NewOutputBuilder() *OutputBuilder { @@ -28,6 +55,14 @@ func (s *OutputBuilder) WithMaterializedParameters(parameters map[string]any) *O return s } +// WithTargetGraph renders persistent node and edge references against the +// concrete target partitions. Graph-local IDs are not globally unique, and a +// concrete relation also lets PostgreSQL avoid planning unrelated partitions. +func (s *OutputBuilder) WithTargetGraph(graphID int32) *OutputBuilder { + s.TargetGraphID = graphID + return s +} + func (s *OutputBuilder) HasOutput() bool { return s.builder.Len() != 0 } @@ -51,6 +86,7 @@ func (s *OutputBuilder) Build() string { return s.builder.String() } +// formatSlice writes a typed PostgreSQL array literal from a Go slice. func formatSlice[T any, TS []T](builder *OutputBuilder, slice TS, dataType pgsql.DataType) error { builder.Write("array [") @@ -68,6 +104,7 @@ func formatSlice[T any, TS []T](builder *OutputBuilder, slice TS, dataType pgsql return nil } +// formatValue writes a supported scalar or slice value as a PostgreSQL literal. func formatValue(builder *OutputBuilder, value any) error { switch typedValue := value.(type) { case uint: @@ -134,6 +171,7 @@ func formatValue(builder *OutputBuilder, value any) error { return nil } +// formatLiteral writes a literal value and its explicit PostgreSQL cast when required. func formatLiteral(builder *OutputBuilder, literal pgsql.Literal) error { if literal.Null { builder.Write("null") @@ -148,6 +186,7 @@ func formatLiteral(builder *OutputBuilder, literal pgsql.Literal) error { return formatValue(builder, literal.Value) } +// formatCase validates paired conditions and results before writing a CASE expression in clause order. func formatCase(builder *OutputBuilder, caseExpr pgsql.Case) error { if len(caseExpr.Conditions) != len(caseExpr.Then) { return fmt.Errorf("case expression has %d conditions and %d then expressions", len(caseExpr.Conditions), len(caseExpr.Then)) @@ -190,6 +229,7 @@ func formatCase(builder *OutputBuilder, caseExpr pgsql.Case) error { return nil } +// formatNode dispatches a PostgreSQL syntax node to the formatter for its concrete AST type. func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { exprStack := []pgsql.SyntaxNode{ rootExpr, @@ -254,6 +294,15 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { if !typedNextExpr.Bare { exprStack = append(exprStack, pgsql.FormattingLiteral(")")) } + if len(typedNextExpr.OrderBy) > 0 { + for idx := len(typedNextExpr.OrderBy) - 1; idx >= 0; idx-- { + exprStack = append(exprStack, typedNextExpr.OrderBy[idx]) + if idx > 0 { + exprStack = append(exprStack, pgsql.FormattingLiteral(", ")) + } + } + exprStack = append(exprStack, pgsql.FormattingLiteral(" order by ")) + } for idx := len(typedNextExpr.Parameters) - 1; idx >= 0; idx-- { exprStack = append(exprStack, typedNextExpr.Parameters[idx]) @@ -277,7 +326,7 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { builder.Write(typedNextExpr.String()) case pgsql.Identifier: - builder.Write(typedNextExpr) + builder.Write(formatIdentifier(typedNextExpr)) case pgsql.CompoundIdentifier: for idx := len(typedNextExpr) - 1; idx >= 0; idx-- { @@ -329,7 +378,13 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { exprStack = append(exprStack, typedNextExpr.Binding.Value, pgsql.FormattingLiteral(" ")) } - exprStack = append(exprStack, typedNextExpr.Name) + tableName := typedNextExpr.Name + if builder.TargetGraphID != 0 && len(tableName) == 1 && + (tableName[0] == pgsql.TableNode || tableName[0] == pgsql.TableEdge) { + tableName = pgsql.CompoundIdentifier{pgsql.Identifier(fmt.Sprintf("%s_%d", tableName[0], builder.TargetGraphID))} + } + + exprStack = append(exprStack, tableName) case pgsql.LateralSubquery: if typedNextExpr.Binding.Set { @@ -438,7 +493,7 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { return fmt.Errorf("conflict target has both columns and an 'on constraint' expression set") } - exprStack = append(exprStack, typedNextExpr.Constraint, pgsql.FormattingLiteral("on constraint ")) + exprStack = append(exprStack, pgsql.FormattingLiteral(typedNextExpr.Constraint.String()), pgsql.FormattingLiteral("on constraint ")) } case *pgsql.AliasedExpression: @@ -538,12 +593,23 @@ func formatNode(builder *OutputBuilder, rootExpr pgsql.SyntaxNode) error { return fmt.Errorf("edge array from path IDs has no path expression") } - exprStack = append( - exprStack, - pgsql.FormattingLiteral(") with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id)"), - typedNextExpr.PathIDs, - pgsql.FormattingLiteral("(select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest("), - ) + if typedNextExpr.GraphID == nil { + exprStack = append( + exprStack, + pgsql.FormattingLiteral(") with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id)"), + typedNextExpr.PathIDs, + pgsql.FormattingLiteral("(select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest("), + ) + } else { + exprStack = append( + exprStack, + pgsql.FormattingLiteral(")"), + typedNextExpr.GraphID, + pgsql.FormattingLiteral(") with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = "), + typedNextExpr.PathIDs, + pgsql.FormattingLiteral("(select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest("), + ) + } case pgsql.Parameter: if builder.MaterializeParameters { @@ -619,6 +685,7 @@ func Expression(expression pgsql.SyntaxNode, builder *OutputBuilder) (string, er return builder.Build(), nil } +// formatSelect writes a SELECT expression with its projection, sources, predicates, grouping, and ordering. func formatSelect(builder *OutputBuilder, selectStmt pgsql.Select) error { builder.Write("select ") @@ -660,9 +727,18 @@ func formatSelect(builder *OutputBuilder, selectStmt pgsql.Select) error { } } + if selectStmt.Having != nil { + builder.Write(" having ") + + if err := formatNode(builder, selectStmt.Having); err != nil { + return err + } + } + return nil } +// formatGroupBy writes comma-separated GROUP BY expressions when grouping is present. func formatGroupBy(builder *OutputBuilder, groupByExpressions []pgsql.Expression) error { for idx, groupByExpression := range groupByExpressions { if idx > 0 { @@ -677,6 +753,7 @@ func formatGroupBy(builder *OutputBuilder, groupByExpressions []pgsql.Expression return nil } +// formatFromClauses writes comma-separated FROM sources and their joins. func formatFromClauses(builder *OutputBuilder, fromClauses []pgsql.FromClause) error { for idx, fromClause := range fromClauses { if idx > 0 { @@ -726,6 +803,7 @@ func formatFromClauses(builder *OutputBuilder, fromClauses []pgsql.FromClause) e return nil } +// formatTableAlias writes an alias and its optional record-shape column list. func formatTableAlias(builder *OutputBuilder, tableAlias pgsql.TableAlias) error { builder.Write(tableAlias.Name) @@ -748,6 +826,7 @@ func formatTableAlias(builder *OutputBuilder, tableAlias pgsql.TableAlias) error return nil } +// formatCommonTableExpressions writes a WITH clause and each materialization-qualified CTE. func formatCommonTableExpressions(builder *OutputBuilder, commonTableExpressions pgsql.With) error { // Only write "with" if there are actually expressions if len(commonTableExpressions.Expressions) == 0 { @@ -794,6 +873,7 @@ func formatCommonTableExpressions(builder *OutputBuilder, commonTableExpressions return nil } +// formatSetExpression dispatches rendering for SELECT, nested query, values, and set-operation operands. func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) error { switch typedSetExpression := expression.(type) { case pgsql.Query: @@ -850,7 +930,7 @@ func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) return fmt.Errorf("set operation for query may not be both ALL and DISTINCT") } - if err := formatSetExpression(builder, typedSetExpression.LOperand); err != nil { + if err := formatSetOperationOperand(builder, typedSetExpression.LOperand); err != nil { return err } @@ -870,7 +950,7 @@ func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) builder.Write("distinct ") } - if err := formatSetExpression(builder, typedSetExpression.ROperand); err != nil { + if err := formatSetOperationOperand(builder, typedSetExpression.ROperand); err != nil { return err } @@ -890,6 +970,20 @@ func formatSetExpression(builder *OutputBuilder, expression pgsql.SetExpression) return nil } +// formatSetOperationOperand parenthesizes query operands so their WITH, ORDER BY, and limits remain scoped to the operand. +func formatSetOperationOperand(builder *OutputBuilder, operand pgsql.SetExpression) error { + if _, isQuery := operand.(pgsql.Query); !isQuery { + return formatSetExpression(builder, operand) + } + builder.Write("(") + if err := formatSetExpression(builder, operand); err != nil { + return err + } + builder.Write(")") + return nil +} + +// formatMergeStatement writes a MERGE statement with matched and unmatched actions. func formatMergeStatement(builder *OutputBuilder, merge pgsql.Merge) error { builder.Write("merge ") @@ -999,6 +1093,7 @@ func formatMergeStatement(builder *OutputBuilder, merge pgsql.Merge) error { return nil } +// formatInsertStatement writes an INSERT source, conflict action, and optional RETURNING projection. func formatInsertStatement(builder *OutputBuilder, insert pgsql.Insert) error { builder.Write("insert into ") @@ -1061,6 +1156,7 @@ func formatInsertStatement(builder *OutputBuilder, insert pgsql.Insert) error { return nil } +// formatUpdateStatement writes an UPDATE target, assignments, sources, predicate, and optional RETURNING projection. func formatUpdateStatement(builder *OutputBuilder, update pgsql.Update) error { builder.Write("update ") @@ -1113,6 +1209,7 @@ func formatUpdateStatement(builder *OutputBuilder, update pgsql.Update) error { return nil } +// formatDeleteStatement writes a DELETE target, USING sources, predicate, and optional RETURNING projection. func formatDeleteStatement(builder *OutputBuilder, sqlDelete pgsql.Delete) error { builder.Write("delete from ") diff --git a/cypher/models/pgsql/format/format_test.go b/cypher/models/pgsql/format/format_test.go index 86b9629d..22ebcdec 100644 --- a/cypher/models/pgsql/format/format_test.go +++ b/cypher/models/pgsql/format/format_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" ) +// mustAsLiteral converts value to a PostgreSQL literal and panics if the value type is unsupported. func mustAsLiteral(value any) pgsql.Literal { if literal, err := pgsql.AsLiteral(value); err != nil { panic(fmt.Sprintf("%v", err)) @@ -26,6 +27,17 @@ func TestFormat_TypeCastedParenthetical(t *testing.T) { require.Equal(t, "('str')::text", formattedQuery) } +// TestFormat_QuotesExpressionShapedIdentifiers verifies that identifier text resembling an expression remains an identifier. +func TestFormat_QuotesExpressionShapedIdentifiers(t *testing.T) { + formatted, err := format.Expression( + pgsql.CompoundIdentifier{"s0", "id(n)"}, + format.NewOutputBuilder(), + ) + + require.NoError(t, err) + require.Equal(t, `s0."id(n)"`, formatted) +} + func TestFormat_Case(t *testing.T) { formattedQuery, err := format.Expression(pgsql.Case{ Conditions: []pgsql.Expression{ @@ -70,6 +82,26 @@ func TestFormat_SelectDistinct(t *testing.T) { require.Equal(t, "select distinct id from node;", formattedQuery) } +func TestFormat_SelectHaving(t *testing.T) { + formattedQuery, err := format.Statement(pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Identifier("depth")}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: pgsql.CompoundIdentifier{"frontier"}}, + }}, + GroupBy: []pgsql.Expression{pgsql.Identifier("depth")}, + Having: pgsql.NewBinaryExpression( + pgsql.FunctionCall{Function: pgsql.FunctionCount, Parameters: []pgsql.Expression{pgsql.Wildcard{}}}, + pgsql.OperatorGreaterThan, + pgsql.NewLiteral(int64(100), pgsql.Int8), + ), + }, + }, format.NewOutputBuilder()) + + require.NoError(t, err) + require.Equal(t, "select depth from frontier group by depth having count(*) > 100;", formattedQuery) +} + func TestFormat_LateralSubqueryJoin(t *testing.T) { formattedQuery, err := format.Statement(pgsql.Query{ Body: pgsql.Select{ @@ -118,6 +150,27 @@ func TestFormat_LateralSubqueryJoin(t *testing.T) { require.Equal(t, "select n.id, e.id from node n join lateral (select e.id from edge e where e.start_id = n.id offset 0) e on true;", formattedQuery) } +// TestFormat_FunctionAggregateOrderBy verifies that aggregate input ordering renders inside the function call. +func TestFormat_FunctionAggregateOrderBy(t *testing.T) { + formattedQuery, err := format.Statement(pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{"edge", "id"}}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.Identifier("ordinality"), + Ascending: true, + }}, + }, + }, + }, + }, format.NewOutputBuilder()) + + require.NoError(t, err) + require.Equal(t, "select array_agg(edge.id order by ordinality);", formattedQuery) +} + func TestFormat_Delete(t *testing.T) { formattedQuery, err := format.Statement(pgsql.Delete{ From: []pgsql.TableReference{{ @@ -664,6 +717,44 @@ func TestFormat_CTEs(t *testing.T) { require.Equal(t, "with recursive expansion_1(root_id, next_id, depth, stop, is_cycle, path) as materialized (select r.start_id, r.end_id, 1, false, r.start_id = r.end_id, array [r.id] from edge r join node a on a.id = r.start_id where a.kind_ids operator (pg_catalog.&&) array [23]::int2[] union all select expansion_1.root_id, r.end_id, expansion_1.depth + 1, b.kind_ids operator (pg_catalog.&&) array [24]::int2[], r.id = any(expansion_1.path), expansion_1.path || r.id from expansion_1 join edge r on r.start_id = expansion_1.next_id join node b on b.id = r.end_id where not expansion_1.is_cycle and not expansion_1.stop) select a.properties, b.properties from expansion_1 join node a on a.id = expansion_1.root_id join node b on b.id = expansion_1.next_id where not expansion_1.is_cycle and expansion_1.stop;", formattedQuery) } +// TestFormat_SetOperationParenthesizesQueryOperand verifies that a query operand retains its WITH clause under a set operation. +func TestFormat_SetOperationParenthesizesQueryOperand(t *testing.T) { + formattedQuery, err := format.Statement(pgsql.Query{ + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: pgsql.Select{ + Projection: pgsql.Projection{mustAsLiteral(1)}, + }, + ROperand: pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Expressions: []pgsql.CommonTableExpression{{ + Alias: pgsql.TableAlias{ + Name: "value", + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{mustAsLiteral(2)}, + }, + }, + }}, + }, + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: pgsql.CompoundIdentifier{"value"}, + }, + }}, + }, + }, + }, + }, format.NewOutputBuilder()) + + require.NoError(t, err) + require.Equal(t, "select 1 union all (with value as (select 2) select * from value);", formattedQuery) +} + func TestFormat_QueryInjection(t *testing.T) { query := pgsql.Query{ Body: pgsql.Select{ diff --git a/cypher/models/pgsql/functions.go b/cypher/models/pgsql/functions.go index 3d0b5f4e..32c5c73c 100644 --- a/cypher/models/pgsql/functions.go +++ b/cypher/models/pgsql/functions.go @@ -1,57 +1,185 @@ package pgsql const ( - FunctionUnidirectionalASPHarness Identifier = "unidirectional_asp_harness" - FunctionUnidirectionalSPHarness Identifier = "unidirectional_sp_harness" - FunctionBidirectionalASPHarness Identifier = "bidirectional_asp_harness" - FunctionBidirectionalSPHarness Identifier = "bidirectional_sp_harness" + // FunctionUnidirectionalASPHarness identifies the SQL harness for unidirectional all-shortest-path search. + FunctionUnidirectionalASPHarness Identifier = "unidirectional_asp_harness" + + // FunctionUnidirectionalSPHarness identifies the SQL harness for unidirectional single-shortest-path search. + FunctionUnidirectionalSPHarness Identifier = "unidirectional_sp_harness" + + // FunctionBidirectionalASPHarness identifies the SQL harness for bidirectional all-shortest-path search. + FunctionBidirectionalASPHarness Identifier = "bidirectional_asp_harness" + + // FunctionBidirectionalSPHarness identifies the SQL harness for bidirectional single-shortest-path search. + FunctionBidirectionalSPHarness Identifier = "bidirectional_sp_harness" + + // FunctionAllShortestPathsDAG identifies the SQL helper that materializes every shortest path from a predecessor DAG. + FunctionAllShortestPathsDAG Identifier = "all_shortest_paths_dag" + + // FunctionAllShortestPathsNoPathProbe identifies the bounded target-side + // reachability preflight. It returns an exact empty result only after the + // reverse component is exhausted; every positive or capped probe delegates + // to the A1 predecessor-DAG helper. + FunctionAllShortestPathsNoPathProbe Identifier = "all_shortest_paths_no_path_probe" + + // FunctionShortestPathCompact identifies the SQL helper that materializes one compact shortest-path witness. + FunctionShortestPathCompact Identifier = "shortest_path_compact" + + // FunctionShortestPathB1StrictAlternating identifies compact bidirectional search with strict node alternation. + FunctionShortestPathB1StrictAlternating Identifier = "shortest_path_b1_strict_alternating" + + // FunctionShortestPathB2SmallerCurrentLevel identifies compact bidirectional search that expands the smaller current level. + FunctionShortestPathB2SmallerCurrentLevel Identifier = "shortest_path_b2_smaller_current_level" + + // FunctionAllShortestPathsB1StrictAlternating identifies two-sided predecessor-DAG enumeration with strict node alternation. + FunctionAllShortestPathsB1StrictAlternating Identifier = "all_shortest_paths_b1_strict_alternating" + + // FunctionAllShortestPathsB2SmallerCurrentLevel identifies two-sided predecessor-DAG enumeration that expands the smaller current level. + FunctionAllShortestPathsB2SmallerCurrentLevel Identifier = "all_shortest_paths_b2_smaller_current_level" + + // FunctionShortestPathSelfEndpointError identifies the SQL helper that raises an invalid self-endpoint error. FunctionShortestPathSelfEndpointError Identifier = "shortest_path_self_endpoint_error" - FunctionIntArrayUnique Identifier = "uniq" - FunctionIntArraySort Identifier = "sort" - FunctionJSONBToTextArray Identifier = "jsonb_to_text_array" - FunctionJSONBArrayElementsText Identifier = "jsonb_array_elements_text" - FunctionJSONBBuildObject Identifier = "jsonb_build_object" - FunctionJSONBArrayLength Identifier = "jsonb_array_length" - FunctionJSONBTypeof Identifier = "jsonb_typeof" - FunctionToJSONB Identifier = "to_jsonb" - FunctionCypherContains Identifier = "cypher_contains" - FunctionCypherStartsWith Identifier = "cypher_starts_with" - FunctionCypherEndsWith Identifier = "cypher_ends_with" - FunctionCypherMin Identifier = "cypher_min" - FunctionCypherMax Identifier = "cypher_max" - FunctionArrayLength Identifier = "array_length" - FunctionCardinality Identifier = "cardinality" - FunctionArrayAggregate Identifier = "array_agg" - FunctionArrayRemove Identifier = "array_remove" - FunctionMin Identifier = "min" - FunctionMax Identifier = "max" - FunctionSum Identifier = "sum" - FunctionAvg Identifier = "avg" - FunctionLocalTimestamp Identifier = "localtimestamp" - FunctionLocalTime Identifier = "localtime" - FunctionCurrentTime Identifier = "current_time" - FunctionCurrentDate Identifier = "current_date" - FunctionNow Identifier = "now" - FunctionToLower Identifier = "lower" - FunctionToUpper Identifier = "upper" - FunctionCoalesce Identifier = "coalesce" - FunctionReplace Identifier = "replace" - FunctionUnnest Identifier = "unnest" - FunctionNextValue Identifier = "nextval" - FunctionPGGetSerialSequence Identifier = "pg_get_serial_sequence" - FunctionJSONBSet Identifier = "jsonb_set" - FunctionCount Identifier = "count" - FunctionStringToArray Identifier = "string_to_array" - FunctionEdgesToPath Identifier = "edges_to_path" - FunctionOrderedEdgesToPath Identifier = "ordered_edges_to_path" - FunctionNodesToPath Identifier = "nodes_to_path" - FunctionKindName Identifier = "kind_name" - FunctionStartNode Identifier = "start_node" - FunctionEndNode Identifier = "end_node" - FunctionExtract Identifier = "extract" - FunctionGenerateSubscripts Identifier = "generate_subscripts" + + // FunctionIntArrayUnique identifies the SQL helper that removes duplicate integer-array values. + FunctionIntArrayUnique Identifier = "uniq" + + // FunctionIntArraySort identifies the SQL helper that orders integer-array values. + FunctionIntArraySort Identifier = "sort" + + // FunctionJSONBToTextArray identifies the SQL helper that converts a JSONB array to text[]. + FunctionJSONBToTextArray Identifier = "jsonb_to_text_array" + + // FunctionJSONBArrayElementsText identifies PostgreSQL's JSONB array-element text expansion function. + FunctionJSONBArrayElementsText Identifier = "jsonb_array_elements_text" + + // FunctionJSONBBuildObject identifies PostgreSQL's JSONB object constructor. + FunctionJSONBBuildObject Identifier = "jsonb_build_object" + + // FunctionJSONBArrayLength identifies PostgreSQL's JSONB array-length function. + FunctionJSONBArrayLength Identifier = "jsonb_array_length" + + // FunctionJSONBTypeof identifies PostgreSQL's JSONB type-inspection function. + FunctionJSONBTypeof Identifier = "jsonb_typeof" + + // FunctionToJSONB identifies PostgreSQL's conversion to JSONB. + FunctionToJSONB Identifier = "to_jsonb" + + // FunctionCypherContains identifies the SQL helper implementing Cypher CONTAINS semantics. + FunctionCypherContains Identifier = "cypher_contains" + + // FunctionCypherStartsWith identifies the SQL helper implementing Cypher STARTS WITH semantics. + FunctionCypherStartsWith Identifier = "cypher_starts_with" + + // FunctionCypherEndsWith identifies the SQL helper implementing Cypher ENDS WITH semantics. + FunctionCypherEndsWith Identifier = "cypher_ends_with" + + // FunctionCypherMin identifies the SQL aggregate implementing Cypher minimum semantics. + FunctionCypherMin Identifier = "cypher_min" + + // FunctionCypherMax identifies the SQL aggregate implementing Cypher maximum semantics. + FunctionCypherMax Identifier = "cypher_max" + + // FunctionArrayLength identifies PostgreSQL's dimension-aware array-length function. + FunctionArrayLength Identifier = "array_length" + + // FunctionCardinality identifies PostgreSQL's total array-element count function. + FunctionCardinality Identifier = "cardinality" + + // FunctionArrayAggregate identifies PostgreSQL's array aggregation function. + FunctionArrayAggregate Identifier = "array_agg" + + // FunctionArrayRemove identifies PostgreSQL's array element-removal function. + FunctionArrayRemove Identifier = "array_remove" + + // FunctionMin identifies PostgreSQL's minimum aggregate. + FunctionMin Identifier = "min" + + // FunctionMax identifies PostgreSQL's maximum aggregate. + FunctionMax Identifier = "max" + + // FunctionSum identifies PostgreSQL's sum aggregate. + FunctionSum Identifier = "sum" + + // FunctionAvg identifies PostgreSQL's average aggregate. + FunctionAvg Identifier = "avg" + + // FunctionLocalTimestamp identifies PostgreSQL's local timestamp constructor. + FunctionLocalTimestamp Identifier = "localtimestamp" + + // FunctionLocalTime identifies PostgreSQL's local time constructor. + FunctionLocalTime Identifier = "localtime" + + // FunctionCurrentTime identifies PostgreSQL's current zoned time value. + FunctionCurrentTime Identifier = "current_time" + + // FunctionCurrentDate identifies PostgreSQL's current date value. + FunctionCurrentDate Identifier = "current_date" + + // FunctionNow identifies PostgreSQL's current transaction timestamp function. + FunctionNow Identifier = "now" + + // FunctionToLower identifies PostgreSQL's lowercase text function. + FunctionToLower Identifier = "lower" + + // FunctionToUpper identifies PostgreSQL's uppercase text function. + FunctionToUpper Identifier = "upper" + + // FunctionCoalesce identifies PostgreSQL's first-non-null expression. + FunctionCoalesce Identifier = "coalesce" + + // FunctionNullIf identifies PostgreSQL's NULLIF function for nulling matching scalar values. + FunctionNullIf Identifier = "nullif" + + // FunctionReplace identifies PostgreSQL's substring-replacement function. + FunctionReplace Identifier = "replace" + + // FunctionUnnest identifies PostgreSQL's array-to-row expansion function. + FunctionUnnest Identifier = "unnest" + + // FunctionNextValue identifies PostgreSQL's sequence increment function. + FunctionNextValue Identifier = "nextval" + + // FunctionPGGetSerialSequence identifies PostgreSQL's serial-sequence lookup function. + FunctionPGGetSerialSequence Identifier = "pg_get_serial_sequence" + + // FunctionJSONBSet identifies PostgreSQL's JSONB path-update function. + FunctionJSONBSet Identifier = "jsonb_set" + + // FunctionCount identifies PostgreSQL's count aggregate. + FunctionCount Identifier = "count" + + // FunctionStringToArray identifies PostgreSQL's delimiter-based text-to-array function. + FunctionStringToArray Identifier = "string_to_array" + + // FunctionEdgesToPath identifies the SQL helper that builds a path from unordered edge composites. + FunctionEdgesToPath Identifier = "edges_to_path" + + // FunctionOrderedEdgesToPath identifies the SQL helper that builds a path from ordered edge composites. + FunctionOrderedEdgesToPath Identifier = "ordered_edges_to_path" + + // FunctionOrderedEdgeIDsToPath identifies the SQL helper that hydrates an ordered edge-ID array into a path. + FunctionOrderedEdgeIDsToPath Identifier = "ordered_edge_ids_to_path" + + // FunctionNodesToPath identifies the SQL helper that builds a path from ordered node composites. + FunctionNodesToPath Identifier = "nodes_to_path" + + // FunctionKindName identifies the SQL helper that resolves a kind ID to its name. + FunctionKindName Identifier = "kind_name" + + // FunctionStartNode identifies the SQL helper that hydrates a relationship's start node. + FunctionStartNode Identifier = "start_node" + + // FunctionEndNode identifies the SQL helper that hydrates a relationship's end node. + FunctionEndNode Identifier = "end_node" + + // FunctionExtract identifies PostgreSQL's temporal component-extraction function. + FunctionExtract Identifier = "extract" + + // FunctionGenerateSubscripts identifies PostgreSQL's array-index generation function. + FunctionGenerateSubscripts Identifier = "generate_subscripts" ) +// IsAggregateFunction supports benchmark evidence processing for is aggregate function. func IsAggregateFunction(function Identifier) bool { switch function { case FunctionCount, FunctionArrayAggregate, FunctionMin, FunctionMax, FunctionCypherMin, FunctionCypherMax, FunctionSum, FunctionAvg: diff --git a/cypher/models/pgsql/model.go b/cypher/models/pgsql/model.go index 5c7096f4..cddcefd8 100644 --- a/cypher/models/pgsql/model.go +++ b/cypher/models/pgsql/model.go @@ -404,8 +404,12 @@ func (s *Parenthetical) AsExpression() Expression { return s } +// EdgeArrayFromPathIDs hydrates edge composites from a path's ordered edge identifiers. type EdgeArrayFromPathIDs struct { + // PathIDs is the ordered edge-ID array to hydrate. PathIDs Expression + // GraphID identifies the graph whose edge IDs are hydrated into edge composites. + GraphID Expression } func (s *EdgeArrayFromPathIDs) NodeType() string { @@ -419,9 +423,16 @@ func (s *EdgeArrayFromPathIDs) AsExpression() Expression { type JoinType int const ( + // JoinTypeInner retains rows that satisfy the join constraint on both sides. JoinTypeInner JoinType = iota + + // JoinTypeLeftOuter retains every left row even when no right row matches. JoinTypeLeftOuter + + // JoinTypeRightOuter retains every right row even when no left row matches. JoinTypeRightOuter + + // JoinTypeFullOuter retains unmatched rows from both sides. JoinTypeFullOuter ) @@ -456,16 +467,26 @@ func (s OrderBy) NodeType() string { type WindowFrameUnit int const ( + // WindowFrameUnitRows measures frame boundaries in physical rows. WindowFrameUnitRows WindowFrameUnit = iota + + // WindowFrameUnitRange measures frame boundaries by ordering-key value ranges. WindowFrameUnitRange + + // WindowFrameUnitGroups measures frame boundaries in peer groups. WindowFrameUnitGroups ) type WindowFrameBoundaryType int const ( + // WindowFrameBoundaryTypeCurrentRow anchors a window boundary at the current row or peer group. WindowFrameBoundaryTypeCurrentRow WindowFrameBoundaryType = iota + + // WindowFrameBoundaryTypePreceding places a window boundary before the current row. WindowFrameBoundaryTypePreceding + + // WindowFrameBoundaryTypeFollowing places a window boundary after the current row. WindowFrameBoundaryTypeFollowing ) @@ -568,13 +589,22 @@ func AsParameter(identifier Identifier, value any) (*Parameter, error) { return parameter, nil } +// FunctionCall represents a PostgreSQL function invocation and its aggregate or window options. type FunctionCall struct { - Bare bool - Distinct bool - Function Identifier + // Bare omits the usual argument parentheses for SQL keyword-like functions. + Bare bool + // Distinct deduplicates argument rows before aggregate evaluation. + Distinct bool + // Function identifies the PostgreSQL function to invoke. + Function Identifier + // Parameters contains the function arguments in call order. Parameters []Expression - Over *Window - CastType DataType + // OrderBy orders aggregate inputs before the function is evaluated. + OrderBy []*OrderBy + // Over supplies the window specification for a window-function call. + Over *Window + // CastType records the function result type known to the translator. + CastType DataType } func (s FunctionCall) AsAssignment() Assignment { diff --git a/cypher/models/pgsql/optimize/analysis_test.go b/cypher/models/pgsql/optimize/analysis_test.go index e55dfab7..95eb37f1 100644 --- a/cypher/models/pgsql/optimize/analysis_test.go +++ b/cypher/models/pgsql/optimize/analysis_test.go @@ -21,6 +21,20 @@ AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p1, p2 ` +// fixedSuffixExpansionQuery exercises one variable expansion followed by a three-edge typed suffix. +const fixedSuffixExpansionQuery = ` +MATCH (root:ExpansionRoot) +WHERE root.root_key = 'root' +MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) +MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) +WHERE predicate.eligible = true +AND predicate.requires_review = false +AND predicate.allows_direct = true +AND (predicate.version = 1 OR predicate.required_approvals = 0) +RETURN p1, p2 +` + +// analyzeCypher parses query, runs optimizer analysis, and requires both stages to succeed. func analyzeCypher(t *testing.T, query string) Analysis { t.Helper() @@ -30,6 +44,7 @@ func analyzeCypher(t *testing.T, query string) Analysis { return Analyze(regularQuery) } +// requireBinding requires an analyzed binding with the expected symbol and kind. func requireBinding(t *testing.T, bindings []Binding, symbol string, kind BindingKind) { t.Helper() @@ -42,6 +57,7 @@ func requireBinding(t *testing.T, bindings []Binding, symbol string, kind Bindin t.Fatalf("expected binding %s:%s in %#v", symbol, kind, bindings) } +// requirePathVariable requires a path variable with the expected relationship count and range shape. func requirePathVariable(t *testing.T, pathVariables []PathVariable, symbol string, relationshipCount int, expectedVariableLength bool) { t.Helper() @@ -56,10 +72,11 @@ func requirePathVariable(t *testing.T, pathVariables []PathVariable, symbol stri t.Fatalf("expected path variable %s in %#v", symbol, pathVariables) } -func TestAnalyzeIdentifiesEligibleADCSRegion(t *testing.T) { +// TestAnalyzeIdentifiesEligibleFixedSuffixExpansionRegion verifies that analysis isolates the variable expansion and its fixed suffix. +func TestAnalyzeIdentifiesEligibleFixedSuffixExpansionRegion(t *testing.T) { t.Parallel() - analysis := analyzeCypher(t, adcsQuery) + analysis := analyzeCypher(t, fixedSuffixExpansionQuery) require.Len(t, analysis.QueryParts, 1) @@ -77,13 +94,13 @@ func TestAnalyzeIdentifiesEligibleADCSRegion(t *testing.T) { require.Len(t, region.Clauses, 3) require.Len(t, region.BindingOccurrences, 10) require.Len(t, region.Predicates, 2) - require.Equal(t, []string{"n"}, region.Predicates[0].Dependencies) - require.Equal(t, []string{"ct"}, region.Predicates[1].Dependencies) + require.Equal(t, []string{"root"}, region.Predicates[0].Dependencies) + require.Equal(t, []string{"predicate"}, region.Predicates[1].Dependencies) - requireBinding(t, region.Bindings, "n", BindingKindNode) - requireBinding(t, region.Bindings, "ca", BindingKindNode) - requireBinding(t, region.Bindings, "ct", BindingKindNode) - requireBinding(t, region.Bindings, "d", BindingKindNode) + requireBinding(t, region.Bindings, "root", BindingKindNode) + requireBinding(t, region.Bindings, "head", BindingKindNode) + requireBinding(t, region.Bindings, "predicate", BindingKindNode) + requireBinding(t, region.Bindings, "terminal", BindingKindNode) requireBinding(t, region.Bindings, "p1", BindingKindPath) requireBinding(t, region.Bindings, "p2", BindingKindPath) @@ -132,18 +149,19 @@ func TestAnalyzeSegmentsRegionsAtSemanticBarriers(t *testing.T) { require.Equal(t, []string{"m"}, secondPart.ProjectionDependencies) } +// TestAnalysisDiagnosticsAreStable verifies that diagnostic ordering and query coordinates remain deterministic. func TestAnalysisDiagnosticsAreStable(t *testing.T) { t.Parallel() var ( - analysis = analyzeCypher(t, adcsQuery) + analysis = analyzeCypher(t, fixedSuffixExpansionQuery) diagnostics = strings.Join(analysis.Diagnostics(), "\n") ) require.Contains(t, diagnostics, "query_part[0] kind=single projection_deps=p1,p2") require.Contains(t, diagnostics, "region[0] part=0 clauses=0..2 matches=3") - require.Contains(t, diagnostics, "bindings=n:node,p1:path,ca:node,d:node,p2:path,ct:node") + require.Contains(t, diagnostics, "bindings=root:node,p1:path,head:node,terminal:node,p2:path,predicate:node") require.Contains(t, diagnostics, "paths=p1,p2") - require.Contains(t, diagnostics, "predicates=n,ct") + require.Contains(t, diagnostics, "predicates=root,predicate") require.Contains(t, diagnostics, "barrier[0] part=0 clause=3 kind=return deps=p1,p2") } diff --git a/cypher/models/pgsql/optimize/expansion_orientation.go b/cypher/models/pgsql/optimize/expansion_orientation.go new file mode 100644 index 00000000..a46209dc --- /dev/null +++ b/cypher/models/pgsql/optimize/expansion_orientation.go @@ -0,0 +1,127 @@ +package optimize + +// contiguousExpansionOrientationCandidate contains the common typed metadata +// for one variable expansion and an adjacent fixed seed region. Prefix- and +// suffix-specific analyzers retain their own correctness facts and fallback +// reasons, then use this type to produce a consistent public decision. +type contiguousExpansionOrientationCandidate struct { + // Target supplies the target input to the contiguousExpansionOrientationCandidate contract. + Target TraversalStepTarget + // Family supplies the family input to the contiguousExpansionOrientationCandidate contract. + Family string + // PlannedPolicy identifies the planned policy. + PlannedPolicy ExpansionSearchPolicy + // EmittedPolicy identifies the emitted policy. + EmittedPolicy ExpansionSearchPolicy + // PlannedCandidates supplies the planned candidates input to the contiguousExpansionOrientationCandidate contract. + PlannedCandidates []ExpansionSearchStrategy + // EmittedCandidates supplies the emitted candidates input to the contiguousExpansionOrientationCandidate contract. + EmittedCandidates []ExpansionSearchStrategy + // CandidateStrategy supplies the candidate strategy input to the contiguousExpansionOrientationCandidate contract. + CandidateStrategy ExpansionSearchStrategy + // ProbeCaps supplies the probe caps input to the contiguousExpansionOrientationCandidate contract. + ProbeCaps ExpansionSearchProbeCaps + // Admission supplies the admission input to the contiguousExpansionOrientationCandidate contract. + Admission ExpansionSearchAdmission + // PrefixStartStep supplies the prefix start step input to the contiguousExpansionOrientationCandidate contract. + PrefixStartStep int + // PrefixEndStep supplies the prefix end step input to the contiguousExpansionOrientationCandidate contract. + PrefixEndStep int + // PrefixLength supplies the prefix length input to the contiguousExpansionOrientationCandidate contract. + PrefixLength int + // SuffixStartStep supplies the suffix start step input to the contiguousExpansionOrientationCandidate contract. + SuffixStartStep int + // SuffixEndStep supplies the suffix end step input to the contiguousExpansionOrientationCandidate contract. + SuffixEndStep int + // SuffixLength supplies the suffix length input to the contiguousExpansionOrientationCandidate contract. + SuffixLength int + // SeedPredicateClass supplies the seed predicate class input to the contiguousExpansionOrientationCandidate contract. + SeedPredicateClass string + // EndpointLimit supplies the endpoint limit input to the contiguousExpansionOrientationCandidate contract. + EndpointLimit int64 +} + +// contiguousExpansionOrientationQualification contains analysis results that +// remain specific to the fixed-prefix or fixed-suffix correctness envelope. +type contiguousExpansionOrientationQualification struct { + // SelectedStrategy supplies the selected strategy input to the contiguousExpansionOrientationQualification contract. + SelectedStrategy ExpansionSearchStrategy + // StructurallyEligible indicates whether structurally eligible applies. + StructurallyEligible bool + // StaticallyEligible indicates whether statically eligible applies. + StaticallyEligible bool + // EligibilityFacts supplies the eligibility facts input to the contiguousExpansionOrientationQualification contract. + EligibilityFacts []ExpansionSearchEligibilityFact + // HasFinalLimit indicates whether has final limit applies. + HasFinalLimit bool + // ObservationMode identifies the observation mode. + ObservationMode ExpansionSearchObservationMode + // LogicalDirection supplies the logical direction input to the contiguousExpansionOrientationQualification contract. + LogicalDirection string + // MinimumDepth sets the inclusive lower traversal-depth bound. + MinimumDepth int64 + // MaximumDepth sets the inclusive upper traversal-depth bound. + MaximumDepth int64 + // SelectionMode identifies the selection mode. + SelectionMode string + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string + // FallbackReason supplies the fallback reason input to the contiguousExpansionOrientationQualification contract. + FallbackReason string +} + +// decision combines common orientation metadata with family-specific +// qualification without conflating a planned policy with emitted SQL. +func (s contiguousExpansionOrientationCandidate) decision(qualification contiguousExpansionOrientationQualification) ExpansionSearchStrategyDecision { + return ExpansionSearchStrategyDecision{ + Target: s.Target, + Family: s.Family, + PlannedPolicy: s.PlannedPolicy, + EmittedPolicy: s.EmittedPolicy, + PlannedCandidates: s.PlannedCandidates, + EmittedCandidates: s.EmittedCandidates, + CandidateStrategy: s.CandidateStrategy, + SelectedStrategy: qualification.SelectedStrategy, + StructurallyEligible: qualification.StructurallyEligible, + StaticallyEligible: qualification.StaticallyEligible, + EligibilityFacts: qualification.EligibilityFacts, + ProbeCaps: s.ProbeCaps, + Admission: s.Admission, + SuffixStartStep: s.SuffixStartStep, + SuffixEndStep: s.SuffixEndStep, + SuffixLength: s.SuffixLength, + PrefixStartStep: s.PrefixStartStep, + PrefixEndStep: s.PrefixEndStep, + PrefixLength: s.PrefixLength, + SeedPredicateClass: s.SeedPredicateClass, + EndpointLimit: s.EndpointLimit, + StateLimit: s.Admission.StateLimit, + HasFinalLimit: qualification.HasFinalLimit, + ObservationMode: qualification.ObservationMode, + LogicalDirection: qualification.LogicalDirection, + MinimumDepth: qualification.MinimumDepth, + MaximumDepth: qualification.MaximumDepth, + SelectionMode: qualification.SelectionMode, + SelectorVersion: qualification.SelectorVersion, + FallbackStrategy: s.Admission.FallbackStrategy, + FallbackReason: qualification.FallbackReason, + } +} + +// setExpansionSearchExpectedEmission keeps compile-time emission metadata in +// sync after statement-wide safety and observation checks change selection. +// It describes statement shape only; execution telemetry records the arm that +// actually ran. +func setExpansionSearchExpectedEmission(decision *ExpansionSearchStrategyDecision) { + decision.EmittedPolicy = "" + decision.EmittedCandidates = []ExpansionSearchStrategy{decision.SelectedStrategy} + decision.ExecutionBoundary = ExpansionSearchExecutionBoundaryInlineStatement + if decision.SelectedStrategy == ExpansionSearchEndpointSeededReverse && decision.StructurallyEligible { + decision.EmittedPolicy = ExpansionSearchPolicyEndpointGuardV1 + decision.EmittedCandidates = []ExpansionSearchStrategy{ + ExpansionSearchStepwiseForward, + ExpansionSearchEndpointSeededReverse, + } + decision.ExecutionBoundary = ExpansionSearchExecutionBoundaryGuardedDualArm + } +} diff --git a/cypher/models/pgsql/optimize/lowering.go b/cypher/models/pgsql/optimize/lowering.go index 20b3b2dc..d709604c 100644 --- a/cypher/models/pgsql/optimize/lowering.go +++ b/cypher/models/pgsql/optimize/lowering.go @@ -6,34 +6,85 @@ import ( ) const ( - LoweringProjectionPruning = "ProjectionPruning" - LoweringLatePathMaterialization = "LatePathMaterialization" - LoweringExpandIntoDetection = "ExpandIntoDetection" - LoweringTraversalDirection = "TraversalDirectionSelection" - LoweringShortestPathStrategy = "ShortestPathStrategySelection" - LoweringShortestPathFilter = "ShortestPathFilterMaterialization" - LoweringLimitPushdown = "LimitPushdown" - LoweringExpansionSuffixPushdown = "ExpansionSuffixPushdown" - LoweringPredicatePlacement = "PredicatePlacement" - LoweringCountStoreFastPath = "CountStoreFastPath" - LoweringCollectIDMembership = "CollectIDMembership" - LoweringAggregateTraversalCount = "AggregateTraversalCount" - LoweringExactRangeExpansion = "ExactRangeExpansion" + // LoweringProjectionPruning identifies removal of traversal fields that downstream clauses do not consume. + LoweringProjectionPruning = "ProjectionPruning" + + // LoweringLatePathMaterialization identifies deferral of path hydration until a consumer requires it. + LoweringLatePathMaterialization = "LatePathMaterialization" + + // LoweringExpandIntoDetection identifies traversal steps whose two endpoints are already bound. + LoweringExpandIntoDetection = "ExpandIntoDetection" + + // LoweringTraversalDirection identifies selection of the lower-cost logical traversal direction. + LoweringTraversalDirection = "TraversalDirectionSelection" + + // LoweringShortestPathStrategy identifies unidirectional or bidirectional shortest-path selection. + LoweringShortestPathStrategy = "ShortestPathStrategySelection" + + // LoweringShortestPathFilter identifies materialization of reusable shortest-path endpoint filters. + LoweringShortestPathFilter = "ShortestPathFilterMaterialization" + + // LoweringLimitPushdown identifies limits moved into a traversal or shortest-path harness. + LoweringLimitPushdown = "LimitPushdown" + + // LoweringExpansionSuffixPushdown identifies fixed-suffix predicates moved closer to variable expansion. + LoweringExpansionSuffixPushdown = "ExpansionSuffixPushdown" + + // LoweringPredicatePlacement identifies attachment of predicates to the earliest safe traversal step. + LoweringPredicatePlacement = "PredicatePlacement" + + // LoweringCountStoreFastPath identifies count queries satisfied directly from graph statistics. + LoweringCountStoreFastPath = "CountStoreFastPath" + + // LoweringCollectIDMembership identifies membership checks rewritten over collected scalar IDs. + LoweringCollectIDMembership = "CollectIDMembership" + + // LoweringAggregateTraversalCount identifies traversal counts lowered without materializing result rows. + LoweringAggregateTraversalCount = "AggregateTraversalCount" + + // LoweringExactRangeExpansion identifies short exact ranges expanded into fixed traversal steps. + LoweringExactRangeExpansion = "ExactRangeExpansion" + + // LoweringPathRelationshipPredicate identifies relationship quantifiers attached to path state. LoweringPathRelationshipPredicate = "PathRelationshipPredicate" + + // LoweringFieldRequirements identifies analysis that records which representation each binding consumer needs. + LoweringFieldRequirements = "FieldRequirements" + + // LoweringShortestPathExecutor identifies selection of a physical shortest-path executor. + LoweringShortestPathExecutor = "ShortestPathExecutorDecision" + + // LoweringExpansionSearchStrategy identifies selection of a physical variable-expansion search strategy. + LoweringExpansionSearchStrategy = "ExpansionSearchStrategyDecision" + + // LoweringEndpointResolution identifies planned bounded endpoint-resolution analysis. + LoweringEndpointResolution = "EndpointResolutionDecision" + + // LoweringTraversalPredicateClassification identifies planned traversal-predicate locality analysis. + LoweringTraversalPredicateClassification = "TraversalPredicateClassificationDecision" ) +// LoweringDecision records the planner choice made for lowering. type LoweringDecision struct { + // Name identifies the name. Name string `json:"name"` } +// PatternTarget locates the query element affected by pattern. type PatternTarget struct { - QueryPartIndex int `json:"query_part_index"` - ClauseIndex int `json:"clause_index"` - PatternIndex int `json:"pattern_index"` - Predicate bool `json:"predicate,omitempty"` - PredicateIndex int `json:"predicate_index,omitempty"` + // QueryPartIndex supplies the query part index input to the PatternTarget contract. + QueryPartIndex int `json:"query_part_index"` + // ClauseIndex supplies the clause index input to the PatternTarget contract. + ClauseIndex int `json:"clause_index"` + // PatternIndex supplies the pattern index input to the PatternTarget contract. + PatternIndex int `json:"pattern_index"` + // Predicate indicates whether predicate applies. + Predicate bool `json:"predicate,omitempty"` + // PredicateIndex supplies the predicate index input to the PatternTarget contract. + PredicateIndex int `json:"predicate_index,omitempty"` } +// TraversalStep evaluates planner state needed for traversal step. func (s PatternTarget) TraversalStep(stepIndex int) TraversalStepTarget { return TraversalStepTarget{ QueryPartIndex: s.QueryPartIndex, @@ -45,189 +96,1105 @@ func (s PatternTarget) TraversalStep(stepIndex int) TraversalStepTarget { } } +// TraversalStepTarget locates the query element affected by traversal step. type TraversalStepTarget struct { - QueryPartIndex int `json:"query_part_index"` - ClauseIndex int `json:"clause_index"` - PatternIndex int `json:"pattern_index"` - Predicate bool `json:"predicate,omitempty"` - PredicateIndex int `json:"predicate_index,omitempty"` - StepIndex int `json:"step_index"` + // QueryPartIndex supplies the query part index input to the TraversalStepTarget contract. + QueryPartIndex int `json:"query_part_index"` + // ClauseIndex supplies the clause index input to the TraversalStepTarget contract. + ClauseIndex int `json:"clause_index"` + // PatternIndex supplies the pattern index input to the TraversalStepTarget contract. + PatternIndex int `json:"pattern_index"` + // Predicate indicates whether predicate applies. + Predicate bool `json:"predicate,omitempty"` + // PredicateIndex supplies the predicate index input to the TraversalStepTarget contract. + PredicateIndex int `json:"predicate_index,omitempty"` + // StepIndex supplies the step index input to the TraversalStepTarget contract. + StepIndex int `json:"step_index"` } +// QuantifierTarget locates the query element affected by quantifier. type QuantifierTarget struct { - QueryPartIndex int `json:"query_part_index"` + // QueryPartIndex supplies the query part index input to the QuantifierTarget contract. + QueryPartIndex int `json:"query_part_index"` + // QuantifierIndex supplies the quantifier index input to the QuantifierTarget contract. QuantifierIndex int `json:"quantifier_index"` } +// ProjectionPruningDecision records the planner choice made for projection pruning. type ProjectionPruningDecision struct { - Target TraversalStepTarget `json:"target"` - ReferencedSymbols []string `json:"referenced_symbols,omitempty"` - PatternBindingReferenced bool `json:"pattern_binding_referenced,omitempty"` - OmitLeftNode bool `json:"omit_left_node,omitempty"` - OmitRelationship bool `json:"omit_relationship,omitempty"` - OmitRightNode bool `json:"omit_right_node,omitempty"` - OmitPathBinding bool `json:"omit_path_binding,omitempty"` + // Target supplies the target input to the ProjectionPruningDecision contract. + Target TraversalStepTarget `json:"target"` + // ReferencedSymbols supplies the referenced symbols input to the ProjectionPruningDecision contract. + ReferencedSymbols []string `json:"referenced_symbols,omitempty"` + // PatternBindingReferenced indicates whether pattern binding referenced applies. + PatternBindingReferenced bool `json:"pattern_binding_referenced,omitempty"` + // OmitLeftNode indicates whether omit left node applies. + OmitLeftNode bool `json:"omit_left_node,omitempty"` + // OmitRelationship indicates whether omit relationship applies. + OmitRelationship bool `json:"omit_relationship,omitempty"` + // OmitRightNode indicates whether omit right node applies. + OmitRightNode bool `json:"omit_right_node,omitempty"` + // OmitPathBinding indicates whether omit path binding applies. + OmitPathBinding bool `json:"omit_path_binding,omitempty"` } type LatePathMaterializationMode string const ( - LatePathMaterializationPathEdgeID LatePathMaterializationMode = "path_edge_id" + // LatePathMaterializationPathEdgeID carries a path as ordered edge IDs until hydration. + LatePathMaterializationPathEdgeID LatePathMaterializationMode = "path_edge_id" + + // LatePathMaterializationExpansionPath carries recursive expansion path state until hydration. LatePathMaterializationExpansionPath LatePathMaterializationMode = "expansion_path" + + // LatePathMaterializationEdgeComposite defers hydration of an edge composite. LatePathMaterializationEdgeComposite LatePathMaterializationMode = "edge_composite" ) +// LatePathMaterializationDecision records the planner choice made for late path materialization. type LatePathMaterializationDecision struct { - Target TraversalStepTarget `json:"target"` - Mode LatePathMaterializationMode `json:"mode"` + // Target supplies the target input to the LatePathMaterializationDecision contract. + Target TraversalStepTarget `json:"target"` + // Mode identifies the mode. + Mode LatePathMaterializationMode `json:"mode"` } +// ExpandIntoDecision records the planner choice made for expand into. type ExpandIntoDecision struct { + // Target supplies the target input to the ExpandIntoDecision contract. Target TraversalStepTarget `json:"target"` } +// TraversalDirectionDecision records the planner choice made for traversal direction. type TraversalDirectionDecision struct { + // Target supplies the target input to the TraversalDirectionDecision contract. Target TraversalStepTarget `json:"target"` - Flip bool `json:"flip,omitempty"` - Reason string `json:"reason,omitempty"` + // Flip indicates whether flip applies. + Flip bool `json:"flip,omitempty"` + // Reason supplies the reason input to the TraversalDirectionDecision contract. + Reason string `json:"reason,omitempty"` } type ShortestPathStrategy string const ( - ShortestPathStrategyBidirectional ShortestPathStrategy = "bidirectional" + // ShortestPathStrategyBidirectional searches simultaneously from both endpoints. + ShortestPathStrategyBidirectional ShortestPathStrategy = "bidirectional" + + // ShortestPathStrategyUnidirectional searches from one endpoint toward the other. ShortestPathStrategyUnidirectional ShortestPathStrategy = "unidirectional" ) +// ShortestPathStrategyDecision records the planner choice made for shortest path strategy. type ShortestPathStrategyDecision struct { - Target TraversalStepTarget `json:"target"` + // Target supplies the target input to the ShortestPathStrategyDecision contract. + Target TraversalStepTarget `json:"target"` + // Strategy supplies the strategy input to the ShortestPathStrategyDecision contract. Strategy ShortestPathStrategy `json:"strategy"` - Reason string `json:"reason,omitempty"` + // Reason supplies the reason input to the ShortestPathStrategyDecision contract. + Reason string `json:"reason,omitempty"` +} + +type ShortestPathExecutor string + +const ( + // ShortestPathPolicyASPI1GuardedV1 identifies the bounded inline + // predecessor-DAG candidate with an exact A1 fallback. + ShortestPathPolicyASPI1GuardedV1 = "asp-i1-guarded-v1" + + // ShortestPathPolicyI1CanonicalGuardedV1 identifies the bounded inline + // canonical-witness candidate with an exact compact S4 fallback. + ShortestPathPolicyI1CanonicalGuardedV1 = "sp-i1-canonical-guarded-v1" + + // ShortestPathPolicyI2DistanceGuardedV1 identifies reverse-physical inline + // distance discovery with independent state/frontier gates and S4 fallback. + ShortestPathPolicyI2DistanceGuardedV1 = "sp-i2-distance-guarded-v1" + + // ShortestPathPolicyI2DistanceGuardedV2 identifies the independently + // qualified tail-stabilized generation. It does not alias V1 evidence. + ShortestPathPolicyI2DistanceGuardedV2 = "sp-i2-distance-guarded-v2" + + // ShortestPathI2QualifiedStateLimit is the immutable total-state ceiling + // preregistered by the SP-I2 production-form protocol and policy seam. + ShortestPathI2QualifiedStateLimit int64 = 100_000 + + // ShortestPathI2QualifiedFrontierLimit is the immutable per-depth frontier + // ceiling preregistered by the SP-I2 production-form protocol and policy seam. + ShortestPathI2QualifiedFrontierLimit int64 = 100_000 + + // ShortestPathSelectorStaticV6 identifies the evidence-gated production + // selector for the qualified inbound, typed, single-kind canonical witness + // envelope. The automatic selector remains sp-static-v5-contained until a + // complete production evidence manifest activates this version. + ShortestPathSelectorStaticV6 = "sp-static-v6" + + // ShortestPathSelectorStaticV7Contained extends contained S3/S4 selection + // to syntax-open SP ranges using the existing effective depth-15 policy. + ShortestPathSelectorStaticV7Contained = "sp-static-v7-contained" + + // ShortestPathSelectorStaticV8HiddenFanIn identifies the exact-bucket, + // evidence-gated distance canary for asymmetric physical topology. + ShortestPathSelectorStaticV8HiddenFanIn = "sp-static-v8-hidden-fanin" + + // ShortestPathSelectorStaticV9HiddenFanInTail is the default-off V2 + // selector identity. Production activation requires a V2 manifest. + ShortestPathSelectorStaticV9HiddenFanInTail = "sp-static-v9-hidden-fanin-tail" + + // ShortestPathExecutorIncumbentWorkspace selects the existing workspace-table executor. + ShortestPathExecutorIncumbentWorkspace ShortestPathExecutor = "SP-S0" + + // ShortestPathExecutorS1ArrayBFS selects breadth-first search with path state held in arrays. + ShortestPathExecutorS1ArrayBFS ShortestPathExecutor = "SP-S1" + + // ShortestPathExecutorS2TraceRelation selects breadth-first search backed by a trace relation. + ShortestPathExecutorS2TraceRelation ShortestPathExecutor = "SP-S2" + + // ShortestPathExecutorS3Unidirectional selects the unidirectional scalar-distance executor. + ShortestPathExecutorS3Unidirectional ShortestPathExecutor = "SP-S3-U-D" + + // ShortestPathExecutorS3EdgeM0 selects unidirectional edge-trail search with deferred path materialization. + ShortestPathExecutorS3EdgeM0 ShortestPathExecutor = "SP-S3-U-E+MAT-M0" + + // ShortestPathExecutorS0Direct selects the direct preflight executor with workspace fallback. + ShortestPathExecutorS0Direct ShortestPathExecutor = "SP-S0-DIRECT" + + // ShortestPathExecutorS4CanonicalDistance selects canonical compact search for distance-only observations. + ShortestPathExecutorS4CanonicalDistance ShortestPathExecutor = "SP-S4-C-D" + + // ShortestPathExecutorS4CanonicalWitness selects canonical compact search with witness materialization. + ShortestPathExecutorS4CanonicalWitness ShortestPathExecutor = "SP-S4-C-WE+MAT-M0" + + // ShortestPathExecutorASPA1DAG selects all-shortest-path enumeration from a predecessor DAG. + ShortestPathExecutorASPA1DAG ShortestPathExecutor = "ASP-A1-DAG" + + // ShortestPathExecutorASPN1NegativeExhaustion performs a bounded reverse + // reachability preflight for no-path results and otherwise delegates to A1. + // It is deliberately a distinct, default-off executor identity. + ShortestPathExecutorASPN1NegativeExhaustion ShortestPathExecutor = "ASP-N1-NEGATIVE-EXHAUSTION" + + // ShortestPathExecutorI1CanonicalDistance selects an inline recursive SQL + // distance search. The distinct identity prevents evidence collected at an + // inline statement boundary from being attributed to a helper function. + ShortestPathExecutorI1CanonicalDistance ShortestPathExecutor = "SP-I1-C-D" + + // ShortestPathExecutorI2GuardedDistance selects reverse-physical inline + // distance discovery with exact compact S4 fallback. + ShortestPathExecutorI2GuardedDistance ShortestPathExecutor = "SP-I2-C-D" + + // ShortestPathExecutorI2GuardedDistanceV2 selects the E1 V2 statement with + // a single materialized admission decision and exact compact-S4 fallback. + ShortestPathExecutorI2GuardedDistanceV2 ShortestPathExecutor = "SP-I2-C-D-V2" + + // Development identities are non-promotional and remain distinguishable in + // diagnostic artifacts. Only E1 is emitted by the switch-free V2 builder. + ShortestPathExecutorI2GuardedDistanceV2E0 ShortestPathExecutor = "SP-I2-C-D-V2-E0" + ShortestPathExecutorI2GuardedDistanceV2E1 ShortestPathExecutor = "SP-I2-C-D-V2-E1" + ShortestPathExecutorI2GuardedDistanceV2E1D ShortestPathExecutor = "SP-I2-C-D-V2-E1D" + ShortestPathExecutorI2GuardedDistanceV2E1P ShortestPathExecutor = "SP-I2-C-D-V2-E1P" + ShortestPathExecutorI2GuardedDistanceV2E1DP ShortestPathExecutor = "SP-I2-C-D-V2-E1DP" + + // ShortestPathExecutorI1CanonicalWitness selects inline recursive SQL with + // ordered edge-ID witness state and late M0 path materialization. + ShortestPathExecutorI1CanonicalWitness ShortestPathExecutor = "SP-I1-U-E+MAT-M0" + + // ShortestPathExecutorI1CanonicalPredecessorWitness selects guarded inline + // minimum-distance/predecessor discovery, one deterministic witness, and an + // exact compact S4 fallback. It is intentionally distinct from the legacy + // unguarded relationship-trail I1 identity above. + ShortestPathExecutorI1CanonicalPredecessorWitness ShortestPathExecutor = "SP-I1-C-WE+MAT-M0" + + // ShortestPathExecutorASPI1DAG selects inline predecessor-DAG discovery and + // late M0 materialization for all shortest paths. + ShortestPathExecutorASPI1DAG ShortestPathExecutor = "ASP-I1-U-DAG+MAT-M0" + + // ShortestPathExecutorB1AlternatingNodeDistance reserves compact bidirectional + // distance search with strict node-at-a-time alternation. + ShortestPathExecutorB1AlternatingNodeDistance ShortestPathExecutor = "SP-B1-C-ALT-NODE-D" + + // ShortestPathExecutorB1AlternatingNodeWitness reserves compact bidirectional + // witness search with strict node-at-a-time alternation and deferred materialization. + ShortestPathExecutorB1AlternatingNodeWitness ShortestPathExecutor = "SP-B1-C-ALT-NODE-WE+MAT-M0" + + // ShortestPathExecutorB2SmallerCurrentLevelDistance reserves compact bidirectional + // distance search that expands the smaller current level. + ShortestPathExecutorB2SmallerCurrentLevelDistance ShortestPathExecutor = "SP-B2-C-MIN-LEVEL-D" + + // ShortestPathExecutorB2SmallerCurrentLevelWitness reserves compact bidirectional + // witness search that expands the smaller current level and defers materialization. + ShortestPathExecutorB2SmallerCurrentLevelWitness ShortestPathExecutor = "SP-B2-C-MIN-LEVEL-WE+MAT-M0" + + // ShortestPathExecutorASPB1AlternatingNodeDAG reserves all-shortest-path DAG + // enumeration with strict node-at-a-time alternation. + ShortestPathExecutorASPB1AlternatingNodeDAG ShortestPathExecutor = "ASP-B1-DAG-ALT-NODE" + + // ShortestPathExecutorASPB2SmallerCurrentLevelDAG reserves all-shortest-path DAG + // enumeration that expands the smaller current level. + ShortestPathExecutorASPB2SmallerCurrentLevelDAG ShortestPathExecutor = "ASP-B2-DAG-MIN-LEVEL" +) + +// ShortestPathScheduler identifies the frontier scheduling policy used by a +// shortest-path executor independently of its result-observation contract. +type ShortestPathScheduler string + +const ( + // ShortestPathSchedulerSingleEndedLevel expands one complete level from a single frontier. + ShortestPathSchedulerSingleEndedLevel ShortestPathScheduler = "single_ended_level" + + // ShortestPathSchedulerStrictAlternatingNode alternates one node expansion from each frontier. + ShortestPathSchedulerStrictAlternatingNode ShortestPathScheduler = "strict_alternating_node" + + // ShortestPathSchedulerSmallerCurrentLevel expands the smaller of the two current frontier levels. + ShortestPathSchedulerSmallerCurrentLevel ShortestPathScheduler = "smaller_current_level" +) + +// Scheduler reports the stable frontier scheduler associated with this executor. +func (s ShortestPathExecutor) Scheduler() ShortestPathScheduler { + switch s { + case ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorS4CanonicalDistance, + ShortestPathExecutorS4CanonicalWitness, + ShortestPathExecutorASPA1DAG, + ShortestPathExecutorASPN1NegativeExhaustion, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI2GuardedDistance, + ShortestPathExecutorI2GuardedDistanceV2, + ShortestPathExecutorI2GuardedDistanceV2E0, + ShortestPathExecutorI2GuardedDistanceV2E1, + ShortestPathExecutorI2GuardedDistanceV2E1D, + ShortestPathExecutorI2GuardedDistanceV2E1P, + ShortestPathExecutorI2GuardedDistanceV2E1DP, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorASPI1DAG: + return ShortestPathSchedulerSingleEndedLevel + case ShortestPathExecutorB1AlternatingNodeDistance, + ShortestPathExecutorB1AlternatingNodeWitness, + ShortestPathExecutorASPB1AlternatingNodeDAG: + return ShortestPathSchedulerStrictAlternatingNode + case ShortestPathExecutorB2SmallerCurrentLevelDistance, + ShortestPathExecutorB2SmallerCurrentLevelWitness, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return ShortestPathSchedulerSmallerCurrentLevel + default: + return "" + } +} + +// ExecutionBoundary reports the SQL boundary represented by the executor +// identity. Benchmark and promotion artifacts must match this value. +func (s ShortestPathExecutor) ExecutionBoundary() string { + switch s { + case ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI2GuardedDistance, + ShortestPathExecutorI2GuardedDistanceV2, + ShortestPathExecutorI2GuardedDistanceV2E0, + ShortestPathExecutorI2GuardedDistanceV2E1, + ShortestPathExecutorI2GuardedDistanceV2E1D, + ShortestPathExecutorI2GuardedDistanceV2E1P, + ShortestPathExecutorI2GuardedDistanceV2E1DP, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorASPI1DAG: + return "inline_statement" + default: + return "stored_helper" + } +} + +type ShortestPathObservationMode string + +const ( + // ShortestPathObservationDistance indicates that only shortest-path length is consumed. + ShortestPathObservationDistance ShortestPathObservationMode = "distance" + + // ShortestPathObservationOnePath indicates that one shortest-path witness is consumed. + ShortestPathObservationOnePath ShortestPathObservationMode = "one_path" + + // ShortestPathObservationAllPaths indicates that every shortest-path witness is consumed. + ShortestPathObservationAllPaths ShortestPathObservationMode = "all_paths" + + // ShortestPathObservationUnknown indicates that analysis could not classify downstream path use. + ShortestPathObservationUnknown ShortestPathObservationMode = "unknown" +) + +// ShortestPathMaximumDepthSource distinguishes an explicit Cypher upper bound +// from the repository's existing effective cap for a syntax-open range. +type ShortestPathMaximumDepthSource string + +const ( + // ShortestPathMaximumDepthExplicit identifies an upper bound written in the query. + ShortestPathMaximumDepthExplicit ShortestPathMaximumDepthSource = "explicit" + + // ShortestPathMaximumDepthPolicyDefault identifies the effective depth-15 + // cap already applied by PostgreSQL translation to an omitted upper bound. + ShortestPathMaximumDepthPolicyDefault ShortestPathMaximumDepthSource = "policy_default" +) + +const ( + // ShortestPathFallbackAllShortestPaths records an all-shortest-path query lacking singleton endpoints required by specialized execution. + ShortestPathFallbackAllShortestPaths = "all_shortest_paths" + + // ShortestPathFallbackCorrelatedEndpoints rejects endpoint sources not proven uncorrelated, such as UNWIND or later query parts. + ShortestPathFallbackCorrelatedEndpoints = "correlated_endpoints" + + // ShortestPathFallbackMultipleEndpointPairs rejects specialized execution when additional row sources prevent proving one endpoint pair. + ShortestPathFallbackMultipleEndpointPairs = "multiple_endpoint_pairs" + + // ShortestPathFallbackNonSingletonID rejects an endpoint whose ID is not statically singleton. + ShortestPathFallbackNonSingletonID = "non_singleton_id" + + // ShortestPathFallbackMultipleIDEqualities rejects an endpoint constrained by competing ID equalities. + ShortestPathFallbackMultipleIDEqualities = "multiple_id_equalities" + + // ShortestPathFallbackPathPredicate rejects a predicate that observes the materialized path. + ShortestPathFallbackPathPredicate = "path_predicate" + + // ShortestPathFallbackRelationshipPredicate rejects a predicate on the traversed relationship. + ShortestPathFallbackRelationshipPredicate = "relationship_predicate" + + // ShortestPathFallbackRelationshipVariable rejects an observed relationship binding. + ShortestPathFallbackRelationshipVariable = "relationship_variable" + + // ShortestPathFallbackDirectionless rejects a directionless shortest-path expansion. + ShortestPathFallbackDirectionless = "directionless" + + // ShortestPathFallbackOptionalMatch rejects shortest-path work under OPTIONAL MATCH semantics. + ShortestPathFallbackOptionalMatch = "optional_match" + + // ShortestPathFallbackUnsupportedDepth rejects a depth range unsupported by the candidate executor. + ShortestPathFallbackUnsupportedDepth = "unsupported_depth" + + // ShortestPathFallbackMutation rejects specialized execution for a statement containing updates. + ShortestPathFallbackMutation = "mutation" + + // ShortestPathFallbackMultiplePathCalls rejects statements containing more than one shortest-path pattern. + ShortestPathFallbackMultiplePathCalls = "multiple_path_calls" + + // ShortestPathFallbackDeepInboundUnqualified rejects an unqualified deep inbound traversal. + ShortestPathFallbackDeepInboundUnqualified = "deep_inbound_unqualified" + + // ShortestPathFallbackNonSingleKindPathState rejects compact path state without one relationship kind. + ShortestPathFallbackNonSingleKindPathState = "non_single_kind_path_state_unqualified" + + // ShortestPathFallbackTournamentUnqualified records that no experimental candidate won qualification. + ShortestPathFallbackTournamentUnqualified = "tournament_unqualified" +) + +type ShortestPathPhysicalExpansion string + +const ( + // ShortestPathPhysicalExpansionStartID joins recursive expansion through each edge's start ID. + ShortestPathPhysicalExpansionStartID ShortestPathPhysicalExpansion = "start_id" + + // ShortestPathPhysicalExpansionEndID joins recursive expansion through each edge's end ID. + ShortestPathPhysicalExpansionEndID ShortestPathPhysicalExpansion = "end_id" +) + +type ShortestPathTopologyClassification string + +const ( + // ShortestPathTopologyPhysicalOutbound classifies traversal aligned with stored edge direction. + ShortestPathTopologyPhysicalOutbound ShortestPathTopologyClassification = "physical_outbound" + + // ShortestPathTopologyPhysicalInboundShallow classifies a shallow traversal against stored edge direction. + ShortestPathTopologyPhysicalInboundShallow ShortestPathTopologyClassification = "physical_inbound_shallow" + + // ShortestPathTopologyPhysicalInboundDeep classifies a deep traversal against stored edge direction. + ShortestPathTopologyPhysicalInboundDeep ShortestPathTopologyClassification = "physical_inbound_deep" + + // ShortestPathTopologyDirectionless classifies traversal that may follow either stored direction. + ShortestPathTopologyDirectionless ShortestPathTopologyClassification = "directionless" +) + +// ShortestPathEligibilityFact records one named qualification check for an executor candidate. +type ShortestPathEligibilityFact struct { + // Name identifies the qualification check. + Name string `json:"name"` + // Eligible reports whether the candidate passed the named check. + Eligible bool `json:"eligible"` +} + +// ShortestPathExecutorDecision records either a qualified static executor or +// the incumbent fallback, keeping every eligibility and fallback fact visible. +type ShortestPathExecutorDecision struct { + // Target locates the traversal step governed by this decision. + Target TraversalStepTarget `json:"target"` + // Family names the executor-selection family that produced the decision. + Family string `json:"family"` + // PlannedCandidates lists the executors considered in preference order. + PlannedCandidates []ShortestPathExecutor `json:"planned_candidates"` + // SelectedExecutor is the executor chosen after qualification. + SelectedExecutor ShortestPathExecutor `json:"selected_executor"` + // ExecutionBoundary distinguishes inline statement SQL from stored helper + // execution. Promotion evidence must match this boundary exactly. + ExecutionBoundary string `json:"execution_boundary"` + // Scheduler identifies the selected executor's frontier scheduling policy. + Scheduler ShortestPathScheduler `json:"scheduler,omitempty"` + // ObservationMode describes how downstream clauses consume the shortest path. + ObservationMode ShortestPathObservationMode `json:"observation_mode"` + // Direction is the logical direction of the traversal. + Direction graph.Direction `json:"direction"` + // PhysicalExpansion identifies which stored edge endpoint advances the search. + PhysicalExpansion ShortestPathPhysicalExpansion `json:"physical_expansion"` + // RelationshipKindCount is the number of statically resolved relationship kinds. + RelationshipKindCount int `json:"relationship_kind_count"` + // UntypedRelationship reports whether the pattern omitted relationship kinds. + UntypedRelationship bool `json:"untyped_relationship"` + // TopologyClassification summarizes logical direction, physical direction, and depth. + TopologyClassification ShortestPathTopologyClassification `json:"topology_classification"` + // Eligibility records each qualification check and its result. + Eligibility []ShortestPathEligibilityFact `json:"eligibility"` + // StructurallyEligible reports whether the query shape can use the candidate executor. + StructurallyEligible bool `json:"structurally_eligible"` + // StaticallyEligible reports whether known literals and kinds satisfy executor constraints. + StaticallyEligible bool `json:"statically_eligible"` + // MinimumDepth is the inclusive lower traversal-depth bound. + MinimumDepth int64 `json:"minimum_depth"` + // MaximumDepth is the inclusive upper traversal-depth bound. + MaximumDepth int64 `json:"maximum_depth"` + // MaximumDepthSource records whether MaximumDepth was explicit syntax or + // supplied by the existing traversal policy. + MaximumDepthSource ShortestPathMaximumDepthSource `json:"maximum_depth_source"` + // StateLimit caps state admitted by bounded experimental executors. + StateLimit int64 `json:"state_limit,omitempty"` + // FrontierLimit caps current and queued frontier rows independently of seen state. + FrontierLimit int64 `json:"frontier_limit,omitempty"` + // PredecessorLimit caps retained witness predecessor rows independently of discovery state. + PredecessorLimit int64 `json:"predecessor_limit,omitempty"` + // EnumerationLimit caps distinct ordered all-shortest-path arrays before exact fallback. + EnumerationLimit int64 `json:"enumeration_limit,omitempty"` + // OutputBytesLimit caps staged all-shortest-path array bytes before exact fallback. + OutputBytesLimit int64 `json:"output_bytes_limit,omitempty"` + // SelectorVersion identifies the policy version that ranked the candidates. + SelectorVersion string `json:"selector_version"` + // SelectionMode records whether selection was automatic or forced by tooling. + SelectionMode string `json:"selection_mode"` + // FallbackExecutor is used when the preferred candidate cannot be applied. + FallbackExecutor ShortestPathExecutor `json:"fallback_executor"` + // FallbackReason explains why the preferred candidate was not selected. + FallbackReason string `json:"fallback_reason"` + // ExperimentalWinner reports whether an experimental candidate beat the incumbent. + ExperimentalWinner bool `json:"experimental_winner,omitempty"` } type ShortestPathFilterMode string const ( - ShortestPathFilterTerminal ShortestPathFilterMode = "terminal" + // ShortestPathFilterTerminal materializes candidate terminal IDs independently of roots. + ShortestPathFilterTerminal ShortestPathFilterMode = "terminal" + + // ShortestPathFilterEndpointPair materializes admissible root-terminal ID pairs. ShortestPathFilterEndpointPair ShortestPathFilterMode = "endpoint_pair" ) +// ShortestPathFilterDecision records the planner choice made for shortest path filter. type ShortestPathFilterDecision struct { - Target TraversalStepTarget `json:"target"` - Mode ShortestPathFilterMode `json:"mode"` - Reason string `json:"reason,omitempty"` + // Target supplies the target input to the ShortestPathFilterDecision contract. + Target TraversalStepTarget `json:"target"` + // Mode identifies the mode. + Mode ShortestPathFilterMode `json:"mode"` + // Reason supplies the reason input to the ShortestPathFilterDecision contract. + Reason string `json:"reason,omitempty"` } type LimitPushdownMode string const ( - LimitPushdownTraversalCTE LimitPushdownMode = "traversal_cte" + // LimitPushdownTraversalCTE applies a row limit inside an ordinary traversal CTE. + LimitPushdownTraversalCTE LimitPushdownMode = "traversal_cte" + + // LimitPushdownShortestPathHarness applies a row limit inside a shortest-path harness. LimitPushdownShortestPathHarness LimitPushdownMode = "shortest_path_harness" ) +// LimitPushdownDecision records the planner choice made for limit pushdown. type LimitPushdownDecision struct { + // Target supplies the target input to the LimitPushdownDecision contract. Target TraversalStepTarget `json:"target"` - Mode LimitPushdownMode `json:"mode"` + // Mode identifies the mode. + Mode LimitPushdownMode `json:"mode"` } +// ExpansionSuffixPushdownDecision describes a fixed traversal suffix evaluated for supplemental search. type ExpansionSuffixPushdownDecision struct { - Target TraversalStepTarget `json:"target"` - SuffixLength int `json:"suffix_length"` - SuffixStartStep int `json:"suffix_start_step"` - SuffixEndStep int `json:"suffix_end_step"` + // Target locates the variable expansion followed by the fixed suffix. + Target TraversalStepTarget `json:"target"` + // SuffixLength is the number of fixed traversal steps eligible for pushdown. + SuffixLength int `json:"suffix_length"` + // SuffixStartStep identifies the first fixed traversal step in the suffix. + SuffixStartStep int `json:"suffix_start_step"` + // SuffixEndStep identifies the final fixed traversal step in the suffix. + SuffixEndStep int `json:"suffix_end_step"` + // ApplySupplemental reports whether translation should emit the supplemental suffix-search branch. + ApplySupplemental bool `json:"apply_supplemental"` + // Reason explains why supplemental suffix search was enabled or withheld. + Reason string `json:"reason,omitempty"` + // PredicateAttachments lists predicates assigned to scopes within the fixed suffix. PredicateAttachments []PredicateAttachment `json:"predicate_attachments,omitempty"` } +type ExpansionSearchStrategy string + +const ( + // ExpansionSearchStepwiseForward selects the incumbent left-to-right expansion plan. + ExpansionSearchStepwiseForward ExpansionSearchStrategy = "EXPANSION-STEPWISE-FORWARD" + + // ExpansionSearchLateHydratedForward selects forward search with deferred entity hydration. + ExpansionSearchLateHydratedForward ExpansionSearchStrategy = "EXPANSION-LATE-HYDRATED-FORWARD" + + // ExpansionSearchFactoredSuffixForward selects forward search with a factored fixed suffix. + ExpansionSearchFactoredSuffixForward ExpansionSearchStrategy = "EXPANSION-FACTORED-SUFFIX-FORWARD" + + // ExpansionSearchSuffixSeededReverse selects reverse probing seeded from a selective fixed suffix. + ExpansionSearchSuffixSeededReverse ExpansionSearchStrategy = "EXPANSION-SUFFIX-SEEDED-REVERSE" + + // ExpansionSearchEndpointSeededReverse selects reverse probing seeded from selective terminal endpoints. + ExpansionSearchEndpointSeededReverse ExpansionSearchStrategy = "EXPANSION-ENDPOINT-SEEDED-REVERSE" + + // ExpansionSearchBackwardViabilityForward selects forward expansion gated by backward reachability. + ExpansionSearchBackwardViabilityForward ExpansionSearchStrategy = "EXPANSION-BACKWARD-VIABILITY-FORWARD" +) + +// ExpansionSearchPolicy identifies a runtime policy independently of the +// expansion arm that the policy may execute. +type ExpansionSearchPolicy string + +const ( + // ExpansionSearchPolicyEndpointGuardV1 identifies the shipped endpoint and + // reverse-state sentinel policy. It is distinct from topology orientation, + // which requires root, suffix, and directional-degree probes. + ExpansionSearchPolicyEndpointGuardV1 ExpansionSearchPolicy = "endpoint-state-guard-v1" + + // ExpansionSearchPolicyOrientationProbeV1 selects an ordinary-expansion + // orientation from bounded, same-statement topology probes. + ExpansionSearchPolicyOrientationProbeV1 ExpansionSearchPolicy = "orientation-probe-v1" + + // ExpansionSearchPolicyOrientationProbeV2 selects an ordinary-expansion + // orientation using depth-weighted forward work and the same bounded, + // same-statement topology probes as v1. + ExpansionSearchPolicyOrientationProbeV2 ExpansionSearchPolicy = "orientation-probe-v2" + + // ExpansionSearchPolicySuffixReverseGuardV1 identifies bounded fixed-suffix + // reverse execution with exact stepwise-forward fallback. Unlike the + // orientation policies, it performs no topology scoring or directional + // degree probes. + ExpansionSearchPolicySuffixReverseGuardV1 ExpansionSearchPolicy = "suffix-reverse-guard-v1" + + // ExpansionSearchPolicySuffixReverseRetryV1 identifies the probe-free, + // reverse-only fixed-suffix candidate whose exact forward fallback executes + // as a second statement in the same stable-snapshot transaction. + ExpansionSearchPolicySuffixReverseRetryV1 ExpansionSearchPolicy = "suffix-reverse-retry-v1" + + // ExpansionSearchPolicyTopologyFixedSuffixV1 identifies the production + // topology-selected fixed-suffix candidate. It is distinct from every + // tool-only suffix guard, retry, and component identity. + ExpansionSearchPolicyTopologyFixedSuffixV1 ExpansionSearchPolicy = "topology-fixed-suffix-v1" + + // ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1 identifies the + // separately qualified first-use topology selector. Its manifest and route + // protocol are intentionally distinct from v1's cache-hit-only contract. + ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1 ExpansionSearchPolicy = "topology-fixed-suffix-first-use-v1" + + // ExpansionSearchSelectorFixedSuffixPathV1 identifies the tool-only static + // selector for full-path fixed-suffix observations. + ExpansionSearchSelectorFixedSuffixPathV1 = "fixed-suffix-path-static-v1" + + // ExpansionSearchSelectorSuffixRouteComponentV1 identifies the default-off + // GraphBench component arm for one exact suffix-seeded reverse statement. + // It is not a production selector, cache identity, or routing policy. + ExpansionSearchSelectorSuffixRouteComponentV1 = "suffix-route-component-v1" + + // ExpansionSearchSuffixReverseGuardSuffixRowLimit caps complete fixed-suffix + // payload rows before reverse execution is admitted. + ExpansionSearchSuffixReverseGuardSuffixRowLimit int64 = 512 + + // ExpansionSearchSuffixReverseGuardStateLimit caps complete reverse recursive + // state before exact forward fallback is selected. + ExpansionSearchSuffixReverseGuardStateLimit int64 = 512 + + // ExpansionSearchSuffixReverseRetryOutputRowLimit caps candidate rows buffered + // before the transaction-local retry boundary exposes any result. + ExpansionSearchSuffixReverseRetryOutputRowLimit int64 = 4_096 + + // ExpansionSearchSuffixReverseRetryOutputBytesLimit caps the encoded candidate + // payload buffered by the PostgreSQL driver before exact forward retry. + ExpansionSearchSuffixReverseRetryOutputBytesLimit int64 = 16 * 1024 * 1024 + + // ExpansionSearchOrientationRootRowLimit caps complete forward-root evidence + // for the initial fixed-suffix orientation tournament. + ExpansionSearchOrientationRootRowLimit int64 = 512 + + // ExpansionSearchOrientationReverseSeedRowLimit caps complete fixed-suffix + // row evidence while preserving duplicate suffix paths. + ExpansionSearchOrientationReverseSeedRowLimit int64 = 512 + + // ExpansionSearchOrientationDirectionalDegreeRowLimit caps each typed + // directional adjacency probe independently. + ExpansionSearchOrientationDirectionalDegreeRowLimit int64 = 16_384 + + // ExpansionSearchOrientationStateLimit caps admitted reverse recursive state. + ExpansionSearchOrientationStateLimit int64 = 4_096 + + // ExpansionSearchOrientationReverseScoreMultiplier is the reverse side of + // orientation-probe-v1's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationReverseScoreMultiplier int64 = 4 + + // ExpansionSearchOrientationForwardScoreMultiplier is the incumbent side + // of orientation-probe-v1's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationForwardScoreMultiplier int64 = 3 + + // ExpansionSearchOrientationV2ReverseScoreMultiplier is the reverse side + // of orientation-probe-v2's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationV2ReverseScoreMultiplier int64 = 4 + + // ExpansionSearchOrientationV2ForwardScoreMultiplier is the incumbent side + // of orientation-probe-v2's strict 3/4 hysteresis comparison. + ExpansionSearchOrientationV2ForwardScoreMultiplier int64 = 3 + + // ExpansionSearchExecutionBoundaryInlineStatement identifies one emitted + // expansion traversal arm in the translated statement. + ExpansionSearchExecutionBoundaryInlineStatement = "inline_statement" + + // ExpansionSearchExecutionBoundaryGuardedDualArm identifies a + // same-statement expansion policy with exact candidate and fallback arms. + ExpansionSearchExecutionBoundaryGuardedDualArm = "guarded_dual_arm" + + // ExpansionSearchExecutionBoundaryTransactionRetry identifies a reverse-only + // statement plus exact incumbent retry inside one stable-snapshot transaction. + ExpansionSearchExecutionBoundaryTransactionRetry = "transaction_retry" +) + +// ExpansionSearchProbeCaps records the maximum complete evidence admitted by +// an orientation policy. SQL probes use cap+1 sentinels to detect overflow. +type ExpansionSearchProbeCaps struct { + // RootRowLimit caps forward-root evidence. + RootRowLimit int64 `json:"root_row_limit,omitempty"` + // ReverseSeedRowLimit caps terminal or fixed-suffix seed evidence. + ReverseSeedRowLimit int64 `json:"reverse_seed_row_limit,omitempty"` + // DirectionalDegreeRowLimit caps typed first-hop adjacency evidence. + DirectionalDegreeRowLimit int64 `json:"directional_degree_row_limit,omitempty"` + // SurvivalRowLimit caps optional one-level survival evidence. + SurvivalRowLimit int64 `json:"survival_row_limit,omitempty"` +} + +// ExpansionSearchAdmission records the exact gate and fallback for a +// specialized orientation arm. +type ExpansionSearchAdmission struct { + // StateLimit caps specialized search state before incumbent fallback. + StateLimit int64 `json:"state_limit,omitempty"` + // OutputRowLimit caps candidate rows buffered before publication. + OutputRowLimit int64 `json:"output_row_limit,omitempty"` + // OutputBytesLimit caps the encoded candidate payload buffered before publication. + OutputBytesLimit int64 `json:"output_bytes_limit,omitempty"` + // RequiresCompleteProbes requires every candidate input probe to remain at + // or below its declared cap before specialized rows may be exposed. + RequiresCompleteProbes bool `json:"requires_complete_probes,omitempty"` + // FallbackStrategy names the exact incumbent used when admission fails. + FallbackStrategy ExpansionSearchStrategy `json:"fallback_strategy,omitempty"` +} + +type ExpansionSearchObservationMode string + +const ( + // ExpansionSearchObservationEndpointIDs indicates that downstream clauses consume only endpoint IDs. + ExpansionSearchObservationEndpointIDs ExpansionSearchObservationMode = "endpoint_ids" + + // ExpansionSearchObservationOrderedPathIDs indicates that downstream clauses consume ordered path IDs. + ExpansionSearchObservationOrderedPathIDs ExpansionSearchObservationMode = "ordered_path_ids" + + // ExpansionSearchObservationFullPath indicates that downstream clauses consume hydrated path values. + ExpansionSearchObservationFullPath ExpansionSearchObservationMode = "full_path" + + // ExpansionSearchObservationUnsupported indicates an observation pattern unsupported by specialized search. + ExpansionSearchObservationUnsupported ExpansionSearchObservationMode = "unsupported" +) + +// ExpansionSearchEligibilityFact records one named qualification check for a search strategy. +type ExpansionSearchEligibilityFact struct { + // Name identifies the qualification check. + Name string `json:"name"` + // Eligible reports whether the strategy passed the named check. + Eligible bool `json:"eligible"` +} + +const ( + // ExpansionSearchFallbackNoFixedSuffix rejects a strategy that requires a fixed suffix when none exists. + ExpansionSearchFallbackNoFixedSuffix = "no_fixed_suffix" + + // ExpansionSearchFallbackSuffixTooShort rejects a fixed suffix below the strategy's minimum length. + ExpansionSearchFallbackSuffixTooShort = "suffix_too_short" + + // ExpansionSearchFallbackOptionalMatch rejects a rewrite that would alter OPTIONAL MATCH behavior. + ExpansionSearchFallbackOptionalMatch = "optional_match" + + // ExpansionSearchFallbackShortestPath rejects ordinary-expansion strategies for shortestPath patterns. + ExpansionSearchFallbackShortestPath = "shortest_path" + + // ExpansionSearchFallbackAllShortestPaths rejects ordinary-expansion strategies for allShortestPaths patterns. + ExpansionSearchFallbackAllShortestPaths = "all_shortest_paths" + + // ExpansionSearchFallbackDirectionlessExpansion rejects a directionless variable expansion. + ExpansionSearchFallbackDirectionlessExpansion = "directionless_expansion" + + // ExpansionSearchFallbackDirectionlessSuffix rejects a directionless edge in the fixed suffix. + ExpansionSearchFallbackDirectionlessSuffix = "directionless_suffix" + + // ExpansionSearchFallbackUnboundedDepth rejects an expansion without a finite maximum depth. + ExpansionSearchFallbackUnboundedDepth = "unbounded_depth" + + // ExpansionSearchFallbackUnsupportedDepth rejects a depth range the candidate cannot preserve. + ExpansionSearchFallbackUnsupportedDepth = "unsupported_depth" + + // ExpansionSearchFallbackMultipleVariableExpansions rejects regions containing more than one variable expansion. + ExpansionSearchFallbackMultipleVariableExpansions = "multiple_variable_expansions" + + // ExpansionSearchFallbackCorrelatedSuffix rejects a fixed suffix that reuses an outer binding. + ExpansionSearchFallbackCorrelatedSuffix = "correlated_suffix" + + // ExpansionSearchFallbackCrossRegionPredicate rejects predicates spanning the variable and fixed regions. + ExpansionSearchFallbackCrossRegionPredicate = "cross_region_predicate" + + // ExpansionSearchFallbackPathDependentPredicate rejects predicates that depend on accumulated path state. + ExpansionSearchFallbackPathDependentPredicate = "path_dependent_predicate" + + // ExpansionSearchFallbackRelationshipVariable rejects an observed relationship binding in the variable expansion or fixed suffix. + ExpansionSearchFallbackRelationshipVariable = "relationship_variable" + + // ExpansionSearchFallbackRelationshipPredicate rejects relationship predicates in the variable expansion or fixed suffix. + ExpansionSearchFallbackRelationshipPredicate = "relationship_predicate" + + // ExpansionSearchFallbackLimitPushdownConflict rejects a rewrite that conflicts with an existing limit pushdown. + ExpansionSearchFallbackLimitPushdownConflict = "limit_pushdown_conflict" + + // ExpansionSearchFallbackUnsupportedObservation rejects downstream uses the candidate cannot reconstruct. + ExpansionSearchFallbackUnsupportedObservation = "unsupported_observation" + + // ExpansionSearchFallbackMutation rejects specialized search for a statement containing updates. + ExpansionSearchFallbackMutation = "mutation" + + // ExpansionSearchFallbackNonDeterministicPredicate rejects a seed predicate that cannot be safely reordered. + ExpansionSearchFallbackNonDeterministicPredicate = "non_deterministic_predicate" + + // ExpansionSearchFallbackUnboundRoot rejects a strategy that requires a previously bound expansion root. + ExpansionSearchFallbackUnboundRoot = "unbound_root" + + // ExpansionSearchFallbackTournamentUnqualified records that no specialized strategy passed qualification. + ExpansionSearchFallbackTournamentUnqualified = "tournament_unqualified" + + // ExpansionSearchFallbackNoFixedPrefix rejects a strategy that requires a fixed prefix when none exists. + ExpansionSearchFallbackNoFixedPrefix = "no_fixed_prefix" + + // ExpansionSearchFallbackExpansionNotTerminal rejects endpoint seeding when the expansion is not terminal. + ExpansionSearchFallbackExpansionNotTerminal = "expansion_not_terminal" + + // ExpansionSearchFallbackPrefixTooLong rejects a prefix that is not exactly one fixed hop. + ExpansionSearchFallbackPrefixTooLong = "prefix_too_long" + + // ExpansionSearchFallbackDirectionlessPrefix rejects a directionless edge in the fixed prefix. + ExpansionSearchFallbackDirectionlessPrefix = "directionless_prefix" + + // ExpansionSearchFallbackTerminalNotSelective rejects endpoint seeding without a selective terminal predicate. + ExpansionSearchFallbackTerminalNotSelective = "terminal_not_selective" + + // ExpansionSearchFallbackCorrelatedTerminal rejects a pre-bound terminal or a terminal predicate that depends on another binding. + ExpansionSearchFallbackCorrelatedTerminal = "correlated_terminal" + + // ExpansionSearchFallbackZeroDepth rejects a rewrite that cannot preserve zero-length paths. + ExpansionSearchFallbackZeroDepth = "zero_depth" +) + +// ExpansionSearchStrategyDecision records qualification and selection details for one variable expansion. +type ExpansionSearchStrategyDecision struct { + // Target locates the variable-expansion step governed by this decision. + Target TraversalStepTarget `json:"target"` + // Family names the search-strategy family that produced the decision. + Family string `json:"family"` + // PlannedPolicy identifies the runtime policy intended for this candidate + // family, whether or not translation currently emits it. + PlannedPolicy ExpansionSearchPolicy `json:"planned_policy,omitempty"` + // EmittedPolicy identifies the runtime policy actually present in emitted + // SQL. It remains empty for a single forced arm or incumbent-only SQL. + EmittedPolicy ExpansionSearchPolicy `json:"emitted_policy,omitempty"` + // PlannedCandidates lists the strategies considered in preference order. + PlannedCandidates []ExpansionSearchStrategy `json:"planned_candidates"` + // EmittedCandidates lists the arms present in the translated statement. + // Runtime telemetry, not this field, records which arm executed. + EmittedCandidates []ExpansionSearchStrategy `json:"emitted_candidates,omitempty"` + // ExecutionBoundary describes the SQL boundary that contains the emitted + // expansion arm or guarded policy. + ExecutionBoundary string `json:"execution_boundary,omitempty"` + // ProbeCaps records bounded evidence inputs for the planned policy. + ProbeCaps ExpansionSearchProbeCaps `json:"probe_caps"` + // Admission supplies the admission input to the ExpansionSearchStrategyDecision contract. + Admission ExpansionSearchAdmission `json:"admission"` + // CandidateStrategy is the specialized strategy proposed by structural analysis. + CandidateStrategy ExpansionSearchStrategy `json:"candidate_strategy,omitempty"` + // SelectedStrategy is the strategy chosen after all qualification checks. + SelectedStrategy ExpansionSearchStrategy `json:"selected_strategy"` + // StructurallyEligible reports whether the traversal shape supports the candidate. + StructurallyEligible bool `json:"structurally_eligible"` + // StaticallyEligible reports whether known bounds and predicates support the candidate. + StaticallyEligible bool `json:"statically_eligible"` + // EligibilityFacts records each qualification check and its result. + EligibilityFacts []ExpansionSearchEligibilityFact `json:"eligibility_facts"` + // SuffixStartStep is the first traversal step in the fixed suffix. + SuffixStartStep int `json:"suffix_start_step,omitempty"` + // SuffixEndStep is the last traversal step in the fixed suffix. + SuffixEndStep int `json:"suffix_end_step,omitempty"` + // SuffixLength is the number of traversal steps in the fixed suffix. + SuffixLength int `json:"suffix_length,omitempty"` + // PrefixStartStep is the first traversal step in the fixed prefix. + PrefixStartStep int `json:"prefix_start_step,omitempty"` + // PrefixEndStep is the last traversal step in the fixed prefix. + PrefixEndStep int `json:"prefix_end_step,omitempty"` + // PrefixLength is the number of traversal steps in the fixed prefix. + PrefixLength int `json:"prefix_length,omitempty"` + // SeedPredicateClass describes the predicate used to bound reverse search seeds. + SeedPredicateClass string `json:"seed_predicate_class,omitempty"` + // EndpointLimit caps terminal endpoints admitted into endpoint-seeded search. + EndpointLimit int64 `json:"endpoint_limit,omitempty"` + // StateLimit caps reverse-search states admitted before falling back. + StateLimit int64 `json:"state_limit,omitempty"` + // HasFinalLimit reports whether the terminal projection has a row limit. + HasFinalLimit bool `json:"has_final_limit,omitempty"` + // ObservationMode describes the representation required by downstream consumers. + ObservationMode ExpansionSearchObservationMode `json:"observation_mode"` + // LogicalDirection supplies the logical direction input to the ExpansionSearchStrategyDecision contract. + LogicalDirection string `json:"logical_direction"` + // MinimumDepth is the inclusive lower expansion-depth bound. + MinimumDepth int64 `json:"minimum_depth"` + // MaximumDepth is the inclusive upper expansion-depth bound, or zero when unbounded. + MaximumDepth int64 `json:"maximum_depth,omitempty"` + // SelectionMode records whether selection was automatic or forced by tooling. + SelectionMode string `json:"selection_mode"` + // SelectorVersion identifies the policy version that ranked the candidates. + SelectorVersion string `json:"selector_version"` + // FallbackStrategy is used when the specialized candidate cannot be applied. + FallbackStrategy ExpansionSearchStrategy `json:"fallback_strategy"` + // FallbackReason explains why the specialized candidate was not selected. + FallbackReason string `json:"fallback_reason"` +} + +// PredicatePlacementDecision records the planner choice made for predicate placement. type PredicatePlacementDecision struct { - Target TraversalStepTarget `json:"target"` - Attachment PredicateAttachment `json:"attachment"` - Placement PredicateAttachmentScope `json:"placement"` + // Target supplies the target input to the PredicatePlacementDecision contract. + Target TraversalStepTarget `json:"target"` + // Attachment supplies the attachment input to the PredicatePlacementDecision contract. + Attachment PredicateAttachment `json:"attachment"` + // Placement supplies the placement input to the PredicatePlacementDecision contract. + Placement PredicateAttachmentScope `json:"placement"` } type PatternPredicatePlacementMode string const ( + // PatternPredicatePlacementExistence lowers a pattern predicate as an existence test. PatternPredicatePlacementExistence PatternPredicatePlacementMode = "existence" ) +// PatternPredicatePlacementDecision records the planner choice made for pattern predicate placement. type PatternPredicatePlacementDecision struct { - Target TraversalStepTarget `json:"target"` - Mode PatternPredicatePlacementMode `json:"mode"` + // Target supplies the target input to the PatternPredicatePlacementDecision contract. + Target TraversalStepTarget `json:"target"` + // Mode identifies the mode. + Mode PatternPredicatePlacementMode `json:"mode"` } type CountStoreFastPathTarget string const ( + // CountStoreFastPathNode reads a node count directly from graph statistics. CountStoreFastPathNode CountStoreFastPathTarget = "node" + + // CountStoreFastPathEdge reads a relationship count directly from graph statistics. CountStoreFastPathEdge CountStoreFastPathTarget = "edge" ) +// CountStoreFastPathDecision records the planner choice made for count store fast path. type CountStoreFastPathDecision struct { - QueryPartIndex int `json:"query_part_index"` - ClauseIndex int `json:"clause_index"` - PatternIndex int `json:"pattern_index"` - BindingSymbol string `json:"binding_symbol,omitempty"` - Target CountStoreFastPathTarget `json:"target"` - KindSymbols []string `json:"kind_symbols,omitempty"` + // QueryPartIndex supplies the query part index input to the CountStoreFastPathDecision contract. + QueryPartIndex int `json:"query_part_index"` + // ClauseIndex supplies the clause index input to the CountStoreFastPathDecision contract. + ClauseIndex int `json:"clause_index"` + // PatternIndex supplies the pattern index input to the CountStoreFastPathDecision contract. + PatternIndex int `json:"pattern_index"` + // BindingSymbol supplies the binding symbol input to the CountStoreFastPathDecision contract. + BindingSymbol string `json:"binding_symbol,omitempty"` + // Target supplies the target input to the CountStoreFastPathDecision contract. + Target CountStoreFastPathTarget `json:"target"` + // KindSymbols supplies the kind symbols input to the CountStoreFastPathDecision contract. + KindSymbols []string `json:"kind_symbols,omitempty"` } +// ExactRangeExpansionDecision records the planner choice made for exact range expansion. type ExactRangeExpansionDecision struct { + // Target supplies the target input to the ExactRangeExpansionDecision contract. Target TraversalStepTarget `json:"target"` - Depth int64 `json:"depth"` + // Depth supplies the depth input to the ExactRangeExpansionDecision contract. + Depth int64 `json:"depth"` } +// PathRelationshipPredicateDecision records the planner choice made for path relationship predicate. type PathRelationshipPredicateDecision struct { - Target QuantifierTarget `json:"target"` - PathSymbol string `json:"path_symbol"` - BindingSymbol string `json:"binding_symbol"` + // Target supplies the target input to the PathRelationshipPredicateDecision contract. + Target QuantifierTarget `json:"target"` + // PathSymbol supplies the path symbol input to the PathRelationshipPredicateDecision contract. + PathSymbol string `json:"path_symbol"` + // BindingSymbol supplies the binding symbol input to the PathRelationshipPredicateDecision contract. + BindingSymbol string `json:"binding_symbol"` } +// AggregateTraversalCountDecision records the planner choice made for aggregate traversal count. type AggregateTraversalCountDecision struct { - QueryPartIndex int `json:"query_part_index"` - SourceSymbol string `json:"source_symbol"` - TerminalSymbol string `json:"terminal_symbol"` - CountAlias string `json:"count_alias"` - Limit int64 `json:"limit,omitempty"` - Target TraversalStepTarget `json:"target"` + // QueryPartIndex supplies the query part index input to the AggregateTraversalCountDecision contract. + QueryPartIndex int `json:"query_part_index"` + // SourceSymbol supplies the source symbol input to the AggregateTraversalCountDecision contract. + SourceSymbol string `json:"source_symbol"` + // TerminalSymbol supplies the terminal symbol input to the AggregateTraversalCountDecision contract. + TerminalSymbol string `json:"terminal_symbol"` + // CountAlias supplies the count alias input to the AggregateTraversalCountDecision contract. + CountAlias string `json:"count_alias"` + // Limit supplies the limit input to the AggregateTraversalCountDecision contract. + Limit int64 `json:"limit,omitempty"` + // Target supplies the target input to the AggregateTraversalCountDecision contract. + Target TraversalStepTarget `json:"target"` +} + +type FieldRequirement string + +const ( + // FieldRequirementEntityID requires only the scalar entity identifier. + FieldRequirementEntityID FieldRequirement = "entity_id" + + // FieldRequirementKinds requires the entity kind array in addition to identity. + FieldRequirementKinds FieldRequirement = "kinds" + + // FieldRequirementProperties requires the entity property document in addition to identity. + FieldRequirementProperties FieldRequirement = "properties" + + // FieldRequirementFullEntity requires the complete node or relationship composite. + FieldRequirementFullEntity FieldRequirement = "full_entity" + + // FieldRequirementRelationshipIDs requires relationship IDs without hydrated relationship composites. + FieldRequirementRelationshipIDs FieldRequirement = "relationship_ids" + + // FieldRequirementOrderedPathEdgeIDs requires edge IDs in path traversal order. + FieldRequirementOrderedPathEdgeIDs FieldRequirement = "ordered_path_edge_ids" + + // FieldRequirementFullPath requires the complete hydrated path composite. + FieldRequirementFullPath FieldRequirement = "full_path" +) + +// FieldRequirementUse groups planner state that must remain consistent while analyzing field requirement use. +type FieldRequirementUse struct { + // Ordinal orders this use relative to the other uses in its query part. + Ordinal int `json:"ordinal"` + // Fields lists the binding components consumed at this use. + Fields []FieldRequirement `json:"fields"` + // Internal reports whether the requirement is internal to translation rather than an external consumer. + Internal bool `json:"internal,omitempty"` +} + +// FieldRequirementDecision is analysis metadata only. Phase 6B consumes this +// staged information when it is safe to lower a composite binding to scalar +// state; recording it here intentionally does not change SQL semantics. +type FieldRequirementDecision struct { + // QueryPartIndex identifies the query part containing the analyzed binding. + QueryPartIndex int `json:"query_part_index"` + // Symbol is the Cypher binding whose representation requirements were analyzed. + Symbol string `json:"symbol"` + // Fields is the union of binding components required by all uses. + Fields []FieldRequirement `json:"fields"` + // Uses preserves the ordered evidence contributing to Fields. + Uses []FieldRequirementUse `json:"uses"` + // LastUse is the greatest use ordinal observed for the binding. + LastUse int `json:"last_use"` } +// AggregateTraversalCountShape groups planner state that must remain consistent while analyzing aggregate traversal count shape. type AggregateTraversalCountShape struct { - QueryPartIndex int - SourceSymbol string - TerminalSymbol string - CountAlias string + // QueryPartIndex supplies the query part index input to the AggregateTraversalCountShape contract. + QueryPartIndex int + // SourceSymbol supplies the source symbol input to the AggregateTraversalCountShape contract. + SourceSymbol string + // TerminalSymbol supplies the terminal symbol input to the AggregateTraversalCountShape contract. + TerminalSymbol string + // CountAlias supplies the count alias input to the AggregateTraversalCountShape contract. + CountAlias string + // ReturnSourceAlias supplies the return source alias input to the AggregateTraversalCountShape contract. ReturnSourceAlias string - ReturnCountAlias string - ReturnCount bool - Limit int64 - SourceMatch *cypher.Match - TerminalMatch *cypher.Match - SourceKinds graph.Kinds - TerminalKinds graph.Kinds + // ReturnCountAlias supplies the return count alias input to the AggregateTraversalCountShape contract. + ReturnCountAlias string + // ReturnCount records the number of return count. + ReturnCount bool + // Limit supplies the limit input to the AggregateTraversalCountShape contract. + Limit int64 + // SourceMatch supplies the source match input to the AggregateTraversalCountShape contract. + SourceMatch *cypher.Match + // TerminalMatch supplies the terminal match input to the AggregateTraversalCountShape contract. + TerminalMatch *cypher.Match + // SourceKinds supplies the source kinds input to the AggregateTraversalCountShape contract. + SourceKinds graph.Kinds + // TerminalKinds supplies the terminal kinds input to the AggregateTraversalCountShape contract. + TerminalKinds graph.Kinds + // RelationshipKinds supplies the relationship kinds input to the AggregateTraversalCountShape contract. RelationshipKinds graph.Kinds - Direction graph.Direction - MinDepth int64 - MaxDepth int64 - Target TraversalStepTarget + // Direction selects the traversal orientation covered by the contract. + Direction graph.Direction + // MinDepth supplies the min depth input to the AggregateTraversalCountShape contract. + MinDepth int64 + // MaxDepth supplies the max depth input to the AggregateTraversalCountShape contract. + MaxDepth int64 + // Target supplies the target input to the AggregateTraversalCountShape contract. + Target TraversalStepTarget } +// LoweringPlan records lowering analyses and semantic or physical decisions for a query. type LoweringPlan struct { - ProjectionPruning []ProjectionPruningDecision `json:"projection_pruning,omitempty"` - LatePathMaterialization []LatePathMaterializationDecision `json:"late_path_materialization,omitempty"` - ExpandInto []ExpandIntoDecision `json:"expand_into,omitempty"` - TraversalDirection []TraversalDirectionDecision `json:"traversal_direction,omitempty"` - ShortestPathStrategy []ShortestPathStrategyDecision `json:"shortest_path_strategy,omitempty"` - ShortestPathFilter []ShortestPathFilterDecision `json:"shortest_path_filter,omitempty"` - LimitPushdown []LimitPushdownDecision `json:"limit_pushdown,omitempty"` - ExpansionSuffixPushdown []ExpansionSuffixPushdownDecision `json:"expansion_suffix_pushdown,omitempty"` - PredicatePlacement []PredicatePlacementDecision `json:"predicate_placement,omitempty"` - PatternPredicate []PatternPredicatePlacementDecision `json:"pattern_predicate_placement,omitempty"` - CountStoreFastPath []CountStoreFastPathDecision `json:"count_store_fast_path,omitempty"` - ExactRangeExpansion []ExactRangeExpansionDecision `json:"exact_range_expansion,omitempty"` + // ProjectionPruning records traversal fields that downstream clauses do not require. + ProjectionPruning []ProjectionPruningDecision `json:"projection_pruning,omitempty"` + // LatePathMaterialization records path values whose hydration can be deferred. + LatePathMaterialization []LatePathMaterializationDecision `json:"late_path_materialization,omitempty"` + // ExpandInto records traversal steps whose endpoints are both already bound. + ExpandInto []ExpandIntoDecision `json:"expand_into,omitempty"` + // TraversalDirection records planned logical direction changes. + TraversalDirection []TraversalDirectionDecision `json:"traversal_direction,omitempty"` + // ShortestPathStrategy records directional search choices for shortest-path steps. + ShortestPathStrategy []ShortestPathStrategyDecision `json:"shortest_path_strategy,omitempty"` + // ShortestPathFilter records endpoint filters selected for materialization. + ShortestPathFilter []ShortestPathFilterDecision `json:"shortest_path_filter,omitempty"` + // LimitPushdown records row limits that may safely constrain traversal work. + LimitPushdown []LimitPushdownDecision `json:"limit_pushdown,omitempty"` + // ExpansionSuffixPushdown records fixed suffixes considered for supplemental filtering, including withheld candidates. + ExpansionSuffixPushdown []ExpansionSuffixPushdownDecision `json:"expansion_suffix_pushdown,omitempty"` + // PredicatePlacement supplies the predicate placement input to the LoweringPlan contract. + PredicatePlacement []PredicatePlacementDecision `json:"predicate_placement,omitempty"` + // PatternPredicate records existence lowering selected for pattern predicates. + PatternPredicate []PatternPredicatePlacementDecision `json:"pattern_predicate_placement,omitempty"` + // CountStoreFastPath records counts answerable directly from graph statistics. + CountStoreFastPath []CountStoreFastPathDecision `json:"count_store_fast_path,omitempty"` + // ExactRangeExpansion records short fixed-depth ranges selected for unrolling. + ExactRangeExpansion []ExactRangeExpansionDecision `json:"exact_range_expansion,omitempty"` + // PathRelationshipPredicate records relationship quantifiers attached to carried path state. PathRelationshipPredicate []PathRelationshipPredicateDecision `json:"path_relationship_predicate,omitempty"` - AggregateTraversalCount []AggregateTraversalCountDecision `json:"aggregate_traversal_count,omitempty"` + // AggregateTraversalCount records traversals lowered directly to aggregate counts. + AggregateTraversalCount []AggregateTraversalCountDecision `json:"aggregate_traversal_count,omitempty"` + // FieldRequirements records downstream representation needs for each analyzed binding. + FieldRequirements []FieldRequirementDecision `json:"field_requirements,omitempty"` + // ShortestPathExecutor records physical executor choices for shortest-path steps. + ShortestPathExecutor []ShortestPathExecutorDecision `json:"shortest_path_executor,omitempty"` + // ExpansionSearchStrategy records physical search choices for variable expansions. + ExpansionSearchStrategy []ExpansionSearchStrategyDecision `json:"expansion_search_strategy,omitempty"` + // EndpointResolution records planned-only bounded endpoint materialization for SP/ASP traversals. + EndpointResolution []EndpointResolutionDecision `json:"endpoint_resolution,omitempty"` + // TraversalPredicate records conservative locality and universality classifications. + TraversalPredicate []TraversalPredicateDecision `json:"traversal_predicate,omitempty"` } +// Empty reports whether the plan contains no lowering-analysis or decision entries. func (s LoweringPlan) Empty() bool { return len(s.ProjectionPruning) == 0 && len(s.LatePathMaterialization) == 0 && @@ -242,9 +1209,15 @@ func (s LoweringPlan) Empty() bool { len(s.CountStoreFastPath) == 0 && len(s.ExactRangeExpansion) == 0 && len(s.PathRelationshipPredicate) == 0 && - len(s.AggregateTraversalCount) == 0 + len(s.AggregateTraversalCount) == 0 && + len(s.FieldRequirements) == 0 && + len(s.ShortestPathExecutor) == 0 && + len(s.ExpansionSearchStrategy) == 0 && + len(s.EndpointResolution) == 0 && + len(s.TraversalPredicate) == 0 } +// Decisions returns one summary entry for each lowering category present in the plan. func (s LoweringPlan) Decisions() []LoweringDecision { var decisions []LoweringDecision add := func(name string, applied bool) { @@ -266,10 +1239,16 @@ func (s LoweringPlan) Decisions() []LoweringDecision { add(LoweringExactRangeExpansion, len(s.ExactRangeExpansion) > 0) add(LoweringPathRelationshipPredicate, len(s.PathRelationshipPredicate) > 0) add(LoweringAggregateTraversalCount, len(s.AggregateTraversalCount) > 0) + add(LoweringFieldRequirements, len(s.FieldRequirements) > 0) + add(LoweringShortestPathExecutor, len(s.ShortestPathExecutor) > 0) + add(LoweringExpansionSearchStrategy, len(s.ExpansionSearchStrategy) > 0) + add(LoweringEndpointResolution, len(s.EndpointResolution) > 0) + add(LoweringTraversalPredicateClassification, len(s.TraversalPredicate) > 0) return decisions } +// IndexPatternTargets evaluates planner state needed for index pattern targets. func IndexPatternTargets(query *cypher.RegularQuery) map[*cypher.PatternPart]PatternTarget { targets := map[*cypher.PatternPart]PatternTarget{} @@ -296,6 +1275,7 @@ func IndexPatternTargets(query *cypher.RegularQuery) map[*cypher.PatternPart]Pat return targets } +// IndexPatternPredicateTargets evaluates planner state needed for index pattern predicate targets. func IndexPatternPredicateTargets(query *cypher.RegularQuery) map[*cypher.PatternPredicate]PatternTarget { targets := map[*cypher.PatternPredicate]PatternTarget{} @@ -322,6 +1302,7 @@ func IndexPatternPredicateTargets(query *cypher.RegularQuery) map[*cypher.Patter return targets } +// indexReadingClauseTargets maps each pattern in readingClauses to stable source coordinates. func indexReadingClauseTargets(targets map[*cypher.PatternPart]PatternTarget, queryPartIndex int, readingClauses []*cypher.ReadingClause) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil { @@ -338,6 +1319,7 @@ func indexReadingClauseTargets(targets map[*cypher.PatternPart]PatternTarget, qu } } +// indexQueryPartPatternPredicateTargets assigns stable target coordinates to pattern predicates in one query part. func indexQueryPartPatternPredicateTargets(targets map[*cypher.PatternPredicate]PatternTarget, queryPartIndex int, queryPart cypher.SyntaxNode) { for _, indexedPredicate := range indexedPatternPredicatesInQueryPart(queryPart) { targets[indexedPredicate.Predicate] = PatternTarget{ diff --git a/cypher/models/pgsql/optimize/lowering_plan.go b/cypher/models/pgsql/optimize/lowering_plan.go index d5038dab..d44d4821 100644 --- a/cypher/models/pgsql/optimize/lowering_plan.go +++ b/cypher/models/pgsql/optimize/lowering_plan.go @@ -1,6 +1,7 @@ package optimize import ( + "slices" "strings" "github.com/specterops/dawgs/cypher/models/cypher" @@ -8,39 +9,92 @@ import ( "github.com/specterops/dawgs/graph" ) +// sourceTraversalStep groups the node and relationship patterns that make up one analyzed traversal step. type sourceTraversalStep struct { - LeftNode *cypher.NodePattern + // LeftNode is the node pattern immediately preceding Relationship in source syntax. + LeftNode *cypher.NodePattern + // Relationship is the edge pattern connecting the two endpoints. Relationship *cypher.RelationshipPattern - RightNode *cypher.NodePattern + // RightNode is the node pattern immediately following Relationship in source syntax. + RightNode *cypher.NodePattern } +// boundSourceSelectivity ranks how strongly known constraints bound a traversal source. type boundSourceSelectivity int const ( - traversalDirectionReasonRightBound = "right_bound" - traversalDirectionReasonRightConstrained = "right_constrained" - traversalDirectionReasonRightPredicate = "right_predicate" + // traversalDirectionReasonRightBound explains a direction flip toward an already bound right endpoint. + traversalDirectionReasonRightBound = "right_bound" + + // traversalDirectionReasonRightConstrained explains a direction flip toward a constrained right endpoint. + traversalDirectionReasonRightConstrained = "right_constrained" + + // traversalDirectionReasonRightPredicate explains a direction flip toward a right endpoint with a selective predicate. + traversalDirectionReasonRightPredicate = "right_predicate" + + // traversalDirectionReasonTerminalKindOnlyEstimateWide explains rejection of a terminal kind whose estimate is too broad. traversalDirectionReasonTerminalKindOnlyEstimateWide = "terminal kind-only estimate too broad" - traversalDirectionReasonBoundSourceSelective = "bound source estimate selective" + // traversalDirectionReasonBoundSourceSelective explains retention of a sufficiently selective bound source. + traversalDirectionReasonBoundSourceSelective = "bound source estimate selective" + + // shortestPathStrategyReasonBoundEndpointPairs selects bidirectional search for materialized endpoint pairs. shortestPathStrategyReasonBoundEndpointPairs = "bound_endpoint_pairs" + + // shortestPathStrategyReasonEndpointPredicates selects bidirectional search for predicates on both endpoints. shortestPathStrategyReasonEndpointPredicates = "endpoint_predicates" - shortestPathFilterReasonTerminalPredicate = "terminal_predicate" + // shortestPathFilterReasonTerminalPredicate materializes a filter for a selective terminal predicate. + shortestPathFilterReasonTerminalPredicate = "terminal_predicate" + + // shortestPathFilterReasonEndpointPairPredicates materializes a filter for correlated endpoint-pair predicates. shortestPathFilterReasonEndpointPairPredicates = "endpoint_pair_predicates" ) const ( + // boundSourceSelectivityNone indicates that no useful source constraint was found. boundSourceSelectivityNone boundSourceSelectivity = iota + + // boundSourceSelectivityKindOnly indicates that only a node-kind predicate constrains the source. boundSourceSelectivityKindOnly + + // boundSourceSelectivityPredicate indicates that a non-unique predicate constrains the source. boundSourceSelectivityPredicate + + // boundSourceSelectivityUnique indicates that a unique lookup constrains the source. boundSourceSelectivityUnique + + // boundSourceSelectivityLimited indicates that a row limit bounds the source. boundSourceSelectivityLimited + + // boundSourceSelectivityTopN indicates that an ordered or aggregate projection with a limit bounds the source. boundSourceSelectivityTopN ) -const maxExactRangeExpansionDepth int64 = 2 +const ( + // maxExactRangeExpansionDepth is the largest exact range expanded into fixed traversal steps. + maxExactRangeExpansionDepth int64 = 2 + + // defaultShortestPathExpansionDepth supplies the maximum depth for an otherwise open shortest-path range. + defaultShortestPathExpansionDepth int64 = 15 + + // defaultShortestPathStateLimit caps intermediate states admitted by guarded experimental executors. + defaultShortestPathStateLimit int64 = ShortestPathI2QualifiedStateLimit + // defaultShortestPathFrontierLimit independently caps queued/current frontier state. + defaultShortestPathFrontierLimit int64 = ShortestPathI2QualifiedFrontierLimit + + // defaultShortestPathPredecessorLimit independently caps retained witness predecessors. + defaultShortestPathPredecessorLimit int64 = 100_000 + + // defaultAllShortestPathsEnumerationLimit independently caps staged distinct path arrays. + defaultAllShortestPathsEnumerationLimit int64 = 100_000 + + // defaultAllShortestPathsOutputBytesLimit independently caps staged ordered edge-array bytes. + defaultAllShortestPathsOutputBytesLimit int64 = 64 * 1024 * 1024 +) + +// BuildLoweringPlan analyzes a query and selects safe semantic and physical lowering decisions. func BuildLoweringPlan(query *cypher.RegularQuery, predicateAttachments []PredicateAttachment) (LoweringPlan, error) { if query == nil || query.SingleQuery == nil { return LoweringPlan{}, nil @@ -93,9 +147,13 @@ func BuildLoweringPlan(query *cypher.RegularQuery, predicateAttachments []Predic attachPredicatePlacementsToSuffixPushdowns(&plan) appendCountStoreFastPathDecisions(&plan, query) appendAggregateTraversalCountDecisions(&plan, query) + finalizeShortestPathExecutorDecisions(&plan, query) + finalizeExpansionSearchStrategyDecisions(&plan, query) + finalizeTraversalEnvelopeDecisions(&plan, query) return plan, nil } +// appendQueryPartLowerings runs every lowering analysis for one query part and appends its decisions to plan. func appendQueryPartLowerings( plan *LoweringPlan, queryPartIndex int, @@ -118,16 +176,1300 @@ func appendQueryPartLowerings( appendLatePathMaterializationDecisions(plan, queryPartIndex, readingClauses, sourceReferences) appendPatternPredicateProjectionLowerings(plan, queryPartIndex, indexedPatternPredicates, sourceReferences) appendPatternPredicatePlacementDecisions(plan, queryPartIndex, indexedPatternPredicates) - appendExpandIntoDecisions(plan, queryPartIndex, readingClauses) + appendExpandIntoDecisions(plan, queryPartIndex, readingClauses, initialDeclaredSymbols) appendTraversalDirectionDecisions(plan, queryPartIndex, readingClauses, bindingPredicateSymbols(predicateAttachments, queryPartIndex), initialDeclaredSymbols, initialSelectivity) shortestPathSearchSymbols := shortestPathSearchPredicateSymbols(readingClauses) appendShortestPathStrategyDecisions(plan, queryPartIndex, readingClauses, shortestPathSearchSymbols) appendShortestPathFilterDecisions(plan, queryPartIndex, readingClauses, shortestPathSearchSymbols) + appendShortestPathExecutorDecisions(plan, queryPartIndex, queryPart, readingClauses, sourceReferences) + appendEndpointResolutionDecisions(plan, queryPartIndex, queryPart, readingClauses, initialDeclaredSymbols) + appendTraversalPredicateDecisions(plan, queryPartIndex, queryPart, readingClauses) appendLimitPushdownDecisions(plan, queryPartIndex, queryPart, readingClauses) - appendExpansionSuffixPushdownDecisions(plan, queryPartIndex, readingClauses) + appendExpansionSuffixPushdownDecisions(plan, queryPartIndex, readingClauses, sourceReferences) + appendEndpointSeededExpansionDecisions(plan, queryPartIndex, queryPart, readingClauses, sourceReferences, initialDeclaredSymbols) + appendExpansionSearchStrategyDecisions(plan, queryPartIndex, queryPart, readingClauses, sourceReferences, initialDeclaredSymbols) + fieldRequirements, err := collectFieldRequirements(queryPartIndex, queryPart) + if err != nil { + return err + } + plan.FieldRequirements = append(plan.FieldRequirements, fieldRequirements...) + applyShortestPathObservationModes(plan, queryPartIndex, readingClauses, fieldRequirements) + applyExpansionSearchObservationModes(plan, queryPartIndex, readingClauses, fieldRequirements) return nil } +// appendEndpointSeededExpansionDecisions qualifies terminal expansions with a fixed prefix for guarded reverse search. +func appendEndpointSeededExpansionDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}, initialDeclaredSymbols map[string]struct{}) { + _, updatingClauses := queryPartProjection(queryPart) + declaredSymbols := copyStringSet(initialDeclaredSymbols) + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + searchSymbols := shortestPathSearchPredicateSymbols([]*cypher.ReadingClause{readingClause}) + idEqualities := singletonIDEqualityCounts(readingClause.Match.Where) + for patternIndex, patternPart := range readingClause.Match.Pattern { + steps := traversalStepsForPattern(patternPart) + variableExpansions := 0 + for _, step := range steps { + if step.Relationship != nil && step.Relationship.Range != nil { + variableExpansions++ + } + } + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil || stepIndex == 0 { + continue + } + target := PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex) + prefixLength := stepIndex + terminal := stepIndex == len(steps)-1 + directedPrefix := true + prefixFixed := true + for _, prefixStep := range steps[:stepIndex] { + directedPrefix = directedPrefix && prefixStep.Relationship != nil && prefixStep.Relationship.Direction != graph.DirectionBoth + prefixFixed = prefixFixed && prefixStep.Relationship != nil && prefixStep.Relationship.Range == nil + } + minDepth := int64(1) + if step.Relationship.Range.StartIndex != nil { + minDepth = *step.Relationship.Range.StartIndex + } + maxDepth := int64(15) + if step.Relationship.Range.EndIndex != nil { + maxDepth = *step.Relationship.Range.EndIndex + } + terminalSymbol := variableSymbol(step.RightNode.Variable) + _, propertySearch := searchSymbols[terminalSymbol] + idSearch := idEqualities[terminalSymbol] == 1 + seedClass := "" + if idSearch { + seedClass = "id_equality" + } else if propertySearch { + seedClass = endpointSeedPredicateClass(readingClause.Match.Where, terminalSymbol) + } + terminalSelective := idSearch || propertySearch + terminalCorrelated := symbolDeclared(declaredSymbols, terminalSymbol) + terminalPredicateLocal := predicateTermsForSymbolAreLocal(readingClause.Match.Where, terminalSymbol) + relationshipPredicate := step.Relationship.Properties != nil || syntaxDependsOn(readingClause.Match.Where, variableSymbol(step.Relationship.Variable)) + pathDependentPredicate := patternPart != nil && patternPart.Variable != nil && syntaxDependsOn(readingClause.Match.Where, patternPart.Variable.Symbol) + deterministicPredicates := !syntaxContainsNonIdentityFunctionInvocation(patternPart) && !syntaxContainsNonIdentityFunctionInvocation(readingClause.Match.Where) + observation := ExpansionSearchObservationEndpointIDs + if patternPart != nil && patternPart.Variable != nil && referencesSourceIdentifier(sourceReferences, patternPart.Variable.Symbol) { + observation = ExpansionSearchObservationFullPath + } + facts := []ExpansionSearchEligibilityFact{ + { + Name: "read_only", + Eligible: updatingClauses == 0, + }, + { + Name: "non_optional", + Eligible: !readingClause.Match.Optional, + }, + { + Name: "ordinary_path", + Eligible: patternPart != nil && !patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern, + }, + { + Name: "single_variable_expansion_in_region", + Eligible: variableExpansions == 1, + }, + { + Name: "terminal_expansion", + Eligible: terminal, + }, + { + Name: "exact_one_hop_prefix", + Eligible: prefixLength == 1 && prefixFixed, + }, + { + Name: "directed_prefix", + Eligible: directedPrefix, + }, + { + Name: "directed_expansion", + Eligible: step.Relationship.Direction != graph.DirectionBoth, + }, + { + Name: "supported_effective_depth", + Eligible: maxDepth >= minDepth && maxDepth <= 64, + }, + { + Name: "minimum_depth_one", + Eligible: minDepth >= 1, + }, + { + Name: "terminal_unbound", + Eligible: !terminalCorrelated, + }, + { + Name: "selective_terminal_predicate", + Eligible: terminalSelective, + }, + { + Name: "terminal_predicate_local", + Eligible: terminalPredicateLocal, + }, + { + Name: "single_relationship_kind", + Eligible: len(step.Relationship.Kinds) == 1, + }, + { + Name: "no_relationship_variable", + Eligible: step.Relationship.Variable == nil, + }, + { + Name: "no_relationship_predicate", + Eligible: !relationshipPredicate, + }, + { + Name: "no_path_dependent_predicate", + Eligible: !pathDependentPredicate, + }, + { + Name: "deterministic_predicates", + Eligible: deterministicPredicates, + }, + { + Name: "supported_observation", + Eligible: observation != ExpansionSearchObservationUnsupported, + }, + } + eligible := expansionSearchFactsEligible(facts) + fallbackReason := ExpansionSearchFallbackTournamentUnqualified + switch { + case updatingClauses > 0: + fallbackReason = ExpansionSearchFallbackMutation + case readingClause.Match.Optional: + fallbackReason = ExpansionSearchFallbackOptionalMatch + case !terminal: + fallbackReason = ExpansionSearchFallbackExpansionNotTerminal + case prefixLength == 0: + fallbackReason = ExpansionSearchFallbackNoFixedPrefix + case prefixLength != 1 || !prefixFixed: + fallbackReason = ExpansionSearchFallbackPrefixTooLong + case !directedPrefix: + fallbackReason = ExpansionSearchFallbackDirectionlessPrefix + case step.Relationship.Direction == graph.DirectionBoth: + fallbackReason = ExpansionSearchFallbackDirectionlessExpansion + case minDepth < 1: + fallbackReason = ExpansionSearchFallbackZeroDepth + case maxDepth < minDepth || maxDepth > 64: + fallbackReason = ExpansionSearchFallbackUnsupportedDepth + case variableExpansions != 1: + fallbackReason = ExpansionSearchFallbackMultipleVariableExpansions + case terminalCorrelated || !terminalPredicateLocal: + fallbackReason = ExpansionSearchFallbackCorrelatedTerminal + case !terminalSelective: + fallbackReason = ExpansionSearchFallbackTerminalNotSelective + case len(step.Relationship.Kinds) != 1: + fallbackReason = ExpansionSearchFallbackTournamentUnqualified + case step.Relationship.Variable != nil: + fallbackReason = ExpansionSearchFallbackRelationshipVariable + case relationshipPredicate: + fallbackReason = ExpansionSearchFallbackRelationshipPredicate + case pathDependentPredicate: + fallbackReason = ExpansionSearchFallbackPathDependentPredicate + case !deterministicPredicates: + fallbackReason = ExpansionSearchFallbackNonDeterministicPredicate + } + selected := ExpansionSearchStepwiseForward + selectionMode := "incumbent_default" + if eligible { + selected = ExpansionSearchEndpointSeededReverse + selectionMode = "static_guarded" + fallbackReason = "" + } + projection, _ := queryPartProjection(queryPart) + candidate := contiguousExpansionOrientationCandidate{ + Target: target, + Family: "fixed_prefix_terminal_expansion", + PlannedPolicy: ExpansionSearchPolicyEndpointGuardV1, + PlannedCandidates: []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse}, + EmittedCandidates: []ExpansionSearchStrategy{ExpansionSearchStepwiseForward}, + CandidateStrategy: ExpansionSearchEndpointSeededReverse, + ProbeCaps: ExpansionSearchProbeCaps{ + ReverseSeedRowLimit: 32, + }, + Admission: ExpansionSearchAdmission{ + StateLimit: 4096, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, + PrefixStartStep: 0, + PrefixEndStep: stepIndex - 1, + PrefixLength: prefixLength, + SeedPredicateClass: seedClass, + EndpointLimit: 32, + } + if eligible { + candidate.EmittedPolicy = ExpansionSearchPolicyEndpointGuardV1 + candidate.EmittedCandidates = []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse} + } + plan.ExpansionSearchStrategy = append(plan.ExpansionSearchStrategy, candidate.decision(contiguousExpansionOrientationQualification{ + SelectedStrategy: selected, + StructurallyEligible: eligible, + StaticallyEligible: eligible, + EligibilityFacts: facts, + HasFinalLimit: projection != nil && projection.Limit != nil, + ObservationMode: observation, + LogicalDirection: step.Relationship.Direction.String(), + MinimumDepth: minDepth, + MaximumDepth: maxDepth, + SelectionMode: selectionMode, + SelectorVersion: "endpoint-seeded-guarded-v1", + FallbackReason: fallbackReason, + })) + } + declarePatternSymbols(declaredSymbols, patternPart) + } + declareWhereSymbols(declaredSymbols, readingClause.Match) + } +} + +// predicateTermsForSymbolAreLocal reports whether every predicate mentioning symbol depends on no other binding. +func predicateTermsForSymbolAreLocal(where *cypher.Where, symbol string) bool { + if where == nil || symbol == "" { + return true + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + dependencies := sortedDependencies(term) + if !slices.Contains(dependencies, symbol) { + continue + } + for _, dependency := range dependencies { + if dependency != symbol { + return false + } + } + } + } + return true +} + +// endpointSeedPredicateClass classifies a terminal property comparison as equality, suffix matching, or generic search. +func endpointSeedPredicateClass(where *cypher.Where, symbol string) string { + if where == nil { + return "" + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + comparison, ok := term.(*cypher.Comparison) + if !ok || comparison == nil || len(comparison.Partials) != 1 { + continue + } + partial := comparison.Partials[0] + leftSymbol, leftOK := propertyLookupVariableSymbol(comparison.Left) + rightSymbol, rightOK := propertyLookupVariableSymbol(partial.Right) + if (leftOK && leftSymbol == symbol && !expressionReferencesAnySource(partial.Right)) || (rightOK && rightSymbol == symbol && !expressionReferencesAnySource(comparison.Left)) { + switch partial.Operator { + case cypher.OperatorEquals: + return "property_equality" + case cypher.OperatorEndsWith: + return "property_ends_with" + default: + return "property_search" + } + } + } + } + return "" +} + +// hasExpansionSearchDecision reports whether plan already contains a search decision for target. +func hasExpansionSearchDecision(plan *LoweringPlan, target TraversalStepTarget) bool { + for _, decision := range plan.ExpansionSearchStrategy { + if decision.Target == target { + return true + } + } + return false +} + +// appendExpansionSearchStrategyDecisions qualifies variable expansions for fixed-suffix search strategies. +func appendExpansionSearchStrategyDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}, initialDeclaredSymbols map[string]struct{}) { + _, updatingClauses := queryPartProjection(queryPart) + declaredSymbols := copyStringSet(initialDeclaredSymbols) + queryPartVariableExpansions := 0 + for _, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + for _, step := range traversalStepsForPattern(patternPart) { + if step.Relationship != nil && step.Relationship.Range != nil { + queryPartVariableExpansions++ + } + } + } + } + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + steps := traversalStepsForPattern(patternPart) + deterministicPredicates := !syntaxContainsFunctionInvocation(patternPart) && !syntaxContainsFunctionInvocation(readingClause.Match.Where) + pathDependentPredicate := patternPart != nil && patternPart.Variable != nil && syntaxDependsOn(readingClause.Match.Where, patternPart.Variable.Symbol) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil { + continue + } + target := PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex) + if hasExpansionSearchDecision(plan, target) { + continue + } + limitConflict := hasLimitPushdownForTarget(plan, target) + suffixLength := fixedSuffixLength(steps[stepIndex+1:]) + suffixEnd := stepIndex + suffixLength + minDepth := int64(1) + if step.Relationship.Range.StartIndex != nil { + minDepth = *step.Relationship.Range.StartIndex + } + maxDepth := int64(0) + boundedDepth := step.Relationship.Range.EndIndex != nil + if boundedDepth { + maxDepth = *step.Relationship.Range.EndIndex + } + directedExpansion := step.Relationship.Direction != graph.DirectionBoth + directedSuffix := suffixLength > 0 + noSuffixRelationshipVariables := true + noRelationshipPredicates := step.Relationship.Properties == nil && !syntaxDependsOn(readingClause.Match.Where, variableSymbol(step.Relationship.Variable)) + suffixSteps := steps[stepIndex+1 : stepIndex+1+suffixLength] + uncorrelatedSuffix := true + for _, suffixStep := range suffixSteps { + directedSuffix = directedSuffix && suffixStep.Relationship.Direction != graph.DirectionBoth + noSuffixRelationshipVariables = noSuffixRelationshipVariables && suffixStep.Relationship.Variable == nil + noRelationshipPredicates = noRelationshipPredicates && suffixStep.Relationship.Properties == nil && !syntaxDependsOn(readingClause.Match.Where, variableSymbol(suffixStep.Relationship.Variable)) + uncorrelatedSuffix = uncorrelatedSuffix && !symbolDeclared(declaredSymbols, variableSymbol(suffixStep.Relationship.Variable)) && !symbolDeclared(declaredSymbols, variableSymbol(suffixStep.RightNode.Variable)) + } + noCrossRegionPredicate := !hasCrossRegionPredicate(readingClause.Match.Where, step, suffixSteps) + boundRoot := symbolDeclared(declaredSymbols, variableSymbol(step.LeftNode.Variable)) + observation := ExpansionSearchObservationEndpointIDs + if patternPart != nil && patternPart.Variable != nil && referencesSourceIdentifier(sourceReferences, patternPart.Variable.Symbol) { + observation = ExpansionSearchObservationFullPath + } + facts := []ExpansionSearchEligibilityFact{ + { + Name: "read_only", + Eligible: updatingClauses == 0, + }, + { + Name: "non_optional", + Eligible: !readingClause.Match.Optional, + }, + { + Name: "ordinary_path", + Eligible: patternPart != nil && !patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern, + }, + { + Name: "single_variable_expansion", + Eligible: queryPartVariableExpansions == 1, + }, + { + Name: "bound_root", + Eligible: boundRoot, + }, + { + Name: "initial_variable_expansion", + Eligible: stepIndex == 0, + }, + { + Name: "directed_expansion", + Eligible: directedExpansion, + }, + { + Name: "bounded_supported_depth", + Eligible: boundedDepth && maxDepth >= minDepth && maxDepth <= 64, + }, + { + Name: "exact_three_hop_suffix", + Eligible: suffixLength == 3, + }, + { + Name: "qualified_fixed_suffix_topology", + Eligible: qualifiedFixedSuffixTopology(step, suffixSteps), + }, + { + Name: "directed_suffix", + Eligible: directedSuffix, + }, + { + Name: "no_relationship_variable", + Eligible: step.Relationship.Variable == nil && noSuffixRelationshipVariables, + }, + { + Name: "no_relationship_predicate", + Eligible: noRelationshipPredicates, + }, + { + Name: "uncorrelated_suffix", + Eligible: uncorrelatedSuffix, + }, + { + Name: "no_cross_region_predicate", + Eligible: noCrossRegionPredicate, + }, + { + Name: "no_path_dependent_predicate", + Eligible: !pathDependentPredicate, + }, + { + Name: "deterministic_predicates", + Eligible: deterministicPredicates, + }, + { + Name: "no_limit_pushdown_conflict", + Eligible: !limitConflict, + }, + { + Name: "supported_observation", + Eligible: observation != ExpansionSearchObservationUnsupported, + }, + } + eligible := true + for _, fact := range facts { + eligible = eligible && fact.Eligible + } + fallbackReason := ExpansionSearchFallbackTournamentUnqualified + switch { + case updatingClauses > 0: + fallbackReason = ExpansionSearchFallbackMutation + case readingClause.Match.Optional: + fallbackReason = ExpansionSearchFallbackOptionalMatch + case patternPart != nil && patternPart.AllShortestPathsPattern: + fallbackReason = ExpansionSearchFallbackAllShortestPaths + case patternPart != nil && patternPart.ShortestPathPattern: + fallbackReason = ExpansionSearchFallbackShortestPath + case queryPartVariableExpansions > 1: + fallbackReason = ExpansionSearchFallbackMultipleVariableExpansions + case stepIndex != 0: + fallbackReason = ExpansionSearchFallbackTournamentUnqualified + case !directedExpansion: + fallbackReason = ExpansionSearchFallbackDirectionlessExpansion + case !boundedDepth: + fallbackReason = ExpansionSearchFallbackUnboundedDepth + case maxDepth < minDepth || maxDepth > 64: + fallbackReason = ExpansionSearchFallbackUnsupportedDepth + case suffixLength == 0: + fallbackReason = ExpansionSearchFallbackNoFixedSuffix + case suffixLength < 3: + fallbackReason = ExpansionSearchFallbackSuffixTooShort + case suffixLength != 3: + fallbackReason = ExpansionSearchFallbackTournamentUnqualified + case !directedSuffix: + fallbackReason = ExpansionSearchFallbackDirectionlessSuffix + case !noRelationshipPredicates: + fallbackReason = ExpansionSearchFallbackRelationshipPredicate + case !uncorrelatedSuffix: + fallbackReason = ExpansionSearchFallbackCorrelatedSuffix + case !noCrossRegionPredicate: + fallbackReason = ExpansionSearchFallbackCrossRegionPredicate + case step.Relationship.Variable != nil || !noSuffixRelationshipVariables: + fallbackReason = ExpansionSearchFallbackRelationshipVariable + case pathDependentPredicate: + fallbackReason = ExpansionSearchFallbackPathDependentPredicate + case !deterministicPredicates: + fallbackReason = ExpansionSearchFallbackNonDeterministicPredicate + case limitConflict: + fallbackReason = ExpansionSearchFallbackLimitPushdownConflict + case !boundRoot && qualifiedFixedSuffixTopology(step, suffixSteps): + fallbackReason = ExpansionSearchFallbackUnboundRoot + } + candidate := contiguousExpansionOrientationCandidate{ + Target: target, + Family: "fixed_suffix_expansion", + PlannedPolicy: ExpansionSearchPolicyOrientationProbeV1, + PlannedCandidates: []ExpansionSearchStrategy{ + ExpansionSearchStepwiseForward, + ExpansionSearchLateHydratedForward, + ExpansionSearchFactoredSuffixForward, + ExpansionSearchSuffixSeededReverse, + ExpansionSearchBackwardViabilityForward, + }, + EmittedCandidates: []ExpansionSearchStrategy{ExpansionSearchStepwiseForward}, + CandidateStrategy: ExpansionSearchSuffixSeededReverse, + ProbeCaps: ExpansionSearchProbeCaps{ + RootRowLimit: ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, + Admission: ExpansionSearchAdmission{ + StateLimit: ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, + SuffixStartStep: stepIndex + 1, + SuffixEndStep: suffixEnd, + SuffixLength: suffixLength, + } + plan.ExpansionSearchStrategy = append(plan.ExpansionSearchStrategy, candidate.decision(contiguousExpansionOrientationQualification{ + SelectedStrategy: ExpansionSearchStepwiseForward, + StructurallyEligible: eligible, + StaticallyEligible: eligible, + EligibilityFacts: facts, + ObservationMode: observation, + LogicalDirection: step.Relationship.Direction.String(), + MinimumDepth: minDepth, + MaximumDepth: maxDepth, + SelectionMode: "incumbent_default", + SelectorVersion: "fixed-suffix-static-v1", + FallbackReason: fallbackReason, + })) + } + declarePatternSymbols(declaredSymbols, patternPart) + } + declareWhereSymbols(declaredSymbols, readingClause.Match) + } +} + +// syntaxContainsFunctionInvocation reports whether node contains any function invocation. +func syntaxContainsFunctionInvocation(node cypher.SyntaxNode) bool { + if node == nil { + return false + } + found := false + _ = walk.Cypher(node, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if _, isFunction := node.(*cypher.FunctionInvocation); isFunction { + found = true + } + })) + return found +} + +// syntaxContainsNonIdentityFunctionInvocation reports whether node invokes a function other than id. +func syntaxContainsNonIdentityFunctionInvocation(node cypher.SyntaxNode) bool { + if node == nil { + return false + } + found := false + _ = walk.Cypher(node, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + if function, isFunction := node.(*cypher.FunctionInvocation); isFunction && function != nil && !strings.EqualFold(function.Name, cypher.IdentityFunction) { + found = true + } + })) + return found +} + +// symbolDeclared reports whether a non-empty symbol is present in the declaration set. +func symbolDeclared(declared map[string]struct{}, symbol string) bool { + if symbol == "" { + return false + } + _, found := declared[symbol] + return found +} + +// hasCrossRegionPredicate reports whether one predicate depends on both expansion and suffix bindings. +func hasCrossRegionPredicate(where *cypher.Where, expansion sourceTraversalStep, suffix []sourceTraversalStep) bool { + if where == nil { + return false + } + prefixSymbols := map[string]struct{}{} + suffixSymbols := map[string]struct{}{} + addSymbol(prefixSymbols, variableSymbol(expansion.LeftNode.Variable)) + addSymbol(prefixSymbols, variableSymbol(expansion.Relationship.Variable)) + addSymbol(prefixSymbols, variableSymbol(expansion.RightNode.Variable)) + for _, step := range suffix { + addSymbol(suffixSymbols, variableSymbol(step.Relationship.Variable)) + addSymbol(suffixSymbols, variableSymbol(step.RightNode.Variable)) + } + for _, expression := range where.Expressions { + var hasPrefix, hasSuffix bool + for _, dependency := range sortedDependencies(expression) { + if _, found := prefixSymbols[dependency]; found { + hasPrefix = true + } + if _, found := suffixSymbols[dependency]; found { + hasSuffix = true + } + } + if hasPrefix && hasSuffix { + return true + } + } + return false +} + +// fixedSuffixLength counts consecutive fixed relationship steps before the next range expansion. +func fixedSuffixLength(steps []sourceTraversalStep) int { + length := 0 + for _, step := range steps { + if step.Relationship == nil || step.Relationship.Range != nil { + break + } + length++ + } + return length +} + +// hasLimitPushdownForTarget reports whether target already has a planned limit pushdown. +func hasLimitPushdownForTarget(plan *LoweringPlan, target TraversalStepTarget) bool { + for _, decision := range plan.LimitPushdown { + if decision.Target == target { + return true + } + } + return false +} + +// qualifiedFixedSuffixTopology reports whether an outbound single-kind expansion has the required three-step typed suffix. +func qualifiedFixedSuffixTopology(expansion sourceTraversalStep, suffix []sourceTraversalStep) bool { + if len(suffix) != 3 || expansion.Relationship == nil || len(expansion.Relationship.Kinds) != 1 || expansion.Relationship.Direction != graph.DirectionOutbound { + return false + } + for _, step := range suffix { + if step.Relationship == nil || step.RightNode == nil || step.Relationship.Direction != graph.DirectionOutbound || len(step.Relationship.Kinds) != 1 || len(step.RightNode.Kinds) != 1 { + return false + } + } + return true +} + +// applyExpansionSearchObservationModes classifies each expansion by the fields its external consumers require. +func applyExpansionSearchObservationModes(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, requirements []FieldRequirementDecision) { + externalFieldsBySymbol := map[string]map[FieldRequirement]struct{}{} + for _, requirement := range requirements { + fields := map[FieldRequirement]struct{}{} + for _, use := range requirement.Uses { + if use.Internal { + continue + } + for _, field := range use.Fields { + fields[field] = struct{}{} + } + } + externalFieldsBySymbol[requirement.Symbol] = fields + } + for idx := range plan.ExpansionSearchStrategy { + decision := &plan.ExpansionSearchStrategy[idx] + if decision.Target.QueryPartIndex != queryPartIndex || decision.Target.Predicate || decision.Target.ClauseIndex >= len(readingClauses) { + continue + } + clause := readingClauses[decision.Target.ClauseIndex] + if clause == nil || clause.Match == nil || decision.Target.PatternIndex >= len(clause.Match.Pattern) { + continue + } + pattern := clause.Match.Pattern[decision.Target.PatternIndex] + if pattern == nil || pattern.Variable == nil { + decision.ObservationMode = ExpansionSearchObservationEndpointIDs + setExpansionSearchEligibilityFact(decision, "supported_observation", true) + continue + } + fields := externalFieldsBySymbol[pattern.Variable.Symbol] + switch { + case hasFieldRequirement(fields, FieldRequirementFullPath): + decision.ObservationMode = ExpansionSearchObservationFullPath + case hasFieldRequirement(fields, FieldRequirementOrderedPathEdgeIDs), hasFieldRequirement(fields, FieldRequirementRelationshipIDs): + decision.ObservationMode = ExpansionSearchObservationOrderedPathIDs + case hasFieldRequirement(fields, FieldRequirementFullEntity): + decision.ObservationMode = ExpansionSearchObservationFullPath + case len(fields) == 0: + decision.ObservationMode = ExpansionSearchObservationEndpointIDs + default: + decision.ObservationMode = ExpansionSearchObservationUnsupported + } + supported := decision.ObservationMode != ExpansionSearchObservationUnsupported + setExpansionSearchEligibilityFact(decision, "supported_observation", supported) + if !supported { + decision.StructurallyEligible = false + decision.StaticallyEligible = false + decision.SelectedStrategy = decision.FallbackStrategy + decision.FallbackReason = ExpansionSearchFallbackUnsupportedObservation + } + } +} + +// hasFieldRequirement reports whether fields contains the requested binding representation. +func hasFieldRequirement(fields map[FieldRequirement]struct{}, field FieldRequirement) bool { + _, found := fields[field] + return found +} + +// setExpansionSearchEligibilityFact updates a named qualification result +// already present on decision and reports whether that fact belongs to this +// candidate family. +func setExpansionSearchEligibilityFact(decision *ExpansionSearchStrategyDecision, name string, eligible bool) bool { + for idx := range decision.EligibilityFacts { + if decision.EligibilityFacts[idx].Name == name { + decision.EligibilityFacts[idx].Eligible = eligible + return true + } + } + return false +} + +// expansionSearchFactsEligible reports whether every recorded expansion-search qualification passed. +func expansionSearchFactsEligible(facts []ExpansionSearchEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +// applyShortestPathObservationModes classifies shortest-path consumers and updates their known-observation qualification. +func applyShortestPathObservationModes(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, requirements []FieldRequirementDecision) { + fieldsBySymbol := map[string]map[FieldRequirement]struct{}{} + for _, requirement := range requirements { + fields := map[FieldRequirement]struct{}{} + for _, field := range requirement.Fields { + fields[field] = struct{}{} + } + fieldsBySymbol[requirement.Symbol] = fields + } + for idx := range plan.ShortestPathExecutor { + decision := &plan.ShortestPathExecutor[idx] + if decision.Target.QueryPartIndex != queryPartIndex || decision.Target.Predicate { + continue + } + if decision.Target.ClauseIndex >= len(readingClauses) { + continue + } + clause := readingClauses[decision.Target.ClauseIndex] + if clause == nil || clause.Match == nil || decision.Target.PatternIndex >= len(clause.Match.Pattern) { + continue + } + pattern := clause.Match.Pattern[decision.Target.PatternIndex] + if pattern == nil || pattern.Variable == nil { + continue + } + fields := fieldsBySymbol[pattern.Variable.Symbol] + if pattern.AllShortestPathsPattern { + if _, fullPath := fields[FieldRequirementFullPath]; fullPath { + decision.ObservationMode = ShortestPathObservationAllPaths + } else if _, orderedIDs := fields[FieldRequirementOrderedPathEdgeIDs]; orderedIDs { + decision.ObservationMode = ShortestPathObservationAllPaths + } + } else if _, fullPath := fields[FieldRequirementFullPath]; fullPath { + decision.ObservationMode = ShortestPathObservationOnePath + } else if _, orderedIDs := fields[FieldRequirementOrderedPathEdgeIDs]; orderedIDs { + decision.ObservationMode = ShortestPathObservationDistance + } + setShortestPathEligibilityFact(decision, "known_observation_mode", decision.ObservationMode != ShortestPathObservationUnknown) + } +} + +// appendShortestPathExecutorDecisions records eligibility facts and incumbent executor decisions for shortest-path expansions. +func appendShortestPathExecutorDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { + var ( + shortestCalls int + patternSources int + hasUnwind bool + ) + for _, readingClause := range readingClauses { + if readingClause == nil { + continue + } + if readingClause.Unwind != nil { + hasUnwind = true + } + if readingClause.Match == nil { + continue + } + patternSources += len(readingClause.Match.Pattern) + for _, patternPart := range readingClause.Match.Pattern { + if patternPart != nil && (patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern) { + shortestCalls++ + } + } + } + _, updatingClauses := queryPartProjection(queryPart) + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) { + continue + } + steps := traversalStepsForPattern(patternPart) + idEqualities := singletonIDEqualityCounts(readingClause.Match.Where) + pathPredicate := syntaxDependsOn(readingClause.Match.Where, variableSymbol(patternPart.Variable)) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil { + continue + } + minDepth := int64(1) + if step.Relationship.Range.StartIndex != nil { + minDepth = *step.Relationship.Range.StartIndex + } + maxDepth := defaultShortestPathExpansionDepth + boundedDepth := step.Relationship.Range.EndIndex != nil + maximumDepthSource := ShortestPathMaximumDepthPolicyDefault + if boundedDepth { + maxDepth = *step.Relationship.Range.EndIndex + maximumDepthSource = ShortestPathMaximumDepthExplicit + } + // PostgreSQL already caps syntax-open recursive traversal at + // defaultShortestPathExpansionDepth. Preserve that public behavior + // while allowing the same effective finite bound to use S3/S4. + supportedDepth := (minDepth == 0 || minDepth == 1) && maxDepth >= minDepth && maxDepth <= 64 + directionSupported := step.Relationship.Direction != graph.DirectionBoth + relationshipVariableObserved := step.Relationship.Variable != nil && referencesSourceIdentifier(sourceReferences, step.Relationship.Variable.Symbol) + noRelationshipVariable := step.Relationship.Variable == nil || (patternPart.AllShortestPathsPattern && !relationshipVariableObserved) + leftIDCount := idEqualities[variableSymbol(step.LeftNode.Variable)] + rightIDCount := idEqualities[variableSymbol(step.RightNode.Variable)] + singletonIDs := leftIDCount == 1 && rightIDCount == 1 + uncorrelatedSource := queryPartIndex == 0 && !hasUnwind + singleEndpointPair := patternSources == 1 + physicalExpansion := ShortestPathPhysicalExpansionStartID + topologyClassification := ShortestPathTopologyPhysicalOutbound + if step.Relationship.Direction == graph.DirectionInbound { + physicalExpansion = ShortestPathPhysicalExpansionEndID + if maxDepth <= 1 { + topologyClassification = ShortestPathTopologyPhysicalInboundShallow + } else { + topologyClassification = ShortestPathTopologyPhysicalInboundDeep + } + } else if step.Relationship.Direction == graph.DirectionBoth { + topologyClassification = ShortestPathTopologyDirectionless + } + facts := []ShortestPathEligibilityFact{ + { + Name: "supported_shortest_path_mode", + Eligible: patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern, + }, + { + Name: "single_three_element_traversal", + Eligible: len(patternPart.PatternElements) == 3 && len(steps) == 1, + }, + { + Name: "non_optional", + Eligible: !readingClause.Match.Optional, + }, + { + Name: "directed", + Eligible: directionSupported, + }, + { + Name: "bounded_supported_depth", + Eligible: supportedDepth, + }, + { + Name: "no_relationship_variable", + Eligible: noRelationshipVariable, + }, + { + Name: "no_relationship_predicate", + Eligible: step.Relationship.Properties == nil, + }, + { + Name: "single_path_call", + Eligible: shortestCalls == 1, + }, + { + Name: "read_only", + Eligible: updatingClauses == 0, + }, + { + Name: "one_static_id_equality_per_endpoint", + Eligible: singletonIDs, + }, + { + Name: "no_path_predicate", + Eligible: !pathPredicate, + }, + { + Name: "uncorrelated_endpoint_source", + Eligible: uncorrelatedSource, + }, + { + Name: "single_endpoint_pair", + Eligible: singleEndpointPair, + }, + { + Name: "known_observation_mode", + Eligible: false, + }, + } + reason := ShortestPathFallbackTournamentUnqualified + switch { + case patternPart.AllShortestPathsPattern && !singletonIDs: + reason = ShortestPathFallbackAllShortestPaths + case readingClause.Match.Optional: + reason = ShortestPathFallbackOptionalMatch + case !directionSupported: + reason = ShortestPathFallbackDirectionless + case pathPredicate: + reason = ShortestPathFallbackPathPredicate + case !noRelationshipVariable: + reason = ShortestPathFallbackRelationshipVariable + case step.Relationship.Properties != nil: + reason = ShortestPathFallbackRelationshipPredicate + case !supportedDepth: + reason = ShortestPathFallbackUnsupportedDepth + case shortestCalls != 1: + reason = ShortestPathFallbackMultiplePathCalls + case updatingClauses != 0: + reason = ShortestPathFallbackMutation + case !uncorrelatedSource: + reason = ShortestPathFallbackCorrelatedEndpoints + case !singleEndpointPair: + reason = ShortestPathFallbackMultipleEndpointPairs + case leftIDCount > 1 || rightIDCount > 1: + reason = ShortestPathFallbackMultipleIDEqualities + case !singletonIDs: + reason = ShortestPathFallbackNonSingletonID + } + family := "SP" + plannedCandidates := []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorS0Direct, + ShortestPathExecutorS1ArrayBFS, + ShortestPathExecutorS2TraceRelation, + ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorS4CanonicalDistance, + ShortestPathExecutorS4CanonicalWitness, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI2GuardedDistance, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorB1AlternatingNodeDistance, + ShortestPathExecutorB1AlternatingNodeWitness, + ShortestPathExecutorB2SmallerCurrentLevelDistance, + ShortestPathExecutorB2SmallerCurrentLevelWitness, + } + if patternPart.AllShortestPathsPattern { + family = "ASP" + plannedCandidates = []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorASPA1DAG, + ShortestPathExecutorASPI1DAG, + ShortestPathExecutorASPB1AlternatingNodeDAG, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + } + } + plan.ShortestPathExecutor = append(plan.ShortestPathExecutor, ShortestPathExecutorDecision{ + Target: PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex), + Family: family, + PlannedCandidates: plannedCandidates, + SelectedExecutor: ShortestPathExecutorIncumbentWorkspace, + ExecutionBoundary: "stored_helper", + ObservationMode: ShortestPathObservationUnknown, + Direction: step.Relationship.Direction, + PhysicalExpansion: physicalExpansion, + RelationshipKindCount: len(step.Relationship.Kinds), + UntypedRelationship: len(step.Relationship.Kinds) == 0, + TopologyClassification: topologyClassification, + Eligibility: facts, + StructurallyEligible: shortestPathFactsEligible(facts), + StaticallyEligible: false, + MinimumDepth: minDepth, + MaximumDepth: maxDepth, + MaximumDepthSource: maximumDepthSource, + StateLimit: defaultShortestPathStateLimit, + FrontierLimit: defaultShortestPathFrontierLimit, + PredecessorLimit: defaultShortestPathPredecessorLimit, + EnumerationLimit: defaultAllShortestPathsEnumerationLimit, + OutputBytesLimit: defaultAllShortestPathsOutputBytesLimit, + SelectorVersion: "sp-static-v3", + SelectionMode: "incumbent_default", + FallbackExecutor: ShortestPathExecutorIncumbentWorkspace, + FallbackReason: reason, + }) + } + } + } +} + +// shortestPathFactsEligible reports whether every recorded shortest-path qualification passed. +func shortestPathFactsEligible(facts []ShortestPathEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +// setShortestPathEligibilityFact replaces or appends one named executor qualification result. +func setShortestPathEligibilityFact(decision *ShortestPathExecutorDecision, name string, eligible bool) { + for idx := range decision.Eligibility { + if decision.Eligibility[idx].Name == name { + decision.Eligibility[idx].Eligible = eligible + return + } + } + decision.Eligibility = append(decision.Eligibility, ShortestPathEligibilityFact{ + Name: name, + Eligible: eligible, + }) +} + +// finalizeShortestPathExecutorDecisions applies statement-wide safety facts +// after every query part has been analyzed. Per-part counting can otherwise +// misclassify two shortest calls separated by WITH, or a shortest read followed +// by a mutation, as eligible singleton read-only execution. +func finalizeShortestPathExecutorDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { + if plan == nil || query == nil || query.SingleQuery == nil { + return + } + defer func() { + for idx := range plan.ShortestPathExecutor { + decision := &plan.ShortestPathExecutor[idx] + decision.Scheduler = decision.SelectedExecutor.Scheduler() + decision.ExecutionBoundary = decision.SelectedExecutor.ExecutionBoundary() + } + }() + + var ( + shortestCalls int + updatingClauses int + ) + visitPart := func(part cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) { + for _, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + if patternPart != nil && (patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern) { + shortestCalls++ + } + } + } + _, partUpdatingClauses := queryPartProjection(part) + updatingClauses += partUpdatingClauses + } + + if multiPart := query.SingleQuery.MultiPartQuery; multiPart != nil { + for _, part := range multiPart.Parts { + if part != nil { + visitPart(part, part.ReadingClauses) + } + } + if finalPart := multiPart.SinglePartQuery; finalPart != nil { + visitPart(finalPart, finalPart.ReadingClauses) + } + } else if singlePart := query.SingleQuery.SinglePartQuery; singlePart != nil { + visitPart(singlePart, singlePart.ReadingClauses) + } + + for idx := range plan.ShortestPathExecutor { + decision := &plan.ShortestPathExecutor[idx] + implicitMaximum := decision.MaximumDepthSource == ShortestPathMaximumDepthPolicyDefault + singlePathCall := shortestCalls == 1 + readOnly := updatingClauses == 0 + setShortestPathEligibilityFact(decision, "single_path_call", singlePathCall) + setShortestPathEligibilityFact(decision, "read_only", readOnly) + structurallyEligible := shortestPathFactsEligible(decision.Eligibility) + qualifiedPhysicalDepth := decision.Direction != graph.DirectionInbound || decision.MaximumDepth <= 1 + qualifiedPathKinds := decision.ObservationMode != ShortestPathObservationOnePath || (!decision.UntypedRelationship && decision.RelationshipKindCount == 1) + setShortestPathEligibilityFact(decision, "qualified_physical_expansion_depth", qualifiedPhysicalDepth) + setShortestPathEligibilityFact(decision, "qualified_one_path_kind_state", qualifiedPathKinds) + decision.StructurallyEligible = structurallyEligible + decision.StaticallyEligible = structurallyEligible && qualifiedPhysicalDepth && qualifiedPathKinds + + if !singlePathCall && (decision.FallbackReason == ShortestPathFallbackTournamentUnqualified || decision.FallbackReason == ShortestPathFallbackCorrelatedEndpoints) { + decision.FallbackReason = ShortestPathFallbackMultiplePathCalls + } else if !readOnly && decision.FallbackReason == ShortestPathFallbackTournamentUnqualified { + decision.FallbackReason = ShortestPathFallbackMutation + } + + if structurallyEligible && decision.ObservationMode == ShortestPathObservationAllPaths { + // The compact all-shortest search is deliberately narrower than the + // singleton witness executors. Minimum-depth zero and self-endpoint + // searches can require cyclic relationship-simple paths, which cannot + // use a minimum-node-depth predecessor DAG without changing semantics. + if decision.MinimumDepth != 1 { + decision.FallbackReason = ShortestPathFallbackUnsupportedDepth + continue + } + decision.SelectedExecutor = ShortestPathExecutorASPA1DAG + decision.StaticallyEligible = true + decision.SelectionMode = "static" + decision.SelectorVersion = "asp-static-v1" + decision.FallbackReason = "" + decision.ExperimentalWinner = true + continue + } + + if structurallyEligible { + if !qualifiedPhysicalDepth { + switch decision.ObservationMode { + case ShortestPathObservationDistance: + decision.SelectedExecutor = ShortestPathExecutorS4CanonicalDistance + case ShortestPathObservationOnePath: + decision.SelectedExecutor = ShortestPathExecutorS4CanonicalWitness + default: + decision.FallbackReason = ShortestPathFallbackDeepInboundUnqualified + continue + } + decision.SelectionMode = "static" + decision.SelectorVersion = "sp-static-v5-contained" + if implicitMaximum { + decision.SelectorVersion = ShortestPathSelectorStaticV7Contained + } + decision.StaticallyEligible = true + decision.FallbackReason = "" + decision.ExperimentalWinner = true + continue + } + if !qualifiedPathKinds { + if decision.ObservationMode == ShortestPathObservationOnePath { + decision.SelectedExecutor = ShortestPathExecutorS4CanonicalWitness + decision.SelectionMode = "static" + decision.SelectorVersion = "sp-static-v5-contained" + if implicitMaximum { + decision.SelectorVersion = ShortestPathSelectorStaticV7Contained + } + decision.StaticallyEligible = true + decision.FallbackReason = "" + decision.ExperimentalWinner = true + continue + } + decision.FallbackReason = ShortestPathFallbackNonSingleKindPathState + continue + } + switch decision.ObservationMode { + case ShortestPathObservationDistance: + decision.SelectedExecutor = ShortestPathExecutorS3Unidirectional + decision.SelectorVersion = "sp-static-v3" + case ShortestPathObservationOnePath: + // Restore the former, already-qualified S3 production envelope. + // Deep physical-inbound and non-single-kind witnesses remain on + // S4 above; expanding S3 into either shape would expose its + // unbounded relationship-trail state to a new workload class. + decision.SelectedExecutor = ShortestPathExecutorS3EdgeM0 + decision.SelectorVersion = "sp-static-v5-contained" + default: + continue + } + if implicitMaximum { + decision.SelectorVersion = ShortestPathSelectorStaticV7Contained + } + decision.SelectionMode = "static" + decision.FallbackReason = "" + decision.ExperimentalWinner = true + } + } +} + +// finalizeExpansionSearchStrategyDecisions applies statement-wide safety +// facts after all query parts and field requirements are known. The generic +// orientation tournament has a statement-wide single-expansion envelope; +// endpoint-seeded reverse retains its established per-region fact and guarded +// fallback across independent WITH-separated traversals. +func finalizeExpansionSearchStrategyDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { + if plan == nil || query == nil || query.SingleQuery == nil { + return + } + var variableExpansions, updatingClauses int + visitPart := func(part cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) { + for _, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for _, patternPart := range readingClause.Match.Pattern { + for _, step := range traversalStepsForPattern(patternPart) { + if step.Relationship != nil && step.Relationship.Range != nil { + variableExpansions++ + } + } + } + } + _, partUpdatingClauses := queryPartProjection(part) + updatingClauses += partUpdatingClauses + } + if multiPart := query.SingleQuery.MultiPartQuery; multiPart != nil { + for _, part := range multiPart.Parts { + if part != nil { + visitPart(part, part.ReadingClauses) + } + } + if finalPart := multiPart.SinglePartQuery; finalPart != nil { + visitPart(finalPart, finalPart.ReadingClauses) + } + } else if singlePart := query.SingleQuery.SinglePartQuery; singlePart != nil { + visitPart(singlePart, singlePart.ReadingClauses) + } + + for idx := range plan.ExpansionSearchStrategy { + decision := &plan.ExpansionSearchStrategy[idx] + singleExpansion := variableExpansions == 1 + readOnly := updatingClauses == 0 + hasStatementWideExpansionFact := setExpansionSearchEligibilityFact(decision, "single_variable_expansion", singleExpansion) + setExpansionSearchEligibilityFact(decision, "read_only", readOnly) + decision.StructurallyEligible = expansionSearchFactsEligible(decision.EligibilityFacts) + decision.StaticallyEligible = decision.StructurallyEligible + if !decision.StructurallyEligible && decision.SelectedStrategy == ExpansionSearchEndpointSeededReverse { + decision.SelectedStrategy = decision.FallbackStrategy + decision.SelectionMode = "incumbent_default" + } + if hasStatementWideExpansionFact && !singleExpansion && (decision.FallbackReason == "" || decision.FallbackReason == ExpansionSearchFallbackTournamentUnqualified || decision.FallbackReason == ExpansionSearchFallbackMultipleVariableExpansions || decision.FallbackReason == ExpansionSearchFallbackUnboundRoot) { + decision.FallbackReason = ExpansionSearchFallbackMultipleVariableExpansions + } else if !readOnly && (decision.FallbackReason == "" || decision.FallbackReason == ExpansionSearchFallbackTournamentUnqualified) { + decision.FallbackReason = ExpansionSearchFallbackMutation + } + setExpansionSearchExpectedEmission(decision) + } +} + +// syntaxDependsOn reports whether node references symbol as an external dependency. +func syntaxDependsOn(node cypher.SyntaxNode, symbol string) bool { + if symbol == "" { + return false + } + for _, dependency := range sortedDependencies(node) { + if dependency == symbol { + return true + } + } + return false +} + +// singletonIDEqualityCounts counts constant id(symbol) equalities for each symbol in where. +func singletonIDEqualityCounts(where *cypher.Where) map[string]int { + counts := map[string]int{} + if where == nil { + return counts + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + comparison, ok := term.(*cypher.Comparison) + if !ok || comparison == nil || len(comparison.Partials) != 1 || comparison.Partials[0].Operator != cypher.OperatorEquals { + continue + } + partial := comparison.Partials[0] + if symbol, ok := identityFunctionSymbol(comparison.Left); ok && expressionIsConstant(partial.Right) { + counts[symbol]++ + } + if symbol, ok := identityFunctionSymbol(partial.Right); ok && expressionIsConstant(comparison.Left) { + counts[symbol]++ + } + } + } + return counts +} + +// identityFunctionSymbol returns the variable named by a single-argument id invocation. +func identityFunctionSymbol(expression cypher.Expression) (string, bool) { + function, ok := expression.(*cypher.FunctionInvocation) + if !ok || function == nil || !strings.EqualFold(function.Name, cypher.IdentityFunction) || len(function.Arguments) != 1 { + return "", false + } + variable, ok := function.Arguments[0].(*cypher.Variable) + if !ok || variable == nil || variable.Symbol == "" { + return "", false + } + return variable.Symbol, true +} + +// appendExactRangeExpansionDecisions records safe short fixed-depth ranges throughout the reading clauses. func appendExactRangeExpansionDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -144,6 +1486,7 @@ func appendExactRangeExpansionDecisions(plan *LoweringPlan, queryPartIndex int, } } +// appendPatternPredicateExactRangeExpansionDecisions records exact-range steps nested inside pattern predicates. func appendPatternPredicateExactRangeExpansionDecisions(plan *LoweringPlan, queryPartIndex int, indexedPredicates []indexedPatternPredicate) { for _, indexedPredicate := range indexedPredicates { patternPart := patternPartForPredicate(indexedPredicate.Predicate) @@ -157,6 +1500,7 @@ func appendPatternPredicateExactRangeExpansionDecisions(plan *LoweringPlan, quer } } +// appendPatternExactRangeExpansionDecisions records exact-range steps in one pattern part. func appendPatternExactRangeExpansionDecisions(plan *LoweringPlan, target PatternTarget, patternPart *cypher.PatternPart) { for stepIndex, step := range traversalStepsForPattern(patternPart) { if exactRangeExpansionCandidate(patternPart, step) { @@ -168,6 +1512,7 @@ func appendPatternExactRangeExpansionDecisions(plan *LoweringPlan, target Patter } } +// exactRangeExpansionCandidate reports whether a non-shortest directed step has a small fixed depth safe to unroll. func exactRangeExpansionCandidate(patternPart *cypher.PatternPart, step sourceTraversalStep) bool { if patternPart == nil { return false @@ -185,6 +1530,7 @@ func exactRangeExpansionCandidate(patternPart *cypher.PatternPart, step sourceTr return depth >= 1 && depth <= maxExactRangeExpansionDepth } +// hasExactRangeExpansionDecision reports whether plan already unrolls target's exact range. func hasExactRangeExpansionDecision(plan *LoweringPlan, target TraversalStepTarget) bool { if plan == nil { return false @@ -199,6 +1545,7 @@ func hasExactRangeExpansionDecision(plan *LoweringPlan, target TraversalStepTarg return false } +// ExactPatternRangeDepth evaluates planner state needed for exact pattern range depth. func ExactPatternRangeDepth(patternRange *cypher.PatternRange) int64 { if patternRange == nil || patternRange.StartIndex == nil || patternRange.EndIndex == nil { return 0 @@ -211,16 +1558,23 @@ func ExactPatternRangeDepth(patternRange *cypher.PatternRange) int64 { return *patternRange.StartIndex } +// indexedQuantifier pairs a quantifier with its stable traversal-order index. type indexedQuantifier struct { - Index int + // Index is the quantifier's zero-based position in structural traversal order. + Index int + // Quantifier is the indexed Cypher predicate node. Quantifier *cypher.Quantifier } +// quantifierCollector records quantifiers in syntax traversal order. type quantifierCollector struct { + // VisitorHandler supplies cancellation and error propagation for the syntax walk. walk.VisitorHandler + // quantifiers accumulates visited quantifiers with their stable indexes. quantifiers []indexedQuantifier } +// Enter evaluates planner state needed for enter. func (s *quantifierCollector) Enter(node cypher.SyntaxNode) { if quantifier, isQuantifier := node.(*cypher.Quantifier); isQuantifier { s.quantifiers = append(s.quantifiers, indexedQuantifier{ @@ -230,9 +1584,13 @@ func (s *quantifierCollector) Enter(node cypher.SyntaxNode) { } } +// Visit evaluates planner state needed for visit. func (s *quantifierCollector) Visit(cypher.SyntaxNode) {} -func (s *quantifierCollector) Exit(cypher.SyntaxNode) {} +// Exit evaluates planner state needed for exit. +func (s *quantifierCollector) Exit(cypher.SyntaxNode) {} + +// indexedQuantifiersInQueryPart returns all quantifiers in stable syntax traversal order. func indexedQuantifiersInQueryPart(queryPart cypher.SyntaxNode) []indexedQuantifier { if queryPart == nil { return nil @@ -249,6 +1607,7 @@ func indexedQuantifiersInQueryPart(queryPart cypher.SyntaxNode) []indexedQuantif return collector.quantifiers } +// quantifiersInSyntax returns the quantifier nodes contained in node in traversal order. func quantifiersInSyntax(node cypher.SyntaxNode) []*cypher.Quantifier { if node == nil { return nil @@ -272,6 +1631,7 @@ func quantifiersInSyntax(node cypher.SyntaxNode) []*cypher.Quantifier { return quantifiers } +// pathRelationshipQuantifierCandidate extracts the path and relationship symbols from a supported relationships(path) quantifier. func pathRelationshipQuantifierCandidate(quantifier *cypher.Quantifier) (string, string, bool) { if quantifier == nil || (quantifier.Type != cypher.QuantifierTypeAny && quantifier.Type != cypher.QuantifierTypeNone) || @@ -299,6 +1659,7 @@ func pathRelationshipQuantifierCandidate(quantifier *cypher.Quantifier) (string, return pathVariable.Symbol, bindingSymbol, true } +// appendPathRelationshipPredicateDecisions recognizes supported relationships(path) quantifiers and records their bindings. func appendPathRelationshipPredicateDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode) { quantifierIndexes := map[*cypher.Quantifier]int{} for _, indexed := range indexedQuantifiersInQueryPart(queryPart) { @@ -341,6 +1702,7 @@ func appendPathRelationshipPredicateDecisions(plan *LoweringPlan, queryPartIndex } } +// appendProjectionPruningDecisions computes unused traversal bindings for each non-optional reading-clause pattern. func appendProjectionPruningDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -362,6 +1724,7 @@ func appendProjectionPruningDecisions(plan *LoweringPlan, queryPartIndex int, re } } +// appendPatternProjectionPruningDecisions records node, relationship, and path fields unused after each step in a pattern. func appendPatternProjectionPruningDecisions(plan *LoweringPlan, target PatternTarget, patternPart *cypher.PatternPart, steps []sourceTraversalStep, sourceReferences map[string]struct{}) { pathReferenced := referencesSourceIdentifier(sourceReferences, variableSymbol(patternPart.Variable)) @@ -398,6 +1761,7 @@ func appendPatternProjectionPruningDecisions(plan *LoweringPlan, target PatternT } } +// appendPatternPredicateProjectionLowerings applies projection analysis to traversal patterns nested in predicates. func appendPatternPredicateProjectionLowerings(plan *LoweringPlan, queryPartIndex int, indexedPredicates []indexedPatternPredicate, sourceReferences map[string]struct{}) { for _, indexedPredicate := range indexedPredicates { var ( @@ -423,6 +1787,7 @@ func appendPatternPredicateProjectionLowerings(plan *LoweringPlan, queryPartInde } } +// appendPatternPredicatePlacementDecisions records existence lowering for pattern predicates in one query part. func appendPatternPredicatePlacementDecisions(plan *LoweringPlan, queryPartIndex int, indexedPredicates []indexedPatternPredicate) { for _, indexedPredicate := range indexedPredicates { var ( @@ -463,6 +1828,7 @@ func appendPatternPredicatePlacementDecisions(plan *LoweringPlan, queryPartIndex } } +// appendLatePathMaterializationDecisions identifies path and edge values whose hydration can be deferred. func appendLatePathMaterializationDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -480,6 +1846,7 @@ func appendLatePathMaterializationDecisions(plan *LoweringPlan, queryPartIndex i } } +// appendPatternLatePathMaterializationDecisions records deferred materialization modes for one pattern's bindings. func appendPatternLatePathMaterializationDecisions(plan *LoweringPlan, target PatternTarget, patternPart *cypher.PatternPart, steps []sourceTraversalStep, sourceReferences map[string]struct{}) { pathReferenced := referencesSourceIdentifier(sourceReferences, variableSymbol(patternPart.Variable)) @@ -521,11 +1888,19 @@ func appendPatternLatePathMaterializationDecisions(plan *LoweringPlan, target Pa } } -func appendExpandIntoDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause) { - declaredSymbols := map[string]struct{}{} +// appendExpandIntoDecisions records traversal steps whose left and right endpoints were already declared. +func appendExpandIntoDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, initialDeclaredSymbols map[string]struct{}) { + declaredSymbols := copyStringSet(initialDeclaredSymbols) for clauseIndex, readingClause := range readingClauses { - if readingClause == nil || readingClause.Match == nil { + if readingClause == nil { + continue + } + if readingClause.Unwind != nil { + addSymbol(declaredSymbols, variableSymbol(readingClause.Unwind.Variable)) + continue + } + if readingClause.Match == nil { continue } @@ -578,11 +1953,15 @@ func appendExpandIntoDecisions(plan *LoweringPlan, queryPartIndex int, readingCl } } +// declaredStepEndpoints snapshots visible symbols before each endpoint of a traversal step is declared. type declaredStepEndpoints struct { - BeforeLeftNode map[string]struct{} + // BeforeLeftNode contains symbols visible before the step's left endpoint declaration. + BeforeLeftNode map[string]struct{} + // BeforeRightNode contains symbols visible after the edge but before the right endpoint declaration. BeforeRightNode map[string]struct{} } +// declaredSymbolsBeforeStepEndpoints computes declaration snapshots for every traversal-step endpoint. func declaredSymbolsBeforeStepEndpoints(initial map[string]struct{}, steps []sourceTraversalStep) []declaredStepEndpoints { var ( declared = copyStringSet(initial) @@ -600,6 +1979,7 @@ func declaredSymbolsBeforeStepEndpoints(initial map[string]struct{}, steps []sou return endpoints } +// appendTraversalDirectionDecisions evaluates each step's bound endpoints and selectivity to choose its direction. func appendTraversalDirectionDecisions( plan *LoweringPlan, queryPartIndex int, @@ -669,6 +2049,7 @@ func appendTraversalDirectionDecisions( } } +// bindingPredicateSymbols returns predicate dependencies that reference declared bindings. func bindingPredicateSymbols(predicateAttachments []PredicateAttachment, queryPartIndex int) map[string]struct{} { symbols := map[string]struct{}{} @@ -685,6 +2066,7 @@ func bindingPredicateSymbols(predicateAttachments []PredicateAttachment, queryPa return symbols } +// copyBoundSourceSelectivity returns an independent copy of symbol selectivity rankings. func copyBoundSourceSelectivity(values map[string]boundSourceSelectivity) map[string]boundSourceSelectivity { copied := make(map[string]boundSourceSelectivity, len(values)) for key, value := range values { @@ -694,6 +2076,7 @@ func copyBoundSourceSelectivity(values map[string]boundSourceSelectivity) map[st return copied } +// carryProjectionSelectivity propagates source selectivity through a WITH projection and its aliases. func carryProjectionSelectivity( projection *cypher.Projection, incomingSymbols map[string]struct{}, @@ -734,6 +2117,7 @@ func carryProjectionSelectivity( return carriedSymbols, carriedSelectivity } +// projectionCarriesAllSymbols reports whether a projection uses the greedy asterisk form. func projectionCarriesAllSymbols(projection *cypher.Projection) bool { if projection == nil { return false @@ -754,6 +2138,7 @@ func projectionCarriesAllSymbols(projection *cypher.Projection) bool { return false } +// projectionCardinalitySelectivity classifies limited projections, ranking ordered or aggregate limits as top-N. func projectionCardinalitySelectivity(projection *cypher.Projection) boundSourceSelectivity { if projection == nil || projection.Limit == nil { return boundSourceSelectivityNone @@ -766,6 +2151,7 @@ func projectionCardinalitySelectivity(projection *cypher.Projection) boundSource return boundSourceSelectivityLimited } +// projectionHasAggregate reports whether any projection item contains an aggregate function. func projectionHasAggregate(projection *cypher.Projection) bool { if projection == nil { return false @@ -785,6 +2171,7 @@ func projectionHasAggregate(projection *cypher.Projection) bool { return false } +// expressionHasAggregate reports whether expression invokes a recognized aggregate function. func expressionHasAggregate(expression cypher.Expression) bool { switch typedExpression := expression.(type) { case *cypher.FunctionInvocation: @@ -794,6 +2181,7 @@ func expressionHasAggregate(expression cypher.Expression) bool { } } +// declareSelectiveMatchSymbols merges inferred node-property selectivity for a match into the symbol table. func declareSelectiveMatchSymbols(symbols map[string]boundSourceSelectivity, match *cypher.Match) { if match == nil { return @@ -827,6 +2215,7 @@ func declareSelectiveMatchSymbols(symbols map[string]boundSourceSelectivity, mat } } +// declareReadingClauseSymbols adds pattern bindings and WHERE dependencies from reading clauses. func declareReadingClauseSymbols(symbols map[string]struct{}, readingClauses []*cypher.ReadingClause) { for _, readingClause := range readingClauses { if readingClause != nil { @@ -835,6 +2224,7 @@ func declareReadingClauseSymbols(symbols map[string]struct{}, readingClauses []* } } +// declareReadingClauseSelectivity merges inferred selectivity from non-optional reading clauses. func declareReadingClauseSelectivity(symbols map[string]boundSourceSelectivity, readingClauses []*cypher.ReadingClause) { for _, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil || readingClause.Match.Optional { @@ -845,6 +2235,7 @@ func declareReadingClauseSelectivity(symbols map[string]boundSourceSelectivity, } } +// nodePatternsForPattern returns every node pattern in chain order. func nodePatternsForPattern(patternPart *cypher.PatternPart) []*cypher.NodePattern { if patternPart == nil { return nil @@ -860,12 +2251,14 @@ func nodePatternsForPattern(patternPart *cypher.PatternPart) []*cypher.NodePatte return nodePatterns } +// mergeBoundSourceSelectivity retains the stronger selectivity rank for symbol. func mergeBoundSourceSelectivity(symbols map[string]boundSourceSelectivity, symbol string, selectivity boundSourceSelectivity) { if selectivity > symbols[symbol] { symbols[symbol] = selectivity } } +// propertyPredicateSelectivity returns the strongest property constraint on symbol in where. func propertyPredicateSelectivity(expression cypher.Expression) (string, boundSourceSelectivity, bool) { comparison, isComparison := expression.(*cypher.Comparison) if !isComparison || len(comparison.Partials) != 1 { @@ -888,6 +2281,7 @@ func propertyPredicateSelectivity(expression cypher.Expression) (string, boundSo return "", boundSourceSelectivityNone, false } +// propertyConstraintSelectivity returns the strongest selectivity inferred from constant-valued inline properties. func propertyConstraintSelectivity(expression cypher.Expression) boundSourceSelectivity { properties, ok := expression.(*cypher.Properties) if !ok || properties == nil || properties.Parameter != nil { @@ -904,6 +2298,7 @@ func propertyConstraintSelectivity(expression cypher.Expression) boundSourceSele return highest } +// propertySelectivity treats a constant objectid as unique and other constant property values as selective predicates. func propertySelectivity(property string, value cypher.Expression) boundSourceSelectivity { if strings.EqualFold(property, "objectid") && expressionIsConstant(value) { return boundSourceSelectivityUnique @@ -916,6 +2311,7 @@ func propertySelectivity(property string, value cypher.Expression) boundSourceSe return boundSourceSelectivityNone } +// expressionIsConstant reports whether expression is a non-null literal or parameter independent of row bindings. func expressionIsConstant(expression cypher.Expression) bool { switch typedExpression := expression.(type) { case *cypher.Literal: @@ -927,6 +2323,7 @@ func expressionIsConstant(expression cypher.Expression) bool { } } +// propertyLookupSymbol returns the variable whose property expression reads, when direct. func propertyLookupSymbol(expression cypher.Expression) (string, string, bool) { propertyLookup, isPropertyLookup := expression.(*cypher.PropertyLookup) if !isPropertyLookup || propertyLookup == nil { @@ -941,10 +2338,12 @@ func propertyLookupSymbol(expression cypher.Expression) (string, string, bool) { return variable.Symbol, propertyLookup.Symbol, true } +// nodePatternHasUniquePropertyConstraint reports whether node contains an inline property treated as unique. func nodePatternHasUniquePropertyConstraint(nodePattern *cypher.NodePattern) bool { return nodePattern != nil && propertyConstraintSelectivity(nodePattern.Properties) == boundSourceSelectivityUnique } +// nodePatternSelectivity ranks a node pattern from kind, inline-property, and attached-predicate constraints. func nodePatternSelectivity(nodePattern *cypher.NodePattern, hasAttachedPredicate bool) boundSourceSelectivity { if nodePattern == nil { return boundSourceSelectivityNone @@ -963,12 +2362,14 @@ func nodePatternSelectivity(nodePattern *cypher.NodePattern, hasAttachedPredicat return selectivity } +// mergeSelectivityValue raises current when next is the stronger source-selectivity rank. func mergeSelectivityValue(current *boundSourceSelectivity, next boundSourceSelectivity) { if next > *current { *current = next } } +// shortestPathSearchPredicateSymbols returns bindings constrained by search-compatible predicates in where. func shortestPathSearchPredicateSymbols(readingClauses []*cypher.ReadingClause) map[string]struct{} { symbols := map[string]struct{}{} @@ -985,6 +2386,7 @@ func shortestPathSearchPredicateSymbols(readingClauses []*cypher.ReadingClause) return symbols } +// addShortestPathSearchPredicateSymbols adds search-constrained symbols from one expression to output. func addShortestPathSearchPredicateSymbols(symbols map[string]struct{}, expression cypher.Expression) { for _, term := range cypherConjunctionTerms(expression) { if symbol, ok := shortestPathSearchPredicateSymbol(term); ok { @@ -993,6 +2395,7 @@ func addShortestPathSearchPredicateSymbols(symbols map[string]struct{}, expressi } } +// cypherConjunctionTerms flattens nested Cypher AND expressions into independent terms. func cypherConjunctionTerms(expression cypher.Expression) []cypher.Expression { if conjunction, isConjunction := expression.(*cypher.Conjunction); isConjunction { var terms []cypher.Expression @@ -1006,6 +2409,7 @@ func cypherConjunctionTerms(expression cypher.Expression) []cypher.Expression { return []cypher.Expression{expression} } +// shortestPathSearchPredicateSymbol extracts the endpoint symbol constrained by a supported search comparison. func shortestPathSearchPredicateSymbol(expression cypher.Expression) (string, bool) { comparison, isComparison := expression.(*cypher.Comparison) if !isComparison || len(comparison.Partials) != 1 { @@ -1028,6 +2432,7 @@ func shortestPathSearchPredicateSymbol(expression cypher.Expression) (string, bo return "", false } +// isEndpointSearchOperator reports whether an operator can constrain endpoint seed values. func isEndpointSearchOperator(operator cypher.Operator) bool { switch operator { case cypher.OperatorEquals, @@ -1046,6 +2451,7 @@ func isEndpointSearchOperator(operator cypher.Operator) bool { } } +// propertyLookupVariableSymbol returns the direct variable at the base of a property lookup. func propertyLookupVariableSymbol(expression cypher.Expression) (string, bool) { propertyLookup, isPropertyLookup := expression.(*cypher.PropertyLookup) if !isPropertyLookup || propertyLookup == nil { @@ -1060,6 +2466,7 @@ func propertyLookupVariableSymbol(expression cypher.Expression) (string, bool) { return variable.Symbol, true } +// expressionReferencesAnySource reports whether expression depends on a variable or property binding. func expressionReferencesAnySource(expression cypher.Expression) bool { switch expression.(type) { case nil, *cypher.Literal, *cypher.Parameter: @@ -1070,6 +2477,7 @@ func expressionReferencesAnySource(expression cypher.Expression) bool { return err != nil || len(references) > 0 } +// traversalDirectionDecisionForStep chooses whether to reverse a step based on bound endpoints and estimated selectivity. func traversalDirectionDecisionForStep( target TraversalStepTarget, stepIndex int, @@ -1122,6 +2530,7 @@ func traversalDirectionDecisionForStep( return TraversalDirectionDecision{}, false } +// boundLeftExpansionDirectionDecisionForStep preserves a bound-left expansion unless terminal evidence justifies reversal. func boundLeftExpansionDirectionDecisionForStep( target TraversalStepTarget, patternPart *cypher.PatternPart, @@ -1188,6 +2597,7 @@ func boundLeftExpansionDirectionDecisionForStep( }, true } +// appendShortestPathStrategyDecisions records bidirectional search when endpoint evidence supports it. func appendShortestPathStrategyDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, predicateConstrainedSymbols map[string]struct{}) { declaredSymbols := map[string]struct{}{} @@ -1240,6 +2650,7 @@ func appendShortestPathStrategyDecisions(plan *LoweringPlan, queryPartIndex int, } } +// shortestPathStrategyDecisionForStep chooses bidirectional search when both endpoints provide usable evidence. func shortestPathStrategyDecisionForStep( target TraversalStepTarget, step sourceTraversalStep, @@ -1272,6 +2683,7 @@ func shortestPathStrategyDecisionForStep( return ShortestPathStrategyDecision{}, false } +// endpointHasSearchConstraint reports whether endpoint has an inline property or attached predicate constraint. func endpointHasSearchConstraint(nodePattern *cypher.NodePattern, symbol string, predicateConstrainedSymbols map[string]struct{}) bool { if nodePattern == nil { return false @@ -1280,6 +2692,7 @@ func endpointHasSearchConstraint(nodePattern *cypher.NodePattern, symbol string, return nodePattern.Properties != nil || referencesSourceIdentifier(predicateConstrainedSymbols, symbol) } +// endpointHasTerminalFilterConstraint reports whether endpoint has a kind, property, or attached predicate constraint useful as a terminal filter. func endpointHasTerminalFilterConstraint(nodePattern *cypher.NodePattern, symbol string, predicateConstrainedSymbols map[string]struct{}) bool { if nodePattern == nil { return false @@ -1288,6 +2701,7 @@ func endpointHasTerminalFilterConstraint(nodePattern *cypher.NodePattern, symbol return nodePatternHasConstraints(nodePattern) || referencesSourceIdentifier(predicateConstrainedSymbols, symbol) } +// appendShortestPathFilterDecisions records terminal and endpoint-pair filters worth materializing for shortest paths. func appendShortestPathFilterDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, predicateConstrainedSymbols map[string]struct{}) { declaredSymbols := map[string]struct{}{} @@ -1341,6 +2755,7 @@ func appendShortestPathFilterDecisions(plan *LoweringPlan, queryPartIndex int, r } } +// shortestPathFilterDecisionForStep chooses an endpoint-pair, terminal, or no filter for one shortest-path step. func shortestPathFilterDecisionForStep( plan *LoweringPlan, target TraversalStepTarget, @@ -1383,6 +2798,7 @@ func shortestPathFilterDecisionForStep( }, true } +// hasShortestPathBidirectionalStrategy reports whether target is planned for bidirectional shortest-path search. func hasShortestPathBidirectionalStrategy(plan *LoweringPlan, target TraversalStepTarget) bool { if plan == nil { return false @@ -1397,6 +2813,7 @@ func hasShortestPathBidirectionalStrategy(plan *LoweringPlan, target TraversalSt return false } +// appendLimitPushdownDecisions records a final literal limit that can safely bound traversal work. func appendLimitPushdownDecisions(plan *LoweringPlan, queryPartIndex int, queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) { if !queryPartAllowsLimitPushdown(queryPart, readingClauses) { return @@ -1436,6 +2853,7 @@ func appendLimitPushdownDecisions(plan *LoweringPlan, queryPartIndex int, queryP } } +// queryPartAllowsLimitPushdown reports whether one reading clause with an unordered, non-distinct LIMIT and no SKIP or updates permits early limiting. func queryPartAllowsLimitPushdown(queryPart cypher.SyntaxNode, readingClauses []*cypher.ReadingClause) bool { projection, updatingClauseCount := queryPartProjection(queryPart) if projection == nil || @@ -1451,6 +2869,7 @@ func queryPartAllowsLimitPushdown(queryPart cypher.SyntaxNode, readingClauses [] return true } +// queryPartProjection returns a query part's terminal projection and number of updating clauses. func queryPartProjection(queryPart cypher.SyntaxNode) (*cypher.Projection, int) { switch typedQueryPart := queryPart.(type) { case *cypher.SinglePartQuery: @@ -1472,7 +2891,22 @@ func queryPartProjection(queryPart cypher.SyntaxNode) (*cypher.Projection, int) } } -func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause) { +// suffixBindingsObserved reports whether downstream syntax consumes a binding introduced in the fixed suffix. +func suffixBindingsObserved(patternPart *cypher.PatternPart, steps []sourceTraversalStep, references map[string]struct{}) bool { + if patternPart != nil && patternPart.Variable != nil && referencesSourceIdentifier(references, patternPart.Variable.Symbol) { + return true + } + for _, step := range steps { + if (step.Relationship != nil && step.Relationship.Variable != nil && referencesSourceIdentifier(references, step.Relationship.Variable.Symbol)) || + (step.RightNode != nil && step.RightNode.Variable != nil && referencesSourceIdentifier(references, step.RightNode.Variable.Symbol)) { + return true + } + } + return false +} + +// appendExpansionSuffixPushdownDecisions records fixed-suffix candidates evaluated for supplemental filtering. +func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex int, readingClauses []*cypher.ReadingClause, sourceReferences map[string]struct{}) { declaredSymbols := map[string]struct{}{} for clauseIndex, readingClause := range readingClauses { @@ -1511,11 +2945,22 @@ func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex i } if suffixLength := expansionSuffixPushdownLength(steps[stepIndex+1:]); suffixLength > 0 { + suffixSteps := steps[stepIndex+1 : stepIndex+1+suffixLength] + // Start with the measured fixed-suffix shape: an observed immediate + // continuation of three or more fixed hops. Shorter suffixes retain + // the established prefilter until their own decoy-density A/B exists. + observed := suffixLength >= 3 && suffixBindingsObserved(patternPart, suffixSteps, sourceReferences) + reason := "supplemental suffix prefilter retained for unobserved continuation" + if observed { + reason = "immediate observed continuation produces suffix rows" + } plan.ExpansionSuffixPushdown = append(plan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ - Target: target, - SuffixLength: suffixLength, - SuffixStartStep: stepIndex + 1, - SuffixEndStep: stepIndex + suffixLength, + Target: target, + SuffixLength: suffixLength, + SuffixStartStep: stepIndex + 1, + SuffixEndStep: stepIndex + suffixLength, + ApplySupplemental: !observed, + Reason: reason, }) } } @@ -1527,11 +2972,13 @@ func appendExpansionSuffixPushdownDecisions(plan *LoweringPlan, queryPartIndex i } } +// expansionStepMayFlipForConstraintBalance reports whether reversal can move stronger constraints to the expansion root. func expansionStepMayFlipForConstraintBalance(stepIndex int, step sourceTraversalStep, declaredEndpoints declaredStepEndpoints) bool { _, mayFlip := traversalDirectionDecisionForStep(TraversalStepTarget{}, stepIndex, step, declaredEndpoints, false, false) return mayFlip } +// leftEndpointBoundForStep reports whether the left endpoint is available from prior scope or a preceding step. func leftEndpointBoundForStep(stepIndex int, step sourceTraversalStep, declaredEndpoints declaredStepEndpoints) bool { leftSymbol := variableSymbol(step.LeftNode.Variable) if leftSymbol == "" { @@ -1542,6 +2989,7 @@ func leftEndpointBoundForStep(stepIndex int, step sourceTraversalStep, declaredE return leftBound } +// hasTraversalDirectionFlip reports whether target has a planned logical direction reversal. func hasTraversalDirectionFlip(plan *LoweringPlan, target TraversalStepTarget) bool { if plan == nil { return false @@ -1556,11 +3004,15 @@ func hasTraversalDirectionFlip(plan *LoweringPlan, target TraversalStepTarget) b return false } +// bindingTargetKey uniquely identifies a binding within one query part. type bindingTargetKey struct { + // QueryPartIndex identifies the query part that owns the binding. QueryPartIndex int - Symbol string + // Symbol is the binding's Cypher variable name. + Symbol string } +// appendPredicatePlacementDecisions appends predicate placement decisions. func appendPredicatePlacementDecisions(plan *LoweringPlan, query *cypher.RegularQuery, predicateAttachments []PredicateAttachment) { if len(predicateAttachments) == 0 { return @@ -1591,6 +3043,7 @@ func appendPredicatePlacementDecisions(plan *LoweringPlan, query *cypher.Regular } } +// attachPredicatePlacementsToSuffixPushdowns copies relevant predicate attachments into each suffix-pushdown decision. func attachPredicatePlacementsToSuffixPushdowns(plan *LoweringPlan) { for suffixIdx := range plan.ExpansionSuffixPushdown { suffix := &plan.ExpansionSuffixPushdown[suffixIdx] @@ -1609,12 +3062,14 @@ func attachPredicatePlacementsToSuffixPushdowns(plan *LoweringPlan) { } } +// appendCountStoreFastPathDecisions records a single-part query answerable directly from node or relationship counts. func appendCountStoreFastPathDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { if decision, ok := countStoreFastPathDecision(query); ok { plan.CountStoreFastPath = append(plan.CountStoreFastPath, decision) } } +// appendAggregateTraversalCountDecisions records variable traversals lowered to grouped aggregate counts. func appendAggregateTraversalCountDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { if shape, ok := AggregateTraversalCountShapeForQuery(query); ok { plan.AggregateTraversalCount = append(plan.AggregateTraversalCount, AggregateTraversalCountDecision{ @@ -1628,6 +3083,7 @@ func appendAggregateTraversalCountDecisions(plan *LoweringPlan, query *cypher.Re } } +// AggregateTraversalCountShapeForQuery constructs the SQL model used for aggregate traversal count shape for query. func AggregateTraversalCountShapeForQuery(query *cypher.RegularQuery) (AggregateTraversalCountShape, bool) { if query == nil || query.SingleQuery == nil || query.SingleQuery.MultiPartQuery == nil { return AggregateTraversalCountShape{}, false @@ -1694,6 +3150,7 @@ func AggregateTraversalCountShapeForQuery(query *cypher.RegularQuery) (Aggregate }, true } +// aggregateTraversalSourceMatch returns the match that establishes a traversal count's source binding. func aggregateTraversalSourceMatch(readingClause *cypher.ReadingClause) (*cypher.Match, *cypher.NodePattern, string, bool) { if readingClause == nil || readingClause.Match == nil { return nil, nil, "", false @@ -1719,6 +3176,7 @@ func aggregateTraversalSourceMatch(readingClause *cypher.ReadingClause) (*cypher return match, nodePattern, nodePattern.Variable.Symbol, true } +// aggregateTraversalMatch returns the single variable-length match eligible for aggregate counting. func aggregateTraversalMatch(readingClause *cypher.ReadingClause, sourceSymbol string) (*cypher.Match, *cypher.RelationshipPattern, *cypher.NodePattern, string, bool) { if readingClause == nil || readingClause.Match == nil { return nil, nil, nil, "", false @@ -1762,6 +3220,7 @@ func aggregateTraversalMatch(readingClause *cypher.ReadingClause, sourceSymbol s return match, relationship, rightNode, rightNode.Variable.Symbol, true } +// aggregateTraversalWithProjection validates the WITH projection and returns its count alias. func aggregateTraversalWithProjection(projection *cypher.Projection, sourceSymbol, terminalSymbol string) (string, bool) { if projection == nil || projection.All || projection.Order != nil || projection.Skip != nil || projection.Limit != nil || len(projection.Items) != 2 { return "", false @@ -1779,13 +3238,19 @@ func aggregateTraversalWithProjection(projection *cypher.Projection, sourceSymbo return countAlias, true } +// aggregateTraversalFinalProjectionShape describes the source and count columns required from the final projection. type aggregateTraversalFinalProjectionShape struct { + // SourceAlias is the output name of the traversal's source binding. SourceAlias string - CountAlias string + // CountAlias is the output name of the aggregate count binding. + CountAlias string + // ReturnCount reports whether the final projection includes the count binding. ReturnCount bool - Limit int64 + // Limit is the descending top-count bound applied by the final projection. + Limit int64 } +// aggregateTraversalFinalProjection validates the terminal projection and returns its aggregate-count output shape. func aggregateTraversalFinalProjection(queryPart *cypher.SinglePartQuery, sourceSymbol, countAlias string) (aggregateTraversalFinalProjectionShape, bool) { if queryPart == nil || len(queryPart.ReadingClauses) > 0 || len(queryPart.UpdatingClauses) > 0 || queryPart.Return == nil || queryPart.Return.Projection == nil { return aggregateTraversalFinalProjectionShape{}, false @@ -1849,6 +3314,7 @@ func aggregateTraversalFinalProjection(queryPart *cypher.SinglePartQuery, source return finalProjection, true } +// aggregateTraversalDepthBounds returns finite minimum and maximum depths for a countable relationship range. func aggregateTraversalDepthBounds(patternRange *cypher.PatternRange) (int64, int64, bool) { if patternRange == nil { return 0, 0, false @@ -1873,6 +3339,7 @@ func aggregateTraversalDepthBounds(patternRange *cypher.PatternRange) (int64, in return minDepth, maxDepth, true } +// projectionItemVariableSymbol returns the direct variable projected by item. func projectionItemVariableSymbol(expression cypher.Expression) (string, bool) { projectionItem, ok := expression.(*cypher.ProjectionItem) if !ok || projectionItem == nil || projectionItem.Alias != nil { @@ -1882,6 +3349,7 @@ func projectionItemVariableSymbol(expression cypher.Expression) (string, bool) { return expressionVariableSymbol(projectionItem.Expression) } +// projectionItemVariableSymbolAndAlias returns a projected variable and its effective output name. func projectionItemVariableSymbolAndAlias(expression cypher.Expression) (string, string, bool) { projectionItem, ok := expression.(*cypher.ProjectionItem) if !ok || projectionItem == nil { @@ -1905,6 +3373,7 @@ func projectionItemVariableSymbolAndAlias(expression cypher.Expression) (string, return symbol, alias, true } +// expressionVariableSymbol returns expression's direct variable symbol without following compound syntax. func expressionVariableSymbol(expression cypher.Expression) (string, bool) { variable, ok := expression.(*cypher.Variable) if !ok || variable == nil || variable.Symbol == "" { @@ -1914,6 +3383,7 @@ func expressionVariableSymbol(expression cypher.Expression) (string, bool) { return variable.Symbol, true } +// projectionItemCountAlias returns the alias of a supported count expression. func projectionItemCountAlias(expression cypher.Expression, terminalSymbol string) (string, bool) { projectionItem, ok := expression.(*cypher.ProjectionItem) if !ok || projectionItem == nil || projectionItem.Alias == nil || projectionItem.Alias.Symbol == "" { @@ -1933,6 +3403,7 @@ func projectionItemCountAlias(expression cypher.Expression, terminalSymbol strin return projectionItem.Alias.Symbol, true } +// aggregateTraversalCountArgumentMatches reports whether count observes the expected terminal binding or all rows. func aggregateTraversalCountArgumentMatches(expression cypher.Expression, terminalSymbol string) bool { if symbol, ok := expressionVariableSymbol(expression); ok { return symbol == terminalSymbol @@ -1942,6 +3413,7 @@ func aggregateTraversalCountArgumentMatches(expression cypher.Expression, termin return ok && rangeQuantifier != nil && rangeQuantifier.Value == cypher.TokenLiteralAsterisk } +// literalInt64 converts a non-negative integer literal to int64 when its value is representable. func literalInt64(expression cypher.Expression) (int64, bool) { literal, ok := expression.(*cypher.Literal) if !ok || literal == nil || literal.Null { @@ -1964,6 +3436,7 @@ func literalInt64(expression cypher.Expression) (int64, bool) { } } +// countStoreFastPathDecision recognizes a count query answerable from node or edge statistics. func countStoreFastPathDecision(query *cypher.RegularQuery) (CountStoreFastPathDecision, bool) { if query == nil || query.SingleQuery == nil || query.SingleQuery.SinglePartQuery == nil { return CountStoreFastPathDecision{}, false @@ -2047,6 +3520,7 @@ func countStoreFastPathDecision(query *cypher.RegularQuery) (CountStoreFastPathD }, true } +// simpleCountProjectionArgument extracts the direct variable or wildcard consumed by a lone count projection. func simpleCountProjectionArgument(returnClause *cypher.Return) (string, bool) { if returnClause == nil || returnClause.Projection == nil { return "", false @@ -2084,10 +3558,12 @@ func simpleCountProjectionArgument(returnClause *cypher.Return) (string, bool) { return "", false } +// constrainedCountFastPathEndpoint reports whether a node endpoint has constraints incompatible with count-store lookup. func constrainedCountFastPathEndpoint(nodePattern *cypher.NodePattern) bool { return nodePattern == nil || nodePattern.Variable != nil || len(nodePattern.Kinds) > 0 || nodePattern.Properties != nil } +// kindSymbols returns the string names of all non-nil kinds in declaration order. func kindSymbols(kinds graph.Kinds) []string { if len(kinds) == 0 { return nil @@ -2101,6 +3577,7 @@ func kindSymbols(kinds graph.Kinds) []string { return symbols } +// indexBindingTargets maps traversal-step node and relationship bindings to their first query-part target coordinates. func indexBindingTargets(query *cypher.RegularQuery) map[bindingTargetKey]TraversalStepTarget { targets := map[bindingTargetKey]TraversalStepTarget{} @@ -2127,6 +3604,7 @@ func indexBindingTargets(query *cypher.RegularQuery) map[bindingTargetKey]Traver return targets } +// indexReadingClauseBindingTargets adds first targets for traversal-step node and relationship bindings in readingClauses. func indexReadingClauseBindingTargets(targets map[bindingTargetKey]TraversalStepTarget, queryPartIndex int, readingClauses []*cypher.ReadingClause) { for clauseIndex, readingClause := range readingClauses { if readingClause == nil || readingClause.Match == nil { @@ -2150,6 +3628,7 @@ func indexReadingClauseBindingTargets(targets map[bindingTargetKey]TraversalStep } } +// setBindingTarget records target for a non-empty binding symbol without overwriting its first declaration. func setBindingTarget(targets map[bindingTargetKey]TraversalStepTarget, queryPartIndex int, symbol string, target TraversalStepTarget) { if symbol == "" { return @@ -2164,6 +3643,7 @@ func setBindingTarget(targets map[bindingTargetKey]TraversalStepTarget, queryPar } } +// expansionSuffixPushdownLength counts fixed directed steps following a variable expansion. func expansionSuffixPushdownLength(suffixSteps []sourceTraversalStep) int { var suffixLength int @@ -2178,6 +3658,7 @@ func expansionSuffixPushdownLength(suffixSteps []sourceTraversalStep) int { return suffixLength } +// declareMatchSymbols adds pattern bindings and WHERE dependencies from match to declared. func declareMatchSymbols(declared map[string]struct{}, match *cypher.Match) { if match == nil { return @@ -2190,6 +3671,7 @@ func declareMatchSymbols(declared map[string]struct{}, match *cypher.Match) { declareWhereSymbols(declared, match) } +// declarePatternSymbols adds path, node, and relationship bindings introduced by a pattern part. func declarePatternSymbols(declared map[string]struct{}, patternPart *cypher.PatternPart) { if patternPart == nil { return @@ -2209,26 +3691,31 @@ func declarePatternSymbols(declared map[string]struct{}, patternPart *cypher.Pat } } +// declareWhereSymbols adds variable dependencies referenced by a match predicate. func declareWhereSymbols(declared map[string]struct{}, match *cypher.Match) { for _, dependency := range dependenciesForMatch(match) { addSymbol(declared, dependency) } } +// nodePatternHasConstraints reports whether a node pattern declares kinds or inline properties. func nodePatternHasConstraints(nodePattern *cypher.NodePattern) bool { return nodePattern != nil && (len(nodePattern.Kinds) > 0 || nodePattern.Properties != nil) } +// relationshipPatternHasProperties reports whether a relationship pattern declares inline properties. func relationshipPatternHasProperties(relationshipPattern *cypher.RelationshipPattern) bool { return relationshipPattern != nil && relationshipPattern.Properties != nil } +// addSymbol inserts a non-empty symbol into a declaration set. func addSymbol(symbols map[string]struct{}, symbol string) { if symbol != "" { symbols[symbol] = struct{}{} } } +// copyStringSet returns an independent copy of a string membership set. func copyStringSet(values map[string]struct{}) map[string]struct{} { copied := make(map[string]struct{}, len(values)) for value := range values { @@ -2238,6 +3725,7 @@ func copyStringSet(values map[string]struct{}) map[string]struct{} { return copied } +// traversalStepsForPattern converts a pattern chain into ordered left-edge-right traversal steps. func traversalStepsForPattern(patternPart *cypher.PatternPart) []sourceTraversalStep { if patternPart == nil { return nil @@ -2278,6 +3766,7 @@ func traversalStepsForPattern(patternPart *cypher.PatternPart) []sourceTraversal return steps } +// variableSymbol returns variable's symbol or an empty string for a missing variable. func variableSymbol(variable *cypher.Variable) string { if variable == nil { return "" diff --git a/cypher/models/pgsql/optimize/optimizer_test.go b/cypher/models/pgsql/optimize/optimizer_test.go index 30bb1171..6e9b8ba5 100644 --- a/cypher/models/pgsql/optimize/optimizer_test.go +++ b/cypher/models/pgsql/optimize/optimizer_test.go @@ -1,6 +1,8 @@ package optimize import ( + "encoding/json" + "fmt" "testing" "github.com/specterops/dawgs/cypher/frontend" @@ -11,14 +13,18 @@ import ( "github.com/stretchr/testify/require" ) +// testRule is a configurable optimizer rule used to assert rewrite ordering and error propagation. type testRule struct { + // name is the stable rule name returned to the optimizer. name string } +// Name evaluates planner state needed for name. func (s testRule) Name() string { return s.name } +// Apply evaluates planner state needed for apply. func (s testRule) Apply(plan *Plan) (bool, error) { return false, nil } @@ -34,17 +40,20 @@ func (s analysisMutatingTestRule) Apply(plan *Plan) (bool, error) { return true, nil } +// testBindingLookup supplies deterministic binding resolution to optimizer tests. type testBindingLookup map[pgsql.Identifier]pgsql.DataType +// LookupDataType evaluates planner state needed for lookup data type. func (s testBindingLookup) LookupDataType(identifier pgsql.Identifier) (pgsql.DataType, bool) { dataType, found := s[identifier] return dataType, found } -func TestOptimizePreservesUnchangedQueryAndAnalyzesIt(t *testing.T) { +// TestOptimizeCopiesAndAnalyzesQuery verifies that optimization preserves the input AST and records query-part metadata. +func TestOptimizeCopiesAndAnalyzesQuery(t *testing.T) { t.Parallel() - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), adcsQuery) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fixedSuffixExpansionQuery) require.NoError(t, err) plan, err := Optimize(regularQuery) @@ -54,9 +63,18 @@ func TestOptimizePreservesUnchangedQueryAndAnalyzesIt(t *testing.T) { require.Len(t, plan.Analysis.QueryParts[0].Regions, 1) require.Equal(t, []string{"p1", "p2"}, plan.Analysis.QueryParts[0].ProjectionDependencies) require.Equal(t, []RuleResult{ - {Name: "ConservativePatternReordering", Applied: false}, - {Name: "InboundTraversalReversal", Applied: false}, - {Name: "PredicateAttachment", Applied: true}, + { + Name: "ConservativePatternReordering", + Applied: false, + }, + { + Name: "InboundTraversalReversal", + Applied: false, + }, + { + Name: "PredicateAttachment", + Applied: true, + }, }, plan.Rules) require.Len(t, plan.PredicateAttachments, 2) } @@ -89,23 +107,105 @@ func TestOptimizeCopiesOnlyWhenDefaultRuleMutates(t *testing.T) { require.True(t, plan.Query.SingleQuery.SinglePartQuery.ReadingClauses[0].Match.Pattern[0].PathDirectionReversed) } -func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { +// TestFieldRequirementAnalysisDistinguishesObservationBoundaries verifies that each consumer requests only the binding fields it observes. +func TestFieldRequirementAnalysisDistinguishesObservationBoundaries(t *testing.T) { t.Parallel() - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), adcsQuery) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = (n:Group)-[r:MemberOf*1..]->(ca:EnterpriseCA) + WHERE n.objectid = 'source' + RETURN id(ca), labels(n), length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringFieldRequirements}) + + bySymbol := map[string]FieldRequirementDecision{} + for _, decision := range plan.LoweringPlan.FieldRequirements { + bySymbol[decision.Symbol] = decision + } + + require.Contains(t, bySymbol["ca"].Fields, FieldRequirementEntityID) + require.NotContains(t, bySymbol["ca"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["n"].Fields, FieldRequirementKinds) + require.Contains(t, bySymbol["n"].Fields, FieldRequirementProperties) + require.Contains(t, bySymbol["p"].Fields, FieldRequirementOrderedPathEdgeIDs) + require.NotContains(t, bySymbol["p"].Fields, FieldRequirementFullPath) +} + +// TestFieldRequirementAnalysisExpandsGreedyProjection verifies that RETURN * requires complete representations of visible bindings. +func TestFieldRequirementAnalysisExpandsGreedyProjection(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[r:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN * + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + + bySymbol := map[string]FieldRequirementDecision{} + for _, decision := range plan.LoweringPlan.FieldRequirements { + bySymbol[decision.Symbol] = decision + } + + require.NotContains(t, bySymbol, cypher.TokenLiteralAsterisk) + require.Contains(t, bySymbol["p"].Fields, FieldRequirementFullPath) + require.Contains(t, bySymbol["s"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["e"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["r"].Fields, FieldRequirementFullEntity) + require.Contains(t, bySymbol["r"].Fields, FieldRequirementRelationshipIDs) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + require.Equal(t, ShortestPathObservationOnePath, plan.LoweringPlan.ShortestPathExecutor[0].ObservationMode) +} + +// TestFieldRequirementAnalysisTreatsWithGreedyProjectionAsFullObservation verifies that WITH * prevents scalar-only path state. +func TestFieldRequirementAnalysisTreatsWithGreedyProjectionAsFullObservation(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH * + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + + for _, decision := range plan.LoweringPlan.FieldRequirements { + if decision.QueryPartIndex == 0 && decision.Symbol == "p" { + require.Contains(t, decision.Fields, FieldRequirementFullPath) + return + } + } + require.Fail(t, "missing path field-requirement decision") +} + +// TestOptimizePlansFixedSuffixFanoutRewrite verifies that an eligible terminal suffix receives supplemental pushdown metadata. +func TestOptimizePlansFixedSuffixFanoutRewrite(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fixedSuffixExpansionQuery) require.NoError(t, err) plan, err := Optimize(regularQuery) require.NoError(t, err) - ctPredicate := PredicateAttachment{ + predicateAttachment := PredicateAttachment{ QueryPartIndex: 0, RegionIndex: 0, ClauseIndex: 2, ExpressionIndex: 0, Scope: PredicateAttachmentScopeBinding, - BindingSymbols: []string{"ct"}, - Dependencies: []string{"ct"}, + BindingSymbols: []string{"predicate"}, + Dependencies: []string{"predicate"}, } require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringExpansionSuffixPushdown}) @@ -123,6 +223,7 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { SuffixLength: 3, SuffixStartStep: 1, SuffixEndStep: 3, + Reason: "immediate observed continuation produces suffix rows", }) require.Contains(t, plan.LoweringPlan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ Target: TraversalStepTarget{ @@ -134,7 +235,9 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { SuffixLength: 2, SuffixStartStep: 1, SuffixEndStep: 2, - PredicateAttachments: []PredicateAttachment{ctPredicate}, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", + PredicateAttachments: []PredicateAttachment{predicateAttachment}, }) require.Contains(t, plan.LoweringPlan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ Target: TraversalStepTarget{ @@ -143,9 +246,11 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { PatternIndex: 0, StepIndex: 3, }, - SuffixLength: 1, - SuffixStartStep: 4, - SuffixEndStep: 4, + SuffixLength: 1, + SuffixStartStep: 4, + SuffixEndStep: 4, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }) require.Contains(t, plan.LoweringPlan.ExpandInto, ExpandIntoDecision{ @@ -171,11 +276,12 @@ func TestOptimizePlansADCSFanoutRewrite(t *testing.T) { PatternIndex: 0, StepIndex: 1, }, - Attachment: ctPredicate, + Attachment: predicateAttachment, Placement: PredicateAttachmentScopeBinding, }) } +// TestOptimizerRunsRulesAndRefreshesAnalysis verifies optimizer runs rules and refreshes analysis behavior. func TestOptimizerRunsRulesAndRefreshesAnalysis(t *testing.T) { t.Parallel() @@ -186,7 +292,10 @@ func TestOptimizerRunsRulesAndRefreshesAnalysis(t *testing.T) { name: "test", }).Optimize(regularQuery) require.NoError(t, err) - require.Equal(t, []RuleResult{{Name: "test", Applied: false}}, plan.Rules) + require.Equal(t, []RuleResult{{ + Name: "test", + Applied: false, + }}, plan.Rules) require.Len(t, plan.Analysis.QueryParts, 1) require.Len(t, plan.Analysis.QueryParts[0].Regions, 1) } @@ -266,6 +375,7 @@ func TestExpressionReferencesAnySource(t *testing.T) { } } +// TestDefaultPredicateAttachmentRuleReportsSkippedWhenNoPredicatesExist verifies default predicate attachment rule reports skipped when no predicates exist behavior. func TestDefaultPredicateAttachmentRuleReportsSkippedWhenNoPredicatesExist(t *testing.T) { t.Parallel() @@ -275,13 +385,23 @@ func TestDefaultPredicateAttachmentRuleReportsSkippedWhenNoPredicatesExist(t *te plan, err := Optimize(regularQuery) require.NoError(t, err) require.Equal(t, []RuleResult{ - {Name: "ConservativePatternReordering", Applied: false}, - {Name: "InboundTraversalReversal", Applied: false}, - {Name: "PredicateAttachment", Applied: false}, + { + Name: "ConservativePatternReordering", + Applied: false, + }, + { + Name: "InboundTraversalReversal", + Applied: false, + }, + { + Name: "PredicateAttachment", + Applied: false, + }, }, plan.Rules) require.Empty(t, plan.PredicateAttachments) } +// TestLoweringPlanReportsProjectionPruning verifies that unused traversal bindings produce explicit pruning decisions. func TestLoweringPlanReportsProjectionPruning(t *testing.T) { t.Parallel() @@ -293,7 +413,10 @@ func TestLoweringPlanReportsProjectionPruning(t *testing.T) { plan, err := Optimize(regularQuery) require.NoError(t, err) - require.Equal(t, []LoweringDecision{{Name: LoweringProjectionPruning}}, plan.LoweringPlan.Decisions()) + require.Equal(t, []LoweringDecision{ + {Name: LoweringProjectionPruning}, + {Name: LoweringFieldRequirements}, + }, plan.LoweringPlan.Decisions()) require.Equal(t, []ProjectionPruningDecision{{ Target: TraversalStepTarget{ QueryPartIndex: 0, @@ -307,6 +430,7 @@ func TestLoweringPlanReportsProjectionPruning(t *testing.T) { }}, plan.LoweringPlan.ProjectionPruning) } +// TestLoweringPlanProjectionPruningKeepsUpdateTargets verifies lowering plan projection pruning keeps update targets behavior. func TestLoweringPlanProjectionPruningKeepsUpdateTargets(t *testing.T) { t.Parallel() @@ -330,6 +454,7 @@ func TestLoweringPlanProjectionPruningKeepsUpdateTargets(t *testing.T) { }}, plan.LoweringPlan.ProjectionPruning) } +// TestLoweringPlanReportsPatternPredicateProjectionPruning verifies lowering plan reports pattern predicate projection pruning behavior. func TestLoweringPlanReportsPatternPredicateProjectionPruning(t *testing.T) { t.Parallel() @@ -354,6 +479,7 @@ func TestLoweringPlanReportsPatternPredicateProjectionPruning(t *testing.T) { }) } +// TestLoweringPlanReportsPatternPredicateExistencePlacement verifies lowering plan reports pattern predicate existence placement behavior. func TestLoweringPlanReportsPatternPredicateExistencePlacement(t *testing.T) { t.Parallel() @@ -377,6 +503,7 @@ func TestLoweringPlanReportsPatternPredicateExistencePlacement(t *testing.T) { }}, plan.LoweringPlan.PatternPredicate) } +// TestLoweringPlanReportsTypedPatternPredicateExistencePlacement verifies lowering plan reports typed pattern predicate existence placement behavior. func TestLoweringPlanReportsTypedPatternPredicateExistencePlacement(t *testing.T) { t.Parallel() @@ -400,6 +527,7 @@ func TestLoweringPlanReportsTypedPatternPredicateExistencePlacement(t *testing.T }}, plan.LoweringPlan.PatternPredicate) } +// TestLoweringPlanReportsPatternPredicateClauseIndex verifies lowering plan reports pattern predicate clause index behavior. func TestLoweringPlanReportsPatternPredicateClauseIndex(t *testing.T) { t.Parallel() @@ -435,6 +563,7 @@ func TestLoweringPlanReportsPatternPredicateClauseIndex(t *testing.T) { }) } +// TestSelectivityModelPlansTraversalDirection verifies selectivity model plans traversal direction behavior. func TestSelectivityModelPlansTraversalDirection(t *testing.T) { t.Parallel() @@ -460,6 +589,7 @@ func TestSelectivityModelPlansTraversalDirection(t *testing.T) { require.True(t, shouldFlip) } +// TestLoweringPlanReportsLatePathMaterialization verifies lowering plan reports late path materialization behavior. func TestLoweringPlanReportsLatePathMaterialization(t *testing.T) { t.Parallel() @@ -544,6 +674,7 @@ func TestLoweringPlanReportsLatePathMaterialization(t *testing.T) { }) } +// TestLoweringPlanReportsExactOneHopRangeExpansion verifies lowering plan reports exact one hop range expansion behavior. func TestLoweringPlanReportsExactOneHopRangeExpansion(t *testing.T) { t.Parallel() @@ -578,6 +709,7 @@ func TestLoweringPlanReportsExactOneHopRangeExpansion(t *testing.T) { }) } +// TestLoweringPlanReportsExactTwoHopRangeExpansion verifies lowering plan reports exact two hop range expansion behavior. func TestLoweringPlanReportsExactTwoHopRangeExpansion(t *testing.T) { t.Parallel() @@ -603,6 +735,7 @@ func TestLoweringPlanReportsExactTwoHopRangeExpansion(t *testing.T) { }}, plan.LoweringPlan.ExactRangeExpansion) } +// TestExactRangeDependentPlanningRequiresDecision verifies that downstream planning changes only after exact-range expansion is selected. func TestExactRangeDependentPlanningRequiresDecision(t *testing.T) { t.Parallel() @@ -641,12 +774,14 @@ func TestExactRangeDependentPlanningRequiresDecision(t *testing.T) { Mode: LatePathMaterializationExpansionPath, }) - appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses) + appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses, nil) require.Contains(t, plan.ExpansionSuffixPushdown, ExpansionSuffixPushdownDecision{ - Target: target.TraversalStep(0), - SuffixLength: 1, - SuffixStartStep: 1, - SuffixEndStep: 1, + Target: target.TraversalStep(0), + SuffixLength: 1, + SuffixStartStep: 1, + SuffixEndStep: 1, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }) }) @@ -669,11 +804,12 @@ func TestExactRangeDependentPlanningRequiresDecision(t *testing.T) { Mode: LatePathMaterializationPathEdgeID, }) - appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses) + appendExpansionSuffixPushdownDecisions(&plan, 0, readingClauses, nil) require.Empty(t, plan.ExpansionSuffixPushdown) }) } +// TestLoweringPlanSkipsExactRangeExpansionBeyondDepthCap verifies lowering plan skips exact range expansion beyond depth cap behavior. func TestLoweringPlanSkipsExactRangeExpansionBeyondDepthCap(t *testing.T) { t.Parallel() @@ -691,6 +827,7 @@ func TestLoweringPlanSkipsExactRangeExpansionBeyondDepthCap(t *testing.T) { require.Empty(t, plan.LoweringPlan.ExactRangeExpansion) } +// TestLoweringPlanSkipsUndirectedExactRangeExpansion verifies lowering plan skips undirected exact range expansion behavior. func TestLoweringPlanSkipsUndirectedExactRangeExpansion(t *testing.T) { t.Parallel() @@ -708,6 +845,7 @@ func TestLoweringPlanSkipsUndirectedExactRangeExpansion(t *testing.T) { require.Empty(t, plan.LoweringPlan.ExactRangeExpansion) } +// TestLoweringPlanSkipsExactOneHopRangeExpansionForNamedRelationshipBinding verifies lowering plan skips exact one hop range expansion for named relationship binding behavior. func TestLoweringPlanSkipsExactOneHopRangeExpansionForNamedRelationshipBinding(t *testing.T) { t.Parallel() @@ -734,6 +872,7 @@ func TestLoweringPlanSkipsExactOneHopRangeExpansionForNamedRelationshipBinding(t }) } +// TestLoweringPlanSkipsExactOneHopRangeExpansionForShortestPath verifies lowering plan skips exact one hop range expansion for shortest path behavior. func TestLoweringPlanSkipsExactOneHopRangeExpansionForShortestPath(t *testing.T) { t.Parallel() @@ -751,6 +890,7 @@ func TestLoweringPlanSkipsExactOneHopRangeExpansionForShortestPath(t *testing.T) require.Empty(t, plan.LoweringPlan.ExactRangeExpansion) } +// TestLoweringPlanReportsPathRelationshipPredicate verifies lowering plan reports path relationship predicate behavior. func TestLoweringPlanReportsPathRelationshipPredicate(t *testing.T) { t.Parallel() @@ -776,6 +916,7 @@ func TestLoweringPlanReportsPathRelationshipPredicate(t *testing.T) { }}, plan.LoweringPlan.PathRelationshipPredicate) } +// TestLoweringPlanReportsNonePathRelationshipPredicate verifies lowering plan reports none path relationship predicate behavior. func TestLoweringPlanReportsNonePathRelationshipPredicate(t *testing.T) { t.Parallel() @@ -801,6 +942,7 @@ func TestLoweringPlanReportsNonePathRelationshipPredicate(t *testing.T) { }}, plan.LoweringPlan.PathRelationshipPredicate) } +// TestLoweringPlanSkipsPathRelationshipPredicateForAllQuantifier verifies lowering plan skips path relationship predicate for all quantifier behavior. func TestLoweringPlanSkipsPathRelationshipPredicateForAllQuantifier(t *testing.T) { t.Parallel() @@ -819,6 +961,7 @@ func TestLoweringPlanSkipsPathRelationshipPredicateForAllQuantifier(t *testing.T require.Empty(t, plan.LoweringPlan.PathRelationshipPredicate) } +// TestLoweringPlanSkipsPathRelationshipPredicateAfterWithProjection verifies lowering plan skips path relationship predicate after with projection behavior. func TestLoweringPlanSkipsPathRelationshipPredicateAfterWithProjection(t *testing.T) { t.Parallel() @@ -838,12 +981,13 @@ func TestLoweringPlanSkipsPathRelationshipPredicateAfterWithProjection(t *testin require.Empty(t, plan.LoweringPlan.PathRelationshipPredicate) } +// TestLoweringPlanReportsExpansionSuffixPushdown verifies that an eligible fixed suffix produces a supplemental-search decision. func TestLoweringPlanReportsExpansionSuffixPushdown(t *testing.T) { t.Parallel() regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` - MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ca:EnterpriseCA) - RETURN p + MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->(boundary:ExpansionNode)-[:EnterSuffix]->(head:SuffixHead) + RETURN path `) require.NoError(t, err) @@ -857,19 +1001,430 @@ func TestLoweringPlanReportsExpansionSuffixPushdown(t *testing.T) { PatternIndex: 0, StepIndex: 0, }, - SuffixLength: 1, - SuffixStartStep: 1, - SuffixEndStep: 1, + SuffixLength: 1, + SuffixStartStep: 1, + SuffixEndStep: 1, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }}, plan.LoweringPlan.ExpansionSuffixPushdown) } +// TestLoweringPlanReportsConservativeFixedSuffixSearchStrategy verifies that eligible suffix topology remains on the incumbent strategy unless qualified. +func TestLoweringPlanReportsConservativeFixedSuffixSearchStrategy(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringExpansionSearchStrategy}) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, "fixed_suffix_expansion", decision.Family) + require.Equal(t, ExpansionSearchPolicyOrientationProbeV1, decision.PlannedPolicy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, "incumbent_default", decision.SelectionMode) + require.Equal(t, "fixed-suffix-static-v1", decision.SelectorVersion) + require.Equal(t, []ExpansionSearchStrategy{ + ExpansionSearchStepwiseForward, + ExpansionSearchLateHydratedForward, + ExpansionSearchFactoredSuffixForward, + ExpansionSearchSuffixSeededReverse, + ExpansionSearchBackwardViabilityForward, + }, decision.PlannedCandidates) + require.Equal(t, []ExpansionSearchStrategy{ExpansionSearchStepwiseForward}, decision.EmittedCandidates) + require.Equal(t, ExpansionSearchExecutionBoundaryInlineStatement, decision.ExecutionBoundary) + require.Equal(t, ExpansionSearchProbeCaps{ + RootRowLimit: ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, decision.ProbeCaps) + require.Equal(t, ExpansionSearchAdmission{ + StateLimit: ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, decision.Admission) + require.True(t, decision.StructurallyEligible) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{ + Name: "qualified_fixed_suffix_topology", + Eligible: true, + }) + require.Equal(t, ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, ExpansionSearchStepwiseForward, decision.FallbackStrategy) + require.Equal(t, ExpansionSearchFallbackTournamentUnqualified, decision.FallbackReason) + require.Equal(t, ExpansionSearchObservationFullPath, decision.ObservationMode) + require.Equal(t, int64(0), decision.MinimumDepth) + require.Equal(t, int64(16), decision.MaximumDepth) + require.Equal(t, 3, decision.SuffixLength) + require.Equal(t, "outbound", decision.LogicalDirection) +} + +// TestLoweringPlanSelectsGuardedEndpointSeededExpansion verifies guarded endpoint seeding for one statement-wide variable expansion. +func TestLoweringPlanSelectsGuardedEndpointSeededExpansion(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix + RETURN p + LIMIT 1000 + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, "fixed_prefix_terminal_expansion", decision.Family) + require.Equal(t, ExpansionSearchPolicyEndpointGuardV1, decision.PlannedPolicy) + require.Equal(t, ExpansionSearchPolicyEndpointGuardV1, decision.EmittedPolicy) + require.Equal(t, []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse}, decision.EmittedCandidates) + require.Equal(t, ExpansionSearchExecutionBoundaryGuardedDualArm, decision.ExecutionBoundary) + require.Equal(t, ExpansionSearchProbeCaps{ReverseSeedRowLimit: 32}, decision.ProbeCaps) + require.Equal(t, ExpansionSearchAdmission{ + StateLimit: 4096, + RequiresCompleteProbes: true, + FallbackStrategy: ExpansionSearchStepwiseForward, + }, decision.Admission) + require.True(t, decision.StructurallyEligible) + require.True(t, decision.StaticallyEligible) + require.Equal(t, ExpansionSearchEndpointSeededReverse, decision.SelectedStrategy) + require.Equal(t, ExpansionSearchStepwiseForward, decision.FallbackStrategy) + require.Equal(t, "static_guarded", decision.SelectionMode) + require.Equal(t, "endpoint-seeded-guarded-v1", decision.SelectorVersion) + require.Equal(t, "property_ends_with", decision.SeedPredicateClass) + require.Equal(t, int64(32), decision.EndpointLimit) + require.Equal(t, int64(4096), decision.StateLimit) + require.Equal(t, 1, decision.PrefixLength) + require.Equal(t, int64(1), decision.MinimumDepth) + require.Equal(t, int64(15), decision.MaximumDepth) + require.True(t, decision.HasFinalLimit) + require.Empty(t, decision.FallbackReason) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{ + Name: "single_variable_expansion_in_region", + Eligible: true, + }) +} + +// TestEndpointSeededExpansionKeepsIndependentMultipartRegionQualified verifies +// that an earlier traversal separated by WITH does not invalidate the existing +// guarded fixed-prefix region. +func TestEndpointSeededExpansionKeepsIndependentMultipartRegionQualified(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (s)-[:MemberOf*0..]->(excluded:Group) + WHERE excluded.objectid ENDS WITH '-516' + WITH collect(s) AS exclude + MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix AND NOT c IN exclude + RETURN p + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 2) + decision := plan.LoweringPlan.ExpansionSearchStrategy[1] + require.Equal(t, "fixed_prefix_terminal_expansion", decision.Family) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{ + Name: "single_variable_expansion_in_region", + Eligible: true, + }) + require.True(t, decision.StructurallyEligible) + require.Equal(t, ExpansionSearchEndpointSeededReverse, decision.SelectedStrategy) + require.Equal(t, ExpansionSearchPolicyEndpointGuardV1, decision.EmittedPolicy) + require.Equal(t, []ExpansionSearchStrategy{ExpansionSearchStepwiseForward, ExpansionSearchEndpointSeededReverse}, decision.EmittedCandidates) + require.Equal(t, ExpansionSearchExecutionBoundaryGuardedDualArm, decision.ExecutionBoundary) + require.Empty(t, decision.FallbackReason) +} + +// TestGuardedEndpointSeededExpansionFallbackReasons verifies stable rejection reasons for unsafe endpoint-seeded shapes. +func TestGuardedEndpointSeededExpansionFallbackReasons(t *testing.T) { + for _, testCase := range []struct { + // name labels the structural rejection case. + name string + // query produces the endpoint-seeding candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "terminal not selective", + query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) RETURN p`, + reason: ExpansionSearchFallbackTerminalNotSelective, + }, + { + name: "zero depth", + query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*0..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p`, + reason: ExpansionSearchFallbackZeroDepth, + }, + { + name: "directionless prefix", + query: `MATCH p = (c:Computer)-[:HasSession]-(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p`, + reason: ExpansionSearchFallbackDirectionlessPrefix, + }, + { + name: "correlated terminal", + query: `MATCH (g:Group) MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g) WHERE g.objectid ENDS WITH '-512' RETURN p`, + reason: ExpansionSearchFallbackCorrelatedTerminal, + }, + { + name: "correlated terminal predicate", + query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' AND g.tenant = c.tenant RETURN p`, + reason: ExpansionSearchFallbackCorrelatedTerminal, + }, + { + name: "nonterminal expansion", + query: `MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group)-[:AdminTo]->() WHERE g.objectid ENDS WITH '-512' RETURN p`, + reason: ExpansionSearchFallbackExpansionNotTerminal, + }, + { + name: "mutation", + query: `MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' CREATE (:Computer) RETURN g`, + reason: ExpansionSearchFallbackMutation, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.NotEmpty(t, plan.LoweringPlan.ExpansionSearchStrategy) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, "fixed_prefix_terminal_expansion", decision.Family) + require.False(t, decision.StructurallyEligible) + require.Equal(t, ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, testCase.reason, decision.FallbackReason) + }) + } +} + +// TestGuardedEndpointSeededExpansionAcceptsTerminalIDEquality verifies that a singleton terminal ID is a selective reverse-search seed. +func TestGuardedEndpointSeededExpansionAcceptsTerminalIDEquality(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..8]->(g) + WHERE id(g) = $terminal_id + RETURN id(c), id(g) + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.True(t, decision.StructurallyEligible) + require.Equal(t, "id_equality", decision.SeedPredicateClass) + require.Equal(t, ExpansionSearchEndpointSeededReverse, decision.SelectedStrategy) +} + +// TestFixedSuffixSearchRejectsPredicateFunctionReevaluation verifies that reordered function evaluation disqualifies suffix search. +func TestFixedSuffixSearchRejectsPredicateFunctionReevaluation(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = 'root' + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) + WHERE root.marker = toString(1) + RETURN root + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, ExpansionSearchFallbackNonDeterministicPredicate, decision.FallbackReason) + require.Contains(t, decision.EligibilityFacts, ExpansionSearchEligibilityFact{ + Name: "deterministic_predicates", + Eligible: false, + }) +} + +// TestExpansionSearchObservationUsesExternalFieldRequirements verifies that downstream field requirements select the search observation mode. +func TestExpansionSearchObservationUsesExternalFieldRequirements(t *testing.T) { + for _, testCase := range []struct { + // name labels the downstream observation form. + name string + // projection contains the downstream expression being classified. + projection string + // observation is the expected search-state representation. + observation ExpansionSearchObservationMode + }{ + { + name: "endpoint IDs", + projection: "id(head), id(terminal)", + observation: ExpansionSearchObservationEndpointIDs, + }, + { + name: "ordered IDs", + projection: "length(path)", + observation: ExpansionSearchObservationOrderedPathIDs, + }, + { + name: "full path", + projection: "path", + observation: ExpansionSearchObservationFullPath, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN `+testCase.projection) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + require.Equal(t, testCase.observation, plan.LoweringPlan.ExpansionSearchStrategy[0].ObservationMode) + }) + } +} + +// TestExpansionSearchFinalizationRejectsVariableExpansionAcrossWith verifies that multiple statement-wide expansions prevent specialized search. +func TestExpansionSearchFinalizationRejectsVariableExpansionAcrossWith(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + WITH root, terminal + MATCH (root)-[:Expand*0..4]->(other) + RETURN id(terminal), id(other) + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 2) + require.Equal(t, ExpansionSearchFallbackMultipleVariableExpansions, plan.LoweringPlan.ExpansionSearchStrategy[0].FallbackReason) + require.False(t, plan.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) +} + +// TestLoweringPlanReportsStableFixedSuffixSearchFallbackCodes verifies diagnostic codes for structurally unsafe suffix searches. +func TestLoweringPlanReportsStableFixedSuffixSearchFallbackCodes(t *testing.T) { + t.Parallel() + + for _, testCase := range []struct { + // name labels the structural rejection case. + name string + // query produces the fixed-suffix candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "no fixed suffix", + query: `MATCH (root)-[:Expand*0..16]->(head) RETURN id(head)`, + reason: ExpansionSearchFallbackNoFixedSuffix, + }, + { + name: "unbounded", + query: `MATCH (root)-[:Expand*0..]->()-[:EnterSuffix]->(head) RETURN id(head)`, + reason: ExpansionSearchFallbackUnboundedDepth, + }, + { + name: "short suffix", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head) RETURN id(head)`, + reason: ExpansionSearchFallbackSuffixTooShort, + }, + { + name: "directionless", + query: `MATCH (root)-[:Expand*0..16]-()-[:EnterSuffix]->(head)-[:ContinueSuffix]->()-[:CompleteSuffix]->(terminal) RETURN id(head)`, + reason: ExpansionSearchFallbackDirectionlessExpansion, + }, + { + name: "directionless suffix", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]-(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackDirectionlessSuffix, + }, + { + name: "optional", + query: `OPTIONAL MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackOptionalMatch, + }, + { + name: "shortest path", + query: `MATCH path = shortestPath((root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal)) RETURN path`, + reason: ExpansionSearchFallbackShortestPath, + }, + { + name: "all shortest paths", + query: `MATCH path = allShortestPaths((root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal)) RETURN path`, + reason: ExpansionSearchFallbackAllShortestPaths, + }, + { + name: "unbound root", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)`, + reason: ExpansionSearchFallbackUnboundRoot, + }, + { + name: "unsupported depth", + query: `MATCH (root)-[:Expand*0..65]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackUnsupportedDepth, + }, + { + name: "relationship variable", + query: `MATCH (root)-[edges:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head)`, + reason: ExpansionSearchFallbackRelationshipVariable, + }, + { + name: "relationship predicate", + query: `MATCH (root)-[edges:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WHERE edges.enabled = true RETURN id(head)`, + reason: ExpansionSearchFallbackRelationshipPredicate, + }, + { + name: "correlated suffix", + query: `MATCH (head:SuffixHead) MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(head)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN path`, + reason: ExpansionSearchFallbackCorrelatedSuffix, + }, + { + name: "cross-region predicate", + query: `MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WHERE root.partition = head.partition RETURN path`, + reason: ExpansionSearchFallbackCrossRegionPredicate, + }, + { + name: "path predicate", + query: `MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WHERE length(path) > 0 RETURN path`, + reason: ExpansionSearchFallbackPathDependentPredicate, + }, + { + name: "unsupported observation", + query: `MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(path)`, + reason: ExpansionSearchFallbackUnsupportedObservation, + }, + { + name: "mutation", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) CREATE (created) RETURN id(head)`, + reason: ExpansionSearchFallbackMutation, + }, + { + name: "limit pushdown conflict", + query: `MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head) LIMIT 10`, + reason: ExpansionSearchFallbackLimitPushdownConflict, + }, + { + name: "tournament unqualified", + query: `MATCH (root)-[:Other|Alternate*0..16]->()-[:A]->(head:X)-[:B]->(:Y)-[:C]->(terminal:Z) RETURN id(head)`, + reason: ExpansionSearchFallbackTournamentUnqualified, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + require.Equal(t, testCase.reason, plan.LoweringPlan.ExpansionSearchStrategy[0].FallbackReason) + require.False(t, plan.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) + }) + } +} + +// TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix verifies that a pre-bound terminal remains part of suffix metadata. func TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix(t *testing.T) { t.Parallel() regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` - MATCH (ca) - MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA) - RETURN p + MATCH (terminal) + MATCH path = (root:ExpansionRoot)-[:Expand*0..16]->(boundary:ExpansionNode)-[:EnterSuffix]->(middle:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path `) require.NoError(t, err) @@ -883,18 +1438,24 @@ func TestLoweringPlanIncludesConstrainedBoundEndpointInExpansionSuffix(t *testin PatternIndex: 0, StepIndex: 0, }, - SuffixLength: 2, - SuffixStartStep: 1, - SuffixEndStep: 2, + SuffixLength: 2, + SuffixStartStep: 1, + SuffixEndStep: 2, + ApplySupplemental: true, + Reason: "supplemental suffix prefilter retained for unobserved continuation", }) } +// TestLoweringPlanReportsCountStoreFastPath verifies lowering plan reports count store fast path behavior. func TestLoweringPlanReportsCountStoreFastPath(t *testing.T) { t.Parallel() testCases := []struct { - name string - query string + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. + query string + // expected retains the expected while anonymous record is assembled or evaluated. expected CountStoreFastPathDecision }{ { @@ -960,6 +1521,7 @@ func TestLoweringPlanReportsCountStoreFastPath(t *testing.T) { } } +// TestLoweringPlanPlacesBindingPredicates verifies lowering plan places binding predicates behavior. func TestLoweringPlanPlacesBindingPredicates(t *testing.T) { t.Parallel() @@ -987,6 +1549,7 @@ func TestLoweringPlanPlacesBindingPredicates(t *testing.T) { require.Empty(t, plan.LoweringPlan.ExpansionSuffixPushdown) } +// TestLoweringPlanDoesNotPlaceCrossClauseBindingPredicates verifies lowering plan does not place cross clause binding predicates behavior. func TestLoweringPlanDoesNotPlaceCrossClauseBindingPredicates(t *testing.T) { t.Parallel() @@ -1005,6 +1568,7 @@ func TestLoweringPlanDoesNotPlaceCrossClauseBindingPredicates(t *testing.T) { require.NotContains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringPredicatePlacement}) } +// TestLoweringPlanReportsExpandInto verifies lowering plan reports expand into behavior. func TestLoweringPlanReportsExpandInto(t *testing.T) { t.Parallel() @@ -1029,6 +1593,57 @@ func TestLoweringPlanReportsExpandInto(t *testing.T) { }}, plan.LoweringPlan.ExpandInto) } +// TestLoweringPlanReportsExpandIntoForEndpointsCarriedAcrossWithAndUnwind verifies lowering plan reports expand into for endpoints carried across with and unwind behavior. +func TestLoweringPlanReportsExpandIntoForEndpointsCarriedAcrossWithAndUnwind(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (a:Group), (b:Group) + WITH a, b, [1, 2] AS copies + UNWIND copies AS copy + MATCH (a)-[:MemberOf]->(b) + RETURN copy + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.ExpandInto, ExpandIntoDecision{ + Target: TraversalStepTarget{ + QueryPartIndex: 1, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }, + }) +} + +// TestLoweringPlanReportsExpandIntoForNodeIntroducedByUnwind verifies lowering plan reports expand into for node introduced by unwind behavior. +func TestLoweringPlanReportsExpandIntoForNodeIntroducedByUnwind(t *testing.T) { + t.Parallel() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (a:Group), (b:Group) + WITH b, [a] AS nodes + UNWIND nodes AS source + MATCH (source)-[:MemberOf]->(b) + RETURN source + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Contains(t, plan.LoweringPlan.ExpandInto, ExpandIntoDecision{ + Target: TraversalStepTarget{ + QueryPartIndex: 1, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }, + }) +} + +// TestLoweringPlanReportsExpandIntoForAnonymousContinuationEndpoint verifies lowering plan reports expand into for anonymous continuation endpoint behavior. func TestLoweringPlanReportsExpandIntoForAnonymousContinuationEndpoint(t *testing.T) { t.Parallel() @@ -1051,6 +1666,7 @@ func TestLoweringPlanReportsExpandIntoForAnonymousContinuationEndpoint(t *testin }) } +// TestLoweringPlanReportsTraversalDirectionForConstrainedRightEndpoint verifies lowering plan reports traversal direction for constrained right endpoint behavior. func TestLoweringPlanReportsTraversalDirectionForConstrainedRightEndpoint(t *testing.T) { t.Parallel() @@ -1075,6 +1691,7 @@ func TestLoweringPlanReportsTraversalDirectionForConstrainedRightEndpoint(t *tes }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanReportsTraversalDirectionForBoundRightEndpoint verifies lowering plan reports traversal direction for bound right endpoint behavior. func TestLoweringPlanReportsTraversalDirectionForBoundRightEndpoint(t *testing.T) { t.Parallel() @@ -1100,6 +1717,7 @@ func TestLoweringPlanReportsTraversalDirectionForBoundRightEndpoint(t *testing.T }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanSkipsTraversalDirectionWhenLeftEndpointHasBindingPredicate verifies lowering plan skips traversal direction when left endpoint has binding predicate behavior. func TestLoweringPlanSkipsTraversalDirectionWhenLeftEndpointHasBindingPredicate(t *testing.T) { t.Parallel() @@ -1115,6 +1733,7 @@ func TestLoweringPlanSkipsTraversalDirectionWhenLeftEndpointHasBindingPredicate( require.Empty(t, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanSkipsTraversalDirectionWhenLeftEndpointHasRegionPredicate verifies lowering plan skips traversal direction when left endpoint has region predicate behavior. func TestLoweringPlanSkipsTraversalDirectionWhenLeftEndpointHasRegionPredicate(t *testing.T) { t.Parallel() @@ -1131,6 +1750,7 @@ func TestLoweringPlanSkipsTraversalDirectionWhenLeftEndpointHasRegionPredicate(t require.Empty(t, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanReportsTraversalDirectionForRightEndpointPredicate verifies lowering plan reports traversal direction for right endpoint predicate behavior. func TestLoweringPlanReportsTraversalDirectionForRightEndpointPredicate(t *testing.T) { t.Parallel() @@ -1156,6 +1776,7 @@ func TestLoweringPlanReportsTraversalDirectionForRightEndpointPredicate(t *testi }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanReportsTraversalDirectionForBoundLeftExpansionToConstrainedRightEndpoint verifies lowering plan reports traversal direction for bound left expansion to constrained right endpoint behavior. func TestLoweringPlanReportsTraversalDirectionForBoundLeftExpansionToConstrainedRightEndpoint(t *testing.T) { t.Parallel() @@ -1182,6 +1803,7 @@ func TestLoweringPlanReportsTraversalDirectionForBoundLeftExpansionToConstrained }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanSkipsBoundLeftDirectionForSelectiveSource verifies lowering plan skips bound left direction for selective source behavior. func TestLoweringPlanSkipsBoundLeftDirectionForSelectiveSource(t *testing.T) { t.Parallel() @@ -1207,6 +1829,7 @@ func TestLoweringPlanSkipsBoundLeftDirectionForSelectiveSource(t *testing.T) { }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanSkipsBoundLeftDirectionAfterPriorLimit verifies lowering plan skips bound left direction after prior limit behavior. func TestLoweringPlanSkipsBoundLeftDirectionAfterPriorLimit(t *testing.T) { t.Parallel() @@ -1234,6 +1857,7 @@ func TestLoweringPlanSkipsBoundLeftDirectionAfterPriorLimit(t *testing.T) { }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanSkipsBoundLeftDirectionAfterGreedyProjectionLimit verifies lowering plan skips bound left direction after greedy projection limit behavior. func TestLoweringPlanSkipsBoundLeftDirectionAfterGreedyProjectionLimit(t *testing.T) { t.Parallel() @@ -1261,6 +1885,7 @@ func TestLoweringPlanSkipsBoundLeftDirectionAfterGreedyProjectionLimit(t *testin }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanCarriesBindingsAcrossNilWithPart verifies lowering plan carries bindings across nil with part behavior. func TestLoweringPlanCarriesBindingsAcrossNilWithPart(t *testing.T) { t.Parallel() @@ -1289,6 +1914,7 @@ func TestLoweringPlanCarriesBindingsAcrossNilWithPart(t *testing.T) { }) } +// TestLoweringPlanAllowsUniqueRightEndpointAfterPriorLimit verifies lowering plan allows unique right endpoint after prior limit behavior. func TestLoweringPlanAllowsUniqueRightEndpointAfterPriorLimit(t *testing.T) { t.Parallel() @@ -1317,6 +1943,7 @@ func TestLoweringPlanAllowsUniqueRightEndpointAfterPriorLimit(t *testing.T) { }}, plan.LoweringPlan.TraversalDirection) } +// TestLoweringPlanReportsAggregateTraversalCountForBoundExpansionCount verifies lowering plan reports aggregate traversal count for bound expansion count behavior. func TestLoweringPlanReportsAggregateTraversalCountForBoundExpansionCount(t *testing.T) { t.Parallel() @@ -1359,6 +1986,7 @@ func TestLoweringPlanReportsAggregateTraversalCountForBoundExpansionCount(t *tes }}, plan.LoweringPlan.AggregateTraversalCount) } +// TestLoweringPlanReportsAggregateTraversalCountForRowCount verifies lowering plan reports aggregate traversal count for row count behavior. func TestLoweringPlanReportsAggregateTraversalCountForRowCount(t *testing.T) { t.Parallel() @@ -1379,6 +2007,7 @@ func TestLoweringPlanReportsAggregateTraversalCountForRowCount(t *testing.T) { require.Equal(t, "adminCount", plan.LoweringPlan.AggregateTraversalCount[0].CountAlias) } +// TestLoweringPlanReportsAggregateTraversalCountWhenReturningCountAlias verifies lowering plan reports aggregate traversal count when returning count alias behavior. func TestLoweringPlanReportsAggregateTraversalCountWhenReturningCountAlias(t *testing.T) { t.Parallel() @@ -1404,6 +2033,7 @@ func TestLoweringPlanReportsAggregateTraversalCountWhenReturningCountAlias(t *te require.Equal(t, "privileges", shape.ReturnCountAlias) } +// TestLoweringPlanReportsAggregateTraversalCountWithTerminalFilter verifies lowering plan reports aggregate traversal count with terminal filter behavior. func TestLoweringPlanReportsAggregateTraversalCountWithTerminalFilter(t *testing.T) { t.Parallel() @@ -1424,6 +2054,7 @@ func TestLoweringPlanReportsAggregateTraversalCountWithTerminalFilter(t *testing require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringAggregateTraversalCount}) } +// TestLoweringPlanSkipsAggregateTraversalCountWithCorrelatedTerminalFilter verifies lowering plan skips aggregate traversal count with correlated terminal filter behavior. func TestLoweringPlanSkipsAggregateTraversalCountWithCorrelatedTerminalFilter(t *testing.T) { t.Parallel() @@ -1444,6 +2075,7 @@ func TestLoweringPlanSkipsAggregateTraversalCountWithCorrelatedTerminalFilter(t require.NotContains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringAggregateTraversalCount}) } +// TestLoweringPlanSkipsSuffixPushdownAfterRightEndpointPredicateDirectionFlip verifies lowering plan skips suffix pushdown after right endpoint predicate direction flip behavior. func TestLoweringPlanSkipsSuffixPushdownAfterRightEndpointPredicateDirectionFlip(t *testing.T) { t.Parallel() @@ -1460,6 +2092,7 @@ func TestLoweringPlanSkipsSuffixPushdownAfterRightEndpointPredicateDirectionFlip require.Empty(t, plan.LoweringPlan.ExpansionSuffixPushdown) } +// TestLoweringPlanReportsShortestPathStrategyForEndpointPredicates verifies lowering plan reports shortest path strategy for endpoint predicates behavior. func TestLoweringPlanReportsShortestPathStrategyForEndpointPredicates(t *testing.T) { t.Parallel() @@ -1495,6 +2128,527 @@ func TestLoweringPlanReportsShortestPathStrategyForEndpointPredicates(t *testing }}, plan.LoweringPlan.ShortestPathFilter) } +// TestLoweringPlanSelectsQualifiedSingletonDistanceExecutor verifies scalar-distance selection for a statically bound endpoint pair. +func TestLoweringPlanSelectsQualifiedSingletonDistanceExecutor(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..16]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, "SP", decision.Family) + require.Equal(t, "static", decision.SelectionMode) + require.Equal(t, "sp-static-v3", decision.SelectorVersion) + require.Equal(t, []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorS0Direct, + ShortestPathExecutorS1ArrayBFS, + ShortestPathExecutorS2TraceRelation, + ShortestPathExecutorS3Unidirectional, + ShortestPathExecutorS3EdgeM0, + ShortestPathExecutorS4CanonicalDistance, + ShortestPathExecutorS4CanonicalWitness, + ShortestPathExecutorI1CanonicalDistance, + ShortestPathExecutorI2GuardedDistance, + ShortestPathExecutorI1CanonicalWitness, + ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathExecutorB1AlternatingNodeDistance, + ShortestPathExecutorB1AlternatingNodeWitness, + ShortestPathExecutorB2SmallerCurrentLevelDistance, + ShortestPathExecutorB2SmallerCurrentLevelWitness, + }, decision.PlannedCandidates) + require.Equal(t, ShortestPathExecutorS3Unidirectional, decision.SelectedExecutor) + require.Equal(t, ShortestPathSchedulerSingleEndedLevel, decision.Scheduler) + require.Equal(t, ShortestPathExecutorIncumbentWorkspace, decision.FallbackExecutor) + require.Empty(t, decision.FallbackReason) + require.Equal(t, ShortestPathObservationDistance, decision.ObservationMode) + require.True(t, decision.StructurallyEligible) + require.Equal(t, int64(1), decision.MinimumDepth) + require.Equal(t, int64(16), decision.MaximumDepth) + require.True(t, decision.ExperimentalWinner) + require.Contains(t, plan.LoweringPlan.Decisions(), LoweringDecision{Name: LoweringShortestPathExecutor}) +} + +// TestLoweringPlanSelectsBoundPairAllShortestDAGExecutor verifies predecessor-DAG selection for bound all-shortest-path endpoints. +func TestLoweringPlanSelectsBoundPairAllShortestDAGExecutor(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[*1..]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, "ASP", decision.Family) + require.Equal(t, ShortestPathObservationAllPaths, decision.ObservationMode) + require.Equal(t, ShortestPathExecutorASPA1DAG, decision.SelectedExecutor) + require.Equal(t, []ShortestPathExecutor{ + ShortestPathExecutorIncumbentWorkspace, + ShortestPathExecutorASPA1DAG, + ShortestPathExecutorASPI1DAG, + ShortestPathExecutorASPB1AlternatingNodeDAG, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + }, decision.PlannedCandidates) + require.Equal(t, ShortestPathSchedulerSingleEndedLevel, decision.Scheduler) + require.Equal(t, "asp-static-v1", decision.SelectorVersion) + require.Equal(t, "static", decision.SelectionMode) + require.True(t, decision.StructurallyEligible) + require.True(t, decision.StaticallyEligible) + require.Equal(t, int64(1), decision.MinimumDepth) + require.Equal(t, defaultShortestPathExpansionDepth, decision.MaximumDepth) + require.Equal(t, defaultShortestPathStateLimit, decision.StateLimit) + require.Equal(t, defaultShortestPathFrontierLimit, decision.FrontierLimit) + require.Equal(t, defaultShortestPathPredecessorLimit, decision.PredecessorLimit) + require.Empty(t, decision.FallbackReason) +} + +// TestShortestPathExecutorSchedulersFreezesTournamentSchedulerMetadata verifies +// production controls and reserved bidirectional arms retain distinct policies. +func TestShortestPathExecutorSchedulersFreezesTournamentSchedulerMetadata(t *testing.T) { + t.Parallel() + tests := map[ShortestPathExecutor]ShortestPathScheduler{ + ShortestPathExecutorS3Unidirectional: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorS3EdgeM0: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorS4CanonicalDistance: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorS4CanonicalWitness: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorASPA1DAG: ShortestPathSchedulerSingleEndedLevel, + ShortestPathExecutorB1AlternatingNodeDistance: ShortestPathSchedulerStrictAlternatingNode, + ShortestPathExecutorB1AlternatingNodeWitness: ShortestPathSchedulerStrictAlternatingNode, + ShortestPathExecutorASPB1AlternatingNodeDAG: ShortestPathSchedulerStrictAlternatingNode, + ShortestPathExecutorB2SmallerCurrentLevelDistance: ShortestPathSchedulerSmallerCurrentLevel, + ShortestPathExecutorB2SmallerCurrentLevelWitness: ShortestPathSchedulerSmallerCurrentLevel, + ShortestPathExecutorASPB2SmallerCurrentLevelDAG: ShortestPathSchedulerSmallerCurrentLevel, + } + for executor, scheduler := range tests { + require.Equal(t, scheduler, executor.Scheduler(), executor) + } + require.Empty(t, ShortestPathExecutorIncumbentWorkspace.Scheduler()) +} + +// TestLoweringPlanShortestExecutorV4SelectionMatrix verifies executor selection across direction, depth, kind, and observation combinations. +func TestLoweringPlanShortestExecutorV4SelectionMatrix(t *testing.T) { + t.Parallel() + tests := []struct { + // name labels the executor-selection case. + name string + // pattern is the relationship pattern supplied to shortestPath. + pattern string + // observation is the return expression that consumes the path. + observation string + // executor is the physical implementation expected from selection. + executor ShortestPathExecutor + // reason is the expected fallback code when selection is ineligible. + reason string + // direction is the logical traversal direction recorded in diagnostics. + direction graph.Direction + // physicalExpansion is the edge endpoint used to advance recursive search. + physicalExpansion ShortestPathPhysicalExpansion + // topology is the expected physical topology classification. + topology ShortestPathTopologyClassification + // kindCount is the expected number of statically resolved relationship kinds. + kindCount int + // untyped reports whether the pattern is expected to omit relationship kinds. + untyped bool + // staticEligible is the expected static qualification result. + staticEligible bool + // selector identifies the policy version expected to make the decision. + selector string + }{ + { + name: "outbound distance depth 64 two kinds", + pattern: `(s)-[:MemberOf|Contains*1..64]->(e)`, + observation: `length(p)`, + executor: ShortestPathExecutorS3Unidirectional, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + kindCount: 2, + staticEligible: true, + selector: "sp-static-v3", + }, + { + name: "outbound one path one kind", + pattern: `(s)-[:MemberOf*1..16]->(e)`, + observation: `p`, + executor: ShortestPathExecutorS3EdgeM0, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "outbound one path two kinds", + pattern: `(s)-[:MemberOf|Contains*1..16]->(e)`, + observation: `p`, + executor: ShortestPathExecutorS4CanonicalWitness, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + kindCount: 2, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "outbound one path wildcard", + pattern: `(s)-[*1..16]->(e)`, + observation: `p`, + executor: ShortestPathExecutorS4CanonicalWitness, + direction: graph.DirectionOutbound, + physicalExpansion: ShortestPathPhysicalExpansionStartID, + topology: ShortestPathTopologyPhysicalOutbound, + untyped: true, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "inbound distance depth one", + pattern: `(s)<-[:MemberOf*0..1]-(e)`, + observation: `length(p)`, + executor: ShortestPathExecutorS3Unidirectional, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundShallow, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v3", + }, + { + name: "inbound path depth one", + pattern: `(s)<-[:MemberOf*1..1]-(e)`, + observation: `p`, + executor: ShortestPathExecutorS3EdgeM0, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundShallow, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "inbound distance depth two", + pattern: `(s)<-[:MemberOf*1..2]-(e)`, + observation: `length(p)`, + executor: ShortestPathExecutorS4CanonicalDistance, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundDeep, + kindCount: 1, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + { + name: "inbound path depth 64 two kinds", + pattern: `(s)<-[:MemberOf|Contains*1..64]-(e)`, + observation: `p`, + executor: ShortestPathExecutorS4CanonicalWitness, + direction: graph.DirectionInbound, + physicalExpansion: ShortestPathPhysicalExpansionEndID, + topology: ShortestPathTopologyPhysicalInboundDeep, + kindCount: 2, + staticEligible: true, + selector: "sp-static-v5-contained", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath(%s) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.pattern, test.observation)) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, test.selector, decision.SelectorVersion) + require.True(t, decision.StructurallyEligible) + require.Equal(t, test.staticEligible, decision.StaticallyEligible) + require.Equal(t, test.executor, decision.SelectedExecutor) + require.Equal(t, test.reason, decision.FallbackReason) + require.Equal(t, test.direction, decision.Direction) + require.Equal(t, test.physicalExpansion, decision.PhysicalExpansion) + require.Equal(t, test.topology, decision.TopologyClassification) + require.Equal(t, test.kindCount, decision.RelationshipKindCount) + require.Equal(t, test.untyped, decision.UntypedRelationship) + require.Equal(t, ShortestPathMaximumDepthExplicit, decision.MaximumDepthSource) + }) + } +} + +// TestLoweringPlanShortestExecutorUsesPolicyBoundForOpenMaximum verifies a +// syntax-open SP retains provenance while using the existing effective cap. +func TestLoweringPlanShortestExecutorUsesPolicyBoundForOpenMaximum(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + observation string + executor ShortestPathExecutor + }{ + {name: "distance", observation: "length(p)", executor: ShortestPathExecutorS3Unidirectional}, + {name: "typed witness", observation: "p", executor: ShortestPathExecutorS3EdgeM0}, + } { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.observation)) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.True(t, decision.StructurallyEligible) + require.Equal(t, int64(15), decision.MaximumDepth) + require.Equal(t, ShortestPathMaximumDepthPolicyDefault, decision.MaximumDepthSource) + require.Equal(t, test.executor, decision.SelectedExecutor) + require.Equal(t, ShortestPathSelectorStaticV7Contained, decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + }) + } +} + +// TestLoweringPlanShortestExecutorV3PreservesStructuralReasonPrecedence verifies that directionless topology wins over later static failures. +func TestLoweringPlanShortestExecutorV3PreservesStructuralReasonPrecedence(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf|Contains*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, ShortestPathFallbackDirectionless, decision.FallbackReason) +} + +// TestLoweringPlanShortestExecutorRejectsUnsupportedMinimumDepth verifies rejection of a minimum depth greater than one. +func TestLoweringPlanShortestExecutorRejectsUnsupportedMinimumDepth(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*2..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, int64(2), decision.MinimumDepth) + require.Equal(t, int64(4), decision.MaximumDepth) + require.Equal(t, ShortestPathFallbackUnsupportedDepth, decision.FallbackReason) +} + +// TestLoweringPlanShortestExecutorRetainsZeroMaximumDepthInDiagnostics verifies that an explicit zero maximum is not omitted from JSON. +func TestLoweringPlanShortestExecutorRetainsZeroMaximumDepthInDiagnostics(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*0..0]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.True(t, decision.StructurallyEligible) + require.Zero(t, decision.MinimumDepth) + require.Zero(t, decision.MaximumDepth) + + diagnostic, err := json.Marshal(decision) + require.NoError(t, err) + require.Contains(t, string(diagnostic), `"maximum_depth":0`) +} + +// TestLoweringPlanShortestExecutorUsesStatementWideCallCount verifies that multiple path calls across query parts disqualify static execution. +func TestLoweringPlanShortestExecutorUsesStatementWideCallCount(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p + MATCH q = shortestPath((x)-[:MemberOf*1..4]->(y)) + WHERE id(x) = $other_start_id AND id(y) = $other_end_id + RETURN length(p), length(q) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 2) + for _, decision := range plan.LoweringPlan.ShortestPathExecutor { + require.False(t, decision.StructurallyEligible) + require.Equal(t, ShortestPathFallbackMultiplePathCalls, decision.FallbackReason) + } +} + +// TestLoweringPlanShortestExecutorUsesStatementWideReadOnlyFact verifies that a later mutation disqualifies an earlier shortest-path candidate. +func TestLoweringPlanShortestExecutorUsesStatementWideReadOnlyFact(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p + CREATE (:Group {name: 'updated'}) + RETURN length(p) + `) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, ShortestPathFallbackMutation, decision.FallbackReason) +} + +// TestLoweringPlanShortestExecutorObservationModeRequiresPathForNodes verifies that nodes(path) requires a path witness. +func TestLoweringPlanShortestExecutorObservationModeRequiresPathForNodes(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + RETURN nodes(p) + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Equal(t, ShortestPathObservationOnePath, plan.LoweringPlan.ShortestPathExecutor[0].ObservationMode) +} + +// TestLoweringPlanShortestExecutorRequiresKnownObservationMode verifies that an unbound path result prevents static executor selection. +func TestLoweringPlanShortestExecutorRequiresKnownObservationMode(t *testing.T) { + t.Parallel() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN s + `) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, ShortestPathObservationUnknown, decision.ObservationMode) + require.False(t, decision.StructurallyEligible) +} + +// TestLoweringPlanShortestExecutorRejectsAdditionalRowSources verifies fallback classification for correlated or ambiguous endpoint sources. +func TestLoweringPlanShortestExecutorRejectsAdditionalRowSources(t *testing.T) { + t.Parallel() + tests := []struct { + // name labels the additional-row-source case. + name string + // query produces the shortest-path candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "unwind source", + query: ` + UNWIND [1, 2] AS source + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `, + reason: ShortestPathFallbackCorrelatedEndpoints, + }, + { + name: "additional match pattern", + query: ` + MATCH (source), p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `, + reason: ShortestPathFallbackMultipleEndpointPairs, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), test.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ShortestPathExecutor, 1) + decision := plan.LoweringPlan.ShortestPathExecutor[0] + require.False(t, decision.StructurallyEligible) + require.Equal(t, test.reason, decision.FallbackReason) + }) + } +} + +// TestLoweringPlanRecordsStableShortestExecutorFallbackCodes verifies diagnostic codes for unsupported shortest-path shapes. +func TestLoweringPlanRecordsStableShortestExecutorFallbackCodes(t *testing.T) { + t.Parallel() + tests := []struct { + // name labels the unsupported shortest-path shape. + name string + // query produces the shortest-path candidate under test. + query string + // reason is the expected stable fallback code. + reason string + }{ + { + name: "all shortest", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) RETURN p`, + reason: ShortestPathFallbackAllShortestPaths, + }, + { + name: "directionless", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]-(e)) RETURN p`, + reason: ShortestPathFallbackDirectionless, + }, + { + name: "relationship variable", + query: `MATCH p = shortestPath((s)-[r:MemberOf*1..4]->(e)) RETURN p`, + reason: ShortestPathFallbackRelationshipVariable, + }, + { + name: "non singleton", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) RETURN p`, + reason: ShortestPathFallbackNonSingletonID, + }, + { + name: "multiple id equalities", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = 1 AND id(s) = 2 AND id(e) = 3 RETURN p`, + reason: ShortestPathFallbackMultipleIDEqualities, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), test.query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.NotEmpty(t, plan.LoweringPlan.ShortestPathExecutor) + require.Equal(t, test.reason, plan.LoweringPlan.ShortestPathExecutor[0].FallbackReason) + }) + } +} + +// TestLoweringPlanReportsShortestPathStrategyForBoundEndpointPairs verifies lowering plan reports shortest path strategy for bound endpoint pairs behavior. func TestLoweringPlanReportsShortestPathStrategyForBoundEndpointPairs(t *testing.T) { t.Parallel() @@ -1521,6 +2675,7 @@ func TestLoweringPlanReportsShortestPathStrategyForBoundEndpointPairs(t *testing }}, plan.LoweringPlan.ShortestPathStrategy) } +// TestLoweringPlanSkipsShortestPathStrategyForLabelOnlyEndpoints verifies lowering plan skips shortest path strategy for label only endpoints behavior. func TestLoweringPlanSkipsShortestPathStrategyForLabelOnlyEndpoints(t *testing.T) { t.Parallel() @@ -1535,6 +2690,7 @@ func TestLoweringPlanSkipsShortestPathStrategyForLabelOnlyEndpoints(t *testing.T require.Empty(t, plan.LoweringPlan.ShortestPathStrategy) } +// TestLoweringPlanReportsShortestPathTerminalFilter verifies lowering plan reports shortest path terminal filter behavior. func TestLoweringPlanReportsShortestPathTerminalFilter(t *testing.T) { t.Parallel() @@ -1561,6 +2717,7 @@ func TestLoweringPlanReportsShortestPathTerminalFilter(t *testing.T) { }}, plan.LoweringPlan.ShortestPathFilter) } +// TestLoweringPlanReportsShortestPathTerminalFilterForKindOnlyTerminal verifies lowering plan reports shortest path terminal filter for kind only terminal behavior. func TestLoweringPlanReportsShortestPathTerminalFilterForKindOnlyTerminal(t *testing.T) { t.Parallel() @@ -1588,6 +2745,7 @@ func TestLoweringPlanReportsShortestPathTerminalFilterForKindOnlyTerminal(t *tes }}, plan.LoweringPlan.ShortestPathFilter) } +// TestLoweringPlanReportsTraversalLimitPushdown verifies lowering plan reports traversal limit pushdown behavior. func TestLoweringPlanReportsTraversalLimitPushdown(t *testing.T) { t.Parallel() @@ -1612,6 +2770,7 @@ func TestLoweringPlanReportsTraversalLimitPushdown(t *testing.T) { }}, plan.LoweringPlan.LimitPushdown) } +// TestLoweringPlanReportsShortestPathLimitPushdown verifies lowering plan reports shortest path limit pushdown behavior. func TestLoweringPlanReportsShortestPathLimitPushdown(t *testing.T) { t.Parallel() @@ -1637,6 +2796,7 @@ func TestLoweringPlanReportsShortestPathLimitPushdown(t *testing.T) { }) } +// TestLoweringPlanSkipsAllShortestPathLimitPushdown verifies lowering plan skips all shortest path limit pushdown behavior. func TestLoweringPlanSkipsAllShortestPathLimitPushdown(t *testing.T) { t.Parallel() @@ -1653,6 +2813,7 @@ func TestLoweringPlanSkipsAllShortestPathLimitPushdown(t *testing.T) { require.Empty(t, plan.LoweringPlan.LimitPushdown) } +// TestLoweringPlanSkipsOptionalMatchLimitPushdown verifies lowering plan skips optional match limit pushdown behavior. func TestLoweringPlanSkipsOptionalMatchLimitPushdown(t *testing.T) { t.Parallel() @@ -1670,6 +2831,7 @@ func TestLoweringPlanSkipsOptionalMatchLimitPushdown(t *testing.T) { require.Empty(t, plan.LoweringPlan.LimitPushdown) } +// TestDeclareReadingClauseSelectivitySkipsOptionalMatch verifies declare reading clause selectivity skips optional match behavior. func TestDeclareReadingClauseSelectivitySkipsOptionalMatch(t *testing.T) { t.Parallel() @@ -1687,6 +2849,7 @@ func TestDeclareReadingClauseSelectivitySkipsOptionalMatch(t *testing.T) { require.NotContains(t, selectivity, "m") } +// TestSelectReferencesOnlyLocalIdentifiersValidatesJoinConstraintsIncrementally verifies select references only local identifiers validates join constraints incrementally behavior. func TestSelectReferencesOnlyLocalIdentifiersValidatesJoinConstraintsIncrementally(t *testing.T) { t.Parallel() @@ -1722,6 +2885,7 @@ func TestSelectReferencesOnlyLocalIdentifiersValidatesJoinConstraintsIncremental require.False(t, SelectReferencesOnlyLocalIdentifiers(selectBody, pgsql.NewIdentifierSet())) } +// TestFlattenConjunctionHandlesValueBinaryExpressions verifies flatten conjunction handles value binary expressions behavior. func TestFlattenConjunctionHandlesValueBinaryExpressions(t *testing.T) { t.Parallel() @@ -1742,6 +2906,7 @@ func TestFlattenConjunctionHandlesValueBinaryExpressions(t *testing.T) { require.Equal(t, right, terms[1]) } +// TestQueryReferencesOnlyLocalIdentifiersAllowsEmptyWith verifies query references only local identifiers allows empty with behavior. func TestQueryReferencesOnlyLocalIdentifiersAllowsEmptyWith(t *testing.T) { t.Parallel() @@ -1763,6 +2928,7 @@ func TestQueryReferencesOnlyLocalIdentifiersAllowsEmptyWith(t *testing.T) { require.True(t, QueryReferencesOnlyLocalIdentifiers(query, pgsql.NewIdentifierSet())) } +// TestFromExpressionReferencesOnlyLocalIdentifiersHandlesLateralSubquery verifies from expression references only local identifiers handles lateral subquery behavior. func TestFromExpressionReferencesOnlyLocalIdentifiersHandlesLateralSubquery(t *testing.T) { t.Parallel() @@ -1780,6 +2946,7 @@ func TestFromExpressionReferencesOnlyLocalIdentifiersHandlesLateralSubquery(t *t require.False(t, FromExpressionReferencesOnlyLocalIdentifiers(lateralSubquery, pgsql.NewIdentifierSet())) } +// TestMeasureSelectivityPopReturnsTopFrame verifies measure selectivity pop returns top frame behavior. func TestMeasureSelectivityPopReturnsTopFrame(t *testing.T) { t.Parallel() @@ -1792,6 +2959,7 @@ func TestMeasureSelectivityPopReturnsTopFrame(t *testing.T) { require.Equal(t, 7, visitor.Selectivity()) } +// TestMeasureSelectivityScoresIDBonusOnlyForPointPredicates verifies measure selectivity scores id bonus only for point predicates behavior. func TestMeasureSelectivityScoresIDBonusOnlyForPointPredicates(t *testing.T) { t.Parallel() @@ -1817,6 +2985,7 @@ func TestMeasureSelectivityScoresIDBonusOnlyForPointPredicates(t *testing.T) { require.Equal(t, selectivityWeightNotEquals, notEqualScore) } +// TestCollectReferencedSourceIdentifiersIgnoresMatchDeclarations verifies collect referenced source identifiers ignores match declarations behavior. func TestCollectReferencedSourceIdentifiersIgnoresMatchDeclarations(t *testing.T) { t.Parallel() @@ -1833,6 +3002,7 @@ func TestCollectReferencedSourceIdentifiersIgnoresMatchDeclarations(t *testing.T require.Contains(t, references, "m") } +// TestLoweringPlanSkipsDirectionlessExpansionSuffixPushdown verifies lowering plan skips directionless expansion suffix pushdown behavior. func TestLoweringPlanSkipsDirectionlessExpansionSuffixPushdown(t *testing.T) { t.Parallel() @@ -1847,10 +3017,11 @@ func TestLoweringPlanSkipsDirectionlessExpansionSuffixPushdown(t *testing.T) { require.Empty(t, plan.LoweringPlan.ExpansionSuffixPushdown) } +// TestPredicateAttachmentRuleAssignsSingleBindingPredicates verifies that single-symbol predicates attach to their binding scopes. func TestPredicateAttachmentRuleAssignsSingleBindingPredicates(t *testing.T) { t.Parallel() - regularQuery, err := frontend.ParseCypher(frontend.NewContext(), adcsQuery) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fixedSuffixExpansionQuery) require.NoError(t, err) plan, err := Optimize(regularQuery) @@ -1863,8 +3034,8 @@ func TestPredicateAttachmentRuleAssignsSingleBindingPredicates(t *testing.T) { ClauseIndex: 0, ExpressionIndex: 0, Scope: PredicateAttachmentScopeBinding, - BindingSymbols: []string{"n"}, - Dependencies: []string{"n"}, + BindingSymbols: []string{"root"}, + Dependencies: []string{"root"}, }, plan.PredicateAttachments[0]) require.Equal(t, PredicateAttachment{ @@ -1873,11 +3044,12 @@ func TestPredicateAttachmentRuleAssignsSingleBindingPredicates(t *testing.T) { ClauseIndex: 2, ExpressionIndex: 0, Scope: PredicateAttachmentScopeBinding, - BindingSymbols: []string{"ct"}, - Dependencies: []string{"ct"}, + BindingSymbols: []string{"predicate"}, + Dependencies: []string{"predicate"}, }, plan.PredicateAttachments[1]) } +// TestPredicateAttachmentRuleKeepsMultiBindingPredicatesAtRegionScope verifies predicate attachment rule keeps multi binding predicates at region scope behavior. func TestPredicateAttachmentRuleKeepsMultiBindingPredicatesAtRegionScope(t *testing.T) { t.Parallel() @@ -1903,6 +3075,7 @@ func TestPredicateAttachmentRuleKeepsMultiBindingPredicatesAtRegionScope(t *test }, plan.PredicateAttachments[0]) } +// firstNodeSymbol returns the first node variable encountered during a structural query walk. func firstNodeSymbol(readingClause *cypher.ReadingClause) string { if readingClause == nil || readingClause.Match == nil || len(readingClause.Match.Pattern) == 0 { return "" @@ -1916,6 +3089,7 @@ func firstNodeSymbol(readingClause *cypher.ReadingClause) string { return nodePattern.Variable.Symbol } +// TestConservativePatternReorderingMovesIndependentNodeAnchorsEarlier verifies conservative pattern reordering moves independent node anchors earlier behavior. func TestConservativePatternReorderingMovesIndependentNodeAnchorsEarlier(t *testing.T) { t.Parallel() @@ -1950,6 +3124,7 @@ func TestConservativePatternReorderingMovesIndependentNodeAnchorsEarlier(t *test require.Len(t, readingClauses[2].Match.Pattern[0].PatternElements, 3) } +// TestConservativePatternReorderingKeepsDependentAnchorsInPlace verifies conservative pattern reordering keeps dependent anchors in place behavior. func TestConservativePatternReorderingKeepsDependentAnchorsInPlace(t *testing.T) { t.Parallel() @@ -1983,6 +3158,7 @@ func TestConservativePatternReorderingKeepsDependentAnchorsInPlace(t *testing.T) require.Equal(t, "b", firstNodeSymbol(readingClauses[1])) } +// TestConservativePatternReorderingUsesSelectivityWithinDependencySafeRegion verifies conservative pattern reordering uses selectivity within dependency safe region behavior. func TestConservativePatternReorderingUsesSelectivityWithinDependencySafeRegion(t *testing.T) { t.Parallel() @@ -2017,6 +3193,7 @@ func TestConservativePatternReorderingUsesSelectivityWithinDependencySafeRegion( require.Len(t, readingClauses[2].Match.Pattern[0].PatternElements, 3) } +// TestConservativePatternReorderingPinsUnresolvedExternalDependencies verifies conservative pattern reordering pins unresolved external dependencies behavior. func TestConservativePatternReorderingPinsUnresolvedExternalDependencies(t *testing.T) { t.Parallel() diff --git a/cypher/models/pgsql/optimize/scalar_continuation_test.go b/cypher/models/pgsql/optimize/scalar_continuation_test.go new file mode 100644 index 00000000..8d1ce3fc --- /dev/null +++ b/cypher/models/pgsql/optimize/scalar_continuation_test.go @@ -0,0 +1,59 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// +// SPDX-License-Identifier: Apache-2.0 + +package optimize + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// fieldRequirementForSymbol optimizes cypherQuery and returns the field-requirement decision for symbol. +func fieldRequirementForSymbol(t *testing.T, cypherQuery, symbol string) FieldRequirementDecision { + t.Helper() + + query, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + require.NoError(t, err) + + plan, err := Optimize(query) + require.NoError(t, err) + + for _, decision := range plan.LoweringPlan.FieldRequirements { + if decision.Symbol == symbol { + return decision + } + } + + require.FailNow(t, "field requirement decision not found", symbol) + return FieldRequirementDecision{} +} + +// TestScalarContinuationFieldRequirementAllowsIDOnlyObservation verifies that an ID consumer permits scalar continuation state. +func TestScalarContinuationFieldRequirementAllowsIDOnlyObservation(t *testing.T) { + t.Parallel() + + decision := fieldRequirementForSymbol(t, + `MATCH (s)-[*1..]->(mid)-[]->(e) RETURN id(mid), id(e)`, + "mid", + ) + + require.Contains(t, decision.Fields, FieldRequirementEntityID) + require.NotContains(t, decision.Fields, FieldRequirementFullEntity) +} + +// TestScalarContinuationFieldRequirementRetainsFullEntityForMutation verifies that mutation prevents scalar-only continuation state. +func TestScalarContinuationFieldRequirementRetainsFullEntityForMutation(t *testing.T) { + t.Parallel() + + decision := fieldRequirementForSymbol(t, + `MATCH (s)-[*1..]->(mid)-[]->(e) DELETE mid`, + "mid", + ) + + require.Contains(t, decision.Fields, FieldRequirementFullEntity) +} diff --git a/cypher/models/pgsql/optimize/source_references.go b/cypher/models/pgsql/optimize/source_references.go index 01dde537..ca966cca 100644 --- a/cypher/models/pgsql/optimize/source_references.go +++ b/cypher/models/pgsql/optimize/source_references.go @@ -1,19 +1,256 @@ package optimize import ( + "sort" + "strings" + "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/walk" ) +// sourceReferenceCollector tracks referenced identifiers and repeated pattern declarations during syntax walking. type sourceReferenceCollector struct { + // VisitorHandler supplies cancellation and error propagation for the syntax walk. walk.VisitorHandler - referencedIdentifiers map[string]struct{} - matchPatternDeclarationRefs map[string]int - matchPatternDeclarations map[*cypher.PatternPart]struct{} + // referencedIdentifiers contains bindings consumed outside their defining pattern declarations. + referencedIdentifiers map[string]struct{} + // matchPatternDeclarationRefs counts match-pattern declarations by binding symbol. + matchPatternDeclarationRefs map[string]int + // matchPatternDeclarations identifies pattern parts whose variables are declarations rather than reads. + matchPatternDeclarations map[*cypher.PatternPart]struct{} + // matchPatternDeclarationDepth tracks nesting beneath the declaration currently being visited. matchPatternDeclarationDepth int } +// fieldRequirementCollector accumulates ordered representation requirements for each Cypher binding. +type fieldRequirementCollector struct { + // VisitorHandler supplies cancellation and error propagation for the syntax walk. + walk.VisitorHandler + + // queryPartIndex identifies the query part whose binding uses are being collected. + queryPartIndex int + // ordinal orders binding uses in traversal order. + ordinal int + // patternDepth tracks whether the visitor is currently inside a pattern declaration. + patternDepth int + // propertyDepth tracks nested property lookups so their base binding is classified once. + propertyDepth int + // functionStack identifies the function consuming a visited expression. + functionStack []*cypher.FunctionInvocation + // bindingKinds maps each symbol to its path, relationship, or node representation. + bindingKinds map[string]string + // patternUses counts pattern occurrences of each binding. + patternUses map[string]int + // decisions accumulates representation requirements by binding symbol. + decisions map[string]*FieldRequirementDecision +} + +// newFieldRequirementCollector initializes requirement tracking for one query part. +func newFieldRequirementCollector(queryPartIndex int) *fieldRequirementCollector { + return &fieldRequirementCollector{ + VisitorHandler: walk.NewCancelableErrorHandler(), + queryPartIndex: queryPartIndex, + bindingKinds: map[string]string{}, + patternUses: map[string]int{}, + decisions: map[string]*FieldRequirementDecision{}, + } +} + +// add records one ordered use and merges its required fields into the binding decision. +func (s *fieldRequirementCollector) add(symbol string, internal bool, fields ...FieldRequirement) { + if symbol == "" { + return + } + + s.ordinal++ + decision, found := s.decisions[symbol] + if !found { + decision = &FieldRequirementDecision{ + QueryPartIndex: s.queryPartIndex, + Symbol: symbol, + } + s.decisions[symbol] = decision + } + + useFields := append([]FieldRequirement(nil), fields...) + decision.Uses = append(decision.Uses, FieldRequirementUse{ + Ordinal: s.ordinal, + Fields: useFields, + Internal: internal, + }) + decision.LastUse = s.ordinal + + present := make(map[FieldRequirement]struct{}, len(decision.Fields)) + for _, field := range decision.Fields { + present[field] = struct{}{} + } + for _, field := range fields { + if _, found := present[field]; !found { + decision.Fields = append(decision.Fields, field) + present[field] = struct{}{} + } + } +} + +// patternVariableSymbol returns a pattern variable's symbol or an empty string when no variable is present. +func patternVariableSymbol(variable *cypher.Variable) string { + if variable == nil { + return "" + } + return variable.Symbol +} + +// addFullBinding records the complete representation required for a path, relationship, or node binding. +func (s *fieldRequirementCollector) addFullBinding(symbol, kind string) { + switch kind { + case "path": + s.add(symbol, false, FieldRequirementFullPath) + case "relationship": + s.add(symbol, false, FieldRequirementFullEntity, FieldRequirementRelationshipIDs) + default: + s.add(symbol, false, FieldRequirementFullEntity) + } +} + +// addGreedyProjectionBindings marks every visible binding for full materialization in deterministic symbol order. +func (s *fieldRequirementCollector) addGreedyProjectionBindings() { + symbols := make([]string, 0, len(s.bindingKinds)) + for symbol := range s.bindingKinds { + symbols = append(symbols, symbol) + } + sort.Strings(symbols) + + for _, symbol := range symbols { + s.addFullBinding(symbol, s.bindingKinds[symbol]) + } +} + +// Enter records representation requirements before visiting a syntax node's children. +func (s *fieldRequirementCollector) Enter(node cypher.SyntaxNode) { + switch typedNode := node.(type) { + case *cypher.PatternPart: + s.patternDepth++ + if symbol := patternVariableSymbol(typedNode.Variable); symbol != "" { + s.bindingKinds[symbol] = "path" + s.add(symbol, true, FieldRequirementOrderedPathEdgeIDs) + } + + case *cypher.NodePattern: + if symbol := patternVariableSymbol(typedNode.Variable); symbol != "" { + s.bindingKinds[symbol] = "node" + s.patternUses[symbol]++ + if s.patternUses[symbol] > 1 { + // Reused pattern bindings are consumed by bound-endpoint joins. + // Those joins still expect the entity representation; scalar-ID + // rehydration is a separate lowering capability. + s.add(symbol, true, FieldRequirementFullEntity) + } + if len(typedNode.Kinds) > 0 { + s.add(symbol, true, FieldRequirementEntityID, FieldRequirementKinds) + } + if typedNode.Properties != nil { + s.add(symbol, true, FieldRequirementEntityID, FieldRequirementProperties) + } + } + + case *cypher.RelationshipPattern: + if symbol := patternVariableSymbol(typedNode.Variable); symbol != "" { + s.bindingKinds[symbol] = "relationship" + s.patternUses[symbol]++ + if s.patternUses[symbol] > 1 { + s.add(symbol, true, FieldRequirementFullEntity) + } + s.add(symbol, true, FieldRequirementRelationshipIDs) + if len(typedNode.Kinds) > 0 { + s.add(symbol, true, FieldRequirementKinds) + } + if typedNode.Properties != nil { + s.add(symbol, true, FieldRequirementProperties) + } + } + + case *cypher.PropertyLookup: + s.propertyDepth++ + + case *cypher.FunctionInvocation: + s.functionStack = append(s.functionStack, typedNode) + + case *cypher.Variable: + if s.patternDepth > 0 { + return + } + if typedNode.Symbol == cypher.TokenLiteralAsterisk { + s.addGreedyProjectionBindings() + return + } + + if s.propertyDepth > 0 { + s.add(typedNode.Symbol, false, FieldRequirementEntityID, FieldRequirementProperties) + return + } + + if len(s.functionStack) > 0 { + switch strings.ToLower(s.functionStack[len(s.functionStack)-1].Name) { + case cypher.IdentityFunction: + s.add(typedNode.Symbol, false, FieldRequirementEntityID) + return + case cypher.NodeLabelsFunction, cypher.EdgeTypeFunction: + s.add(typedNode.Symbol, false, FieldRequirementKinds) + return + case cypher.PathLengthFunction: + s.add(typedNode.Symbol, false, FieldRequirementOrderedPathEdgeIDs) + return + case cypher.NodesFunction, cypher.RelationshipsFunction: + s.add(typedNode.Symbol, false, FieldRequirementFullPath) + return + } + } + + s.addFullBinding(typedNode.Symbol, s.bindingKinds[typedNode.Symbol]) + } +} + +// Visit performs no leaf-specific work because Enter classifies every relevant node. +func (s *fieldRequirementCollector) Visit(cypher.SyntaxNode) {} + +// Exit unwinds pattern, property, and function nesting after visiting a syntax node's children. +func (s *fieldRequirementCollector) Exit(node cypher.SyntaxNode) { + switch node.(type) { + case *cypher.PatternPart: + s.patternDepth-- + case *cypher.PropertyLookup: + s.propertyDepth-- + case *cypher.FunctionInvocation: + s.functionStack = s.functionStack[:len(s.functionStack)-1] + } +} + +// collectFieldRequirements walks root and returns normalized representation needs for its bindings. +func collectFieldRequirements(queryPartIndex int, root cypher.SyntaxNode) ([]FieldRequirementDecision, error) { + if root == nil { + return nil, nil + } + + collector := newFieldRequirementCollector(queryPartIndex) + if err := walk.Cypher(root, collector); err != nil { + return nil, err + } + + symbols := make([]string, 0, len(collector.decisions)) + for symbol := range collector.decisions { + symbols = append(symbols, symbol) + } + sort.Strings(symbols) + + decisions := make([]FieldRequirementDecision, 0, len(symbols)) + for _, symbol := range symbols { + decisions = append(decisions, *collector.decisions[symbol]) + } + return decisions, nil +} + +// newSourceReferenceCollector initializes empty reference and match-declaration tracking for a syntax walk. func newSourceReferenceCollector() *sourceReferenceCollector { return &sourceReferenceCollector{ VisitorHandler: walk.NewCancelableErrorHandler(), @@ -23,18 +260,21 @@ func newSourceReferenceCollector() *sourceReferenceCollector { } } +// addVariable records a referenced variable unless it is part of the declaration currently being traversed. func (s *sourceReferenceCollector) addVariable(variable *cypher.Variable) { if variable != nil && variable.Symbol != "" { s.referencedIdentifiers[variable.Symbol] = struct{}{} } } +// addMatchPatternDeclaration counts a non-empty variable declared inside a pattern expression so repeated declarations can be retained as references. func (s *sourceReferenceCollector) addMatchPatternDeclaration(variable *cypher.Variable) { if variable != nil && variable.Symbol != "" { s.matchPatternDeclarationRefs[variable.Symbol] += 1 } } +// collectRepeatedMatchPatternDeclarations marks multiply declared match symbols as source references. func (s *sourceReferenceCollector) collectRepeatedMatchPatternDeclarations() { for identifier, numDeclarations := range s.matchPatternDeclarationRefs { if numDeclarations > 1 { @@ -43,6 +283,7 @@ func (s *sourceReferenceCollector) collectRepeatedMatchPatternDeclarations() { } } +// isMatchPatternDeclaration reports whether node is a variable declaration belonging to a match pattern. func (s *sourceReferenceCollector) isMatchPatternDeclaration(patternPart *cypher.PatternPart) bool { _, isDeclaration := s.matchPatternDeclarations[patternPart] return isDeclaration @@ -97,6 +338,7 @@ func (s *sourceReferenceCollector) Exit(node cypher.SyntaxNode) { } } +// collectReferencedSourceIdentifiers returns identifiers used outside a declaring match pattern or declared repeatedly within one. func collectReferencedSourceIdentifiers(root cypher.SyntaxNode) (map[string]struct{}, error) { if root == nil { return map[string]struct{}{}, nil @@ -104,13 +346,14 @@ func collectReferencedSourceIdentifiers(root cypher.SyntaxNode) (map[string]stru collector := newSourceReferenceCollector() if err := walk.Cypher(root, collector); err != nil { - return collector.referencedIdentifiers, err + return nil, err } collector.collectRepeatedMatchPatternDeclarations() return collector.referencedIdentifiers, nil } +// referencesSourceIdentifier reports whether references contains symbol or the wildcard source marker. func referencesSourceIdentifier(references map[string]struct{}, symbol string) bool { if _, referencesAll := references[cypher.TokenLiteralAsterisk]; referencesAll { return true diff --git a/cypher/models/pgsql/optimize/sp_i2_cap_contract_test.go b/cypher/models/pgsql/optimize/sp_i2_cap_contract_test.go new file mode 100644 index 00000000..657fd184 --- /dev/null +++ b/cypher/models/pgsql/optimize/sp_i2_cap_contract_test.go @@ -0,0 +1,19 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package optimize + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestShortestPathI2QualifiedCapsFreezeProductionContract freezes the shared +// planner and production-authorization cap values. +func TestShortestPathI2QualifiedCapsFreezeProductionContract(t *testing.T) { + require.Equal(t, int64(100_000), ShortestPathI2QualifiedStateLimit) + require.Equal(t, int64(100_000), ShortestPathI2QualifiedFrontierLimit) + require.Equal(t, ShortestPathI2QualifiedStateLimit, defaultShortestPathStateLimit) + require.Equal(t, ShortestPathI2QualifiedFrontierLimit, defaultShortestPathFrontierLimit) +} diff --git a/cypher/models/pgsql/optimize/suffix_reverse_guard_test.go b/cypher/models/pgsql/optimize/suffix_reverse_guard_test.go new file mode 100644 index 00000000..1d1312d0 --- /dev/null +++ b/cypher/models/pgsql/optimize/suffix_reverse_guard_test.go @@ -0,0 +1,60 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package optimize + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/specterops/dawgs/cypher/frontend" +) + +const suffixReverseGuardPathQuery = ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path +` + +func optimizeSuffixReverseGuardQuery(t *testing.T, query string) ExpansionSearchStrategyDecision { + t.Helper() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + plan, err := Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + return plan.LoweringPlan.ExpansionSearchStrategy[0] +} + +// TestFixedSuffixObservationDistinguishesFullPathFromEndpoint verifies the +// static fact consumed by suffix-reverse-guard-v1 is available before tooling +// overrides are applied. +func TestFixedSuffixObservationDistinguishesFullPathFromEndpoint(t *testing.T) { + fullPath := optimizeSuffixReverseGuardQuery(t, suffixReverseGuardPathQuery) + require.True(t, fullPath.StructurallyEligible) + require.True(t, fullPath.StaticallyEligible) + require.Equal(t, ExpansionSearchObservationFullPath, fullPath.ObservationMode) + + endpoint := optimizeSuffixReverseGuardQuery(t, ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(terminal) + `) + require.True(t, endpoint.StructurallyEligible) + require.True(t, endpoint.StaticallyEligible) + require.Equal(t, ExpansionSearchObservationEndpointIDs, endpoint.ObservationMode) +} + +// TestSuffixReverseGuardConstantsAreIndependent verifies the feasibility lane +// cannot silently inherit orientation-v2's policy or state limit identity. +func TestSuffixReverseGuardConstantsAreIndependent(t *testing.T) { + require.Equal(t, ExpansionSearchPolicy("suffix-reverse-guard-v1"), ExpansionSearchPolicySuffixReverseGuardV1) + require.Equal(t, "fixed-suffix-path-static-v1", ExpansionSearchSelectorFixedSuffixPathV1) + require.Positive(t, ExpansionSearchSuffixReverseGuardSuffixRowLimit) + require.Positive(t, ExpansionSearchSuffixReverseGuardStateLimit) + require.NotEqual(t, ExpansionSearchPolicyOrientationProbeV2, ExpansionSearchPolicySuffixReverseGuardV1) + require.NotEqual(t, ExpansionSearchOrientationStateLimit, ExpansionSearchSuffixReverseGuardStateLimit) +} diff --git a/cypher/models/pgsql/optimize/traversal_envelope.go b/cypher/models/pgsql/optimize/traversal_envelope.go new file mode 100644 index 00000000..098d2a26 --- /dev/null +++ b/cypher/models/pgsql/optimize/traversal_envelope.go @@ -0,0 +1,264 @@ +package optimize + +// Endpoint-resolution limits are immutable analysis metadata for the first +// bounded-resolution envelope. Each runtime limit has an explicit cap+1 +// sentinel; this slice records the contract without changing execution. +const ( + // EndpointResolutionSingletonLimit reserves the stable protocol value used to recognize endpoint resolution singleton limit across artifacts and executions. + EndpointResolutionSingletonLimit int64 = 1 + + // EndpointResolutionSingletonSentinel reserves the stable protocol value used to recognize endpoint resolution singleton sentinel across artifacts and executions. + EndpointResolutionSingletonSentinel int64 = 2 + + // EndpointResolutionSmallSetLimit reserves the stable protocol value used to recognize endpoint resolution small set limit across artifacts and executions. + EndpointResolutionSmallSetLimit int64 = 32 + + // EndpointResolutionSmallSetSentinel reserves the stable protocol value used to recognize endpoint resolution small set sentinel across artifacts and executions. + EndpointResolutionSmallSetSentinel int64 = 33 +) + +// EndpointResolutionClass identifies how one traversal endpoint, or a +// correlated endpoint pair, could be resolved before traversal. +type EndpointResolutionClass string + +const ( + // EndpointResolutionClassIDEquality reserves the stable protocol value used to recognize endpoint resolution class id equality across artifacts and executions. + EndpointResolutionClassIDEquality EndpointResolutionClass = "id_equality" + + // EndpointResolutionClassUniquePropertyEquality reserves the stable protocol value used to recognize endpoint resolution class unique property equality across artifacts and executions. + EndpointResolutionClassUniquePropertyEquality EndpointResolutionClass = "unique_property_equality" + + // EndpointResolutionClassNonUniquePropertyEquality reserves the stable protocol value used to recognize endpoint resolution class non unique property equality across artifacts and executions. + EndpointResolutionClassNonUniquePropertyEquality EndpointResolutionClass = "nonunique_property_equality" + + // EndpointResolutionClassExplicitSmallSet reserves the stable protocol value used to recognize endpoint resolution class explicit small set across artifacts and executions. + EndpointResolutionClassExplicitSmallSet EndpointResolutionClass = "explicit_small_set" + + // EndpointResolutionClassCorrelatedPair reserves the stable protocol value used to recognize endpoint resolution class correlated pair across artifacts and executions. + EndpointResolutionClassCorrelatedPair EndpointResolutionClass = "correlated_pair" + + // EndpointResolutionClassUnsupported reserves the stable protocol value used to recognize endpoint resolution class unsupported across artifacts and executions. + EndpointResolutionClassUnsupported EndpointResolutionClass = "unsupported" +) + +// EndpointResolutionPlan identifies the exact incumbent and the planned-only +// bounded resolver independently of any shortest-path executor. +type EndpointResolutionPlan string + +const ( + // EndpointResolutionPlanIncumbent reserves the stable protocol value used to recognize endpoint resolution plan incumbent across artifacts and executions. + EndpointResolutionPlanIncumbent EndpointResolutionPlan = "ENDPOINT-RESOLUTION-INCUMBENT" + + // EndpointResolutionPlanBounded reserves the stable protocol value used to recognize endpoint resolution plan bounded across artifacts and executions. + EndpointResolutionPlanBounded EndpointResolutionPlan = "ENDPOINT-RESOLUTION-BOUNDED" +) + +const ( + // EndpointResolutionFallbackPlannedOnly reserves the stable protocol value used to recognize endpoint resolution fallback planned only across artifacts and executions. + EndpointResolutionFallbackPlannedOnly = "planned_only" + + // EndpointResolutionFallbackMutation reserves the stable protocol value used to recognize endpoint resolution fallback mutation across artifacts and executions. + EndpointResolutionFallbackMutation = "mutation" + + // EndpointResolutionFallbackOptionalMatch reserves the stable protocol value used to recognize endpoint resolution fallback optional match across artifacts and executions. + EndpointResolutionFallbackOptionalMatch = "optional_match" + + // EndpointResolutionFallbackCorrelatedPair reserves the stable protocol value used to recognize endpoint resolution fallback correlated pair across artifacts and executions. + EndpointResolutionFallbackCorrelatedPair = "correlated_pair" + + // EndpointResolutionFallbackUnsupported reserves the stable protocol value used to recognize endpoint resolution fallback unsupported across artifacts and executions. + EndpointResolutionFallbackUnsupported = "unsupported_endpoint_class" + + // EndpointResolutionFallbackSmallSetOverflow reserves the stable protocol value used to recognize endpoint resolution fallback small set overflow across artifacts and executions. + EndpointResolutionFallbackSmallSetOverflow = "explicit_small_set_overflow" +) + +// EndpointResolutionCaps serializes both admitted cardinalities and their +// overflow sentinels so future SQL cannot silently reinterpret the contract. +type EndpointResolutionCaps struct { + // SingletonLimit supplies the singleton limit input to the EndpointResolutionCaps contract. + SingletonLimit int64 `json:"singleton_limit"` + // SingletonSentinel supplies the singleton sentinel input to the EndpointResolutionCaps contract. + SingletonSentinel int64 `json:"singleton_sentinel"` + // SmallSetLimit supplies the small set limit input to the EndpointResolutionCaps contract. + SmallSetLimit int64 `json:"small_set_limit"` + // SmallSetSentinel supplies the small set sentinel input to the EndpointResolutionCaps contract. + SmallSetSentinel int64 `json:"small_set_sentinel"` +} + +// EndpointResolutionInput records one endpoint's statically recognizable +// resolution shape. Cardinality remains runtime evidence. +type EndpointResolutionInput struct { + // Symbol supplies the symbol input to the EndpointResolutionInput contract. + Symbol string `json:"symbol"` + // Class supplies the class input to the EndpointResolutionInput contract. + Class EndpointResolutionClass `json:"class"` + // Property supplies the property input to the EndpointResolutionInput contract. + Property string `json:"property,omitempty"` + // StaticValueCount records the number of static value count. + StaticValueCount int `json:"static_value_count,omitempty"` + // ParameterizedSet indicates whether parameterized set applies. + ParameterizedSet bool `json:"parameterized_set,omitempty"` + // Limit supplies the limit input to the EndpointResolutionInput contract. + Limit int64 `json:"limit,omitempty"` + // Sentinel supplies the sentinel input to the EndpointResolutionInput contract. + Sentinel int64 `json:"sentinel,omitempty"` +} + +// EndpointResolutionEligibilityFact records one conservative qualification +// check for bounded endpoint materialization. +type EndpointResolutionEligibilityFact struct { + // Name identifies the name. + Name string `json:"name"` + // Eligible indicates whether eligible applies. + Eligible bool `json:"eligible"` +} + +// EndpointResolutionDecision is analysis-only metadata for one SP/ASP +// traversal. The exact existing resolver remains selected in this milestone. +type EndpointResolutionDecision struct { + // Target supplies the target input to the EndpointResolutionDecision contract. + Target TraversalStepTarget `json:"target"` + // Family supplies the family input to the EndpointResolutionDecision contract. + Family string `json:"family"` + // Root supplies the root input to the EndpointResolutionDecision contract. + Root EndpointResolutionInput `json:"root"` + // Terminal supplies the terminal input to the EndpointResolutionDecision contract. + Terminal EndpointResolutionInput `json:"terminal"` + // PairClass supplies the pair class input to the EndpointResolutionDecision contract. + PairClass EndpointResolutionClass `json:"pair_class,omitempty"` + // PlannedClasses supplies the planned classes input to the EndpointResolutionDecision contract. + PlannedClasses []EndpointResolutionClass `json:"planned_classes"` + // Caps binds each guarded resource dimension to its enforced limit. + Caps EndpointResolutionCaps `json:"caps"` + // PlannedCandidates supplies the planned candidates input to the EndpointResolutionDecision contract. + PlannedCandidates []EndpointResolutionPlan `json:"planned_candidates"` + // CandidatePlan supplies the candidate plan input to the EndpointResolutionDecision contract. + CandidatePlan EndpointResolutionPlan `json:"candidate_plan"` + // SelectedPlan supplies the selected plan input to the EndpointResolutionDecision contract. + SelectedPlan EndpointResolutionPlan `json:"selected_plan"` + // FallbackPlan supplies the fallback plan input to the EndpointResolutionDecision contract. + FallbackPlan EndpointResolutionPlan `json:"fallback_plan"` + // EligibilityFacts supplies the eligibility facts input to the EndpointResolutionDecision contract. + EligibilityFacts []EndpointResolutionEligibilityFact `json:"eligibility_facts"` + // StructurallyEligible indicates whether structurally eligible applies. + StructurallyEligible bool `json:"structurally_eligible"` + // StaticallyEligible indicates whether statically eligible applies. + StaticallyEligible bool `json:"statically_eligible"` + // SelectionMode identifies the selection mode. + SelectionMode string `json:"selection_mode"` + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string `json:"selector_version"` + // FallbackReason supplies the fallback reason input to the EndpointResolutionDecision contract. + FallbackReason string `json:"fallback_reason"` +} + +// TraversalPredicateClass identifies the strongest safe placement property +// proven from syntax. Unsupported path forms deliberately remain conservative. +type TraversalPredicateClass string + +const ( + // TraversalPredicateClassStepLocalNode reserves the stable protocol value used to recognize traversal predicate class step local node across artifacts and executions. + TraversalPredicateClassStepLocalNode TraversalPredicateClass = "step_local_node" + + // TraversalPredicateClassStepLocalRelationship reserves the stable protocol value used to recognize traversal predicate class step local relationship across artifacts and executions. + TraversalPredicateClassStepLocalRelationship TraversalPredicateClass = "step_local_relationship" + + // TraversalPredicateClassUniversalAllNodes reserves the stable protocol value used to recognize traversal predicate class universal all nodes across artifacts and executions. + TraversalPredicateClassUniversalAllNodes TraversalPredicateClass = "universal_all_nodes" + + // TraversalPredicateClassUniversalNoneNodes reserves the stable protocol value used to recognize traversal predicate class universal none nodes across artifacts and executions. + TraversalPredicateClassUniversalNoneNodes TraversalPredicateClass = "universal_none_nodes" + + // TraversalPredicateClassUniversalAllRelationships reserves the stable protocol value used to recognize traversal predicate class universal all relationships across artifacts and executions. + TraversalPredicateClassUniversalAllRelationships TraversalPredicateClass = "universal_all_relationships" + + // TraversalPredicateClassUniversalNoneRelationships reserves the stable protocol value used to recognize traversal predicate class universal none relationships across artifacts and executions. + TraversalPredicateClassUniversalNoneRelationships TraversalPredicateClass = "universal_none_relationships" + + // TraversalPredicateClassWholePath reserves the stable protocol value used to recognize traversal predicate class whole path across artifacts and executions. + TraversalPredicateClassWholePath TraversalPredicateClass = "whole_path" + + // TraversalPredicateClassUnsupported reserves the stable protocol value used to recognize traversal predicate class unsupported across artifacts and executions. + TraversalPredicateClassUnsupported TraversalPredicateClass = "unsupported" +) + +// TraversalPredicatePlan separates planned step evaluation from the exact +// incumbent predicate placement that remains selected. +type TraversalPredicatePlan string + +const ( + // TraversalPredicatePlanIncumbent reserves the stable protocol value used to recognize traversal predicate plan incumbent across artifacts and executions. + TraversalPredicatePlanIncumbent TraversalPredicatePlan = "TRAVERSAL-PREDICATE-INCUMBENT" + + // TraversalPredicatePlanStep reserves the stable protocol value used to recognize traversal predicate plan step across artifacts and executions. + TraversalPredicatePlanStep TraversalPredicatePlan = "TRAVERSAL-PREDICATE-STEP" +) + +const ( + // TraversalPredicateFallbackPlannedOnly reserves the stable protocol value used to recognize traversal predicate fallback planned only across artifacts and executions. + TraversalPredicateFallbackPlannedOnly = "planned_only" + + // TraversalPredicateFallbackMutation reserves the stable protocol value used to recognize traversal predicate fallback mutation across artifacts and executions. + TraversalPredicateFallbackMutation = "mutation" + + // TraversalPredicateFallbackOptional reserves the stable protocol value used to recognize traversal predicate fallback optional across artifacts and executions. + TraversalPredicateFallbackOptional = "optional_match" + + // TraversalPredicateFallbackCorrelation reserves the stable protocol value used to recognize traversal predicate fallback correlation across artifacts and executions. + TraversalPredicateFallbackCorrelation = "correlated_predicate" + + // TraversalPredicateFallbackWholePath reserves the stable protocol value used to recognize traversal predicate fallback whole path across artifacts and executions. + TraversalPredicateFallbackWholePath = "whole_path" + + // TraversalPredicateFallbackUnsupported reserves the stable protocol value used to recognize traversal predicate fallback unsupported across artifacts and executions. + TraversalPredicateFallbackUnsupported = "unsupported_predicate" +) + +// TraversalPredicateEligibilityFact records one conservative classification +// or placement qualification. +type TraversalPredicateEligibilityFact struct { + // Name identifies the name. + Name string `json:"name"` + // Eligible indicates whether eligible applies. + Eligible bool `json:"eligible"` +} + +// TraversalPredicateDecision records one predicate relevant to a variable +// traversal. It never authorizes placement by itself. +type TraversalPredicateDecision struct { + // Target supplies the target input to the TraversalPredicateDecision contract. + Target TraversalStepTarget `json:"target"` + // PredicateIndex supplies the predicate index input to the TraversalPredicateDecision contract. + PredicateIndex int `json:"predicate_index"` + // Source supplies the source input to the TraversalPredicateDecision contract. + Source string `json:"source"` + // Class supplies the class input to the TraversalPredicateDecision contract. + Class TraversalPredicateClass `json:"class"` + // PathSymbol supplies the path symbol input to the TraversalPredicateDecision contract. + PathSymbol string `json:"path_symbol,omitempty"` + // BindingSymbol supplies the binding symbol input to the TraversalPredicateDecision contract. + BindingSymbol string `json:"binding_symbol,omitempty"` + // ReferencedSymbols supplies the referenced symbols input to the TraversalPredicateDecision contract. + ReferencedSymbols []string `json:"referenced_symbols,omitempty"` + // PlannedCandidates supplies the planned candidates input to the TraversalPredicateDecision contract. + PlannedCandidates []TraversalPredicatePlan `json:"planned_candidates"` + // CandidatePlan supplies the candidate plan input to the TraversalPredicateDecision contract. + CandidatePlan TraversalPredicatePlan `json:"candidate_plan,omitempty"` + // SelectedPlan supplies the selected plan input to the TraversalPredicateDecision contract. + SelectedPlan TraversalPredicatePlan `json:"selected_plan"` + // FallbackPlan supplies the fallback plan input to the TraversalPredicateDecision contract. + FallbackPlan TraversalPredicatePlan `json:"fallback_plan"` + // EligibilityFacts supplies the eligibility facts input to the TraversalPredicateDecision contract. + EligibilityFacts []TraversalPredicateEligibilityFact `json:"eligibility_facts"` + // StructurallyEligible indicates whether structurally eligible applies. + StructurallyEligible bool `json:"structurally_eligible"` + // StaticallyEligible indicates whether statically eligible applies. + StaticallyEligible bool `json:"statically_eligible"` + // SelectionMode identifies the selection mode. + SelectionMode string `json:"selection_mode"` + // ClassifierVersion identifies the schema version for classifier version. + ClassifierVersion string `json:"classifier_version"` + // FallbackReason supplies the fallback reason input to the TraversalPredicateDecision contract. + FallbackReason string `json:"fallback_reason"` +} diff --git a/cypher/models/pgsql/optimize/traversal_envelope_plan.go b/cypher/models/pgsql/optimize/traversal_envelope_plan.go new file mode 100644 index 00000000..74c44ab0 --- /dev/null +++ b/cypher/models/pgsql/optimize/traversal_envelope_plan.go @@ -0,0 +1,779 @@ +package optimize + +import ( + "strings" + + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/cypher/models/walk" +) + +// endpointResolutionCandidate groups planner state that must remain consistent while analyzing endpoint resolution candidate. +type endpointResolutionCandidate struct { + // class retains the class while endpointResolutionCandidate is assembled or evaluated. + class EndpointResolutionClass + // property retains the property while endpointResolutionCandidate is assembled or evaluated. + property string + // staticValueCount records the number of static value count. + staticValueCount int + // parameterizedSet indicates whether parameterized set applies. + parameterizedSet bool + // rank retains the rank while endpointResolutionCandidate is assembled or evaluated. + rank int +} + +// endpointResolutionCaps returns the resource limits enforced for endpoint resolution. +func endpointResolutionCaps() EndpointResolutionCaps { + return EndpointResolutionCaps{ + SingletonLimit: EndpointResolutionSingletonLimit, + SingletonSentinel: EndpointResolutionSingletonSentinel, + SmallSetLimit: EndpointResolutionSmallSetLimit, + SmallSetSentinel: EndpointResolutionSmallSetSentinel, + } +} + +// endpointResolutionInput evaluates planner state needed for endpoint resolution input. +func endpointResolutionInput(symbol string, node *cypher.NodePattern, where *cypher.Where) EndpointResolutionInput { + input := EndpointResolutionInput{ + Symbol: symbol, + Class: EndpointResolutionClassUnsupported, + } + if symbol == "" { + return input + } + + var candidates []endpointResolutionCandidate + if where != nil { + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + if candidate, found := endpointResolutionCandidateForTerm(term, symbol); found { + candidates = append(candidates, candidate) + } + } + } + } + candidates = append(candidates, inlineEndpointResolutionCandidates(node, symbol)...) + if len(candidates) == 0 { + return input + } + + best := candidates[0] + for _, candidate := range candidates[1:] { + if candidate.rank > best.rank { + best = candidate + } + } + input.Class = best.class + input.Property = best.property + input.StaticValueCount = best.staticValueCount + input.ParameterizedSet = best.parameterizedSet + switch best.class { + case EndpointResolutionClassIDEquality, EndpointResolutionClassUniquePropertyEquality: + input.Limit = EndpointResolutionSingletonLimit + input.Sentinel = EndpointResolutionSingletonSentinel + case EndpointResolutionClassNonUniquePropertyEquality, EndpointResolutionClassExplicitSmallSet: + input.Limit = EndpointResolutionSmallSetLimit + input.Sentinel = EndpointResolutionSmallSetSentinel + } + + return input +} + +// endpointResolutionCandidateForTerm evaluates planner state needed for endpoint resolution candidate for term. +func endpointResolutionCandidateForTerm(expression cypher.Expression, symbol string) (endpointResolutionCandidate, bool) { + expression = unwrapCypherParenthetical(expression) + comparison, ok := expression.(*cypher.Comparison) + if !ok || comparison == nil || len(comparison.Partials) != 1 || comparison.Partials[0] == nil { + return endpointResolutionCandidate{}, false + } + partial := comparison.Partials[0] + switch partial.Operator { + case cypher.OperatorEquals: + if identitySymbol, found := identityFunctionSymbol(comparison.Left); found && identitySymbol == symbol && expressionIsConstant(partial.Right) { + return endpointResolutionCandidate{ + class: EndpointResolutionClassIDEquality, + staticValueCount: 1, + rank: 5, + }, true + } + if identitySymbol, found := identityFunctionSymbol(partial.Right); found && identitySymbol == symbol && expressionIsConstant(comparison.Left) { + return endpointResolutionCandidate{ + class: EndpointResolutionClassIDEquality, + staticValueCount: 1, + rank: 5, + }, true + } + if propertySymbol, property, found := propertyLookupSymbol(comparison.Left); found && propertySymbol == symbol && expressionIsConstant(partial.Right) { + return propertyEndpointResolutionCandidate(property), true + } + if propertySymbol, property, found := propertyLookupSymbol(partial.Right); found && propertySymbol == symbol && expressionIsConstant(comparison.Left) { + return propertyEndpointResolutionCandidate(property), true + } + + case cypher.OperatorIn: + values, parameterized, recognized := explicitSetCardinality(partial.Right) + if !recognized { + return endpointResolutionCandidate{}, false + } + if identitySymbol, found := identityFunctionSymbol(comparison.Left); found && identitySymbol == symbol { + return endpointResolutionCandidate{ + class: EndpointResolutionClassExplicitSmallSet, + staticValueCount: values, + parameterizedSet: parameterized, + rank: 4, + }, true + } + if propertySymbol, property, found := propertyLookupSymbol(comparison.Left); found && propertySymbol == symbol { + return endpointResolutionCandidate{ + class: EndpointResolutionClassExplicitSmallSet, + property: property, + staticValueCount: values, + parameterizedSet: parameterized, + rank: 4, + }, true + } + } + + return endpointResolutionCandidate{}, false +} + +// propertyEndpointResolutionCandidate evaluates planner state needed for property endpoint resolution candidate. +func propertyEndpointResolutionCandidate(property string) endpointResolutionCandidate { + // A property name is not a uniqueness proof. Until graph-schema metadata is + // available to the optimizer, every property equality uses the bounded + // non-unique envelope and its cap+1 runtime sentinel. + return endpointResolutionCandidate{ + class: EndpointResolutionClassNonUniquePropertyEquality, + property: property, + staticValueCount: 1, + rank: 2, + } +} + +// inlineEndpointResolutionCandidates evaluates planner state needed for inline endpoint resolution candidates. +func inlineEndpointResolutionCandidates(node *cypher.NodePattern, symbol string) []endpointResolutionCandidate { + if node == nil || variableSymbol(node.Variable) != symbol { + return nil + } + properties, ok := node.Properties.(*cypher.Properties) + if !ok || properties == nil || properties.Parameter != nil { + return nil + } + + candidates := make([]endpointResolutionCandidate, 0, len(properties.Map)) + for property, value := range properties.Map { + if expressionIsConstant(value) { + candidates = append(candidates, propertyEndpointResolutionCandidate(property)) + } + } + return candidates +} + +// constantListCardinality evaluates planner state needed for constant list cardinality. +func constantListCardinality(expression cypher.Expression) (int, bool) { + literal, ok := unwrapCypherParenthetical(expression).(*cypher.ListLiteral) + if !ok || literal == nil || len(*literal) == 0 { + return 0, false + } + for _, value := range *literal { + if !expressionIsConstant(value) { + return 0, false + } + } + return len(*literal), true +} + +// explicitSetCardinality recognizes both statically enumerable list literals +// and parameterized sets. Parameter contents remain runtime evidence and must +// pass the same 32/33 bounded-resolution sentinel as a literal set. +func explicitSetCardinality(expression cypher.Expression) (values int, parameterized, recognized bool) { + expression = unwrapCypherParenthetical(expression) + if _, ok := expression.(*cypher.Parameter); ok { + return 0, true, true + } + values, recognized = constantListCardinality(expression) + return values, false, recognized +} + +// endpointInputWithinStaticCap evaluates planner state needed for endpoint input within static cap. +func endpointInputWithinStaticCap(input EndpointResolutionInput) bool { + return input.Class != EndpointResolutionClassExplicitSmallSet || input.StaticValueCount <= int(EndpointResolutionSmallSetLimit) +} + +// endpointResolutionClassSupported evaluates planner state needed for endpoint resolution class supported. +func endpointResolutionClassSupported(class EndpointResolutionClass) bool { + return class != "" && class != EndpointResolutionClassUnsupported && class != EndpointResolutionClassCorrelatedPair +} + +// endpointPairPredicateCorrelated evaluates planner state needed for endpoint pair predicate correlated. +func endpointPairPredicateCorrelated(where *cypher.Where, leftSymbol, rightSymbol string) bool { + if where == nil || leftSymbol == "" || rightSymbol == "" { + return false + } + for _, expression := range where.Expressions { + for _, term := range cypherConjunctionTerms(expression) { + dependencies := sortedDependencies(term) + if stringSliceContains(dependencies, leftSymbol) && stringSliceContains(dependencies, rightSymbol) { + return true + } + } + } + return false +} + +// appendEndpointResolutionDecisions appends endpoint resolution decisions. +func appendEndpointResolutionDecisions( + plan *LoweringPlan, + queryPartIndex int, + queryPart cypher.SyntaxNode, + readingClauses []*cypher.ReadingClause, + initialDeclaredSymbols map[string]struct{}, +) { + _, updatingClauses := queryPartProjection(queryPart) + declaredSymbols := copyStringSet(initialDeclaredSymbols) + hasUnwind := false + for _, readingClause := range readingClauses { + if readingClause != nil && readingClause.Unwind != nil { + hasUnwind = true + } + } + + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + if patternPart == nil || (!patternPart.ShortestPathPattern && !patternPart.AllShortestPathsPattern) { + declarePatternSymbols(declaredSymbols, patternPart) + continue + } + steps := traversalStepsForPattern(patternPart) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil || step.LeftNode == nil || step.RightNode == nil { + continue + } + leftSymbol := variableSymbol(step.LeftNode.Variable) + rightSymbol := variableSymbol(step.RightNode.Variable) + root := endpointResolutionInput(leftSymbol, step.LeftNode, readingClause.Match.Where) + terminal := endpointResolutionInput(rightSymbol, step.RightNode, readingClause.Match.Where) + _, leftPreviouslyBound := declaredSymbols[leftSymbol] + _, rightPreviouslyBound := declaredSymbols[rightSymbol] + correlated := queryPartIndex > 0 || hasUnwind || len(readingClause.Match.Pattern) != 1 || leftPreviouslyBound || rightPreviouslyBound || endpointPairPredicateCorrelated(readingClause.Match.Where, leftSymbol, rightSymbol) + classesSupported := endpointResolutionClassSupported(root.Class) && endpointResolutionClassSupported(terminal.Class) + withinCaps := endpointInputWithinStaticCap(root) && endpointInputWithinStaticCap(terminal) + facts := []EndpointResolutionEligibilityFact{ + { + Name: "supported_shortest_path_mode", + Eligible: patternPart.ShortestPathPattern || patternPart.AllShortestPathsPattern, + }, + { + Name: "single_traversal_step", + Eligible: len(steps) == 1 && len(patternPart.PatternElements) == 3, + }, + { + Name: "read_only", + Eligible: updatingClauses == 0, + }, + { + Name: "non_optional", + Eligible: !readingClause.Match.Optional, + }, + { + Name: "bounded_endpoint_classes", + Eligible: classesSupported, + }, + { + Name: "within_static_endpoint_caps", + Eligible: withinCaps, + }, + { + Name: "uncorrelated_pair", + Eligible: !correlated, + }, + } + eligible := endpointResolutionFactsEligible(facts) + fallbackReason := EndpointResolutionFallbackPlannedOnly + switch { + case updatingClauses != 0: + fallbackReason = EndpointResolutionFallbackMutation + case readingClause.Match.Optional: + fallbackReason = EndpointResolutionFallbackOptionalMatch + case correlated: + fallbackReason = EndpointResolutionFallbackCorrelatedPair + case !withinCaps: + fallbackReason = EndpointResolutionFallbackSmallSetOverflow + case !classesSupported || len(steps) != 1 || len(patternPart.PatternElements) != 3: + fallbackReason = EndpointResolutionFallbackUnsupported + } + + plannedClasses := []EndpointResolutionClass{root.Class, terminal.Class} + pairClass := EndpointResolutionClass("") + if correlated { + pairClass = EndpointResolutionClassCorrelatedPair + plannedClasses = append(plannedClasses, pairClass) + } + family := "SP" + if patternPart.AllShortestPathsPattern { + family = "ASP" + } + plan.EndpointResolution = append(plan.EndpointResolution, EndpointResolutionDecision{ + Target: PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex), + Family: family, + Root: root, + Terminal: terminal, + PairClass: pairClass, + PlannedClasses: plannedClasses, + Caps: endpointResolutionCaps(), + PlannedCandidates: []EndpointResolutionPlan{EndpointResolutionPlanIncumbent, EndpointResolutionPlanBounded}, + CandidatePlan: EndpointResolutionPlanBounded, + SelectedPlan: EndpointResolutionPlanIncumbent, + FallbackPlan: EndpointResolutionPlanIncumbent, + EligibilityFacts: facts, + StructurallyEligible: eligible, + StaticallyEligible: false, + SelectionMode: "analysis_only", + SelectorVersion: "endpoint-resolution-v1", + FallbackReason: fallbackReason, + }) + } + declarePatternSymbols(declaredSymbols, patternPart) + } + declareWhereSymbols(declaredSymbols, readingClause.Match) + } +} + +// endpointResolutionFactsEligible evaluates planner state needed for endpoint resolution facts eligible. +func endpointResolutionFactsEligible(facts []EndpointResolutionEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +// setEndpointResolutionFact evaluates planner state needed for set endpoint resolution fact. +func setEndpointResolutionFact(decision *EndpointResolutionDecision, name string, eligible bool) { + for idx := range decision.EligibilityFacts { + if decision.EligibilityFacts[idx].Name == name { + decision.EligibilityFacts[idx].Eligible = eligible + return + } + } +} + +// traversalPredicateClassification groups planner state that must remain consistent while analyzing traversal predicate classification. +type traversalPredicateClassification struct { + // class retains the class while traversalPredicateClassification is assembled or evaluated. + class TraversalPredicateClass + // bindingSymbol retains the binding symbol while traversalPredicateClassification is assembled or evaluated. + bindingSymbol string + // relevant indicates whether relevant applies. + relevant bool + // correlated indicates whether correlated applies. + correlated bool +} + +// classifyTraversalPredicate constructs the SQL model used for classify traversal predicate. +func classifyTraversalPredicate( + expression cypher.Expression, + pathSymbol string, + nodeSymbols map[string]struct{}, + relationshipSymbols map[string]struct{}, +) traversalPredicateClassification { + expression = unwrapCypherParenthetical(expression) + if quantifier, ok := expression.(*cypher.Quantifier); ok { + return classifyTraversalQuantifier(quantifier, pathSymbol) + } + + dependencies := sortedDependencies(expression) + if pathSymbol != "" && stringSliceContains(dependencies, pathSymbol) { + return traversalPredicateClassification{ + class: TraversalPredicateClassWholePath, + relevant: true, + correlated: len(dependencies) > 1, + } + } + + var nodeDependencies, relationshipDependencies int + for _, dependency := range dependencies { + if _, found := nodeSymbols[dependency]; found { + nodeDependencies++ + } + if _, found := relationshipSymbols[dependency]; found { + relationshipDependencies++ + } + } + relevantDependencies := nodeDependencies + relationshipDependencies + if relevantDependencies == 0 { + return traversalPredicateClassification{} + } + correlated := len(dependencies) != 1 || relevantDependencies != 1 + // A WHERE reference to an endpoint symbol is a boundary predicate, and a + // variable-length relationship binding can be list/path-valued. Neither + // syntax proves evaluation against every recursive step. Only explicit + // path quantifiers and inline relationship properties are classified as + // step-evaluable below. + return traversalPredicateClassification{ + class: TraversalPredicateClassUnsupported, + relevant: true, + correlated: correlated, + } +} + +// classifyTraversalQuantifier evaluates planner state needed for classify traversal quantifier. +func classifyTraversalQuantifier(quantifier *cypher.Quantifier, pathSymbol string) traversalPredicateClassification { + if quantifier == nil || quantifier.Filter == nil || quantifier.Filter.Specifier == nil || quantifier.Filter.Specifier.Variable == nil { + return traversalPredicateClassification{ + class: TraversalPredicateClassUnsupported, + relevant: true, + } + } + function, ok := quantifier.Filter.Specifier.Expression.(*cypher.FunctionInvocation) + if !ok || function == nil || function.NumArguments() != 1 { + return traversalPredicateClassification{ + class: TraversalPredicateClassUnsupported, + relevant: true, + } + } + pathVariable, ok := function.Arguments[0].(*cypher.Variable) + if !ok || pathVariable == nil || pathSymbol == "" || pathVariable.Symbol != pathSymbol { + return traversalPredicateClassification{} + } + bindingSymbol := quantifier.Filter.Specifier.Variable.Symbol + bodyDependencies := sortedDependencies(quantifier.Filter.Where) + correlated := false + for _, dependency := range bodyDependencies { + if dependency != bindingSymbol { + correlated = true + } + } + collectionNodes := strings.EqualFold(function.Name, cypher.NodesFunction) + collectionRelationships := strings.EqualFold(function.Name, cypher.RelationshipsFunction) + if !collectionNodes && !collectionRelationships { + return traversalPredicateClassification{ + class: TraversalPredicateClassWholePath, + bindingSymbol: bindingSymbol, + relevant: true, + correlated: correlated, + } + } + if correlated || quantifier.Filter.Where == nil || !traversalPredicateUsesOnlySafeFunctions(quantifier.Filter.Where, collectionRelationships) { + return traversalPredicateClassification{ + class: TraversalPredicateClassWholePath, + bindingSymbol: bindingSymbol, + relevant: true, + correlated: correlated, + } + } + + classification := traversalPredicateClassification{ + bindingSymbol: bindingSymbol, + relevant: true, + } + switch { + case collectionNodes && quantifier.Type == cypher.QuantifierTypeAll: + classification.class = TraversalPredicateClassUniversalAllNodes + case collectionNodes && quantifier.Type == cypher.QuantifierTypeNone: + classification.class = TraversalPredicateClassUniversalNoneNodes + case collectionRelationships && quantifier.Type == cypher.QuantifierTypeAll: + classification.class = TraversalPredicateClassUniversalAllRelationships + case collectionRelationships && quantifier.Type == cypher.QuantifierTypeNone: + classification.class = TraversalPredicateClassUniversalNoneRelationships + default: + classification.class = TraversalPredicateClassWholePath + } + return classification +} + +// traversalPredicateUsesOnlySafeFunctions evaluates planner state needed for traversal predicate uses only safe functions. +func traversalPredicateUsesOnlySafeFunctions(node cypher.SyntaxNode, relationshipBinding bool) bool { + safe := true + _ = walk.Cypher(node, walk.NewSimpleVisitor[cypher.SyntaxNode](func(node cypher.SyntaxNode, _ walk.VisitorHandler) { + function, ok := node.(*cypher.FunctionInvocation) + if !ok || function == nil { + return + } + if strings.EqualFold(function.Name, cypher.IdentityFunction) { + return + } + if relationshipBinding && strings.EqualFold(function.Name, cypher.EdgeTypeFunction) { + return + } + safe = false + })) + return safe +} + +// traversalPredicateClassStepEvaluable evaluates planner state needed for traversal predicate class step evaluable. +func traversalPredicateClassStepEvaluable(class TraversalPredicateClass) bool { + switch class { + case TraversalPredicateClassStepLocalNode, + TraversalPredicateClassStepLocalRelationship, + TraversalPredicateClassUniversalAllNodes, + TraversalPredicateClassUniversalNoneNodes, + TraversalPredicateClassUniversalAllRelationships, + TraversalPredicateClassUniversalNoneRelationships: + return true + default: + return false + } +} + +// appendTraversalPredicateDecisions appends traversal predicate decisions. +func appendTraversalPredicateDecisions( + plan *LoweringPlan, + queryPartIndex int, + queryPart cypher.SyntaxNode, + readingClauses []*cypher.ReadingClause, +) { + _, updatingClauses := queryPartProjection(queryPart) + for clauseIndex, readingClause := range readingClauses { + if readingClause == nil || readingClause.Match == nil { + continue + } + for patternIndex, patternPart := range readingClause.Match.Pattern { + if patternPart == nil { + continue + } + pathSymbol := variableSymbol(patternPart.Variable) + steps := traversalStepsForPattern(patternPart) + for stepIndex, step := range steps { + if step.Relationship == nil || step.Relationship.Range == nil { + continue + } + target := PatternTarget{ + QueryPartIndex: queryPartIndex, + ClauseIndex: clauseIndex, + PatternIndex: patternIndex, + }.TraversalStep(stepIndex) + nodeSymbols := map[string]struct{}{} + if symbol := variableSymbol(step.LeftNode.Variable); symbol != "" { + nodeSymbols[symbol] = struct{}{} + } + if symbol := variableSymbol(step.RightNode.Variable); symbol != "" { + nodeSymbols[symbol] = struct{}{} + } + relationshipSymbols := map[string]struct{}{} + if symbol := variableSymbol(step.Relationship.Variable); symbol != "" { + relationshipSymbols[symbol] = struct{}{} + } + + predicateIndex := 0 + if readingClause.Match.Where != nil { + for _, whereExpression := range readingClause.Match.Where.Expressions { + for _, term := range cypherConjunctionTerms(whereExpression) { + classification := classifyTraversalPredicate(term, pathSymbol, nodeSymbols, relationshipSymbols) + if !classification.relevant { + continue + } + appendTraversalPredicateDecision(plan, target, predicateIndex, "where", pathSymbol, classification, sortedDependencies(term), updatingClauses == 0, !readingClause.Match.Optional) + predicateIndex++ + } + } + } + if step.Relationship.Properties != nil { + classification := traversalPredicateClassification{ + class: TraversalPredicateClassStepLocalRelationship, + relevant: true, + } + if !inlinePropertiesStepLocal(step.Relationship.Properties) { + classification.class = TraversalPredicateClassUnsupported + classification.correlated = len(sortedDependencies(step.Relationship.Properties)) > 0 + } + appendTraversalPredicateDecision(plan, target, predicateIndex, "relationship_pattern", pathSymbol, classification, sortedDependencies(step.Relationship.Properties), updatingClauses == 0, !readingClause.Match.Optional) + predicateIndex++ + } + for _, node := range []*cypher.NodePattern{step.LeftNode, step.RightNode} { + if node == nil || node.Properties == nil { + continue + } + // Node-pattern properties constrain the pattern boundary; they + // are not predicates over every node visited by a variable range. + classification := traversalPredicateClassification{ + class: TraversalPredicateClassUnsupported, + relevant: true, + correlated: len(sortedDependencies(node.Properties)) > 0, + } + appendTraversalPredicateDecision(plan, target, predicateIndex, "node_pattern", pathSymbol, classification, sortedDependencies(node.Properties), updatingClauses == 0, !readingClause.Match.Optional) + predicateIndex++ + } + } + } + } +} + +// inlinePropertiesStepLocal evaluates planner state needed for inline properties step local. +func inlinePropertiesStepLocal(expression cypher.Expression) bool { + properties, ok := expression.(*cypher.Properties) + if !ok || properties == nil || properties.Parameter != nil { + return false + } + for _, value := range properties.Map { + if !expressionIsConstant(value) { + return false + } + } + return true +} + +// appendTraversalPredicateDecision appends traversal predicate decision. +func appendTraversalPredicateDecision( + plan *LoweringPlan, + target TraversalStepTarget, + predicateIndex int, + source, pathSymbol string, + classification traversalPredicateClassification, + referencedSymbols []string, + readOnly, nonOptional bool, +) { + stepEvaluable := traversalPredicateClassStepEvaluable(classification.class) + facts := []TraversalPredicateEligibilityFact{ + { + Name: "read_only", + Eligible: readOnly, + }, + { + Name: "non_optional", + Eligible: nonOptional, + }, + { + Name: "step_evaluable", + Eligible: stepEvaluable, + }, + { + Name: "uncorrelated", + Eligible: !classification.correlated, + }, + } + eligible := traversalPredicateFactsEligible(facts) + fallbackReason := TraversalPredicateFallbackPlannedOnly + switch { + case !readOnly: + fallbackReason = TraversalPredicateFallbackMutation + case !nonOptional: + fallbackReason = TraversalPredicateFallbackOptional + case classification.correlated: + fallbackReason = TraversalPredicateFallbackCorrelation + case classification.class == TraversalPredicateClassWholePath: + fallbackReason = TraversalPredicateFallbackWholePath + case !stepEvaluable: + fallbackReason = TraversalPredicateFallbackUnsupported + } + plannedCandidates := []TraversalPredicatePlan{TraversalPredicatePlanIncumbent} + candidatePlan := TraversalPredicatePlan("") + if stepEvaluable { + candidatePlan = TraversalPredicatePlanStep + plannedCandidates = append(plannedCandidates, candidatePlan) + } + plan.TraversalPredicate = append(plan.TraversalPredicate, TraversalPredicateDecision{ + Target: target, + PredicateIndex: predicateIndex, + Source: source, + Class: classification.class, + PathSymbol: pathSymbol, + BindingSymbol: classification.bindingSymbol, + ReferencedSymbols: referencedSymbols, + PlannedCandidates: plannedCandidates, + CandidatePlan: candidatePlan, + SelectedPlan: TraversalPredicatePlanIncumbent, + FallbackPlan: TraversalPredicatePlanIncumbent, + EligibilityFacts: facts, + StructurallyEligible: eligible, + StaticallyEligible: false, + SelectionMode: "analysis_only", + ClassifierVersion: "traversal-predicate-v1", + FallbackReason: fallbackReason, + }) +} + +// traversalPredicateFactsEligible evaluates planner state needed for traversal predicate facts eligible. +func traversalPredicateFactsEligible(facts []TraversalPredicateEligibilityFact) bool { + for _, fact := range facts { + if !fact.Eligible { + return false + } + } + return true +} + +// setTraversalPredicateFact evaluates planner state needed for set traversal predicate fact. +func setTraversalPredicateFact(decision *TraversalPredicateDecision, name string, eligible bool) { + for idx := range decision.EligibilityFacts { + if decision.EligibilityFacts[idx].Name == name { + decision.EligibilityFacts[idx].Eligible = eligible + return + } + } +} + +// finalizeTraversalEnvelopeDecisions evaluates planner state needed for finalize traversal envelope decisions. +func finalizeTraversalEnvelopeDecisions(plan *LoweringPlan, query *cypher.RegularQuery) { + if plan == nil || query == nil || query.SingleQuery == nil { + return + } + readOnly := statementUpdatingClauseCount(query) == 0 + for idx := range plan.EndpointResolution { + decision := &plan.EndpointResolution[idx] + setEndpointResolutionFact(decision, "read_only", readOnly) + decision.StructurallyEligible = endpointResolutionFactsEligible(decision.EligibilityFacts) + decision.StaticallyEligible = false + if !readOnly { + decision.FallbackReason = EndpointResolutionFallbackMutation + } + } + for idx := range plan.TraversalPredicate { + decision := &plan.TraversalPredicate[idx] + setTraversalPredicateFact(decision, "read_only", readOnly) + decision.StructurallyEligible = traversalPredicateFactsEligible(decision.EligibilityFacts) + decision.StaticallyEligible = false + if !readOnly { + decision.FallbackReason = TraversalPredicateFallbackMutation + } + } +} + +// statementUpdatingClauseCount evaluates planner state needed for statement updating clause count. +func statementUpdatingClauseCount(query *cypher.RegularQuery) int { + if query == nil || query.SingleQuery == nil { + return 0 + } + count := 0 + if multiPart := query.SingleQuery.MultiPartQuery; multiPart != nil { + for _, part := range multiPart.Parts { + if part != nil { + count += len(part.UpdatingClauses) + } + } + if finalPart := multiPart.SinglePartQuery; finalPart != nil { + count += len(finalPart.UpdatingClauses) + } + } else if singlePart := query.SingleQuery.SinglePartQuery; singlePart != nil { + count += len(singlePart.UpdatingClauses) + } + return count +} + +// unwrapCypherParenthetical evaluates planner state needed for unwrap cypher parenthetical. +func unwrapCypherParenthetical(expression cypher.Expression) cypher.Expression { + for { + parenthetical, ok := expression.(*cypher.Parenthetical) + if !ok || parenthetical == nil { + return expression + } + expression = parenthetical.Expression + } +} + +// stringSliceContains evaluates planner state needed for string slice contains. +func stringSliceContains(values []string, expected string) bool { + for _, value := range values { + if value == expected { + return true + } + } + return false +} diff --git a/cypher/models/pgsql/optimize/traversal_envelope_test.go b/cypher/models/pgsql/optimize/traversal_envelope_test.go new file mode 100644 index 00000000..2cfd3281 --- /dev/null +++ b/cypher/models/pgsql/optimize/traversal_envelope_test.go @@ -0,0 +1,460 @@ +package optimize + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// optimizeTraversalEnvelope evaluates planner state needed for optimize traversal envelope. +func optimizeTraversalEnvelope(t *testing.T, query string) LoweringPlan { + t.Helper() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + + plan, err := Optimize(regularQuery) + require.NoError(t, err) + return plan.LoweringPlan +} + +// TestEndpointResolutionClassifiesBoundedInputsWithoutSelectingThem verifies endpoint resolution classifies bounded inputs without selecting them behavior. +func TestEndpointResolutionClassifiesBoundedInputsWithoutSelectingThem(t *testing.T) { + t.Parallel() + + testCases := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // where retains the where while anonymous record is assembled or evaluated. + where string + // rootClass retains the root class while anonymous record is assembled or evaluated. + rootClass EndpointResolutionClass + // terminalClass retains the terminal class while anonymous record is assembled or evaluated. + terminalClass EndpointResolutionClass + // valueCount records the number of value count. + valueCount int + // runtimeCount records the number of runtime count. + runtimeCount bool + }{ + { + name: "ID equality", + where: "id(s) = $source_id AND id(e) = $terminal_id", + rootClass: EndpointResolutionClassIDEquality, + terminalClass: EndpointResolutionClassIDEquality, + valueCount: 1, + }, + { + name: "property name is not uniqueness proof", + where: "s.objectid = $source_id AND e.objectid = $terminal_id", + rootClass: EndpointResolutionClassNonUniquePropertyEquality, + terminalClass: EndpointResolutionClassNonUniquePropertyEquality, + valueCount: 1, + }, + { + name: "nonunique property equality", + where: "s.name = $source_name AND e.name = $terminal_name", + rootClass: EndpointResolutionClassNonUniquePropertyEquality, + terminalClass: EndpointResolutionClassNonUniquePropertyEquality, + valueCount: 1, + }, + { + name: "explicit small set", + where: "id(s) IN [1, 2] AND id(e) IN [3, 4]", + rootClass: EndpointResolutionClassExplicitSmallSet, + terminalClass: EndpointResolutionClassExplicitSmallSet, + valueCount: 2, + }, + { + name: "parameterized explicit small set", + where: "id(s) IN $source_ids AND e.name IN $terminal_names", + rootClass: EndpointResolutionClassExplicitSmallSet, + terminalClass: EndpointResolutionClassExplicitSmallSet, + runtimeCount: true, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE %s + RETURN length(p) + `, testCase.where)) + + require.Len(t, plan.EndpointResolution, 1) + decision := plan.EndpointResolution[0] + require.Equal(t, testCase.rootClass, decision.Root.Class) + require.Equal(t, testCase.terminalClass, decision.Terminal.Class) + require.Equal(t, testCase.valueCount, decision.Root.StaticValueCount) + require.Equal(t, testCase.valueCount, decision.Terminal.StaticValueCount) + require.Equal(t, testCase.runtimeCount, decision.Root.ParameterizedSet) + require.Equal(t, testCase.runtimeCount, decision.Terminal.ParameterizedSet) + if testCase.runtimeCount { + require.Equal(t, EndpointResolutionSmallSetLimit, decision.Root.Limit) + require.Equal(t, EndpointResolutionSmallSetSentinel, decision.Root.Sentinel) + } + require.Equal(t, EndpointResolutionPlanBounded, decision.CandidatePlan) + require.Equal(t, EndpointResolutionPlanIncumbent, decision.SelectedPlan) + require.Equal(t, EndpointResolutionPlanIncumbent, decision.FallbackPlan) + require.True(t, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, "analysis_only", decision.SelectionMode) + require.Equal(t, EndpointResolutionFallbackPlannedOnly, decision.FallbackReason) + require.Contains(t, plan.Decisions(), LoweringDecision{Name: LoweringEndpointResolution}) + }) + } +} + +// TestEndpointResolutionRecordsCapsAndSentinelsInJSON verifies endpoint resolution records caps and sentinels in json behavior. +func TestEndpointResolutionRecordsCapsAndSentinelsInJSON(t *testing.T) { + t.Parallel() + + plan := optimizeTraversalEnvelope(t, ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) IN [1, 2] AND id(e) IN [3, 4] + RETURN p + `) + require.Len(t, plan.EndpointResolution, 1) + + diagnostic, err := json.Marshal(plan.EndpointResolution[0]) + require.NoError(t, err) + require.JSONEq(t, `{ + "target":{"query_part_index":0,"clause_index":0,"pattern_index":0,"step_index":0}, + "family":"ASP", + "root":{"symbol":"s","class":"explicit_small_set","static_value_count":2,"limit":32,"sentinel":33}, + "terminal":{"symbol":"e","class":"explicit_small_set","static_value_count":2,"limit":32,"sentinel":33}, + "planned_classes":["explicit_small_set","explicit_small_set"], + "caps":{"singleton_limit":1,"singleton_sentinel":2,"small_set_limit":32,"small_set_sentinel":33}, + "planned_candidates":["ENDPOINT-RESOLUTION-INCUMBENT","ENDPOINT-RESOLUTION-BOUNDED"], + "candidate_plan":"ENDPOINT-RESOLUTION-BOUNDED", + "selected_plan":"ENDPOINT-RESOLUTION-INCUMBENT", + "fallback_plan":"ENDPOINT-RESOLUTION-INCUMBENT", + "eligibility_facts":[ + {"name":"supported_shortest_path_mode","eligible":true}, + {"name":"single_traversal_step","eligible":true}, + {"name":"read_only","eligible":true}, + {"name":"non_optional","eligible":true}, + {"name":"bounded_endpoint_classes","eligible":true}, + {"name":"within_static_endpoint_caps","eligible":true}, + {"name":"uncorrelated_pair","eligible":true} + ], + "structurally_eligible":true, + "statically_eligible":false, + "selection_mode":"analysis_only", + "selector_version":"endpoint-resolution-v1", + "fallback_reason":"planned_only" + }`, string(diagnostic)) +} + +// TestEndpointResolutionReportsConservativeFallbackReasons verifies endpoint resolution reports conservative fallback reasons behavior. +func TestEndpointResolutionReportsConservativeFallbackReasons(t *testing.T) { + t.Parallel() + + values := make([]string, EndpointResolutionSmallSetSentinel) + for index := range values { + values[index] = fmt.Sprint(index + 1) + } + + testCases := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. + query string + // reason retains the reason while anonymous record is assembled or evaluated. + reason string + // pairClass retains the pair class while anonymous record is assembled or evaluated. + pairClass EndpointResolutionClass + // structural indicates whether structural applies. + structural bool + }{ + { + name: "read only remains planned", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 + RETURN p + `, + reason: EndpointResolutionFallbackPlannedOnly, + structural: true, + }, + { + name: "mutation", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 + CREATE (:Audit) + RETURN p + `, + reason: EndpointResolutionFallbackMutation, + }, + { + name: "optional match", + query: ` + OPTIONAL MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 + RETURN p + `, + reason: EndpointResolutionFallbackOptionalMatch, + }, + { + name: "correlated pair", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = 1 AND id(e) = 2 AND s.tenant = e.tenant + RETURN p + `, + reason: EndpointResolutionFallbackCorrelatedPair, + pairClass: EndpointResolutionClassCorrelatedPair, + }, + { + name: "small set cap plus one", + query: fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) IN [%s] AND id(e) IN [100] + RETURN p + `, strings.Join(values, ",")), + reason: EndpointResolutionFallbackSmallSetOverflow, + }, + { + name: "unsupported endpoint syntax", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE s.name STARTS WITH 'source' AND e.name STARTS WITH 'terminal' + RETURN p + `, + reason: EndpointResolutionFallbackUnsupported, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, testCase.query) + require.Len(t, plan.EndpointResolution, 1) + decision := plan.EndpointResolution[0] + require.Equal(t, testCase.reason, decision.FallbackReason) + require.Equal(t, testCase.pairClass, decision.PairClass) + require.Equal(t, testCase.structural, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, EndpointResolutionPlanIncumbent, decision.SelectedPlan) + }) + } +} + +// TestTraversalPredicateClassifiesLocalUniversalAndWholePathForms verifies traversal predicate classifies local universal and whole path forms behavior. +func TestTraversalPredicateClassifiesLocalUniversalAndWholePathForms(t *testing.T) { + t.Parallel() + + testCases := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // predicate retains the predicate while anonymous record is assembled or evaluated. + predicate string + // class retains the class while anonymous record is assembled or evaluated. + class TraversalPredicateClass + // bindingSymbol retains the binding symbol while anonymous record is assembled or evaluated. + bindingSymbol string + // fallback retains the fallback while anonymous record is assembled or evaluated. + fallback string + // structural indicates whether structural applies. + structural bool + }{ + { + name: "endpoint WHERE predicate is not step local", + predicate: "s.enabled = true", + class: TraversalPredicateClassUnsupported, + fallback: TraversalPredicateFallbackUnsupported, + structural: false, + }, + { + name: "range binding WHERE predicate is not step local", + predicate: "rels.enabled = true", + class: TraversalPredicateClassUnsupported, + fallback: TraversalPredicateFallbackUnsupported, + structural: false, + }, + { + name: "all nodes", + predicate: "all(n IN nodes(p) WHERE n.enabled = true)", + class: TraversalPredicateClassUniversalAllNodes, + bindingSymbol: "n", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "none nodes", + predicate: "none(n IN nodes(p) WHERE n.disabled = true)", + class: TraversalPredicateClassUniversalNoneNodes, + bindingSymbol: "n", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "all relationships", + predicate: "all(r IN relationships(p) WHERE type(r) = 'MemberOf')", + class: TraversalPredicateClassUniversalAllRelationships, + bindingSymbol: "r", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "none relationships", + predicate: "none(r IN relationships(p) WHERE type(r) = 'AdminTo')", + class: TraversalPredicateClassUniversalNoneRelationships, + bindingSymbol: "r", + fallback: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "whole path", + predicate: "length(p) > 2", + class: TraversalPredicateClassWholePath, + fallback: TraversalPredicateFallbackWholePath, + structural: false, + }, + { + name: "correlated endpoints", + predicate: "s.tenant = e.tenant", + class: TraversalPredicateClassUnsupported, + fallback: TraversalPredicateFallbackCorrelation, + structural: false, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, fmt.Sprintf(` + MATCH p = shortestPath((s)-[rels:MemberOf*1..4]->(e)) + WHERE %s + RETURN p + `, testCase.predicate)) + + require.Len(t, plan.TraversalPredicate, 1) + decision := plan.TraversalPredicate[0] + require.Equal(t, testCase.class, decision.Class) + require.Equal(t, testCase.bindingSymbol, decision.BindingSymbol) + require.Equal(t, testCase.fallback, decision.FallbackReason) + require.Equal(t, testCase.structural, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, TraversalPredicatePlanIncumbent, decision.SelectedPlan) + require.Equal(t, TraversalPredicatePlanIncumbent, decision.FallbackPlan) + require.Equal(t, "analysis_only", decision.SelectionMode) + require.Contains(t, plan.Decisions(), LoweringDecision{Name: LoweringTraversalPredicateClassification}) + }) + } +} + +// TestTraversalPredicateOnlyClaimsInlineRelationshipPropertiesAsStepLocal verifies traversal predicate only claims inline relationship properties as step local behavior. +func TestTraversalPredicateOnlyClaimsInlineRelationshipPropertiesAsStepLocal(t *testing.T) { + t.Parallel() + + plan := optimizeTraversalEnvelope(t, ` + MATCH p = shortestPath((s {enabled: true})-[rels:MemberOf*1..4{active: true}]->(e)) + RETURN p + `) + require.Len(t, plan.TraversalPredicate, 2) + + require.Equal(t, "relationship_pattern", plan.TraversalPredicate[0].Source) + require.Equal(t, TraversalPredicateClassStepLocalRelationship, plan.TraversalPredicate[0].Class) + require.True(t, plan.TraversalPredicate[0].StructurallyEligible) + require.Equal(t, TraversalPredicateFallbackPlannedOnly, plan.TraversalPredicate[0].FallbackReason) + + require.Equal(t, "node_pattern", plan.TraversalPredicate[1].Source) + require.Equal(t, TraversalPredicateClassUnsupported, plan.TraversalPredicate[1].Class) + require.False(t, plan.TraversalPredicate[1].StructurallyEligible) + require.Equal(t, TraversalPredicateFallbackUnsupported, plan.TraversalPredicate[1].FallbackReason) +} + +// TestTraversalPredicateReportsMutationOptionalAndCorrelationFallbacks verifies traversal predicate reports mutation optional and correlation fallbacks behavior. +func TestTraversalPredicateReportsMutationOptionalAndCorrelationFallbacks(t *testing.T) { + t.Parallel() + + testCases := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. + query string + // reason retains the reason while anonymous record is assembled or evaluated. + reason string + // structural indicates whether structural applies. + structural bool + }{ + { + name: "read only remains planned", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `, + reason: TraversalPredicateFallbackPlannedOnly, + structural: true, + }, + { + name: "mutation", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + CREATE (:Audit) + RETURN p + `, + reason: TraversalPredicateFallbackMutation, + }, + { + name: "optional match", + query: ` + OPTIONAL MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `, + reason: TraversalPredicateFallbackOptional, + }, + { + name: "correlated predicate", + query: ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE s.tenant = e.tenant + RETURN p + `, + reason: TraversalPredicateFallbackCorrelation, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + plan := optimizeTraversalEnvelope(t, testCase.query) + require.Len(t, plan.TraversalPredicate, 1) + decision := plan.TraversalPredicate[0] + require.Equal(t, testCase.reason, decision.FallbackReason) + require.Equal(t, testCase.structural, decision.StructurallyEligible) + require.False(t, decision.StaticallyEligible) + require.Equal(t, TraversalPredicatePlanIncumbent, decision.SelectedPlan) + }) + } +} + +// TestTraversalPredicateJSONKeepsCandidatePlannedOnly verifies traversal predicate json keeps candidate planned only behavior. +func TestTraversalPredicateJSONKeepsCandidatePlannedOnly(t *testing.T) { + t.Parallel() + + plan := optimizeTraversalEnvelope(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `) + require.Len(t, plan.TraversalPredicate, 1) + + diagnostic, err := json.Marshal(plan.TraversalPredicate[0]) + require.NoError(t, err) + require.Contains(t, string(diagnostic), `"class":"universal_all_nodes"`) + require.Contains(t, string(diagnostic), `"planned_candidates":["TRAVERSAL-PREDICATE-INCUMBENT","TRAVERSAL-PREDICATE-STEP"]`) + require.Contains(t, string(diagnostic), `"selected_plan":"TRAVERSAL-PREDICATE-INCUMBENT"`) + require.Contains(t, string(diagnostic), `"statically_eligible":false`) + require.Contains(t, string(diagnostic), `"fallback_reason":"planned_only"`) +} diff --git a/cypher/models/pgsql/test/logical_forms_legacy_builder_test.go b/cypher/models/pgsql/test/logical_forms_legacy_builder_test.go new file mode 100644 index 00000000..ca6f3fdc --- /dev/null +++ b/cypher/models/pgsql/test/logical_forms_legacy_builder_test.go @@ -0,0 +1,163 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// translateLegacyQuery builds legacy criteria, translates the resulting Cypher, and returns formatted SQL and metadata. +func translateLegacyQuery(t *testing.T, criteria ...graph.Criteria) (string, translate.Result) { + t.Helper() + + builder := query.NewBuilderWithCriteria(criteria...) + regularQuery, err := builder.Build(false) + require.NoError(t, err) + + translation, err := translate.Translate(context.Background(), regularQuery, newKindMapper(), nil, translate.DefaultGraphID) + require.NoError(t, err) + + formatted, err := translate.Translated(translation) + require.NoError(t, err) + return formatted, translation +} + +// TestLegacyBuilderPostgreSQL_LogicalForms verifies boolean grouping, typed thresholds, and binding-local predicates in migrated builder queries. +func TestLegacyBuilderPostgreSQL_LogicalForms(t *testing.T) { + t.Run("LOGIC-01 branch-local relationship kinds", func(t *testing.T) { + formatted, _ := translateLegacyQuery(t, + query.Where(query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind01")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind02")), + ), + )), + query.Returning(query.RelationshipID()), + ) + + require.Contains(t, formatted, " or ") + require.Contains(t, formatted, "n0.id = @pi0") + require.Contains(t, formatted, "n1.id = @pi1") + require.Contains(t, formatted, "n0.id = @pi2") + require.Contains(t, formatted, "n1.id = @pi3") + require.Contains(t, formatted, "e0.kind_id = any") + }) + + t.Run("LOGIC-02 cross-binding temporal disjunction", func(t *testing.T) { + formatted, _ := translateLegacyQuery(t, + query.Where(query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + )), + query.Returning(query.RelationshipID()), + ) + + require.Contains(t, formatted, "e0.properties -> 'lastseen'") + require.Contains(t, formatted, "n0.properties -> 'lastcollected'") + require.Contains(t, formatted, "n1.properties -> 'lastcollected'") + require.Contains(t, formatted, " or ") + }) + + t.Run("LOGIC-03 typed threshold and scoped negation", func(t *testing.T) { + threshold := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("RegressionKind03"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + )), + query.Returning(query.NodeID()), + ) + + require.Contains(t, formatted, "not") + require.Contains(t, formatted, " or ") + require.Contains(t, formatted, "n0.properties -> 'lastseen'") + require.Contains(t, formatted, "@pi0") + require.Equal(t, map[string]any{"pi0": threshold}, translation.Parameters) + }) +} + +// TestLegacyBuilderPostgreSQL_LOGIC05ProjectionOrder verifies that migrated projections preserve caller-specified column order. +func TestLegacyBuilderPostgreSQL_LOGIC05ProjectionOrder(t *testing.T) { + testCases := map[string]struct { + // projection supplies the legacy graph criteria for the case. + projection *graphProjection + // columns lists the SQL fragments in their required projection order. + columns []string + }{ + "full opposite node plus relationship": { + projection: projectionOf(query.Relationship(), query.End()), + columns: []string{"select s0.e0 as r", "s0.n1 as e"}, + }, + "opposite ID and kinds plus relationship ID and kind": { + projection: projectionOf(query.EndID(), query.KindsOf(query.End()), query.RelationshipID(), query.KindsOf(query.Relationship())), + columns: []string{"select (s0.n1).id", "(s0.n1).kind_ids", "(s0.e0).id", "kind_name((s0.e0).kind_id)"}, + }, + "start relationship end triple": { + projection: projectionOf(query.Start(), query.Relationship(), query.End()), + columns: []string{"select s0.n0 as s", "s0.e0 as r", "s0.n1 as e"}, + }, + "relationship ID only": { + projection: projectionOf(query.RelationshipID()), + columns: []string{"select (s0.e0).id"}, + }, + "full relationship": { + projection: projectionOf(query.Relationship()), + columns: []string{"select s0.e0 as r"}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, _ := translateLegacyQuery(t, testCase.projection.criteria) + cursor := 0 + for _, column := range testCase.columns { + next := strings.Index(formatted[cursor:], column) + require.NotEqualf(t, -1, next, "missing projection column %q in %s", column, formatted) + cursor += next + len(column) + } + }) + } +} + +// graphProjection keeps the table-driven projection cases strongly typed +// without obscuring that they are legacy query criteria. +type graphProjection struct { + // criteria is the legacy returning criterion represented by this projection. + criteria graph.Criteria +} + +// projectionOf wraps returning criteria in the strongly typed projection used by table-driven cases. +func projectionOf(criteria ...graph.Criteria) *graphProjection { + return &graphProjection{criteria: query.Returning(criteria...)} +} diff --git a/cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go b/cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go new file mode 100644 index 00000000..5b9a42bc --- /dev/null +++ b/cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go @@ -0,0 +1,207 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderPostgreSQL_ReconciliationForms verifies migrated relationship reconciliation reads and deletes across kind-set sizes. +func TestLegacyBuilderPostgreSQL_ReconciliationForms(t *testing.T) { + reconciliationKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("RegressionKind%02d", idx+1)) + } + return kinds + } + + assertRelationshipDelete := func(t *testing.T, formatted string) { + t.Helper() + selection := strings.Index(formatted, "select ") + deletion := strings.Index(formatted, "delete from edge e1 using s0") + require.NotEqual(t, -1, selection) + require.Greater(t, deletion, selection, "selection must precede mutation: %s", formatted) + require.Contains(t, formatted, "where (s0.e0).id = e1.id") + } + + for _, count := range []int{1, 2, 9, 30} { + kinds := reconciliationKinds(count) + + t.Run(fmt.Sprintf("REC-01 inbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.Equals(query.EndProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ) + + assertRelationshipDelete(t, formatted) + require.Contains(t, formatted, "n1.id = e0.end_id") + require.Contains(t, formatted, "n1.properties -> 'objectid'") + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", sequentialKindIDs(33, count))) + require.Equal(t, map[string]any{"pi0": "target-id"}, translation.Parameters) + }) + + t.Run(fmt.Sprintf("REC-02 outbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("RegressionKind31")), + query.Equals(query.StartProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ) + + assertRelationshipDelete(t, formatted) + require.Contains(t, formatted, "n0.id = e0.start_id") + require.Contains(t, formatted, "n0.properties -> 'objectid'") + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", sequentialKindIDs(33, count))) + require.Equal(t, map[string]any{"pi0": "target-id"}, translation.Parameters) + }) + } + + testCases := map[string]struct { + // criteria contains the legacy query-builder inputs for the case. + criteria []graph.Criteria + // fragments lists SQL fragments that the translation must contain. + fragments []string + // parameters is the exact parameter map expected from translation. + parameters map[string]any + // read reports whether the case reads rather than deletes a relationship. + read bool + }{ + "REC-03 inbound primary group": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.Equals(query.EndProperty("objectid"), "group-id"), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Equals(query.RelationshipProperty("isprimarygroup"), false), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = e0.end_id", "e0.properties -> 'isprimarygroup'", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": "group-id", "pi1": false}, + }, + "REC-03 outbound primary group": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("RegressionKind31")), + query.Equals(query.StartProperty("objectid"), "computer-id"), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Equals(query.RelationshipProperty("isprimarygroup"), true), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n0.id = e0.start_id", "e0.properties -> 'isprimarygroup'", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": "computer-id", "pi1": true}, + }, + "REC-04 object ID list relationship delete": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.In(query.EndProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = e0.end_id", "n1.properties ->> 'objectid'", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": []string{"target-1", "target-2"}}, + }, + "REC-05 delegated enrollment discovery": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.In(query.EndProperty("objectid"), []string{"ca-1", "ca-2"}), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind32")), + query.Kind(query.Start(), graph.StringKind("RegressionKind31")), + )), + query.Returning(query.Relationship(), query.Start()), + }, + fragments: []string{"select s0.e0 as r, s0.n0 as s", "n1.properties ->> 'objectid'"}, + parameters: map[string]any{"pi0": []string{"ca-1", "ca-2"}}, + read: true, + }, + "REC-06 delegated enrollment delete by IDs": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.InIDs(query.EndID(), graph.ID(101), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind32")), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = any", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": []uint64{101, 202}}, + }, + "REC-07 HostsCAService relationship delete": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("RegressionKind31")), + query.Equals(query.EndProperty("objectid"), "ca-id"), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind32")), + )), + query.Delete(query.Relationship()), + }, + fragments: []string{"n1.id = e0.end_id", "delete from edge e1 using s0"}, + parameters: map[string]any{"pi0": "ca-id"}, + }, + "REC-08 AD entity detach delete": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("RegressionKind31")), + query.In(query.NodeProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Node()), + }, + fragments: []string{"n0.properties ->> 'objectid'", "delete from node n1 using s0", "where (s0.n0).id = n1.id"}, + parameters: map[string]any{"pi0": []string{"target-1", "target-2"}}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, testCase.criteria...) + for _, fragment := range testCase.fragments { + require.Contains(t, formatted, fragment) + } + if !testCase.read { + selection := strings.Index(formatted, "select ") + deletion := strings.Index(formatted, "delete from ") + require.Greater(t, deletion, selection, "selection must precede mutation: %s", formatted) + } + require.Equal(t, testCase.parameters, translation.Parameters) + }) + } +} + +// sequentialKindIDs formats count consecutive kind IDs beginning at start for SQL-fragment assertions. +func sequentialKindIDs(first, count int) string { + ids := make([]string, count) + for idx := range count { + ids[idx] = fmt.Sprint(first + idx) + } + return strings.Join(ids, ", ") +} diff --git a/cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go b/cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go new file mode 100644 index 00000000..ed5971c3 --- /dev/null +++ b/cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go @@ -0,0 +1,365 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// scanLookupRegressionKinds converts numeric fixture suffixes to their RegressionKind names. +func scanLookupRegressionKinds(numbers ...int) graph.Kinds { + kinds := make(graph.Kinds, len(numbers)) + for idx, number := range numbers { + kinds[idx] = graph.StringKind("RegressionKind" + twoDigitKindSuffix(number)) + } + return kinds +} + +// twoDigitKindSuffix formats a fixture kind number as two decimal digits. +func twoDigitKindSuffix(value int) string { + if value < 10 { + return "0" + string(rune('0'+value)) + } + return string(rune('0'+value/10)) + string(rune('0'+value%10)) +} + +// assertScanLookupTranslation translates criteria and requires every expected SQL fragment to be present. +func assertScanLookupTranslation(t *testing.T, criteria []graph.Criteria, fragments ...string) { + t.Helper() + formatted, _ := translateLegacyQuery(t, criteria...) + for _, fragment := range fragments { + require.Contains(t, formatted, fragment) + } +} + +// TestLegacyBuilderPostgreSQL_RelationshipScans verifies migrated relationship scans preserve endpoint, kind, property, and projection semantics. +func TestLegacyBuilderPostgreSQL_RelationshipScans(t *testing.T) { + t.Run("SCAN-01 base endpoints and relationship ID", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.KindIn(query.Start(), scanLookupRegressionKinds(61, 62)...), + query.Kind(query.Relationship(), scanLookupRegressionKinds(63)[0]), + query.KindIn(query.End(), scanLookupRegressionKinds(61, 62)...), + )), + query.Returning(query.RelationshipID()), + }, "n0.kind_ids", "n1.kind_ids", "array [93, 94]::int2[]", "e0.kind_id = any (array [95]::int2[])", "select (s0.e0).id") + }) + + t.Run("SCAN-02 excludes Meta endpoints and hydrates relationships", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupRegressionKinds(64, 65)...)), + query.KindIn(query.Relationship(), scanLookupRegressionKinds(66, 67)...), + query.Not(query.KindIn(query.End(), scanLookupRegressionKinds(64, 65)...)), + )), + query.Returning(query.Relationship()), + }, "not", "array [96, 97]::int2[]", "array [98, 99]::int2[]", "select s0.e0 as r") + }) + + t.Run("SCAN-03 exists relationship property and ID", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupRegressionKinds(64, 65)...)), + query.Kind(query.Relationship(), scanLookupRegressionKinds(68)[0]), + query.Exists(query.RelationshipProperty("lastseen")), + query.Not(query.KindIn(query.End(), scanLookupRegressionKinds(64, 65)...)), + )), + query.Returning(query.RelationshipID()), + }, "e0.properties ? 'lastseen'", "not (e0.properties -> 'lastseen')", "array [100]::int2[]", "select (s0.e0).id") + }) + + for _, relationshipKind := range []int{70, 71} { + t.Run("SCAN-04 raw ownership representative "+twoDigitKindSuffix(relationshipKind), func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Relationship(), scanLookupRegressionKinds(relationshipKind)[0]), + query.Kind(query.Start(), scanLookupRegressionKinds(69)[0]), + )), + query.Returning(query.Relationship()), + }, "n0.kind_ids", "array [101]::int2[]", "select s0.e0 as r") + }) + } + + nineKinds := scanLookupRegressionKinds(72, 73, 74, 75, 76, 77, 78, 79, 80) + t.Run("SCAN-05 nine relationship kinds bound end", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Kind(query.Start(), scanLookupRegressionKinds(69)[0]), + query.KindIn(query.Relationship(), nineKinds...), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.Relationship(), query.Start()), + ) + require.Contains(t, formatted, "array [104, 105, 106, 107, 108, 109, 110, 111, 112]::int2[]") + require.Contains(t, formatted, "select s0.e0 as r, s0.n0 as s") + require.Equal(t, map[string]any{"pi0": uint64(202)}, translation.Parameters) + }) + + t.Run("SCAN-06 FetchKinds column order", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Relationship(), scanLookupRegressionKinds(82)[0]), + query.Kind(query.End(), scanLookupRegressionKinds(81)[0]), + )), + query.Returning(query.StartID(), query.RelationshipID(), query.KindsOf(query.Relationship()), query.EndID()), + }, "select s0.n0 as \"id(s)\", (s0.e0).id as \"id(r)\", kind_name((s0.e0).kind_id)::text as \"type(r)\", (s0.n1).id as \"id(e)\"") + }) + + t.Run("SCAN-07 directed start and end IDs", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.KindIn(query.Relationship(), scanLookupRegressionKinds(83, 84)...)), + query.Returning(query.StartID(), query.EndID()), + }, "array [115, 116]::int2[]", "select s0.n0 as \"id(s)\", s0.n1 as \"id(e)\"") + }) + + t.Run("SCAN-08 scenario A and B", func(t *testing.T) { + for name, testCase := range map[string]struct { + // endKinds optionally constrains the terminal node kinds. + endKinds graph.Kinds + // relKinds constrains the relationship kinds admitted by the scan. + relKinds graph.Kinds + }{ + "scenario A": {relKinds: scanLookupRegressionKinds(87, 88, 89, 90, 91, 92)}, + "scenario B": { + endKinds: scanLookupRegressionKinds(81), + relKinds: scanLookupRegressionKinds(87, 88, 89, 90, 91), + }, + } { + t.Run(name, func(t *testing.T) { + criteria := []graph.Criteria{ + query.KindIn(query.Start(), scanLookupRegressionKinds(85, 86, 81)...), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + query.KindIn(query.Relationship(), testCase.relKinds...), + } + if len(testCase.endKinds) > 0 { + criteria = append(criteria, query.KindIn(query.End(), testCase.endKinds...)) + } + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And(criteria...)), + query.Returning(query.StartID()), + }, "n0.kind_ids", "n1.id = any", "select (s0.n0).id") + }) + } + }) +} + +// TestLegacyBuilderPostgreSQL_NodeLookups verifies migrated node lookups preserve ID, kind, property, projection, and limit semantics. +func TestLegacyBuilderPostgreSQL_NodeLookups(t *testing.T) { + t.Run("LOOKUP-01 ID and full-node projections", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.KindIn(query.Node(), scanLookupRegressionKinds(85, 86)...)), + query.Returning(query.NodeID()), + }, "array [117, 118]::int2[]", "select (s0.n0).id") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Kind(query.Node(), scanLookupRegressionKinds(93)[0])), + query.Returning(query.Node()), + }, "array [125]::int2[]", "select s0.n0 as n") + }) + + t.Run("LOOKUP-02 equalities and limit", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(81)[0]), + query.Equals(query.NodeProperty("objectid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + query.Limit(1), + }, "n0.properties -> 'objectid'", "select s0.n0 as n", "limit 1") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Equals(query.NodeProperty("name"), "dc.example.test"), + query.Equals(query.NodeProperty("enabled"), true), + )), + query.Returning(query.NodeID()), + }, "n0.properties -> 'name'", "n0.properties -> 'enabled'", "select (s0.n0).id") + }) + + t.Run("LOOKUP-03 boolean two-column projection", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(81)[0]), + query.Equals(query.NodeProperty("hasura"), true), + )), + query.Returning(query.NodeID(), query.NodeProperty("hasura")), + }, "select (s0.n0).id as \"id(n)\", ((s0.n0).properties -> 'hasura') as \"n.hasura\"") + }) + + t.Run("LOOKUP-04 prefix suffix and equality", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(94)[0]), + query.StringStartsWith(query.NodeProperty("distinguishedname"), "CN=ADMINSDHOLDER,"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + }, "cypher_starts_with", "n0.properties -> 'domainsid'") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Or( + query.StringEndsWith(query.NodeProperty("objectid"), "-S-1"), + query.StringEndsWith(query.NodeProperty("objectid"), "-S-2"), + )), + query.Returning(query.NodeID()), + }, "cypher_ends_with", " or ") + }) + + t.Run("LOOKUP-05 case-insensitive strings preserve literals", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.CaseInsensitiveStringStartsWith(query.NodeProperty("name"), "Remote Desktop Users%_")), + query.Returning(query.NodeID()), + ) + require.Contains(t, formatted, "lower") + require.Contains(t, formatted, "cypher_starts_with") + require.Equal(t, map[string]any{"pi0": "remote desktop users%_"}, translation.Parameters) + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.CaseInsensitiveStringContains(query.NodeProperty("objectid"), "Approver_GUID")), + query.Returning(query.Node()), + }, "lower", "cypher_contains") + }) + + t.Run("LOOKUP-06 required and excluded kind groups", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.KindIn(query.Node(), scanLookupRegressionKinds(85, 86)...), + query.Kind(query.Node(), scanLookupRegressionKinds(69)[0]), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + }, "array [117, 118]::int2[]", "array [101]::int2[]", "cypher_ends_with") + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(69)[0]), + query.Not(query.KindIn(query.Node(), scanLookupRegressionKinds(85, 98)...)), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + )), + query.Returning(query.Node()), + }, "not", "array [117, 130]::int2[]") + }) + + t.Run("LOOKUP-07 missing property", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Not(query.Exists(query.NodeProperty("name")))), + query.Returning(query.Node()), + }, "n0.properties ? 'name'", "not (n0.properties -> 'name')", "not") + }) + + t.Run("LOOKUP-08 nullable approver disjunction", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(95)[0]), + query.Equals(query.NodeProperty("tenantid"), "tenant-1"), + query.Equals(query.NodeProperty("approvalrequired"), true), + query.Or( + query.IsNotNull(query.NodeProperty("userapprovers")), + query.IsNotNull(query.NodeProperty("groupapprovers")), + ), + )), + query.Returning(query.Node()), + }, "n0.properties ? 'userapprovers'", "n0.properties ? 'groupapprovers'", " or ") + }) + + t.Run("LOOKUP-09 duplicate ID list hydration", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202), graph.ID(101))), + query.Returning(query.Node()), + ) + require.Contains(t, formatted, "n0.id = any") + require.Contains(t, formatted, "select s0.n0 as n") + require.Equal(t, map[string]any{"pi0": []uint64{101, 202, 101}}, translation.Parameters) + }) + + t.Run("LOOKUP-10 nested negated flags", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Node(), scanLookupRegressionKinds(86)[0]), + query.Not(query.And(query.Exists(query.NodeProperty("gmsa")), query.Equals(query.NodeProperty("gmsa"), true))), + query.Not(query.And(query.Exists(query.NodeProperty("msa")), query.Equals(query.NodeProperty("msa"), true))), + query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202)), + )), + query.Returning(query.Node()), + }, "not", "n0.properties -> 'gmsa'", "n0.properties -> 'msa'", "n0.id = any") + }) + + t.Run("LOOKUP-11 tenant adjacency and endpoint property list", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), scanLookupRegressionKinds(97)[0]), + query.KindIn(query.End(), scanLookupRegressionKinds(95, 96)...), + query.In(query.EndProperty("roletemplateid"), []string{"role-a", "role-b"}), + )), + query.Returning(query.End()), + }, "n0.id = @pi0", "array [127, 128]::int2[]", "n1.properties ->> 'roletemplateid'", "select s0.n1 as e") + }) + + t.Run("LOOKUP-12 exact edge key and First", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), scanLookupRegressionKinds(83)[0]), + )), + query.Returning(query.Relationship()), + query.Limit(1), + }, "n0.id = @pi0", "n1.id = @pi1", "array [115]::int2[]", "select s0.e0 as r", "limit 1") + }) + + t.Run("LOOKUP-13 suffix with bound opposite endpoint projections", func(t *testing.T) { + for _, projection := range []graph.Criteria{query.Returning(query.Start()), query.Returning(query.StartID())} { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), scanLookupRegressionKinds(82)[0]), + query.Equals(query.EndID(), graph.ID(202)), + )), + projection, + }, "cypher_ends_with", "n1.id = @pi1") + } + }) + + t.Run("LOOKUP-14 descending property order", func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.Kind(query.Node(), scanLookupRegressionKinds(99)[0])), + query.Returning(query.Node()), + query.OrderBy(query.Order(query.NodeProperty("name"), query.Descending())), + }, "select s0.n0 as n", "order by", "desc") + }) + + t.Run("LOOKUP-16 typed and untyped four-property equalities", func(t *testing.T) { + for name, kindCriteria := range map[string]graph.Criteria{ + "typed": query.Kind(query.Node(), scanLookupRegressionKinds(81)[0]), + "untyped": query.And(), + } { + t.Run(name, func(t *testing.T) { + assertScanLookupTranslation(t, []graph.Criteria{ + query.Where(query.And( + kindCriteria, + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty("ldapavailable"), true), + query.Equals(query.NodeProperty("ldapsigning"), false), + )), + query.Returning(query.NodeID()), + }, "n0.properties -> 'domainsid'", "n0.properties -> 'isdc'", "n0.properties -> 'ldapavailable'", "n0.properties -> 'ldapsigning'", "select (s0.n0).id") + }) + } + }) +} diff --git a/cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go b/cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go new file mode 100644 index 00000000..879adef8 --- /dev/null +++ b/cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go @@ -0,0 +1,277 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "fmt" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderPostgreSQL_StandaloneHopForms verifies migrated one-hop queries preserve anchors, direction, kinds, and projections. +func TestLegacyBuilderPostgreSQL_StandaloneHopForms(t *testing.T) { + hopKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("RegressionKind%02d", idx+1)) + } + return kinds + } + + t.Run("HOP-01 exact and one-element IN start anchors", func(t *testing.T) { + for name, anchor := range map[string]graph.Criteria{ + "exact": query.Equals(query.StartID(), graph.ID(101)), + "in": query.InIDs(query.StartID(), graph.ID(101)), + } { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And(anchor, query.Kind(query.Relationship(), graph.StringKind("RegressionKind01")))), + query.Returning(query.Relationship(), query.End()), + ) + require.Contains(t, formatted, "n0.id = e0.start_id") + require.Contains(t, formatted, "e0.kind_id = any (array [33]::int2[])") + require.Contains(t, formatted, "select s0.e0 as r, s0.n1 as e") + if name == "exact" { + require.Equal(t, map[string]any{"pi0": uint64(101)}, translation.Parameters) + } else { + require.Equal(t, map[string]any{"pi0": []uint64{101}}, translation.Parameters) + } + }) + } + }) + + t.Run("HOP-02 end anchor and inbound projection", func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind01")), + )), + query.Returning(query.Relationship(), query.Start()), + ) + require.Contains(t, formatted, "n1.id = e0.end_id") + require.Contains(t, formatted, "select s0.e0 as r, s0.n0 as s") + require.Equal(t, map[string]any{"pi0": uint64(202)}, translation.Parameters) + }) + + for _, count := range []int{2, 5, 9, 30} { + kinds := hopKinds(count) + kindIDs := sequentialKindIDs(33, count) + + t.Run(fmt.Sprintf("HOP-03 outbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.End()), + ) + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", kindIDs)) + require.Contains(t, formatted, "n0.id = any") + require.Contains(t, formatted, "select s0.e0 as r, s0.n1 as e") + require.Equal(t, map[string]any{"pi0": []uint64{101}}, translation.Parameters) + }) + + t.Run(fmt.Sprintf("HOP-03 inbound %d kinds", count), func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.Start()), + ) + require.Contains(t, formatted, fmt.Sprintf("array [%s]::int2[]", kindIDs)) + require.Contains(t, formatted, "n1.id = any") + require.Contains(t, formatted, "select s0.e0 as r, s0.n0 as s") + require.Equal(t, map[string]any{"pi0": []uint64{202}}, translation.Parameters) + }) + } + + testCases := map[string]struct { + // criteria contains the legacy query-builder inputs for the case. + criteria []graph.Criteria + // fragments lists SQL fragments that the translation must contain. + fragments []string + // parameters is the exact parameter map expected from translation. + parameters map[string]any + }{ + "HOP-04 endpoint kind disjunction": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind51")), + query.KindIn(query.End(), graph.StringKind("RegressionKind52"), graph.StringKind("RegressionKind53")), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n1.kind_ids operator (pg_catalog.&&) array [84, 85]::int2[]", "e0.kind_id = any (array [83]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101}}, + }, + "HOP-05 endpoint IDs through variable spelling": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind54")), + query.InIDs(query.End(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n0.id = @pi0", "n1.id = any", "e0.kind_id = any (array [86]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": []uint64{202, 303}}, + }, + "HOP-05 endpoint IDs through identity-function spelling": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.Start(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind54")), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n0.id = any", "n1.id = any", "e0.kind_id = any (array [86]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101}, "pi1": []uint64{202, 303}}, + }, + "HOP-06 scalar endpoint properties": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind55")), + query.Equals(query.EndProperty("enabled"), true), + query.Equals(query.EndProperty("score"), 7), + query.Equals(query.EndProperty("name"), "target"), + query.Equals(query.EndProperty("isassignabletorole"), "true"), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n1.properties -> 'enabled'", "n1.properties -> 'score'", "n1.properties -> 'name'", "n1.properties -> 'isassignabletorole'", "e0.kind_id = any (array [87]::int2[])"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": true, "pi2": 7, "pi3": "target", "pi4": "true"}, + }, + "HOP-07 nested production branches": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind56")), + query.Kind(query.End(), graph.StringKind("RegressionKind57")), + query.Or( + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.GreaterThan(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authorizedsignatures"), 0), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.Equals(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + ), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{" or ", "n1.kind_ids operator (pg_catalog.&&) array [89]::int2[]", "n1.properties -> 'schemaversion'", "n1.properties -> 'authorizedsignatures'", "e0.kind_id = any (array [88]::int2[])"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": false, "pi2": 1, "pi3": 0, "pi4": true, "pi5": false, "pi6": 1, "pi7": true}, + }, + "HOP-08 collection and scalar OR": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind58")), + query.Or( + query.Equals(query.EndProperty("schannelauthenticationenabled"), true), + query.Equals(query.Size(query.EndProperty("effectiveekus")), 0), + query.InInverted(query.EndProperty("effectiveekus"), "1.3.6.1.5.5.7.3.2"), + ), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{" or ", "jsonb_array_length", "jsonb_to_text_array", "e0.kind_id = any (array [90]::int2[])"}, + parameters: map[string]any{"pi0": uint64(101), "pi1": true, "pi2": 0, "pi3": "1.3.6.1.5.5.7.3.2"}, + }, + "HOP-09 two-sided ID lists": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101), graph.ID(202)), + query.InIDs(query.EndID(), graph.ID(303), graph.ID(404)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind59")), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n0.id = any", "n1.id = any", "e0.kind_id = any (array [91]::int2[])", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101, 202}, "pi1": []uint64{303, 404}}, + }, + "HOP-10 outbound full direction": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + query.Kind(query.End(), graph.StringKind("RegressionKind52")), + query.Equals(query.EndProperty("active"), true), + )), + query.Returning(query.Relationship(), query.End()), + }, + fragments: []string{"n1.kind_ids operator (pg_catalog.&&) array [84]::int2[]", "n1.properties -> 'active'", "select s0.e0 as r, s0.n1 as e"}, + parameters: map[string]any{"pi0": []uint64{101}, "pi1": true}, + }, + "HOP-10 inbound full direction": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + query.Kind(query.Start(), graph.StringKind("RegressionKind51")), + query.Equals(query.StartProperty("active"), true), + )), + query.Returning(query.Relationship(), query.Start()), + }, + fragments: []string{"n0.kind_ids operator (pg_catalog.&&) array [83]::int2[]", "n0.properties -> 'active'", "select s0.e0 as r, s0.n0 as s"}, + parameters: map[string]any{"pi0": []uint64{202}, "pi1": true}, + }, + "HOP-10 start node projection": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + )), + query.Returning(query.Start()), + }, + fragments: []string{"select s0.n0 as s"}, + parameters: map[string]any{"pi0": []uint64{202}}, + }, + "HOP-10 end ID relationship projection": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("RegressionKind60")), + )), + query.Returning(query.EndID(), query.Relationship()), + }, + fragments: []string{"select s0.n1 as \"id(e)\", s0.e0 as r"}, + parameters: map[string]any{"pi0": []uint64{101}}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, testCase.criteria...) + for _, fragment := range testCase.fragments { + require.Contains(t, formatted, fragment) + } + require.Equal(t, testCase.parameters, translation.Parameters) + }) + } +} diff --git a/cypher/models/pgsql/test/testcase.go b/cypher/models/pgsql/test/testcase.go index 65dcf571..69297e7f 100644 --- a/cypher/models/pgsql/test/testcase.go +++ b/cypher/models/pgsql/test/testcase.go @@ -1,6 +1,7 @@ package test import ( + "bytes" "context" "embed" "encoding/json" @@ -25,12 +26,21 @@ import ( ) const ( - prefixCase = "case:" + // prefixCase introduces a named translation case in a fixture file. + prefixCase = "case:" + + // prefixExclusiveTest marks a fixture case that must run without the other cases. prefixExclusiveTest = "exclusive:" - prefixCypherParams = "cypher_params:" - prefixPgSQLParams = "pgsql_params:" + + // prefixCypherParams introduces the JSON parameter map supplied to Cypher translation. + prefixCypherParams = "cypher_params:" + + // prefixPgSQLParams introduces the JSON parameter map expected in rendered PostgreSQL. + prefixPgSQLParams = "pgsql_params:" ) +// testCaseFiles embeds the translation fixtures consumed by the package test runner. +// //go:embed translation_cases/* var testCaseFiles embed.FS @@ -61,6 +71,7 @@ func (s *TranslationTestCase) Copy() *TranslationTestCase { } } +// writeStrings writes each string to writer in order and returns the first write failure. func writeStrings(output io.Writer, strs ...string) error { for _, str := range strs { if _, err := output.Write([]byte(str)); err != nil { @@ -71,6 +82,7 @@ func writeStrings(output io.Writer, strs ...string) error { return nil } +// licenseHeader is the exact header required at the start of every generated fixture file. var licenseHeader = `-- Copyright %d Specter Ops, Inc. -- -- Licensed under the Apache License, Version 2.0 @@ -147,6 +159,7 @@ func (s *TranslationTestCase) WriteTo(output io.Writer, kindMapper pgsql.KindMap return nil } +// Assert translates the case and compares normalized SQL and parameters with the golden expectations. func (s *TranslationTestCase) Assert(t *testing.T, expectedSQL string, kindMapper pgsql.KindMapper) { if regularQuery, err := frontend.ParseCypher(frontend.NewContext(), s.Cypher); err != nil { t.Fatalf("Failed to compile cypher query: %s - %v", s.Cypher, err) @@ -177,7 +190,15 @@ func (s *TranslationTestCase) Assert(t *testing.T, expectedSQL string, kindMappe require.Equalf(t, expectedSQL, normalizedActual, "Test case for cypher query: '%s' failed to match.", s.Cypher) if s.PgSQLParams != nil { - require.Equal(t, s.PgSQLParams, translation.Parameters) + // Golden parameters are stored as JSON, whose decoder represents + // numbers as float64. Compare the translated bag through the same + // serialization boundary so typed integer parameters do not create a + // false mismatch while their values and emitted casts remain exact. + var normalizedParameters map[string]any + encodedParameters, err := json.Marshal(translation.Parameters) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(encodedParameters, &normalizedParameters)) + require.Equal(t, s.PgSQLParams, normalizedParameters) } } } @@ -323,6 +344,7 @@ func ReadTranslationTestCaseFile(path string, fin fs.File) (TranslationTestCaseF }, err } +// updatedCasesDir returns the configured fixture update directory or an isolated temporary directory. func updatedCasesDir() (string, error) { if workingDir, err := os.Getwd(); err != nil { return "", err @@ -337,6 +359,7 @@ func updatedCasesDir() (string, error) { } } +// UpdateTranslationTestCases regenerates SQL golden files from their embedded Cypher cases. func UpdateTranslationTestCases(mapper pgsql.KindMapper) error { if updatedCasesPath, err := updatedCasesDir(); err != nil { return err @@ -357,20 +380,28 @@ func UpdateTranslationTestCases(mapper pgsql.KindMapper) error { return err } else if nextCases, _, err := caseFile.Load(); err != nil { return err - } else if output, err := os.OpenFile(updatedCaseFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644); err != nil { - return err } else { + var output strings.Builder formattedLicenseHeader := fmt.Sprintf(licenseHeader, time.Now().Year()) - if _, err := io.WriteString(output, formattedLicenseHeader); err != nil { + if _, err := io.WriteString(&output, formattedLicenseHeader); err != nil { return err } for _, nextCase := range nextCases { - nextCase.WriteTo(output, mapper) + if err := nextCase.WriteTo(&output, mapper); err != nil { + return err + } } - output.Close() + trailingNewlines := "\n" + if bytes.HasSuffix(caseFile.content, []byte("\n\n")) { + trailingNewlines = "\n\n" + } + content := strings.TrimRight(output.String(), "\n") + trailingNewlines + if err := os.WriteFile(updatedCaseFilePath, []byte(content), 0644); err != nil { + return err + } } } } diff --git a/cypher/models/pgsql/test/translation_cases/delete.sql b/cypher/models/pgsql/test/translation_cases/delete.sql index c6695b5d..db750976 100644 --- a/cypher/models/pgsql/test/translation_cases/delete.sql +++ b/cypher/models/pgsql/test/translation_cases/delete.sql @@ -21,5 +21,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; -- case: match ()-[]->()-[r:EdgeKind1]->() delete r -with s0 as (select e0.id as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) and e1.id != s0.e0), s2 as (delete from edge e2 using s1 where (s1.e1).id = e2.id) select 1; +with s0 as (select e0.id as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) and e1.id != s0.e0), s2 as (delete from edge e2 using s1 where (s1.e1).id = e2.id) select 1; +-- case: match (s)-[*1..]->(mid)-[]->(e) delete mid +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, true, false, array [e0.id] from edge e0 join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, true, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id)), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)), s3 as (delete from node n3 using s2 where (s2.n1).id = n3.id) select 1; diff --git a/cypher/models/pgsql/test/translation_cases/multipart.sql b/cypher/models/pgsql/test/translation_cases/multipart.sql index 1317707c..294273a9 100644 --- a/cypher/models/pgsql/test/translation_cases/multipart.sql +++ b/cypher/models/pgsql/test/translation_cases/multipart.sql @@ -24,13 +24,13 @@ with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposit with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'value'))::jsonb = to_jsonb((1)::int8)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'me'))) select s3.n1 as n1 from s3), s4 as (select s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s2, node n2 where (n2.id = (s2.n1).id)) select s4.n2 as b from s4; -- case: match (n:NodeKind1)-[:EdgeKind1*1..]->(:NodeKind2)-[:EdgeKind2]->(m:NodeKind1) where (n:NodeKind1 or n:NodeKind2) and n.enabled = true with m, collect(distinct(n)) as p where size(p) >= 10 return m -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, array_remove(coalesce(array_agg(distinct (s3.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n2) select s0.n2 as m from s0 where (cardinality(s0.i0)::int >= 10); +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, array_remove(coalesce(array_agg(distinct (s3.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n2) select s0.n2 as m from s0 where (cardinality(s0.i0)::int >= 10); -- case: match (n:NodeKind1)-[:EdgeKind1*1..]->(:NodeKind2)-[:EdgeKind2]->(m:NodeKind1) where (n:NodeKind1 or n:NodeKind2) and n.enabled = true with m, count(distinct(n)) as p where p >= 10 return m -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); -- case: match (n:NodeKind1)-[:EdgeKind1*1..]->(:NodeKind2)-[:EdgeKind2]->(m:NodeKind1) where (n:NodeKind1 or n:NodeKind2) and n.enabled = true with m, count(distinct(n)) as p where p >= 10 return m -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select s3.n2 as n2, count(distinct (s3.n0))::int8 as i0 from s3 group by n2) select s0.n2 as m from s0 where (s0.i0 >= 10); -- case: with 365 as max_days match (n:NodeKind1) where n.pwdlastset < (datetime().epochseconds - (max_days * 86400)) and not n.pwdlastset IN [-1.0, 0.0] return n limit 100 with s0 as (select 365 as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from s0, node n0 where (not ((n0.properties ->> 'pwdlastset'))::float8 = any (array [- 1, 0]::float8[]) and ((n0.properties ->> 'pwdlastset'))::numeric < (extract(epoch from now()::timestamp with time zone)::numeric - (s0.i0 * 86400))) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n from s1 limit 100; @@ -39,10 +39,10 @@ with s0 as (select 365 as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n with recursive candidate_sources(root_id) as (select source_node.id as root_id from node source_node where (((source_node.properties -> 'hasspn'))::jsonb = to_jsonb((true)::bool)::jsonb and ((source_node.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb and not coalesce((source_node.properties ->> 'objectid'), '')::text like '%-502' and not coalesce(((source_node.properties ->> 'gmsa'))::bool, false)::bool = true and not coalesce(((source_node.properties ->> 'msa'))::bool, false)::bool = true) and source_node.kind_ids operator (pg_catalog.@>) array [1]::int2[]), traversal(root_id, next_id, depth, path) as (select candidate_sources.root_id, e.end_id, 1, array [e.id]::int8[] from candidate_sources join edge e on e.start_id = candidate_sources.root_id where e.kind_id = any (array [3, 4]::int2[]) union all select traversal.root_id, e.end_id, traversal.depth + 1, traversal.path || e.id from traversal join lateral (select e.id, e.start_id, e.end_id from edge e where e.start_id = traversal.next_id and e.id != all (traversal.path) and e.kind_id = any (array [3, 4]::int2[]) offset 0) e on true where traversal.depth < 15), terminal_nodes(id) as materialized (select terminal_node.id from node terminal_node where terminal_node.kind_ids operator (pg_catalog.@>) array [2]::int2[]), terminal_hits(root_id) as (select traversal.root_id from traversal join terminal_nodes on terminal_nodes.id = traversal.next_id), ranked(root_id, adminCount) as (select terminal_hits.root_id, count(*)::int8 as adminCount from terminal_hits group by terminal_hits.root_id order by adminCount desc limit 100) select (source_node.id, source_node.kind_ids, source_node.properties)::nodecomposite as n from ranked join node source_node on source_node.id = ranked.root_id order by ranked.adminCount desc; -- case: match (n:NodeKind1) where n.objectid = 'S-1-5-21-1260426776-3623580948-1897206385-23225' match p = (n)-[:EdgeKind1|EdgeKind2*1..]->(c:NodeKind2) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = 'S-1-5-21-1260426776-3623580948-1897206385-23225')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = 'S-1-5-21-1260426776-3623580948-1897206385-23225')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, s1.ep0, array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (a) with a match (b) with a, b match (a)-[]-(b) return a -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1) select s3.n0 as n0, s3.n1 as n1 from s3), s4 as (select s2.n0 as n0, s2.n1 as n1 from s2 join edge e0 on (((s2.n0).id = e0.start_id and (s2.n1).id = e0.end_id) or ((s2.n1).id = e0.start_id and (s2.n0).id = e0.end_id)) where ((s2.n0).id <> (s2.n1).id)) select s4.n0 as a from s4; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1) select s3.n0 as n0, s3.n1 as n1 from s3), s4 as (select s2.n0 as n0, s2.n1 as n1 from s2 join edge e0 on (((s2.n0).id = e0.start_id and (s2.n1).id = e0.end_id) or ((s2.n1).id = e0.start_id and (s2.n0).id = e0.end_id))) select s4.n0 as a from s4; -- case: match (g1:NodeKind1) where g1.name starts with 'test' with collect (g1.domain) as excludes match (d:NodeKind2) where d.name starts with 'other' and not d.name in excludes return d with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'name') like 'test%') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select array_remove(coalesce(array_agg(((s1.n0).properties ->> 'domain'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1), s2 as (select s0.i0 as i0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (not (n1.properties ->> 'name') = any (s0.i0) and (n1.properties ->> 'name') like 'other%') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s2.n1 as d from s2; @@ -51,25 +51,25 @@ with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposit with s0 as (select 'a' as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from s0, node n0 where ((jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = ' ') and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as o from s1; -- case: match (dc)-[r:EdgeKind1*0..]->(g:NodeKind1) where g.objectid ends with '-516' with collect(dc) as exclude match p = (c:NodeKind2)-[n:EdgeKind2]->(u:NodeKind2)-[:EdgeKind2*1..]->(g:NodeKind1) where g.objectid ends with '-512' and not c in exclude return p limit 100 -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'objectid') like '%-516') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select s2_seed.root_id, s2_seed.root_id, 0, false, false, array []::int8[] from s2_seed union all select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, false, false, e0.id || s2.path from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true where s2.depth < 15 and not s2.is_cycle and s2.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.next_id offset 0) n0 on true) select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i0 from s1), s3 as (select e1.id as e1, s0.i0 as i0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s0, edge e1 join node n3 on n3.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n3.id = e1.end_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.start_id where (not n2.id = any (s0.i0)) and e1.kind_id = any (array [4]::int2[])), s4 as (with recursive s5_seed(root_id) as not materialized (select distinct (s3.n3).id as root_id from s3), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.start_id = e2.end_id, array [e2.id] from s5_seed join edge e2 on e2.start_id = s5_seed.root_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [4]::int2[]) union all select s5.root_id, e2.end_id, s5.depth + 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s5.path || e2.id from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [4]::int2[]) offset 0) e2 on true join node n4 on n4.id = e2.end_id where s5.depth < 15 and not s5.is_cycle) select s3.e1 as e1, s5.path as ep1, s3.i0 as i0, s3.n2 as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s3, s5 join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.root_id offset 0) n3 on true join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.next_id offset 0) n4 on true where s5.satisfied and (s3.n3).id = s5.root_id and s3.e1 != all (s5.path) limit 100) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n2, s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4 limit 100; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'objectid') like '%-516') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select s2_seed.root_id, s2_seed.root_id, 0, false, false, array []::int8[] from s2_seed union all select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, false, false, e0.id || s2.path from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true where s2.depth < 15 and not s2.is_cycle and s2.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.next_id offset 0) n0 on true) select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i0 from s1), s3 as (select e1.id as e1, s0.i0 as i0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s0, edge e1 join node n3 on n3.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n3.id = e1.end_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.start_id where (not n2.id = any (s0.i0)) and e1.kind_id = any (array [4]::int2[])), s4 as (with recursive s4_endpoint_seeded_endpoints as materialized (select n4.id as id, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from node n4 where ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[] limit 33), s4_endpoint_seeded_reverse(root_id, next_id, depth, path) as (select s4_endpoint_seeded_endpoints.id, s4_endpoint_seeded_endpoints.id, 0, array []::int8[] from s4_endpoint_seeded_endpoints union all select s4_endpoint_seeded_reverse.root_id, e2.start_id, s4_endpoint_seeded_reverse.depth + 1, array_prepend(e2.id, s4_endpoint_seeded_reverse.path)::int8[] from s4_endpoint_seeded_reverse join edge e2 on e2.end_id = s4_endpoint_seeded_reverse.next_id where s4_endpoint_seeded_reverse.depth < 15 and e2.id != all (s4_endpoint_seeded_reverse.path) and e2.kind_id = any (array [4]::int2[])), s4_endpoint_seeded_states as materialized (select s4_endpoint_seeded_reverse.root_id, s4_endpoint_seeded_reverse.next_id, s4_endpoint_seeded_reverse.depth, s4_endpoint_seeded_reverse.path from s4_endpoint_seeded_reverse limit 4097), s4_endpoint_seeded_incumbent as materialized (with recursive s5_seed(root_id) as not materialized (select distinct (s3.n3).id as root_id from s3), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.start_id = s5_seed.root_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [4]::int2[]) union all select s5.root_id, e2.end_id, s5.depth + 1, ((n4.properties ->> 'objectid') like '%-512') and n4.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s5.path || e2.id from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [4]::int2[]) offset 0) e2 on true join node n4 on n4.id = e2.end_id where s5.depth < 15 and not s5.is_cycle) select s3.e1 as e1, s5.path as ep1, s3.i0 as i0, s3.n2 as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s3, s5 join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.root_id offset 0) n3 on true join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.next_id offset 0) n4 on true where s5.satisfied and (s3.n3).id = s5.root_id and s3.e1 != all (s5.path) and not s5.path && array [s3.e1]::int8[]) select s3.e1 as e1, s4_endpoint_seeded_states.path as ep1, s3.i0 as i0, s3.n2 as n2, s3.n3 as n3, s4_endpoint_seeded_endpoints.n4 as n4 from s3 join s4_endpoint_seeded_states on (s3.n3).id = s4_endpoint_seeded_states.next_id join s4_endpoint_seeded_endpoints on s4_endpoint_seeded_endpoints.id = s4_endpoint_seeded_states.root_id where not exists (select 1 from s4_endpoint_seeded_endpoints offset 32 limit 1) and not exists (select 1 from s4_endpoint_seeded_states offset 4096 limit 1) and s4_endpoint_seeded_states.depth >= 1 and not s4_endpoint_seeded_states.path && array [s3.e1]::int8[] union all select s4_endpoint_seeded_incumbent.e1 as e1, s4_endpoint_seeded_incumbent.ep1 as ep1, s4_endpoint_seeded_incumbent.i0 as i0, s4_endpoint_seeded_incumbent.n2 as n2, s4_endpoint_seeded_incumbent.n3 as n3, s4_endpoint_seeded_incumbent.n4 as n4 from s4_endpoint_seeded_incumbent where exists (select 1 from s4_endpoint_seeded_endpoints offset 32 limit 1) or exists (select 1 from s4_endpoint_seeded_states offset 4096 limit 1) limit 100) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n2, array [s4.e1]::int8[] || s4.ep1, array [s4.n2, s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4 limit 100; -- case: match (n:NodeKind1)<-[:EdgeKind1]-(:NodeKind2) where n.objectid ends with '-516' with n, count(n) as dc_count where dc_count = 1 return n with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((n0.properties ->> 'objectid') like '%-516') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s1.n0 as n0, count(s1.n0)::int8 as i0 from s1 group by n0) select s0.n0 as n from s0 where (s0.i0 = 1); -- case: match (n:NodeKind1)-[:EdgeKind1]->(m:NodeKind2) where n.enabled = true with n, collect(distinct(n)) as p where size(p) >= 100 match p = (n)-[:EdgeKind1]->(m) return p limit 10 -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on (((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select s1.n0 as n0, array_remove(coalesce(array_agg(distinct (s1.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s1 group by n0), s2 as (select e1.id as e1, s0.i0 as i0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (cardinality(s0.i0)::int >= 100) and (s0.n0).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) limit 10) select case when (s2.n0).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on (((n0.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select s1.n0 as n0, array_remove(coalesce(array_agg(distinct (s1.n0))::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s1 group by n0), s2 as (select e1.id as e1, s0.i0 as i0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (cardinality(s0.i0)::int >= 100) and (s0.n0).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) limit 10) select case when (s2.n0).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e1]::int8[], array [s2.n0, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; -- case: with "a" as check, "b" as ref match p = (u)-[:EdgeKind1]->(g:NodeKind1) where u.name starts with check and u.domain = ref with collect(tolower(g.samaccountname)) as refmembership, tolower(u.samaccountname) as samname return refmembership, samname with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text) select s1.i2 as refmembership, s1.i3 as samname from s1; -- case: with "a" as check, "b" as ref match p = (u)-[:EdgeKind1]->(g:NodeKind1) where u.name starts with check and u.domain = ref with collect(tolower(g.samaccountname)) as refmembership, tolower(u.samaccountname) as samname match (u)-[:EdgeKind2]-(g:NodeKind1) where tolower(u.samaccountname) = samname and not tolower(g.samaccountname) IN refmembership return g -with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text), s3 as (select s1.i2 as i2, s1.i3 as i3, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, edge e1 join node n2 on (n2.id = e1.end_id or n2.id = e1.start_id) join node n3 on n3.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n3.id = e1.end_id or n3.id = e1.start_id) where (n2.id <> n3.id) and (not lower((n3.properties ->> 'samaccountname'))::text = any (s1.i2)) and e1.kind_id = any (array [4]::int2[]) and (lower((n2.properties ->> 'samaccountname'))::text = s1.i3)) select s3.n3 as g from s3; +with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text), s3 as (select s1.i2 as i2, s1.i3 as i3, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, edge e1 join node n2 on (n2.id = e1.end_id or n2.id = e1.start_id) join node n3 on n3.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n3.id = e1.end_id or n3.id = e1.start_id) where ((n2.id = e1.start_id and n3.id = e1.end_id) or (n3.id = e1.start_id and n2.id = e1.end_id)) and (not lower((n3.properties ->> 'samaccountname'))::text = any (s1.i2)) and e1.kind_id = any (array [4]::int2[]) and (lower((n2.properties ->> 'samaccountname'))::text = s1.i3)) select s3.n3 as g from s3; -- case: with "a" as check, "b" as ref match p = (u)-[:EdgeKind1]->(g:NodeKind1) where u.name starts with check and u.domain = ref with collect(tolower(g.samaccountname)) as refmembership, tolower(u.samaccountname) as samname match (u)-[:EdgeKind2]->(g:NodeKind1) where tolower(u.samaccountname) = samname and not tolower(g.samaccountname) IN refmembership return g with s0 as (select 'a' as i0, 'b' as i1), s1 as (with s2 as (select s0.i0 as i0, s0.i1 as i1, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) and ((n0.properties ->> 'domain') = s0.i1 and cypher_starts_with((n0.properties ->> 'name'), (i0)::text)::bool)) select array_remove(coalesce(array_agg(lower(((s2.n1).properties ->> 'samaccountname'))::text)::text[], array []::text[])::text[], null)::text[] as i2, lower(((s2.n0).properties ->> 'samaccountname'))::text as i3 from s2 group by lower(((s2.n0).properties ->> 'samaccountname'))::text), s3 as (select s1.i2 as i2, s1.i3 as i3, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, edge e1 join node n2 on n2.id = e1.start_id join node n3 on n3.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n3.id = e1.end_id where (not lower((n3.properties ->> 'samaccountname'))::text = any (s1.i2)) and e1.kind_id = any (array [4]::int2[]) and (lower((n2.properties ->> 'samaccountname'))::text = s1.i3)) select s3.n3 as g from s3; -- case: match p =(n:NodeKind1)<-[r:EdgeKind1|EdgeKind2*..3]-(u:NodeKind1) where n.domain = 'test' with n, count(r) as incomingCount where incomingCount > 90 with collect(n) as lotsOfAdmins match p =(n:NodeKind1)<-[:EdgeKind1]-() where n in lotsOfAdmins return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = 'test')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.end_id = e0.start_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s2.depth < 3 and not s2.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied) select s1.n0 as n0, count(s1.e0)::int8 as i0 from s1 group by n0), s3 as (select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i1 from s0 where (s0.i0 > 90)), s4 as (select e1.id as e1, s3.i1 as i1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s3, edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id join node n3 on n3.id = e1.start_id where e1.kind_id = any (array [3]::int2[]) and (n2.id = any (s3.i1))) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null then null else ordered_edges_to_path(s4.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n2, s4.n3]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = 'test')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.start_id, s2.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s2.depth < 3 and not s2.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0) as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied) select s1.n0 as n0, count(s1.e0)::int8 as i0 from s1 group by n0), s3 as (select array_remove(coalesce(array_agg((n0).id)::int8[], array []::int8[])::int8[], null)::int8[] as i1 from s0 where (s0.i0 > 90)), s4 as (select e1.id as e1, s3.i1 as i1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s3, edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id join node n3 on n3.id = e1.start_id where e1.kind_id = any (array [3]::int2[]) and (n2.id = any (s3.i1))) select case when (s4.n2).id is null or s4.e1 is null or (s4.n3).id is null then null else ordered_edge_ids_to_path(0, s4.n2, array [s4.e1]::int8[], array [s4.n2, s4.n3]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (u:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) with g match (g)<-[:EdgeKind1]-(u:NodeKind1) return g with s0 as (with s1 as (select (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select s1.n1 as n1 from s1), s2 as (select s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.start_id where e1.kind_id = any (array [3]::int2[])) select s2.n1 as g from s2; @@ -78,19 +78,19 @@ with s0 as (with s1 as (select (n1.id, n1.kind_ids, n1.properties)::nodecomposit with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'name') ~ '.*TT' and (jsonb_typeof((n0.properties -> 'domain')) = 'string' and (n0.properties ->> 'domain') = 'MY DOMAIN')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select array_remove(coalesce(array_agg(((s1.n0).properties ->> 'email'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1), s2 as (select s0.i0 as i0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) and (not (n2.properties ->> 'email') = any (s0.i0) and (n2.properties ->> 'name') like 'blah%')) select s2.n1 as o from s2; -- case: match (e) match p = ()-[]->(e) return p limit 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n1, array [s1.e0]::int8[], array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; -- case: match p = (a)-[]->() match q = ()-[]->(a) return p, q -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n2, array [s1.e1]::int8[], array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; -- case: match (m:NodeKind1)-[*1..]->(g:NodeKind2)-[]->(c3:NodeKind1) where not g.name in ["foo"] with collect(g.name) as bar match p=(m:NodeKind1)-[*1..]->(g:NodeKind2) where g.name in bar return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.end_id = e2.start_id, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n3, s4.ep1, array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (m:NodeKind1)-[:EdgeKind1*1..]->(g:NodeKind2)-[:EdgeKind2]->(c3:NodeKind1) where m.samaccountname =~ '^[A-Z]{1,3}[0-9]{1,3}$' and not m.samaccountname contains "DEX" and not g.name IN ["D"] and not m.samaccountname =~ "^.*$" with collect(g.name) as admingroups match p=(m:NodeKind1)-[:EdgeKind1*1..]->(g:NodeKind2) where m.samaccountname =~ '^[A-Z]{1,3}[0-9]{1,3}$' and g.name in admingroups and not m.samaccountname =~ "^.*$" return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not coalesce((n0.properties ->> 'samaccountname'), '')::text like '%DEX%' and not (n0.properties ->> 'samaccountname') ~ '^.*$') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.end_id = e2.start_id, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id where e2.kind_id = any (array [3]::int2[]) union select s5.root_id, e2.start_id, s5.depth + 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not coalesce((n0.properties ->> 'samaccountname'), '')::text like '%DEX%' and not (n0.properties ->> 'samaccountname') ~ '^.*$') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['D']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [4]::int2[]))), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id where e2.kind_id = any (array [3]::int2[]) union select s5.root_id, e2.start_id, s5.depth + 1, ((n3.properties ->> 'samaccountname') ~ '^[A-Z]{1,3}[0-9]{1,3}$' and not (n3.properties ->> 'samaccountname') ~ '^.*$') and n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n3, s4.ep1, array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (a:NodeKind2)-[:EdgeKind1]->(g:NodeKind1)-[:EdgeKind2]->(s:NodeKind2) with count(a) as uc where uc > 5 match p = (a)-[:EdgeKind1]->(g)-[:EdgeKind2]->(s) return p -with s0 as (with s1 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s1.e0) select count(s2.n0)::int8 as i0 from s2), s3 as (select e2.id as e2, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, edge e2 join node n3 on n3.id = e2.start_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) and (s0.i0 > 5)), s4 as (select s3.e2 as e2, e3.id as e3, s3.i0 as i0, s3.n3 as n3, s3.n4 as n4, (n5.id, n5.kind_ids, n5.properties)::nodecomposite as n5 from s3 join edge e3 on (s3.n4).id = e3.start_id join node n5 on n5.id = e3.end_id where e3.kind_id = any (array [4]::int2[]) and e3.id != s3.e2) select case when (s4.n3).id is null or s4.e2 is null or (s4.n4).id is null or s4.e3 is null or (s4.n5).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e2]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s4.e3]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4, s4.n5]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s1.e0) select count(s2.n0)::int8 as i0 from s2), s3 as (select e2.id as e2, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, edge e2 join node n3 on n3.id = e2.start_id join node n4 on n4.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) and (s0.i0 > 5)), s4 as (select s3.e2 as e2, e3.id as e3, s3.i0 as i0, s3.n3 as n3, s3.n4 as n4, (n5.id, n5.kind_ids, n5.properties)::nodecomposite as n5 from s3 join edge e3 on (s3.n4).id = e3.start_id join node n5 on n5.id = e3.end_id where e3.kind_id = any (array [4]::int2[]) and e3.id != s3.e2) select case when (s4.n3).id is null or s4.e2 is null or (s4.n4).id is null or s4.e3 is null or (s4.n5).id is null then null else ordered_edge_ids_to_path(0, s4.n3, array [s4.e2]::int8[] || array [s4.e3]::int8[], array [s4.n3, s4.n4, s4.n5]::nodecomposite[])::pathcomposite end as p from s4; -- case: match (g:NodeKind1) optional match (g)<-[r:EdgeKind1]-(m:NodeKind2) with g, count(r) as memberCount where memberCount = 0 return g with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, s1.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join edge e0 on (s1.n0).id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])), s3 as (select s1.n0 as n0, s2.e0 as e0, s2.n1 as n1 from s1 left outer join s2 on (s1.n0 = s2.n0)) select s3.n0 as n0, count(s3.e0)::int8 as i0 from s3 group by n0) select s0.n0 as g from s0 where (s0.i0 = 0); diff --git a/cypher/models/pgsql/test/translation_cases/nodes.sql b/cypher/models/pgsql/test/translation_cases/nodes.sql index 5b10c34e..ac29c06e 100644 --- a/cypher/models/pgsql/test/translation_cases/nodes.sql +++ b/cypher/models/pgsql/test/translation_cases/nodes.sql @@ -15,7 +15,7 @@ -- SPDX-License-Identifier: Apache-2.0 -- case: match (n) return labels(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(n)" from s0; -- case: match (n) where 'NodeKind1' in labels(n) return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 where ('NodeKind1' = any ((array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[])); @@ -24,10 +24,10 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 where ((array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] = array ['NodeKind1', 'NodeKind2']::text[]); -- case: match (n) where n.name = 'n3' with labels(n) as labels return labels, size(labels) -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n3'))) select (array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as i0 from s1) select s0.i0 as labels, cardinality(s0.i0)::int from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n3'))) select (array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as i0 from s1) select s0.i0 as labels, cardinality(s0.i0)::int as "size(labels)" from s0; -- case: match (n) with 1 as _kind_idx, n return labels(n), _kind_idx -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select 1 as i0, s1.n0 as n0 from s1) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[], s0.i0 as _kind_idx from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select 1 as i0, s1.n0 as n0 from s1) select (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(n)", s0.i0 as _kind_idx from s0; -- case: match (n:NodeKind1) return n.name as displayname order by displayname with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select ((s0.n0).properties -> 'name') as displayname from s0 order by displayname; @@ -72,7 +72,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 where ((jsonb_typeof((((s0.n0)).properties -> 'a-aaa')) = 'string' and (((s0.n0)).properties ->> 'a-aaa') = '123')); -- case: match ()-[r]-() where startNode(r).`something` = "abc" return r -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where (n0.id <> n1.id)) select s0.e0 as r from s0 where ((jsonb_typeof(((start_node(((s0.e0).id, (s0.e0).start_id, (s0.e0).end_id, (s0.e0).kind_id, (s0.e0).properties)::edgecomposite)::nodecomposite).properties -> 'something')) = 'string' and ((start_node(((s0.e0).id, (s0.e0).start_id, (s0.e0).end_id, (s0.e0).kind_id, (s0.e0).properties)::edgecomposite)::nodecomposite).properties ->> 'something') = 'abc')); +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((n0.id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and n0.id = e0.end_id))) select s0.e0 as r from s0 where ((jsonb_typeof(((start_node(((s0.e0).id, (s0.e0).start_id, (s0.e0).end_id, (s0.e0).kind_id, (s0.e0).properties)::edgecomposite)::nodecomposite).properties -> 'something')) = 'string' and ((start_node(((s0.e0).id, (s0.e0).start_id, (s0.e0).end_id, (s0.e0).kind_id, (s0.e0).properties)::edgecomposite)::nodecomposite).properties ->> 'something') = 'abc')); -- case: match (n:NodeKind1 {name: "SOME NAME"}) return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'SOME NAME')) select s0.n0 as n from s0; @@ -100,7 +100,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]))) select s0.n0 as s from s0; -- case: match (n:NodeKind1), (e) where n.name = e.name return n -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (((s0.n0).properties -> 'name') = (n1.properties -> 'name'))) select s1.n0 as n from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (nullif(((s0.n0).properties -> 'name'), ('null')::jsonb)::jsonb = nullif((n1.properties -> 'name'), ('null')::jsonb)::jsonb)) select s1.n0 as n from s1; -- case: match (s), (e) where id(s) in e.captured_ids return s, e with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((s0.n0).id = any (jsonb_to_text_array((n1.properties -> 'captured_ids'))::int8[]))) select s1.n0 as s, s1.n1 as e from s1; @@ -112,7 +112,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))) select s0.n0 as s from s0; -- case: match (s:NodeKind1), (e:NodeKind2) where s.selected or s.tid = e.tid and e.enabled return s, e -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((((s0.n0).properties ->> 'selected'))::bool or ((s0.n0).properties -> 'tid') = (n1.properties -> 'tid') and ((n1.properties ->> 'enabled'))::bool) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((((s0.n0).properties ->> 'selected'))::bool or nullif(((s0.n0).properties -> 'tid'), ('null')::jsonb)::jsonb = nullif((n1.properties -> 'tid'), ('null')::jsonb)::jsonb and ((n1.properties ->> 'enabled'))::bool) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; -- case: match (s) where s.value + 2 / 3 > 10 return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties ->> 'value'))::int8 + 2 / 3 > 10)) select s0.n0 as s from s0; @@ -127,10 +127,10 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (lower((n0.properties ->> 'name'))::text = '1234')) select distinct s0.n0 as s from s0; -- case: match (s:NodeKind1), (e:NodeKind2) where s.name = e.name return s, e -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (((s0.n0).properties -> 'name') = (n1.properties -> 'name')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (nullif(((s0.n0).properties -> 'name'), ('null')::jsonb)::jsonb = nullif((n1.properties -> 'name'), ('null')::jsonb)::jsonb) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as s, s1.n1 as e from s1; -- case: match (n) where n.system_tags is not null and not (n:NodeKind1 or n:NodeKind2) return id(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ? 'system_tags' and not (n0.properties -> 'system_tags') = ('null')::jsonb) and not (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]))) select (s0.n0).id from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ? 'system_tags' and not (n0.properties -> 'system_tags') = ('null')::jsonb) and not (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]))) select (s0.n0).id as "id(n)" from s0; -- case: match (s), (e) where s.name = '1234' and e.other = 1234 return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (((n1.properties -> 'other'))::jsonb = to_jsonb((1234)::int8)::jsonb)) select s1.n0 as s from s1; @@ -139,7 +139,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof(((s0.n0).properties -> 'name')) = 'string' and ((s0.n0).properties ->> 'name') = '1234') or ((n1.properties -> 'other'))::jsonb = to_jsonb((1234)::int8)::jsonb)) select s1.n0 as s from s1; -- case: match (n), (k) where n.name = '1234' and k.name = '1234' match (e) where e.name = n.name return k, e -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = '1234'))), s2 as (select s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1, node n2 where ((n2.properties -> 'name') = ((s1.n0).properties -> 'name'))) select s2.n1 as k, s2.n2 as e from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = '1234'))), s2 as (select s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s1, node n2 where (nullif((n2.properties -> 'name'), ('null')::jsonb)::jsonb = nullif(((s1.n0).properties -> 'name'), ('null')::jsonb)::jsonb)) select s2.n1 as k, s2.n2 as e from s2; -- case: match (n) return n skip 5 limit 10 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as n from s0 offset 5 limit 10; @@ -190,10 +190,10 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'isassignabletorole'))::jsonb = to_jsonb((true)::bool)::jsonb)) select s0.n0 as s from s0; -- case: match (s) return s.value + 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + 1 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + 1 as "s.value + 1" from s0; -- case: match (s) return (s.value + 1) / 3 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((((s0.n0).properties ->> 'value'))::int8 + 1) / 3 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((((s0.n0).properties ->> 'value'))::int8 + 1) / 3 as "(s.value + 1) / 3" from s0; -- case: match (s) where id(s) in [1, 2, 3, 4] return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.id = any (array [1, 2, 3, 4]::int8[]))) select s0.n0 as s from s0; @@ -241,16 +241,16 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); -- case: match (s) where not (s)-[]->()-[]->() return s -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on n1.id = e0.end_id where (s0.n0).id = e0.start_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select e0.id as e0, s0.n0 as n0, n1.id as n1 from edge e0 join node n1 on n1.id = e0.end_id where (s0.n0).id = e0.start_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on s1.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); -- case: match (s) where ()-[]->()-[]->(s) return s -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from edge e0 join node n1 on n1.id = e0.start_id join node n2 on n2.id = e0.end_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on (s1.n2).id = e1.start_id join node n0 on (s1.n0).id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, n2.id as n2 from edge e0 join node n1 on n1.id = e0.start_id join node n2 on n2.id = e0.end_id), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on s1.n2 = e1.start_id and (s1.n0).id = e1.end_id where e1.id != s1.e0) select count(*) > 0 from s2)); -- case: match (g:Group) where (:User)-[:MemberOf]->(:Group)-[:MemberOf]->(g) return count(g) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]) select count(s0.n0)::int8 from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [6]::int2[] and n1.id = e0.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [13]::int2[] and n2.id = e0.end_id where e0.kind_id = any (array [25]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on (s1.n2).id = e1.start_id join node n0 on (s1.n0).id = e1.end_id where e1.kind_id = any (array [25]::int2[]) and e1.id != s1.e0) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]) select count(s0.n0)::int8 as "count(g)" from s0 where ((with s1 as (select e0.id as e0, s0.n0 as n0, n2.id as n2 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [6]::int2[] and n1.id = e0.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [13]::int2[] and n2.id = e0.end_id where e0.kind_id = any (array [25]::int2[])), s2 as (select s1.e0 as e0, s1.n0 as n0, s1.n2 as n2 from s1 join edge e1 on s1.n2 = e1.start_id and (s1.n0).id = e1.end_id where e1.kind_id = any (array [25]::int2[]) and e1.id != s1.e0) select count(*) > 0 from s2)); -- case: match (s) where not (s)-[{prop: 'a'}]-({name: 'n3'}) return s -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id)) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); -- case: match (s) where not (s)<-[{prop: 'a'}]-({name: 'n3'}) return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and n1.id = e0.start_id where (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and (s0.n0).id = e0.end_id) select count(*) > 0 from s1)); @@ -271,7 +271,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (not (with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and n1.id = e0.end_id where (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and (s0.n0).id = e0.start_id) select count(*) > 0 from s1)); -- case: match (s) where not (s)-[]-() return id(s) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (s0.n0).id from s0 where (not exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (s0.n0).id as "id(s)" from s0 where (not exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); -- case: match (s) where ()--(s) return s with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (exists (select 1 from edge e0 where (e0.start_id = (s0.n0).id or e0.end_id = (s0.n0).id))); @@ -283,19 +283,19 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as s from s0 where (exists (select 1 from edge e0)); -- case: match (g) where ({name: 'n3'})-[{prop: 'a'}]-(g) return g -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as g from s0 where ((with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s0.n0 as g from s0 where ((with s1 as (select s0.n0 as n0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id)) and (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id)) select count(*) > 0 from s1)); -- case: match (a:NodeKind1), (b:NodeKind2) where (a:NodeKind1)-[]-(b:NodeKind2) return a -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as a from s1 where ((with s2 as (select s1.n0 as n0, s1.n1 as n1 from edge e0 where ((s1.n0).id <> (s1.n1).id) and (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select count(*) > 0 from s2)); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s1.n0 as a from s1 where ((with s2 as (select s1.n0 as n0, s1.n1 as n1 from edge e0 where (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select count(*) > 0 from s2)); -- case: match (x:NodeKind1{name:'foo'}) match (x)-[]-(y:NodeKind2{name:'bar'}) return x -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select s1.n0 as x from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select s1.n0 as x from s1; -- case: match (y:NodeKind2{name:'bar'}) match ()-[]-(y) return y -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'bar')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select s1.n0 as y from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'bar')), s1 as (select s0.n0 as n0 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select s1.n0 as y from s1; -- case: match (x:NodeKind1{name:'foo'}) match (y:NodeKind2{name:'bar'}) match (x)-[]-(y) return x -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on ((s1.n0).id = e0.start_id or (s1.n0).id = e0.end_id) and ((s1.n1).id = e0.end_id or (s1.n1).id = e0.start_id) where ((s1.n0).id <> (s1.n1).id)) select s2.n0 as x from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select s2.n0 as x from s2; -- case: match (n) where n.system_tags contains ($param) return n -- pgsql_params:{"pi0":null} @@ -324,7 +324,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties ->> 'pwdlastset'))::numeric < (extract(epoch from now()::timestamp with time zone)::numeric * 1000 - 86400000) and not ((n0.properties ->> 'pwdlastset'))::float8 = any (array [- 1, 0]::float8[])) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as u from s0 limit 100; -- case: match (n:NodeKind1) where size(n.array_value) > 0 return n -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (jsonb_array_length((n0.properties -> 'array_value'))::int > 0) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (case when jsonb_typeof((n0.properties -> 'array_value')) = 'array' then jsonb_array_length((n0.properties -> 'array_value'))::int else null end > 0) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0; -- case: match (n) where 1 in n.array return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (1 = any (jsonb_to_text_array((n0.properties -> 'array'))::int8[]))) select s0.n0 as n from s0; @@ -401,7 +401,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0; -- case: match (n:NodeKind1) optional match (m:NodeKind2) where m.distinguishedname = n.unknown + m.unknown optional match (o:NodeKind2) where o.distinguishedname <> n.otherunknown return n, m, o -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'distinguishedname') = ((s0.n0).properties ->> 'unknown') || (n1.properties ->> 'unknown')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select s0.n0 as n0, s1.n1 as n1 from s0 left outer join s1 on (s0.n0 = s1.n0)), s3 as (select s2.n0 as n0, s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s2, node n2 where ((n2.properties -> 'distinguishedname') <> ((s2.n0).properties -> 'otherunknown')) and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s4 as (select s2.n0 as n0, s2.n1 as n1, s3.n2 as n2 from s2 left outer join s3 on (s2.n1 = s3.n1) and (s2.n0 = s3.n0)) select s4.n0 as n, s4.n1 as m, s4.n2 as o from s4; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'distinguishedname') = ((s0.n0).properties ->> 'unknown') || (n1.properties ->> 'unknown')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select s0.n0 as n0, s1.n1 as n1 from s0 left outer join s1 on (s0.n0 = s1.n0)), s3 as (select s2.n0 as n0, s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s2, node n2 where (nullif((n2.properties -> 'distinguishedname'), ('null')::jsonb)::jsonb <> nullif(((s2.n0).properties -> 'otherunknown'), ('null')::jsonb)::jsonb) and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s4 as (select s2.n0 as n0, s2.n1 as n1, s3.n2 as n2 from s2 left outer join s3 on (s2.n1 = s3.n1) and (s2.n0 = s3.n0)) select s4.n0 as n, s4.n1 as m, s4.n2 as o from s4; -- case: match (n) where n.name = "alpha' || (SELECT inet_server_addr()::text::int) || '" return n with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'alpha'' || (SELECT inet_server_addr()::text::int) || '''))) select s0.n0 as n from s0; diff --git a/cypher/models/pgsql/test/translation_cases/pattern_binding.sql b/cypher/models/pgsql/test/translation_cases/pattern_binding.sql index 370b5caf..e29e5dd2 100644 --- a/cypher/models/pgsql/test/translation_cases/pattern_binding.sql +++ b/cypher/models/pgsql/test/translation_cases/pattern_binding.sql @@ -21,85 +21,85 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'name') like '%test%') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select case when (s0.n0).id is null then null else (array [s0.n0]::nodecomposite[], array []::edgecomposite[])::pathcomposite end as p from s0; -- case: match p = ()-[]->() return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = ()-[]->() return nodes(p) -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select ((case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end).nodes)::nodecomposite[] from s0; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select ((case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end).nodes)::nodecomposite[] as "nodes(p)" from s0; -- case: match p = (:NodeKind1)-[:EdgeKind1|EdgeKind2*1..1]->(:NodeKind2) where any(r in relationships(p) where type(r) STARTS WITH 'EdgeKind') return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((exists (select 1 from edge i0 where (kind_name(i0.kind_id)::text like 'EdgeKind%') and i0.id = any (array [s0.e0]::int8[])))::bool); +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((exists (select 1 from edge i0 where (kind_name(i0.kind_id)::text like 'EdgeKind%') and i0.id = any (array [s0.e0]::int8[])))::bool); -- case: match (a)-[*2..2]->(b)-[]->(c) return a -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n0 as n0, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on (s1.n2).id = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != s1.e0 and e2.id != s1.e1) select s2.n0 as a from s2; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n0 as n0, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on s1.n2 = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != s1.e0 and e2.id != s1.e1) select s2.n0 as a from s2; -- case: match p=(:NodeKind1)-[r]->(:NodeKind1) where r.isacl return p limit 100 with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where (((e0.properties ->> 'isacl'))::bool) limit 100) select case when (s0.n0).id is null or (s0.e0).id is null or (s0.n1).id is null then null else (array [s0.n0, s0.n1]::nodecomposite[], array [s0.e0]::edgecomposite[])::pathcomposite end as p from s0 limit 100; -- case: match p = ()-[r1]->()-[r2]->(e) return e -with s0 as (select e0.id as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0) select s1.n2 as e from s1; +with s0 as (select e0.id as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0) select s1.n2 as e from s1; -- case: match ()-[r1]->()-[r2]->()-[]->() where r1.name = 'a' and r2.name = 'b' return r1 -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where ((jsonb_typeof((e0.properties -> 'name')) = 'string' and (e0.properties ->> 'name') = 'a'))), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where ((jsonb_typeof((e1.properties -> 'name')) = 'string' and (e1.properties ->> 'name') = 'b')) and e1.id != (s0.e0).id), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on (s1.n2).id = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != (s1.e0).id and e2.id != (s1.e1).id) select s2.e0 as r1 from s2; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where ((jsonb_typeof((e0.properties -> 'name')) = 'string' and (e0.properties ->> 'name') = 'a'))), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where ((jsonb_typeof((e1.properties -> 'name')) = 'string' and (e1.properties ->> 'name') = 'b')) and e1.id != (s0.e0).id), s2 as (select s1.e0 as e0, s1.e1 as e1, s1.n1 as n1, s1.n2 as n2 from s1 join edge e2 on s1.n2 = e2.start_id join node n3 on n3.id = e2.end_id where e2.id != (s1.e0).id and e2.id != (s1.e1).id) select s2.e0 as r1 from s2; -- case: match p = (a)-[]->()<-[]-(f) where a.name = 'value' and f.is_target return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'value')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on (((n2.properties ->> 'is_target'))::bool) and n2.id = e1.start_id where e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'value')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on (((n2.properties ->> 'is_target'))::bool) and n2.id = e1.start_id where e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[] || array [s1.e1]::int8[], array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; -- case: match p = ()-[*..]->() return p limit 1 -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 1) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 1) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1; -- case: match p = (s)-[*..]->(i)-[]->() where id(s) = 1 and i.name = 'n3' return p limit 1 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, (n0.id = 1), e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id union all select s1.root_id, e0.start_id, s1.depth + 1, (n0.id = 1), false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0) limit 1) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 1; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n3'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, (n0.id = 1), false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id union all select s1.root_id, e0.start_id, s1.depth + 1, (n0.id = 1), false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0) limit 1) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0 || array [s2.e1]::int8[], array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 1; -- case: match p = ()-[e:EdgeKind1]->()-[:EdgeKind1*..]->() return e, p -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id where e1.kind_id = any (array [3]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [3]::int2[]) offset 0) e1 on true where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where (s0.n1).id = s2.root_id and (s0.e0).id != all (s2.path)) select s1.e0 as e, case when (s1.n0).id is null or (s1.e0).id is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, array [s1.e0]::edgecomposite[] || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, false, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id where e1.kind_id = any (array [3]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [3]::int2[]) offset 0) e1 on true where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where (s0.n1).id = s2.root_id and (s0.e0).id != all (s2.path)) select s1.e0 as e, case when (s1.n0).id is null or (s1.e0).id is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(0, s1.n0, array [s1.e0]::edgecomposite[] || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; -- case: match p = (m:NodeKind1)-[:EdgeKind1]->(c:NodeKind2) where m.objectid ends with "-513" and not toUpper(c.operatingsystem) contains "SERVER" return p limit 1000 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((n0.properties ->> 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on (not upper((n1.properties ->> 'operatingsystem'))::text like '%SERVER%') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) limit 1000) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((n0.properties ->> 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on (not upper((n1.properties ->> 'operatingsystem'))::text like '%SERVER%') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) limit 1000) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; -- case: match p = (:NodeKind1)-[:EdgeKind1|EdgeKind2]->(e:NodeKind2)-[:EdgeKind2]->(:NodeKind1) where 'a' in e.values or 'b' in e.values or size(e.values) = 0 return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('a' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or 'b' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or jsonb_array_length((n1.properties -> 'values'))::int = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('a' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or 'b' = any (jsonb_to_text_array((n1.properties -> 'values'))::text[]) or case when jsonb_typeof((n1.properties -> 'values')) = 'array' then jsonb_array_length((n1.properties -> 'values'))::int else null end = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.e1 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[] || array [s1.e1]::int8[], array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1; -- case: match p = (n:NodeKind1)-[r]-(m:NodeKind1) return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n1.id = e0.end_id or n1.id = e0.start_id) where (n0.id <> n1.id)) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n1.id = e0.end_id or n1.id = e0.start_id) where ((n0.id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and n0.id = e0.end_id))) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = (:NodeKind1)-[:EdgeKind1]->(:NodeKind2)-[:EdgeKind2*1..]->(t:NodeKind2) where coalesce(t.system_tags, '') contains 'admin_tier_0' return p limit 1000 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.end_id where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and (s0.n1).id = s2.root_id and s0.e0 != all (s2.path) limit 1000) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) union all select s2.root_id, e1.end_id, s2.depth + 1, (coalesce((n2.properties ->> 'system_tags'), '')::text like '%admin_tier_0%') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.end_id where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and (s0.n1).id = s2.root_id and s0.e0 != all (s2.path) limit 1000) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[] || s1.ep0, array [s1.n0, s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; -- case: match (u:NodeKind1) where u.samaccountname in ["foo", "bar"] match p = (u)-[:EdgeKind1|EdgeKind2*1..3]->(t) where coalesce(t.system_tags, '') contains 'admin_tier_0' return p limit 1000 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'samaccountname') = any (array ['foo', 'bar']::text[])) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 3 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'samaccountname') = any (array ['foo', 'bar']::text[])) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s2.root_id, e0.end_id, s2.depth + 1, (coalesce((n1.properties ->> 'system_tags'), '')::text like '%admin_tier_0%'), false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 3 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and (s0.n0).id = s2.root_id) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, s1.ep0, array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1 limit 1000; -- case: match (x:NodeKind1) where x.name = 'foo' match (y:NodeKind2) where y.name = 'bar' match p=(x)-[:EdgeKind1]->(y) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[], array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (x:NodeKind1{name:'foo'}) match (y:NodeKind2{name:'bar'}) match p=(x)-[:EdgeKind1]->(y) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[], array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (x:NodeKind1{name:'foo'}) match p=(x)-[:EdgeKind1]->(y:NodeKind2{name:'bar'}) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (x:NodeKind1{name:'foo'}) match p=(x)-[]-(y:NodeKind2{name:'bar'}) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar') and (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (e) match p = ()-[]-(e) return p limit 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((s0.n0).id <> n1.id)) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on ((s0.n0).id = e0.end_id or (s0.n0).id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where (((s0.n0).id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and (s0.n0).id = e0.end_id))) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n1, array [s1.e0]::int8[], array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; -- case: match (x:NodeKind1{name:'foo'}) match (y:NodeKind2{name:'bar'}) match p=(x)-[]-(y) return p -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on ((s1.n0).id = e0.start_id or (s1.n0).id = e0.end_id) and ((s1.n1).id = e0.end_id or (s1.n1).id = e0.start_id) where ((s1.n0).id <> (s1.n1).id)) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'foo')), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'bar')), s2 as (select e0.id as e0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e0 on (((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[], array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (e) match p = ()-[]->(e) return p limit 1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select e0.id as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0 join edge e0 on (s0.n0).id = e0.end_id join node n1 on n1.id = e0.start_id) select case when (s1.n1).id is null or s1.e0 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n1, array [s1.e0]::int8[], array [s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1 limit 1; -- case: match p = (a)-[]->() match q = ()-[]->(a) return p, q -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edges_to_path(s1.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n0).id = e1.end_id join node n2 on n2.id = e1.start_id) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n2).id is null or s1.e1 is null or (s1.n0).id is null then null else ordered_edge_ids_to_path(0, s1.n2, array [s1.e1]::int8[], array [s1.n2, s1.n0]::nodecomposite[])::pathcomposite end as q from s1; -- case: match (m:NodeKind1)-[*1..]->(g:NodeKind2)-[]->(c3:NodeKind1) where not g.name in ["foo"] with collect(g.name) as bar match p=(m:NodeKind1)-[*1..]->(g:NodeKind2) where g.name in bar return p -with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.end_id = e2.start_id, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edges_to_path(s4.n3, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s4.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; +with s0 as (with s1 as (with recursive s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id join node n1 on n1.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, (not (n1.properties ->> 'name') = any (array ['foo']::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s2.depth < 15 and not s2.is_cycle) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s2.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.next_id offset 0) n1 on true where s2.satisfied and exists (select 1 from edge e1 join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where n1.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1 from s1 join edge e1 on (s1.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id = e1.end_id where e1.id != all (s1.ep0)) select array_remove(coalesce(array_agg(((s3.n1).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s3), s4 as (with recursive s5_seed(root_id) as not materialized (select n4.id as root_id from s0, node n4 where n4.kind_ids operator (pg_catalog.@>) array [2]::int2[] and ((n4.properties ->> 'name') = any (s0.i0))), s5(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.end_id, e2.start_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s5_seed join edge e2 on e2.end_id = s5_seed.root_id join node n3 on n3.id = e2.start_id union select s5.root_id, e2.start_id, s5.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e2.id || s5.path from s5 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.end_id = s5.next_id and e2.id != all (s5.path) offset 0) e2 on true join node n3 on n3.id = e2.start_id where s5.depth < 15 and not s5.is_cycle) select s5.path as ep1, s0.i0 as i0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3, (n4.id, n4.kind_ids, n4.properties)::nodecomposite as n4 from s0, s5 join lateral (select n4.id, n4.kind_ids, n4.properties from node n4 where n4.id = s5.root_id offset 0) n4 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s5.next_id offset 0) n3 on true where s5.satisfied) select case when (s4.n3).id is null or s4.ep1 is null or (s4.n4).id is null then null else ordered_edge_ids_to_path(0, s4.n3, s4.ep1, array [s4.n3, s4.n4]::nodecomposite[])::pathcomposite end as p from s4; -- case: MATCH p=(:Computer)-[r:HasSession]->(:User) WHERE r.lastseen >= datetime() - duration('P3D') RETURN p LIMIT 100 with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [5]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [6]::int2[] and n1.id = e0.end_id where (((e0.properties ->> 'lastseen'))::timestamp with time zone >= now()::timestamp with time zone - interval 'P3D') and e0.kind_id = any (array [7]::int2[]) limit 100) select case when (s0.n0).id is null or (s0.e0).id is null or (s0.n1).id is null then null else (array [s0.n0, s0.n1]::nodecomposite[], array [s0.e0]::edgecomposite[])::pathcomposite end as p from s0 limit 100; -- case: MATCH p=(:GPO)-[r:GPLink|Contains*1..]->(:Base) WHERE HEAD(r).enforced OR NONE(n in TAIL(TAIL(NODES(p))) WHERE (n:OU AND n.blocksinheritance)) RETURN p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s2.pc0 as p from s0, lateral (select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as pc0 offset 0) s2 where (((((s0.e0)[1]).properties ->> 'enforced'))::bool or ((select count(*)::int from unnest(coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[]) as i0 where ((i0.kind_ids operator (pg_catalog.@>) array [9]::int2[] and ((i0.properties ->> 'blocksinheritance'))::bool))) = 0 and coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[] is not null)::bool); +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s2.pc0 as p from s0, lateral (select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as pc0 offset 0) s2 where (((((s0.e0)[1]).properties ->> 'enforced'))::bool or ((select count(*)::int from unnest(coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[]) as i0 where ((i0.kind_ids operator (pg_catalog.@>) array [9]::int2[] and ((i0.properties ->> 'blocksinheritance'))::bool))) = 0 and coalesce((coalesce((((s2.pc0).nodes)::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[])[2:], array []::nodecomposite[])::nodecomposite[] is not null)::bool); -- case: MATCH p=(:GPO)-[r:GPLink|Contains*1..]->(:Base) WHERE NONE(x in TAIL(r) WHERE NOT type(x) = 'Contains') RETURN p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where (((select count(*)::int from unnest(coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[]) as i0 where (not i0.kind_id = 12)) = 0 and coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[] is not null)::bool); +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [11, 12]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [10]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [11, 12]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.path) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id and _edge.graph_id = 0) as e0, s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where (((select count(*)::int from unnest(coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[]) as i0 where (not i0.kind_id = 12)) = 0 and coalesce((s0.e0)[2:], array []::edgecomposite[])::edgecomposite[] is not null)::bool); diff --git a/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql b/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql index f01d52a6..0d4e4cc4 100644 --- a/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql +++ b/cypher/models/pgsql/test/translation_cases/pattern_expansion.sql @@ -15,76 +15,79 @@ -- SPDX-License-Identifier: Apache-2.0 -- case: match (n)-[*..]->(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; -- case: match (n)-[*1..2]->(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 2 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 2 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true) select s0.n0 as n, s0.n1 as e from s0; -- case: match (n)-[*3..5]->(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, e0.start_id = e0.end_id, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 3) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.end_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 3) select s0.n0 as n, s0.n1 as e from s0; -- case: match (n)<-[*2..5]-(e) return n, e -with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from edge e0 union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2) select s0.n0 as n, s0.n1 as e from s0; +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from edge e0 union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 5 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2) select s0.n0 as n, s0.n1 as e from s0; -- case: match p = (n)-[*..]->(e:NodeKind1) return p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match (n)-[*..]->(e:NodeKind1) where n.name = 'n1' return e -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n1 as e from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n1 as e from s0; -- case: match (n)-[*..]->(e:NodeKind1) where n.name = 'n2' return n -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n2'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n0 as n from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n2'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select s0.n0 as n from s0; -- case: match (n)-[*..]->(e:NodeKind1)-[]->(l) where n.name = 'n1' return l -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; -- case: match (n)-[*2..3]->(e:NodeKind1)-[]->(l) where n.name = 'n1' return l -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 3 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2 and s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 3 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.depth >= 2 and s1.satisfied), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n2 as l from s2; -- case: match (n)-[]->(e:NodeKind1)-[*2..3]->(l) where n.name = 'n1' return l -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) offset 0) e1 on true where s2.depth < 3 and not s2.is_cycle) select s0.e0 as e0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.depth >= 2 and (s0.n1).id = s2.root_id and s0.e0 != all (s2.path)) select s1.n2 as l from s1; - --- case: match p = (src:NodeKind1)-[:EdgeKind1*1..]->(mid)-[:EdgeKind1]-(dst:NodeKind1) where src.name = 'reuse-source' and dst.name = 'reuse-source' return p -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'reuse-source')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where (n0.id <> n1.id) and e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, ((jsonb_typeof((n2.properties -> 'name')) = 'string' and (n2.properties ->> 'name') = 'reuse-source')) and n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], e1.end_id = e1.start_id, array [e1.id] from s2_seed join edge e1 on e1.end_id = s2_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3]::int2[]) union all select s2.root_id, e1.start_id, s2.depth + 1, ((jsonb_typeof((n2.properties -> 'name')) = 'string' and (n2.properties ->> 'name') = 'reuse-source')) and n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e1.id || s2.path from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [3]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and (s0.n1).id = s2.root_id and s0.e0 != all (s2.path)) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n2, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n2, s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct s0.n1 as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.start_id, e1.end_id, 1, false, false, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id union all select s2.root_id, e1.end_id, s2.depth + 1, false, false, s2.path || e1.id from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.start_id = s2.next_id and e1.id != all (s2.path) offset 0) e1 on true where s2.depth < 3 and not s2.is_cycle) select s0.e0 as e0, s0.n0 as n0, n1.id as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.depth >= 2 and s0.n1 = s2.root_id and s0.e0 != all (s2.path)) select s1.n2 as l from s1; -- case: match (n)-[*..]->(e)-[:EdgeKind1|EdgeKind2]->()-[*..]->(l) where n.name = 'n1' and e.name = 'n2' return l -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [3, 4]::int2[]))), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3, 4]::int2[]) and e1.id != all (s0.ep0)), s3 as (with recursive s4_seed(root_id) as not materialized (select distinct (s2.n2).id as root_id from s2), s4(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, false, e2.start_id = e2.end_id, array [e2.id] from s4_seed join edge e2 on e2.start_id = s4_seed.root_id union all select s4.root_id, e2.end_id, s4.depth + 1, false, false, s4.path || e2.id from s4 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s4.next_id and e2.id != all (s4.path) offset 0) e2 on true where s4.depth < 15 and not s4.is_cycle) select s2.e1 as e1, s2.ep0 as ep0, s2.n0 as n0, s2.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s2, s4 join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s4.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s4.next_id offset 0) n3 on true where (s2.n2).id = s4.root_id and s2.e1 != all (s4.path)) select s3.n3 as l from s3; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1'))), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'n2')), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id and e1.kind_id = any (array [3, 4]::int2[]))), s2 as (select e1.id as e1, s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3, 4]::int2[]) and e1.id != all (s0.ep0)), s3 as (with recursive s4_seed(root_id) as not materialized (select distinct s2.n2 as root_id from s2), s4(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, false, false, array [e2.id] from s4_seed join edge e2 on e2.start_id = s4_seed.root_id union all select s4.root_id, e2.end_id, s4.depth + 1, false, false, s4.path || e2.id from s4 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s4.next_id and e2.id != all (s4.path) offset 0) e2 on true where s4.depth < 15 and not s4.is_cycle) select s2.e1 as e1, s2.ep0 as ep0, s2.n0 as n0, s2.n1 as n1, n2.id as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s2, s4 join lateral (select n2.id from node n2 where n2.id = s4.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s4.next_id offset 0) n3 on true where s2.n2 = s4.root_id and s2.e1 != all (s4.path)) select s3.n3 as l from s3; -- case: match p = (:NodeKind1)-[:EdgeKind1*1..]->(n:NodeKind2) where 'admin_tier_0' in split(n.system_tags, ' ') return p limit 1000 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties ->> 'system_tags'), ' ')::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties ->> 'system_tags'), ' ')::text[])) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; -- case: match p = (s:NodeKind1)-[*..]->(e:NodeKind2) where s <> e return p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and (n0.id <> n1.id)) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and (n0.id <> n1.id)) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = (g:NodeKind1)-[:EdgeKind1|EdgeKind2*]->(target:NodeKind1) where g.objectid ends with '1234' and target.objectid ends with '4567' return p -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%1234') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%1234') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((n1.properties ->> 'objectid') like '%4567') and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = (m:NodeKind2)-[:EdgeKind1*1..]->(n:NodeKind1) where n.objectid = '1234' return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = '1234')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = '1234')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match p = (:NodeKind1)<-[:EdgeKind1|EdgeKind2*..]-() return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, false, false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, false, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match p = (:NodeKind1)<-[:EdgeKind1|EdgeKind2*..]-(:NodeKind2)<-[:EdgeKind1|EdgeKind2*2..]-(:NodeKind1) return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], e1.end_id = e1.start_id, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.depth >= 2 and s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.depth >= 2 and s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0 || s2.ep1, array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; -- case: match p = (:NodeKind1)<-[:EdgeKind1|EdgeKind2*..]-(:NodeKind2)<-[:EdgeKind1|EdgeKind2*..]-(:NodeKind1) return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], e1.end_id = e1.start_id, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep1) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n1.kind_ids operator (pg_catalog.@>) array [2]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e1.id] from s3_seed join edge e1 on e1.end_id = s3_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3, 4]::int2[]) union all select s3.root_id, e1.start_id, s3.depth + 1, n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e1.id from s3 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s3.next_id and e1.id != all (s3.path) and e1.kind_id = any (array [3, 4]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s3.depth < 15 and not s3.is_cycle) select s0.ep0 as ep0, s3.path as ep1, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s3 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s3.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.next_id offset 0) n2 on true where s3.satisfied and (s0.n1).id = s3.root_id limit 10) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null or s2.ep1 is null or (s2.n2).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0 || s2.ep1, array [s2.n0, s2.n1, s2.n2]::nodecomposite[])::pathcomposite end as p from s2 limit 10; -- case: match p = (n:NodeKind1)-[:EdgeKind1|EdgeKind2*1..2]->(r:NodeKind2) where r.name =~ '(?i)Global Administrator.*|User Administrator.*|Cloud Application Administrator.*|Authentication Policy Administrator.*|Exchange Administrator.*|Helpdesk Administrator.*|Privileged Authentication Administrator.*' return p limit 10 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'name') ~ '(?i)Global Administrator.*|User Administrator.*|Cloud Application Administrator.*|Authentication Policy Administrator.*|Exchange Administrator.*|Helpdesk Administrator.*|Privileged Authentication Administrator.*') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 2 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((n1.properties ->> 'name') ~ '(?i)Global Administrator.*|User Administrator.*|Cloud Application Administrator.*|Authentication Policy Administrator.*|Exchange Administrator.*|Helpdesk Administrator.*|Privileged Authentication Administrator.*') and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, n0.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e0.id || s1.path from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3, 4]::int2[]) offset 0) e0 on true join node n0 on n0.id = e0.start_id where s1.depth < 2 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.root_id offset 0) n1 on true join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.next_id offset 0) n0 on true where s1.satisfied limit 10) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match p = (t:NodeKind2)<-[:EdgeKind1*1..]-(a) where (a:NodeKind1 or a:NodeKind2) and t.objectid ends with '-512' return p limit 1000 -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%-512') and n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), e0.end_id = e0.start_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties ->> 'objectid') like '%-512') and n0.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.end_id, e0.start_id, 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), false, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.start_id, s1.depth + 1, ((n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [2]::int2[])), false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.end_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.start_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied limit 1000) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 1000; -- case: match p=(n:NodeKind1)-[:EdgeKind1|EdgeKind2]->(g:NodeKind1)-[:EdgeKind2]->(:NodeKind2)-[:EdgeKind1*1..]->(m:NodeKind1) where n.objectid = m.objectid return p limit 100 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s1.n2).id as root_id from s1), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], e2.start_id = e2.end_id, array [e2.id] from s3_seed join edge e2 on e2.start_id = s3_seed.root_id join node n3 on n3.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) union all select s3.root_id, e2.end_id, s3.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e2.id from s3 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s3.next_id and e2.id != all (s3.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.end_id where s3.depth < 15 and not s3.is_cycle) select s1.e0 as e0, s1.e1 as e1, s3.path as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, s3 join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s3.next_id offset 0) n3 on true where s3.satisfied and (s1.n2).id = s3.root_id and (((s1.n0).properties -> 'objectid') = (n3.properties -> 'objectid')) and s1.e0 != all (s3.path) and s1.e1 != all (s3.path) limit 100) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null or s2.ep0 is null or (s2.n3).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s2.e1]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id) || (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1, s2.n2, s2.n3]::nodecomposite[])::pathcomposite end as p from s2 limit 100; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, e1.id as e1, s0.n0 as n0, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e1.end_id where e1.kind_id = any (array [4]::int2[]) and e1.id != s0.e0), s2 as (with recursive s3_seed(root_id) as not materialized (select distinct (s1.n2).id as root_id from s1), s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select e2.start_id, e2.end_id, 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e2.id] from s3_seed join edge e2 on e2.start_id = s3_seed.root_id join node n3 on n3.id = e2.end_id where e2.kind_id = any (array [3]::int2[]) union all select s3.root_id, e2.end_id, s3.depth + 1, n3.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s3.path || e2.id from s3 join lateral (select e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties from edge e2 where e2.start_id = s3.next_id and e2.id != all (s3.path) and e2.kind_id = any (array [3]::int2[]) offset 0) e2 on true join node n3 on n3.id = e2.end_id where s3.depth < 15 and not s3.is_cycle) select s1.e0 as e0, s1.e1 as e1, s3.path as ep0, s1.n0 as n0, s1.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1, s3 join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s3.root_id offset 0) n2 on true join lateral (select n3.id, n3.kind_ids, n3.properties from node n3 where n3.id = s3.next_id offset 0) n3 on true where s3.satisfied and (s1.n2).id = s3.root_id and (nullif(((s1.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif((n3.properties -> 'objectid'), ('null')::jsonb)::jsonb) and s1.e0 != all (s3.path) and s1.e1 != all (s3.path) limit 100) select case when (s2.n0).id is null or s2.e0 is null or (s2.n1).id is null or s2.e1 is null or (s2.n2).id is null or s2.ep0 is null or (s2.n3).id is null then null else ordered_edge_ids_to_path(0, s2.n0, array [s2.e0]::int8[] || array [s2.e1]::int8[] || s2.ep0, array [s2.n0, s2.n1, s2.n2, s2.n3]::nodecomposite[])::pathcomposite end as p from s2 limit 100; -- case: match (a:NodeKind1)-[:EdgeKind1*0..]->(b:NodeKind1) where a.name = 'solo' and b.name = 'solo' return a.name, b.name -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'solo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'solo')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'solo')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select ((s0.n0).properties -> 'name') as "a.name", ((s0.n1).properties -> 'name') as "b.name" from s0; -- case: match (a:NodeKind1)-[:EdgeKind1*0..]->(b:NodeKind1) where a.name = 'zero-source' and b.name = 'zero-target' return count(b) -with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'zero-source')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select count(s0.n1)::int8 from s0; +with s0 as (with recursive s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'zero-source')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select s1_seed.root_id, s1_seed.root_id, 0, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array []::int8[] from s1_seed join node n1 on n1.id = s1_seed.root_id union all select e0.start_id, e0.end_id, 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]) union all select s1.root_id, e0.end_id, s1.depth + 1, ((jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = 'zero-target')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) and e0.kind_id = any (array [3]::int2[]) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle and s1.depth > 0) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied) select count(s0.n1)::int8 as "count(b)" from s0; + +-- case: match (s)-[*1..]->(mid)-[]->(e) return id(mid), id(e) +with s0 as (with recursive s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select e0.start_id, e0.end_id, 1, true, false, array [e0.id] from edge e0 join node n1 on n1.id = e0.end_id union all select s1.root_id, e0.end_id, s1.depth + 1, true, false, s1.path || e0.id from s1 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s1.next_id and e0.id != all (s1.path) offset 0) e0 on true join node n1 on n1.id = e0.end_id where s1.depth < 15 and not s1.is_cycle) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from s1 join lateral (select n0.id, n0.kind_ids, n0.properties from node n0 where n0.id = s1.root_id offset 0) n0 on true join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true where s1.satisfied and exists (select 1 from edge e1 join node n2 on n2.id = e1.end_id where n1.id = e1.start_id)), s2 as (select s0.ep0 as ep0, s0.n0 as n0, s0.n1 as n1, n2.id as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != all (s0.ep0)) select s2.n1 as "id(mid)", s2.n2 as "id(e)" from s2; + +-- case: match p = (src:NodeKind1)-[:EdgeKind1*1..]->(mid)-[:EdgeKind1]-(dst:NodeKind1) where src.name = 'reuse-source' and dst.name = 'reuse-source' return p +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'reuse-source')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (n0.id = e0.end_id or n0.id = e0.start_id) join node n1 on (n1.id = e0.end_id or n1.id = e0.start_id) where ((n0.id = e0.start_id and n1.id = e0.end_id) or (n1.id = e0.start_id and n0.id = e0.end_id)) and e0.kind_id = any (array [3]::int2[])), s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n1).id as root_id from s0), s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select e1.end_id, e1.start_id, 1, ((jsonb_typeof((n2.properties -> 'name')) = 'string' and (n2.properties ->> 'name') = 'reuse-source')) and n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, array [e1.id] from s2_seed join edge e1 on e1.end_id = s2_seed.root_id join node n2 on n2.id = e1.start_id where e1.kind_id = any (array [3]::int2[]) union all select s2.root_id, e1.start_id, s2.depth + 1, ((jsonb_typeof((n2.properties -> 'name')) = 'string' and (n2.properties ->> 'name') = 'reuse-source')) and n2.kind_ids operator (pg_catalog.@>) array [1]::int2[], false, e1.id || s2.path from s2 join lateral (select e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties from edge e1 where e1.end_id = s2.next_id and e1.id != all (s2.path) and e1.kind_id = any (array [3]::int2[]) offset 0) e1 on true join node n2 on n2.id = e1.start_id where s2.depth < 15 and not s2.is_cycle) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and (s0.n1).id = s2.root_id and s0.e0 != all (s2.path)) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n2, s1.ep0 || array [s1.e0]::int8[], array [s1.n2, s1.n1, s1.n0]::nodecomposite[])::pathcomposite end as p from s1; -- case: match (n) match (a)-[*0..]->()-[]->(ct1) where n.gmsa = true or ct1.subjectaltrequiredns = false return ct1 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (with recursive s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select n1.id, n1.id, 0, true, false, array []::int8[] from node n1 union all select e0.start_id, e0.end_id, 1, true, e0.start_id = e0.end_id, array [e0.id] from edge e0 join node n2 on n2.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, true, false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n2 on n2.id = e0.end_id where s2.depth < 15 and not s2.is_cycle and s2.depth > 0) select s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id, n2.kind_ids, n2.properties from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and exists (select 1 from edge e1 join node n3 on ((((s0.n0).properties -> 'gmsa'))::jsonb = to_jsonb((true)::bool)::jsonb or ((n3.properties -> 'subjectaltrequiredns'))::jsonb = to_jsonb((false)::bool)::jsonb) and n3.id = e1.end_id where n2.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, s1.n2 as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1 join edge e1 on (s1.n2).id = e1.start_id join node n3 on ((((s1.n0).properties -> 'gmsa'))::jsonb = to_jsonb((true)::bool)::jsonb or ((n3.properties -> 'subjectaltrequiredns'))::jsonb = to_jsonb((false)::bool)::jsonb) and n3.id = e1.end_id where e1.id != all (s1.ep0)) select s3.n3 as ct1 from s3; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (with recursive s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select n1.id, n1.id, 0, true, false, array []::int8[] from node n1 union all select e0.start_id, e0.end_id, 1, true, false, array [e0.id] from edge e0 join node n2 on n2.id = e0.end_id union all select s2.root_id, e0.end_id, s2.depth + 1, true, false, s2.path || e0.id from s2 join lateral (select e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties from edge e0 where e0.start_id = s2.next_id and e0.id != all (s2.path) offset 0) e0 on true join node n2 on n2.id = e0.end_id where s2.depth < 15 and not s2.is_cycle and s2.depth > 0) select s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, n2.id as n2 from s0, s2 join lateral (select n1.id, n1.kind_ids, n1.properties from node n1 where n1.id = s2.root_id offset 0) n1 on true join lateral (select n2.id from node n2 where n2.id = s2.next_id offset 0) n2 on true where s2.satisfied and exists (select 1 from edge e1 join node n3 on ((((s0.n0).properties -> 'gmsa'))::jsonb = to_jsonb((true)::bool)::jsonb or ((n3.properties -> 'subjectaltrequiredns'))::jsonb = to_jsonb((false)::bool)::jsonb) and n3.id = e1.end_id where n2.id = e1.start_id)), s3 as (select s1.ep0 as ep0, s1.n0 as n0, s1.n1 as n1, s1.n2 as n2, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n3 from s1 join edge e1 on s1.n2 = e1.start_id join node n3 on ((((s1.n0).properties -> 'gmsa'))::jsonb = to_jsonb((true)::bool)::jsonb or ((n3.properties -> 'subjectaltrequiredns'))::jsonb = to_jsonb((false)::bool)::jsonb) and n3.id = e1.end_id where e1.id != all (s1.ep0)) select s3.n3 as ct1 from s3; diff --git a/cypher/models/pgsql/test/translation_cases/post_processing.sql b/cypher/models/pgsql/test/translation_cases/post_processing.sql new file mode 100644 index 00000000..41dcae59 --- /dev/null +++ b/cypher/models/pgsql/test/translation_cases/post_processing.sql @@ -0,0 +1,45 @@ +-- Copyright 2026 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- case: match (n) where not n:RegressionKind03 and (n.lastseen is null or n.lastseen < datetime($threshold)) return id(n) +-- cypher_params: {"threshold":"2026-01-02T03:04:05Z"} +-- pgsql_params:{"pi0":"2026-01-02T03:04:05Z"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not n0.kind_ids operator (pg_catalog.@>) array [35]::int2[] and ((not n0.properties ? 'lastseen' or (n0.properties -> 'lastseen') = ('null')::jsonb) or ((n0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone))) select (s0.n0).id as "id(n)" from s0; + +-- case: match ()-[r]->() where not r:RegressionKind45 and r.lastseen < datetime($threshold) return id(r) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (not e0.kind_id = any (array [77]::int2[]) and ((e0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone)) select (s0.e0).id as "id(r)" from s0; + +-- case: match ()-[r]->() where not (r:RegressionKind45 or r:RegressionKind46) and r.lastseen < datetime($threshold) return id(r) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (not (e0.kind_id = any (array [77]::int2[]) or e0.kind_id = any (array [78]::int2[])) and ((e0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone)) select (s0.e0).id as "id(r)" from s0; + +-- case: match ()-[r:HasSession]->() where r.lastseen is null or r.lastseen < datetime($threshold) return id(r) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where ((not e0.properties ? 'lastseen' or (e0.properties -> 'lastseen') = ('null')::jsonb) or ((e0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone) and e0.kind_id = any (array [7]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (n) where not (n:RegressionKind48 or n:RegressionKind49) and (n.lastseen is null or n.lastseen < datetime($threshold)) return id(n) +-- cypher_params: {"threshold":"2026-01-03T00:00:00Z"} +-- pgsql_params:{"pi0":"2026-01-03T00:00:00Z"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not (n0.kind_ids operator (pg_catalog.@>) array [80]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [81]::int2[]) and ((not n0.properties ? 'lastseen' or (n0.properties -> 'lastseen') = ('null')::jsonb) or ((n0.properties ->> 'lastseen'))::timestamp with time zone < (@pi0::text)::timestamp with time zone))) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where not (n:RegressionKind48 or n:RegressionKind49) and n.name is null and n.objectid starts with $sid_prefix return id(n) +-- cypher_params: {"sid_prefix":"S-1-5"} +-- pgsql_params:{"pi0":"S-1-5"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not (n0.kind_ids operator (pg_catalog.@>) array [80]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [81]::int2[]) and (not n0.properties ? 'name' or (n0.properties -> 'name') = ('null')::jsonb) and cypher_starts_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool)) select (s0.n0).id as "id(n)" from s0; diff --git a/cypher/models/pgsql/test/translation_cases/quantifiers.sql b/cypher/models/pgsql/test/translation_cases/quantifiers.sql index c4d2f249..5b342396 100644 --- a/cypher/models/pgsql/test/translation_cases/quantifiers.sql +++ b/cypher/models/pgsql/test/translation_cases/quantifiers.sql @@ -30,20 +30,20 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties ->> 'usedeskeyonly'))::bool or ((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'supportedencryptiontypes'))) as i0 where (i0 like '%DES%')) >= 1)::bool or ((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i1 where (lower(i1)::text like '%mssqlservercluster%' or lower(i1)::text like '%mssqlserverclustermgmtapi%' or lower(i1)::text like '%msclustervirtualserver%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s0.n0 as n from s0 limit 100; -- case: MATCH (m:NodeKind1) WHERE m.unconstraineddelegation = true WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-516' WITH m, COLLECT(n) AS matchingNs WHERE NONE(n IN matchingNs WHERE n.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where ((i1.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = 0 and s2.i0 is not null)::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where (nullif((i1.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = 0 and s2.i0 is not null)::bool); -- case: MATCH (m:NodeKind1) WHERE m.unconstraineddelegation = true WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-516' WITH m, COLLECT(n) AS matchingNs WHERE ALL(n IN matchingNs WHERE n.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where ((i1.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = cardinality(s2.i0))::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where (nullif((i1.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = cardinality(s2.i0))::bool); -- case: MATCH (m:NodeKind1) WHERE ANY(name in m.serviceprincipalnames WHERE name CONTAINS "PHANTOM") WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-525' WITH m, COLLECT(n) AS matchingNs WHERE NONE(t IN matchingNs WHERE t.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where ((i2.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = 0 and s2.i1 is not null)::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where (nullif((i2.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = 0 and s2.i1 is not null)::bool); -- case: WITH [1, 2] AS nums MATCH (n:NodeKind1) WHERE ANY(num IN nums + [3] WHERE num = 3) RETURN n with s0 as (select array [1, 2]::int8[] as i0), s1 as (select s0.i0 as i0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from s0, node n0 where (((select count(*)::int from unnest(s0.i0 || array [3]::int8[]) as i1 where (i1 = 3)) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n from s1; -- case: MATCH (m:NodeKind1) WHERE m.unconstraineddelegation = true WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-516' WITH m, COLLECT(n) AS matchingNs WHERE ALL(n IN matchingNs WHERE n.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where ((i1.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = cardinality(s2.i0))::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'unconstraineddelegation'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-516') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i0 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i0) as i1 where (nullif((i1.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = cardinality(s2.i0))::bool); -- case: MATCH (m:NodeKind1) WHERE ANY(name in m.serviceprincipalnames WHERE name CONTAINS "PHANTOM") WITH m MATCH (n:NodeKind1)-[:EdgeKind1]->(g:NodeKind2) WHERE g.objectid ENDS WITH '-525' WITH m, COLLECT(n) AS matchingNs WHERE NONE(t IN matchingNs WHERE t.objectid = m.objectid) RETURN m -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where ((i2.properties -> 'objectid') = ((s2.n0).properties -> 'objectid'))) = 0 and s2.i1 is not null)::bool); +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((select count(*)::int from unnest(jsonb_to_text_array((n0.properties -> 'serviceprincipalnames'))) as i0 where (i0 like '%PHANTOM%')) >= 1)::bool) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select s1.n0 as n0 from s1), s2 as (with s3 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, edge e0 join node n2 on ((n2.properties ->> 'objectid') like '%-525') and n2.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n2.id = e0.end_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select s3.n0 as n0, array_remove(coalesce(array_agg(s3.n1)::nodecomposite[], array []::nodecomposite[])::nodecomposite[], null)::nodecomposite[] as i1 from s3 group by n0) select s2.n0 as m from s2 where (((select count(*)::int from unnest(s2.i1) as i2 where (nullif((i2.properties -> 'objectid'), ('null')::jsonb)::jsonb = nullif(((s2.n0).properties -> 'objectid'), ('null')::jsonb)::jsonb)) = 0 and s2.i1 is not null)::bool); diff --git a/cypher/models/pgsql/test/translation_cases/reconciliation.sql b/cypher/models/pgsql/test/translation_cases/reconciliation.sql new file mode 100644 index 00000000..3a42470a --- /dev/null +++ b/cypher/models/pgsql/test/translation_cases/reconciliation.sql @@ -0,0 +1,139 @@ +-- Copyright 2026 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- case: match (s)-[r]->(e) where (id(s) = $forward_start and id(e) = $forward_end and r:RegressionKind01) or (id(s) = $forward_end and id(e) = $forward_start and r:RegressionKind02) return id(r) +-- cypher_params: {"forward_end":202,"forward_start":101} +-- pgsql_params:{"pi0":101,"pi1":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n1 on n1.id = e0.end_id join node n0 on n0.id = e0.start_id where ((n0.id = @pi0::float8 and n1.id = @pi1::float8 and e0.kind_id = any (array [33]::int2[])) or (n0.id = @pi1::float8 and n1.id = @pi0::float8 and e0.kind_id = any (array [34]::int2[])))) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind03)-[r:RegressionKind04]->(e:RegressionKind03) where r.lastseen < s.lastcollected or r.lastseen < e.lastcollected return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [35]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [35]::int2[] and n1.id = e0.end_id where (nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n0.properties -> 'lastcollected'), ('null')::jsonb)::jsonb or nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n1.properties -> 'lastcollected'), ('null')::jsonb)::jsonb) and e0.kind_id = any (array [36]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind05)-[r:RegressionKind06]->(e:RegressionKind07) where e.objectid = $object_id and r.shoulddelete = $should_delete delete r +-- cypher_params: {"object_id":"delete-edge","should_delete":true} +-- pgsql_params:{"pi0":"delete-edge","pi1":true} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [39]::int2[] and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [37]::int2[] and n0.id = e0.start_id where (((e0.properties -> 'shoulddelete'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb) and e0.kind_id = any (array [38]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (n:RegressionKind08) where n.objectid = $object_id detach delete n +-- cypher_params: {"object_id":"delete-node"} +-- pgsql_params:{"pi0":"delete-node"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [40]::int2[]), s1 as (delete from node n1 using s0 where (s0.n0).id = n1.id) select 1; + +-- case: match ()-[r:RegressionKind09]->(e) return r, e +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match ()-[r:RegressionKind09]->(e) return id(e), labels(e), id(r), type(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select (s0.n1).id as "id(e)", (array(select _kind.name from generate_subscripts((s0.n1).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n1).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(e)", (s0.e0).id as "id(r)", kind_name((s0.e0).kind_id)::text as "type(r)" from s0; + +-- case: match (s)-[r:RegressionKind09]->(e) return s, r, e +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select s0.n0 as s, s0.e0 as r, s0.n1 as e from s0; + +-- case: match ()-[r:RegressionKind09]->() return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match ()-[r:RegressionKind09]->() return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [41]::int2[])) select s0.e0 as r from s0; + +-- case: match ()-[r:RegressionKind01]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind01|RegressionKind02]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e:RegressionKind31) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-01"} +-- pgsql_params:{"pi0":"rec-01"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->() where s.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-02"} +-- pgsql_params:{"pi0":"rec-02"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid = $object_id and r.isprimarygroup = $flag delete r +-- cypher_params: {"flag":false,"object_id":"rec-03-in"} +-- pgsql_params:{"pi0":"rec-03-in","pi1":false} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where (((e0.properties -> 'isprimarygroup'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb) and e0.kind_id = any (array [64]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind31)-[r:RegressionKind32]->() where s.objectid = $object_id and r.isprimarygroup = $flag delete r +-- cypher_params: {"flag":true,"object_id":"rec-03-out"} +-- pgsql_params:{"pi0":"rec-03-out","pi1":true} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (((e0.properties -> 'isprimarygroup'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb) and e0.kind_id = any (array [64]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid in $object_ids delete r +-- cypher_params: {"object_ids":["rec-04-a","rec-04-b"]} +-- pgsql_params:{"pi0":["rec-04-a","rec-04-b"]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((n1.properties ->> 'objectid') = any (@pi0::text[])) and n1.kind_ids operator (pg_catalog.@>) array [63]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [64]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind34]->(e:RegressionKind33) where e.objectid in $object_ids delete r +-- cypher_params: {"object_ids":["rec-04-azure-a","rec-04-azure-b"]} +-- pgsql_params:{"pi0":["rec-04-azure-a","rec-04-azure-b"]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((n1.properties ->> 'objectid') = any (@pi0::text[])) and n1.kind_ids operator (pg_catalog.@>) array [65]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [66]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (s:RegressionKind35)-[r:RegressionKind36]->(e) where e.objectid in $ca_ids return r, s +-- cypher_params: {"ca_ids":["ca-a","ca-b"]} +-- pgsql_params:{"pi0":["ca-a","ca-b"]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((n1.properties ->> 'objectid') = any (@pi0::text[])) and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [67]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [68]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match ()-[r:RegressionKind37]->(e:RegressionKind35) where id(e) in $template_ids delete r +-- cypher_params: {"template_ids":[101,202]} +-- pgsql_params:{"pi0":[101,202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.kind_ids operator (pg_catalog.@>) array [67]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [69]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match ()-[r:RegressionKind39]->(e:RegressionKind38) where e.objectid = $object_id delete r +-- cypher_params: {"object_id":"rec-07"} +-- pgsql_params:{"pi0":"rec-07"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on ((jsonb_typeof((n1.properties -> 'objectid')) = 'string' and (n1.properties ->> 'objectid') = @pi0::text)) and n1.kind_ids operator (pg_catalog.@>) array [70]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [71]::int2[])), s1 as (delete from edge e1 using s0 where (s0.e0).id = e1.id) select 1; + +-- case: match (n:RegressionKind31) where n.objectid in $object_ids detach delete n +-- cypher_params: {"object_ids":["rec-08-a","rec-08-b"]} +-- pgsql_params:{"pi0":["rec-08-a","rec-08-b"]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.properties ->> 'objectid') = any (@pi0::text[])) and n0.kind_ids operator (pg_catalog.@>) array [63]::int2[]), s1 as (delete from node n1 using s0 where (s0.n0).id = n1.id) select 1; + +-- case: match (s:RegressionKind40)-[r:RegressionKind41]->(e:RegressionKind40) where r.lastseen < s.lastcollected or r.lastseen < e.lastcollected return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n1.id = e0.end_id where (nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n0.properties -> 'lastcollected'), ('null')::jsonb)::jsonb or nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n1.properties -> 'lastcollected'), ('null')::jsonb)::jsonb) and e0.kind_id = any (array [73]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind40)-[r:RegressionKind42]->(e:RegressionKind40) where r.lastseen < s.lastcollected or r.lastseen < e.lastcollected return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n1.id = e0.end_id where (nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n0.properties -> 'lastcollected'), ('null')::jsonb)::jsonb or nullif((e0.properties -> 'lastseen'), ('null')::jsonb)::jsonb < nullif((n1.properties -> 'lastcollected'), ('null')::jsonb)::jsonb) and e0.kind_id = any (array [74]::int2[])) select s0.e0 as r from s0; + +-- case: match (s:RegressionKind40)-[r]->(e:RegressionKind40) where (id(s) = $forward_start and id(e) = $forward_end and r:RegressionKind43) or (id(s) = $forward_end and id(e) = $forward_start and r:RegressionKind44) return id(r) +-- cypher_params: {"forward_end":202,"forward_start":101} +-- pgsql_params:{"pi0":101,"pi1":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [72]::int2[] and n0.id = e0.start_id where ((n0.id = @pi0::float8 and n1.id = @pi1::float8 and e0.kind_id = any (array [75]::int2[])) or (n0.id = @pi1::float8 and n1.id = @pi0::float8 and e0.kind_id = any (array [76]::int2[])))) select (s0.e0).id as "id(r)" from s0; diff --git a/cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql b/cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql new file mode 100644 index 00000000..2dd82d4d --- /dev/null +++ b/cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql @@ -0,0 +1,164 @@ +-- Copyright 2026 Specter Ops, Inc. +-- +-- Licensed under the Apache License, Version 2.0 +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +-- SPDX-License-Identifier: Apache-2.0 + +-- case: match (s)-[r:RegressionKind63]->(e) where (s:RegressionKind61 or s:RegressionKind62) and (e:RegressionKind61 or e:RegressionKind62) return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((n0.kind_ids operator (pg_catalog.@>) array [93]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [94]::int2[])) and n0.id = e0.start_id join node n1 on ((n1.kind_ids operator (pg_catalog.@>) array [93]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [94]::int2[])) and n1.id = e0.end_id where e0.kind_id = any (array [95]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s)-[r:RegressionKind66|RegressionKind67]->(e) where not (s:RegressionKind64 or s:RegressionKind65) and not (e:RegressionKind64 or e:RegressionKind65) return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (not (n0.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n0.id = e0.start_id join node n1 on (not (n1.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n1.id = e0.end_id where e0.kind_id = any (array [98, 99]::int2[])) select s0.e0 as r from s0; + +-- case: match (s)-[r:RegressionKind68]->(e) where not (s:RegressionKind64 or s:RegressionKind65) and r.lastseen is not null and not (e:RegressionKind64 or e:RegressionKind65) return id(r) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (not (n0.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n0.id = e0.start_id join node n1 on (not (n1.kind_ids operator (pg_catalog.@>) array [96]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [97]::int2[])) and n1.id = e0.end_id where ((e0.properties ? 'lastseen' and not (e0.properties -> 'lastseen') = ('null')::jsonb)) and e0.kind_id = any (array [100]::int2[])) select (s0.e0).id as "id(r)" from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind70]->() return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [102]::int2[])) select s0.e0 as r from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind71]->() return r +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [103]::int2[])) select s0.e0 as r from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind72]->(e) where id(e) = $end_id return r, s +-- cypher_params: {"end_id":202} +-- pgsql_params:{"pi0":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi0::float8) and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [104]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s:RegressionKind69)-[r:RegressionKind72|RegressionKind73|RegressionKind74|RegressionKind75|RegressionKind76|RegressionKind77|RegressionKind78|RegressionKind79|RegressionKind80]->(e) where id(e) = $end_id return r, s +-- cypher_params: {"end_id":202} +-- pgsql_params:{"pi0":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi0::float8) and n1.id = e0.end_id join node n0 on n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [104, 105, 106, 107, 108, 109, 110, 111, 112]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind82]->(e:RegressionKind81) return id(s), id(r), type(r), id(e) +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n1 on n1.kind_ids operator (pg_catalog.@>) array [113]::int2[] and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [114]::int2[])) select s0.n0 as "id(s)", (s0.e0).id as "id(r)", kind_name((s0.e0).kind_id)::text as "type(r)", s0.n1 as "id(e)" from s0; + +-- case: match (s)-[r:RegressionKind83]->(e) return id(s), id(e) +with s0 as (select n0.id as n0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [115]::int2[])) select s0.n0 as "id(s)", s0.n1 as "id(e)" from s0; + +-- case: match (s)-[r:RegressionKind83|RegressionKind84]->(e) return id(s), id(e) +with s0 as (select n0.id as n0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [115, 116]::int2[])) select s0.n0 as "id(s)", s0.n1 as "id(e)" from s0; + +-- case: match (s)-[r:RegressionKind87|RegressionKind88|RegressionKind89|RegressionKind90|RegressionKind91|RegressionKind92]->(e) where (s:RegressionKind85 or s:RegressionKind86 or s:RegressionKind81) and id(e) in $end_ids return id(s) +-- cypher_params: {"end_ids":[202,303]} +-- pgsql_params:{"pi0":[202,303]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on ((n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [113]::int2[])) and n0.id = e0.start_id where e0.kind_id = any (array [119, 120, 121, 122, 123, 124]::int2[])) select (s0.n0).id as "id(s)" from s0; + +-- case: match (s)-[r:RegressionKind87|RegressionKind88|RegressionKind89|RegressionKind90|RegressionKind91]->(e:RegressionKind81) where (s:RegressionKind85 or s:RegressionKind86 or s:RegressionKind81) and id(e) in $end_ids return id(s) +-- cypher_params: {"end_ids":[202,303]} +-- pgsql_params:{"pi0":[202,303]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.kind_ids operator (pg_catalog.@>) array [113]::int2[] and n1.id = e0.end_id join node n0 on ((n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [113]::int2[])) and n0.id = e0.start_id where e0.kind_id = any (array [119, 120, 121, 122, 123]::int2[])) select (s0.n0).id as "id(s)" from s0; + +-- case: match (n) where n:RegressionKind85 or n:RegressionKind86 return id(n) +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[])) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n:RegressionKind93) return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [125]::int2[]) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind81) where n.objectid = $objectid return n limit 1 +-- cypher_params: {"objectid":"S-1-5-21"} +-- pgsql_params:{"pi0":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'objectid')) = 'string' and (n0.properties ->> 'objectid') = @pi0::text)) and n0.kind_ids operator (pg_catalog.@>) array [113]::int2[]) select s0.n0 as n from s0 limit 1; + +-- case: match (n) where n.name = $name and n.enabled = $enabled return id(n) +-- cypher_params: {"enabled":true,"name":"dc.example.test"} +-- pgsql_params:{"pi0":"dc.example.test","pi1":true} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = @pi0::text) and ((n0.properties -> 'enabled'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb)) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n:RegressionKind81) where n.hasura = true return id(n), n.hasura +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (((n0.properties -> 'hasura'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [113]::int2[]) select (s0.n0).id as "id(n)", ((s0.n0).properties -> 'hasura') as "n.hasura" from s0; + +-- case: match (n:RegressionKind94) where n.distinguishedname starts with $prefix and n.domainsid = $domain return n +-- cypher_params: {"domain":"S-1-5-21","prefix":"CN=ADMINSDHOLDER,CN=SYSTEM,"} +-- pgsql_params:{"pi0":"CN=ADMINSDHOLDER,CN=SYSTEM,","pi1":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_starts_with((n0.properties ->> 'distinguishedname'), (@pi0::text)::text)::bool and (jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi1::text)) and n0.kind_ids operator (pg_catalog.@>) array [126]::int2[]) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind85) where n.objectid ends with $suffix_a or n.objectid ends with $suffix_b return id(n) +-- cypher_params: {"suffix_a":"-S-1","suffix_b":"-S-2"} +-- pgsql_params:{"pi0":"-S-1","pi1":"-S-2"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool or cypher_ends_with((n0.properties ->> 'objectid'), (@pi1::text)::text)::bool) and n0.kind_ids operator (pg_catalog.@>) array [117]::int2[]) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where toLower(n.name) starts with $prefix return id(n) +-- cypher_params: {"prefix":"remote desktop users%_"} +-- pgsql_params:{"pi0":"remote desktop users%_"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_starts_with((lower((n0.properties ->> 'name'))::text)::text, (@pi0::text)::text)::bool)) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where toLower(n.objectid) contains $fragment return n +-- cypher_params: {"fragment":"approver_guid"} +-- pgsql_params:{"pi0":"approver_guid"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (cypher_contains((lower((n0.properties ->> 'objectid'))::text)::text, (@pi0::text)::text)::bool)) select s0.n0 as n from s0; + +-- case: match (n) where (n:RegressionKind85 or n:RegressionKind86) and n:RegressionKind69 and n.objectid ends with $suffix and n.domainsid = $domain return n +-- cypher_params: {"domain":"S-1-5-21","suffix":"-512"} +-- pgsql_params:{"pi0":"-512","pi1":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [118]::int2[]) and n0.kind_ids operator (pg_catalog.@>) array [101]::int2[] and cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool and (jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi1::text))) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind69) where not (n:RegressionKind85 or n:RegressionKind98) and n.objectid ends with $suffix return n +-- cypher_params: {"suffix":"-512"} +-- pgsql_params:{"pi0":"-512"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not (n0.kind_ids operator (pg_catalog.@>) array [117]::int2[] or n0.kind_ids operator (pg_catalog.@>) array [130]::int2[]) and cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool) and n0.kind_ids operator (pg_catalog.@>) array [101]::int2[]) select s0.n0 as n from s0; + +-- case: match (n) where n.name is null return n +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((not n0.properties ? 'name' or (n0.properties -> 'name') = ('null')::jsonb))) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind95) where n.tenantid = $tenant and n.approvalrequired = true and (n.userapprovers is not null or n.groupapprovers is not null) return n +-- cypher_params: {"tenant":"tenant-1"} +-- pgsql_params:{"pi0":"tenant-1"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'tenantid')) = 'string' and (n0.properties ->> 'tenantid') = @pi0::text) and ((n0.properties -> 'approvalrequired'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties ? 'userapprovers' and not (n0.properties -> 'userapprovers') = ('null')::jsonb) or (n0.properties ? 'groupapprovers' and not (n0.properties -> 'groupapprovers') = ('null')::jsonb))) and n0.kind_ids operator (pg_catalog.@>) array [127]::int2[]) select s0.n0 as n from s0; + +-- case: match (n) where id(n) in $ids return n +-- cypher_params: {"ids":[101,202,101]} +-- pgsql_params:{"pi0":[101,202,101]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.id = any (@pi0::float8[]))) select s0.n0 as n from s0; + +-- case: match (n:RegressionKind86) where not (n.gmsa is not null and n.gmsa = true) and not (n.msa is not null and n.msa = true) and id(n) in $ids return n +-- cypher_params: {"ids":[101,202]} +-- pgsql_params:{"pi0":[101,202]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (not ((n0.properties ? 'gmsa' and not (n0.properties -> 'gmsa') = ('null')::jsonb) and ((n0.properties -> 'gmsa'))::jsonb = to_jsonb((true)::bool)::jsonb) and not ((n0.properties ? 'msa' and not (n0.properties -> 'msa') = ('null')::jsonb) and ((n0.properties -> 'msa'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.id = any (@pi0::float8[])) and n0.kind_ids operator (pg_catalog.@>) array [118]::int2[]) select s0.n0 as n from s0; + +-- case: match (s)-[:RegressionKind97]->(e) where id(s) = $tenant_id and (e:RegressionKind95 or e:RegressionKind96) and e.roletemplateid in $role_ids return e +-- cypher_params: {"role_ids":["role-a","role-b"],"tenant_id":101} +-- pgsql_params:{"pi0":101,"pi1":["role-a","role-b"]} +with s0 as (select n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on ((n1.kind_ids operator (pg_catalog.@>) array [127]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [128]::int2[]) and (n1.properties ->> 'roletemplateid') = any (@pi1::text[])) and n1.id = e0.end_id where e0.kind_id = any (array [129]::int2[])) select s0.n1 as e from s0; + +-- case: match (s)-[:RegressionKind97]->(e:RegressionKind95) where id(s) = $tenant_id and e.enabled = true return e +-- cypher_params: {"tenant_id":101} +-- pgsql_params:{"pi0":101} +with s0 as (select n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (((n1.properties -> 'enabled'))::jsonb = to_jsonb((true)::bool)::jsonb) and n1.kind_ids operator (pg_catalog.@>) array [127]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [129]::int2[])) select s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind83]->(e) where id(s) = $start_id and id(e) = $end_id return r limit 1 +-- cypher_params: {"end_id":202,"start_id":101} +-- pgsql_params:{"pi0":101,"pi1":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (n1.id = @pi1::float8) and n1.id = e0.end_id where e0.kind_id = any (array [115]::int2[]) limit 1) select s0.e0 as r from s0 limit 1; + +-- case: match (s)-[:RegressionKind82]->(e) where s.objectid ends with $suffix and id(e) = $end_id return s +-- cypher_params: {"end_id":202,"suffix":"-555"} +-- pgsql_params:{"pi0":"-555","pi1":202} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi1::float8) and n1.id = e0.end_id join node n0 on (cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool) and n0.id = e0.start_id where e0.kind_id = any (array [114]::int2[])) select s0.n0 as s from s0; + +-- case: match (s)-[:RegressionKind82]->(e) where s.objectid ends with $suffix and id(e) = $end_id return id(s) +-- cypher_params: {"end_id":202,"suffix":"-555"} +-- pgsql_params:{"pi0":"-555","pi1":202} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi1::float8) and n1.id = e0.end_id join node n0 on (cypher_ends_with((n0.properties ->> 'objectid'), (@pi0::text)::text)::bool) and n0.id = e0.start_id where e0.kind_id = any (array [114]::int2[])) select (s0.n0).id as "id(s)" from s0; + +-- case: match (n:RegressionKind99) return n order by n.name desc +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [131]::int2[]) select s0.n0 as n from s0 order by ((s0.n0).properties -> 'name') desc; + +-- case: match (n:RegressionKind81) where n.domainsid = $domain and n.isdc = true and n.ldapavailable = true and n.ldapsigning = false return id(n) +-- cypher_params: {"domain":"S-1-5-21"} +-- pgsql_params:{"pi0":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi0::text) and ((n0.properties -> 'isdc'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'ldapavailable'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'ldapsigning'))::jsonb = to_jsonb((false)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [113]::int2[]) select (s0.n0).id as "id(n)" from s0; + +-- case: match (n) where n.domainsid = $domain and n.isdc = true and n.ldapsavailable = true and n.epa = false return n +-- cypher_params: {"domain":"S-1-5-21"} +-- pgsql_params:{"pi0":"S-1-5-21"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'domainsid')) = 'string' and (n0.properties ->> 'domainsid') = @pi0::text) and ((n0.properties -> 'isdc'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'ldapsavailable'))::jsonb = to_jsonb((true)::bool)::jsonb and ((n0.properties -> 'epa'))::jsonb = to_jsonb((false)::bool)::jsonb)) select s0.n0 as n from s0; diff --git a/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql b/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql index 21458c72..98a9830e 100644 --- a/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql +++ b/cypher/models/pgsql/test/translation_cases/scalar_aggregation.sql @@ -15,58 +15,58 @@ -- SPDX-License-Identifier: Apache-2.0 -- case: MATCH (n) RETURN sum(n.age) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select sum((((s0.n0).properties ->> 'age'))::float8)::numeric from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select sum((((s0.n0).properties ->> 'age'))::float8)::numeric as "sum(n.age)" from s0; -- case: MATCH (n) RETURN avg(n.salary) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select avg((((s0.n0).properties ->> 'salary'))::float8)::numeric from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select avg((((s0.n0).properties ->> 'salary'))::float8)::numeric as "avg(n.salary)" from s0; -- case: MATCH (n) RETURN min(n.created_date) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'created_date'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'created_date'))::jsonb as "min(n.created_date)" from s0; -- case: MATCH (n) RETURN max(n.updated_date) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'updated_date'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'updated_date'))::jsonb as "max(n.updated_date)" from s0; -- case: MATCH (n) RETURN min(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'name'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_min(((s0.n0).properties -> 'name'))::jsonb as "min(n.name)" from s0; -- case: MATCH (n) RETURN max(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'name'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cypher_max(((s0.n0).properties -> 'name'))::jsonb as "max(n.name)" from s0; -- case: MATCH (n) RETURN n.department, sum(n.salary) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), sum((((s0.n0).properties ->> 'salary'))::float8)::numeric from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", sum((((s0.n0).properties ->> 'salary'))::float8)::numeric as "sum(n.salary)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN n.department, avg(n.age) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), avg((((s0.n0).properties ->> 'age'))::float8)::numeric from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", avg((((s0.n0).properties ->> 'age'))::float8)::numeric as "avg(n.age)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN count(n), sum(n.age), avg(n.age), min(n.age), max(n.age) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8, sum((((s0.n0).properties ->> 'age'))::float8)::numeric, avg((((s0.n0).properties ->> 'age'))::float8)::numeric, cypher_min(((s0.n0).properties -> 'age'))::jsonb, cypher_max(((s0.n0).properties -> 'age'))::jsonb from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 as "count(n)", sum((((s0.n0).properties ->> 'age'))::float8)::numeric as "sum(n.age)", avg((((s0.n0).properties ->> 'age'))::float8)::numeric as "avg(n.age)", cypher_min(((s0.n0).properties -> 'age'))::jsonb as "min(n.age)", cypher_max(((s0.n0).properties -> 'age'))::jsonb as "max(n.age)" from s0; -- case: RETURN 'hello world' -select 'hello world'; +select 'hello world' as "'hello world'"; -- case: RETURN 2 + 3 -select 2 + 3; +select 2 + 3 as "2 + 3"; -- case: MATCH (n) RETURN n.department, collect(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as "collect(n.name)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN collect(n.name) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as "collect(n.name)" from s0; -- case: MATCH (n) RETURN n.department, collect(n.name), count(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department'), array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray, count(s0.n0)::int8 from s0 group by ((s0.n0).properties -> 'department'); +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select ((s0.n0).properties -> 'department') as "n.department", array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as "collect(n.name)", count(s0.n0)::int8 as "count(n)" from s0 group by ((s0.n0).properties -> 'department'); -- case: MATCH (n) RETURN size(n.tags) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select jsonb_array_length(((s0.n0).properties -> 'tags'))::int from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select case when jsonb_typeof(((s0.n0).properties -> 'tags')) = 'array' then jsonb_array_length(((s0.n0).properties -> 'tags'))::int else null end as "size(n.tags)" from s0; -- case: MATCH (n) RETURN size(collect(n.name)) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cardinality(array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray)::int from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select cardinality(array_remove(coalesce(array_agg(((s0.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray)::int as "size(collect(n.name))" from s0; -- case: MATCH (n) WITH collect(labels(n)) as label_sets RETURN size(label_sets) -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(to_jsonb((array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[])::jsonb)::jsonb[], array []::jsonb[])::jsonb[], null)::jsonb[] as i0 from s1) select cardinality(s0.i0)::int from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select array_remove(coalesce(array_agg(to_jsonb((array(select _kind.name from generate_subscripts((s1.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s1.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[])::jsonb)::jsonb[], array []::jsonb[])::jsonb[], null)::jsonb[] as i0 from s1) select cardinality(s0.i0)::int as "size(label_sets)" from s0; -- case: MATCH (n) WHERE size(n.permissions) > 2 RETURN n -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (jsonb_array_length((n0.properties -> 'permissions'))::int > 2)) select s0.n0 as n from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (case when jsonb_typeof((n0.properties -> 'permissions')) = 'array' then jsonb_array_length((n0.properties -> 'permissions'))::int else null end > 2)) select s0.n0 as n from s0; -- case: MATCH (n) WITH n, collect(n.prop) as props WHERE size(props) > 1 RETURN n, props with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select s1.n0 as n0, array_remove(coalesce(array_agg(((s1.n0).properties ->> 'prop'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1 group by n0) select s0.n0 as n, s0.i0 as props from s0 where (cardinality(s0.i0)::int > 1); @@ -84,19 +84,19 @@ with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposit with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s1.n0)::int8 as i0 from s1), s2 as (select s0.i0 as i0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1) select s2.n1 as o from s2; -- case: MATCH (n) RETURN count(n) + count(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 + count(s0.n0)::int8 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 + count(s0.n0)::int8 as "count(n) + count(n)" from s0; -- case: MATCH (n) RETURN count(n) * 2 -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 * 2 from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 * 2 as "count(n) * 2" from s0; -- case: MATCH (n) RETURN count(n) AS total ORDER BY total DESC with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select count(s0.n0)::int8 as total from s0 order by total desc; -- case: MATCH (n) RETURN toInteger(n.value) + count(n) -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + count(s0.n0)::int8 from s0 group by (((s0.n0).properties ->> 'value'))::int8; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s0.n0).properties ->> 'value'))::int8 + count(s0.n0)::int8 as "toInteger(n.value) + count(n)" from s0 group by (((s0.n0).properties ->> 'value'))::int8; -- case: MATCH (n) WITH toInteger(n.value) AS value, count(n) AS node_count RETURN value + node_count -with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s1.n0).properties ->> 'value'))::int8 as i0, count(s1.n0)::int8 as i1 from s1 group by (((s1.n0).properties ->> 'value'))::int8) select s0.i0 + s0.i1 from s0; +with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s1.n0).properties ->> 'value'))::int8 as i0, count(s1.n0)::int8 as i1 from s1 group by (((s1.n0).properties ->> 'value'))::int8) select s0.i0 + s0.i1 as "value + node_count" from s0; -- case: MATCH (n) WITH toInteger(n.value) + count(n) AS score RETURN score with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0) select (((s1.n0).properties ->> 'value'))::int8 + count(s1.n0)::int8 as i0 from s1 group by (((s1.n0).properties ->> 'value'))::int8) select s0.i0 as score from s0; diff --git a/cypher/models/pgsql/test/translation_cases/shortest_paths.sql b/cypher/models/pgsql/test/translation_cases/shortest_paths.sql index d2439bdd..8d0265e3 100644 --- a/cypher/models/pgsql/test/translation_cases/shortest_paths.sql +++ b/cypher/models/pgsql/test/translation_cases/shortest_paths.sql @@ -16,81 +16,81 @@ -- case: match p = allShortestPaths((s:NodeKind1)-[*..]->()) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from edge where end_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = allShortestPaths((s:NodeKind1)-[*..]->({name: "123"})) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where (jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = '123')) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id) = 0 then true else shortest_path_self_endpoint_error(e0.end_id, e0.end_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p = allShortestPaths((s:NodeKind1)-[*..]->(e)) where e.name = '123' return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = '123'))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id) = 0 then true else shortest_path_self_endpoint_error(e0.end_id, e0.end_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n0.id from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p=shortestPath((n:NodeKind1)-[:EdgeKind1*1..]->(m)) where 'admin_tier_0' in split(m.system_tags, ' ') and n.objectid ends with '-513' and n<>m return p limit 1000 --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, (1000)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id) limit 1000; +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from pg_temp.bsp_forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s1.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from pg_temp.bsp_backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s1.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, false, (1000)::int8) limit 1000) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id) limit 1000; -- case: match p=shortestPath((n:NodeKind1)-[:EdgeKind1*1..]->(m)) where 'admin_tier_0' in split(m.system_tags, ' ') and n.objectid ends with '-513' and m<>n return p limit 1000 --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, (1000)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((n0.properties -\u003e\u003e 'objectid') like '%-513') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from pg_temp.bsp_forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s1.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ('admin_tier_0' = any (string_to_array((n1.properties -\u003e\u003e 'system_tags'), ' ')::text[]))) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from pg_temp.bsp_backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s1.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((n0.properties ->> ''objectid'') like ''%-513'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and (''admin_tier_0'' = any (string_to_array((n1.properties ->> ''system_tags''), '' '')::text[])) and n0.id is not null and n1.id is not null;')::text, false, (1000)::int8) limit 1000) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; -- case: match p=shortestPath((t:NodeKind1)<-[:EdgeKind1|EdgeKind2*1..]-(s:NodeKind2)) where coalesce(t.system_tags, '') contains 'admin_tier_0' and t.name =~ 'name.*' and s<>t return p limit 1000 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (coalesce((n0.properties -\u003e\u003e 'system_tags'), '')::text like '%admin_tier_0%' and (n0.properties -\u003e\u003e 'name') ~ 'name.*') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3, 4]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3, 4]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (1000)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (1000)::int8) limit 1000) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id) limit 1000; -- case: match p=shortestPath((a)-[:EdgeKind1*]->(b)) where id(a) = 1 and id(b) = 2 return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (n0.id = 1)) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where (n1.id = 2)) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where (n0.id = 1) and (n1.id = 2) and n0.id is not null and n1.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +-- pgsql_params:{"pi0":1,"pi1":2} +with s0 as (with recursive singleton_endpoints as (select n0.id as root_id, n1.id as terminal_id from node n0, node n1 where (n0.id = @pi0::int8) and (n1.id = @pi1::int8) and case when n0.id != n1.id then true else shortest_path_self_endpoint_error(n0.id, n1.id) end), s1(next_id, depth, path) as (select singleton_endpoints.root_id, 0, array []::int8[] from singleton_endpoints union all select e0.end_id, s1.depth + 1, s1.path || array [e0.id]::int8[] from s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and s1.depth < 15 and e0.id != all (s1.path)) select (array [(n0.id, n0.kind_ids, n0.properties)::nodecomposite]::nodecomposite[] || coalesce(m0_hydrated.nodes, array []::nodecomposite[]), coalesce(m0_hydrated.edges, array []::edgecomposite[]))::pathcomposite as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join singleton_endpoints on s1.next_id = singleton_endpoints.terminal_id join node n0 on n0.id = singleton_endpoints.root_id join node n1 on n1.id = s1.next_id join lateral (select array_agg((m0_terminal.id, m0_terminal.kind_ids, m0_terminal.properties)::nodecomposite order by m0_path_index)::nodecomposite[] as nodes, array_agg((m0_edge.id, m0_edge.start_id, m0_edge.end_id, m0_edge.kind_id, m0_edge.properties)::edgecomposite order by m0_path_index)::edgecomposite[] as edges, count(*)::int8 as hydrated_count from generate_subscripts(s1.path, 1) as m0_path_index join edge m0_edge on m0_edge.id = (s1.path)[m0_path_index] join node m0_terminal on m0_terminal.id = m0_edge.end_id) m0_hydrated on true where s1.depth >= 1 and m0_hydrated.hydrated_count = cardinality(s1.path) order by s1.depth, s1.path limit 1) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else s0.ep0 end as p from s0; -- case: match p=shortestPath((a)-[:EdgeKind1*]->(b:NodeKind1)) where a <> b return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from edge where end_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.end_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); -- case: match p=shortestPath((a:NodeKind2)-[:EdgeKind1*]->(b)) where a <> b return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [2]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from edge where end_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.end_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); -- case: match p=shortestPath((b)<-[:EdgeKind1*]-(a)) where id(a) = 1 and id(b) = 2 return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (n0.id = 2)) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = e0.end_id) = 0 then true else shortest_path_self_endpoint_error(e0.end_id, e0.end_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s1.root_id and forward_visited.id = e0.start_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where (n1.id = 1)) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.end_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s1.root_id and backward_visited.id = e0.end_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where (n0.id = 2) and (n1.id = 1) and n0.id is not null and n1.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +-- pgsql_params:{"pi0":2,"pi1":1} +with s0 as (with singleton_endpoints as (select n0.id as root_id, n1.id as terminal_id from node n0, node n1 where (n0.id = @pi0::int8) and (n1.id = @pi1::int8) and case when n0.id != n1.id then true else shortest_path_self_endpoint_error(n0.id, n1.id) end), s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select shortest_path_compact.* from singleton_endpoints, shortest_path_compact(0, singleton_endpoints.root_id, singleton_endpoints.terminal_id, 1, 15, array [3]::int2[], true, 100000)) select (array [(n0.id, n0.kind_ids, n0.properties)::nodecomposite]::nodecomposite[] || coalesce(m0_hydrated.nodes, array []::nodecomposite[]), coalesce(m0_hydrated.edges, array []::edgecomposite[]))::pathcomposite as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id join lateral (select array_agg((m0_terminal.id, m0_terminal.kind_ids, m0_terminal.properties)::nodecomposite order by m0_path_index)::nodecomposite[] as nodes, array_agg((m0_edge.id, m0_edge.start_id, m0_edge.end_id, m0_edge.kind_id, m0_edge.properties)::edgecomposite order by m0_path_index)::edgecomposite[] as edges, count(*)::int8 as hydrated_count from generate_subscripts(s1.path, 1) as m0_path_index join edge m0_edge on m0_edge.id = (s1.path)[m0_path_index] join node m0_terminal on m0_terminal.id = m0_edge.start_id) m0_hydrated on true where m0_hydrated.hydrated_count = cardinality(s1.path)) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else s0.ep0 end as p from s0; -- case: match p = allShortestPaths((m:NodeKind1)<-[:EdgeKind1*..]-(n)) where coalesce(m.system_tags, '') contains 'admin_tier_0' and n.name = '123' and n <> m return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = '123'))) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s1.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where (coalesce((n0.properties -\u003e\u003e 'system_tags'), '')::text like '%admin_tier_0%') and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s1.root_id), false, e0.id || s1.path from backward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n1.id, n0.id from node n1, node n0 where ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''123'')) and (coalesce((n0.properties ->> ''system_tags''), '''')::text like ''%admin_tier_0%'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id is not null and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n1.id, n0.id from node n1, node n0 where ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''123'')) and (coalesce((n0.properties ->> ''system_tags''), '''')::text like ''%admin_tier_0%'') and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n1.id is not null and n0.id is not null;')::text)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n1).id <> (s0.n0).id); -- case: match p=shortestPath((a)-[:EdgeKind1*]->(b:NodeKind1)) where a <> b return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n1.id as root_id from node n1 where n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from edge where end_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.end_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.start_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.end_id), false, e0.id || s1.path from forward_front s1 join edge e0 on e0.end_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.start_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n1 on n1.id = s1.root_id join node n0 on n0.id = s1.next_id) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 where ((s0.n0).id <> (s0.n1).id); -- case: match p=(c:NodeKind1)-[]->(u:NodeKind2) match p2=shortestPath((u:NodeKind2)-[*1..]->(d:NodeKind1)) return p, p2 limit 500 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select distinct n1.id as root_id from traversal_root_filter s2_seed_filter join node n1 on n1.id = s2_seed_filter.id where n1.kind_ids operator (pg_catalog.@\u003e) array [2]::int2[]) select e1.start_id, e1.end_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e1.end_id), e1.start_id = e1.end_id, array [e1.id] from s2_seed join edge e1 on e1.start_id = s2_seed.root_id where case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e1.start_id) = 0 then true else shortest_path_self_endpoint_error(e1.start_id, e1.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e1.end_id, s2.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e1.end_id), false, s2.path || e1.id from forward_front s2 join edge e1 on e1.start_id = s2.next_id where e1.id != all (s2.path) and not exists (select 1 from visited where visited.root_id = s2.root_id and visited.id = e1.end_id);"} -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id), s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('insert into traversal_root_filter (id) select distinct (s0.n1).id from s0 where (s0.n1).id is not null;')::text, ('insert into traversal_terminal_filter (id) select distinct n2.id from node n2 where n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id is not null;')::text)) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join node n1 on n1.id = s2.root_id join node n2 on n2.id = s2.next_id where (s0.n1).id = s2.root_id and case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s1.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edges_to_path(s1.n1, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p2 from s1 limit 500; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id), s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('insert into traversal_root_filter (id) select distinct (s0.n1).id from s0 where (s0.n1).id is not null;')::text, ('insert into traversal_terminal_filter (id) select distinct n2.id from node n2 where n2.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n2.id is not null;')::text)) select s0.e0 as e0, s2.path as ep0, s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0, s2 join node n1 on n1.id = s2.root_id join node n2 on n2.id = s2.next_id where (s0.n1).id = s2.root_id and case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.e0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, array [s1.e0]::int8[], array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as p, case when (s1.n1).id is null or s1.ep0 is null or (s1.n2).id is null then null else ordered_edge_ids_to_path(0, s1.n1, s1.ep0, array [s1.n1, s1.n2]::nodecomposite[])::pathcomposite end as p2 from s1 limit 500; -- case: match p = allShortestPaths((a)-[:EdgeKind1*..]->()) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select e0.start_id, e0.end_id, 1, exists (select 1 from edge where end_id = e0.start_id), e0.start_id = e0.end_id, array [e0.id] from edge e0 where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from edge where end_id = e0.start_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_asp_harness(@pi0::text, @pi1::text, 15)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0; -- case: match p=shortestPath((n:NodeKind1)-[:EdgeKind1*1..]->(m:NodeKind2)) return p limit 10 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s1_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s1_seed join edge e0 on e0.start_id = s1_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s1.root_id, e0.end_id, s1.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), false, s1.path || e0.id from forward_front s1 join edge e0 on e0.start_id = s1.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s1.path) and not exists (select 1 from visited where visited.root_id = s1.root_id and visited.id = e0.end_id);"} -with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (10)::int8)) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s0.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (with s1(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text, (10)::int8) limit 10) select s1.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1 join node n0 on n0.id = s1.root_id join node n1 on n1.id = s1.next_id where case when s1.root_id != s1.next_id then true else shortest_path_self_endpoint_error(s1.root_id, s1.next_id) end) select case when (s0.n0).id is null or s0.ep0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, s0.ep0, array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match (a:NodeKind1), (b:NodeKind2) match p=shortestPath((a)-[:EdgeKind1*]->(b)) return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s3.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s3.root_id and backward_visited.id = e0.start_id);"} -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from pg_temp.bsp_forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s3.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from pg_temp.bsp_backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s3.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text, false)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0, array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match (a:NodeKind1), (b:NodeKind2) match p=allShortestPaths((a)-[:EdgeKind1*..]->(b)) return p -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [3]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s3.path);"} -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0, array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2; -- case: match p=shortestPath((u:NodeKind1)-[:EdgeKind1*1..]->(g:NodeKind2)) with distinct g as Group, count(u) as UserCount return Group.name, UserCount order by UserCount desc limit 5 -- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id where e0.kind_id = any (array [3]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.end_id, s2.depth + 1, exists (select 1 from traversal_terminal_filter where traversal_terminal_filter.id = e0.end_id), false, s2.path || e0.id from forward_front s2 join edge e0 on e0.start_id = s2.next_id where e0.kind_id = any (array [3]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from visited where visited.root_id = s2.root_id and visited.id = e0.end_id);"} -with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text)) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select distinct s1.n1 as n2, count(s1.n0)::int8 as i0 from s1 group by n1) select ((s0.n2).properties -> 'name'), s0.i0 as UserCount from s0 order by s0.i0 desc limit 5; +with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from unidirectional_sp_harness(@pi0::text, @pi1::text, 15, ('')::text, ('insert into traversal_terminal_filter (id) select distinct n1.id from node n1 where n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id is not null;')::text)) select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select distinct s1.n1 as n2, count(s1.n0)::int8 as i0 from s1 group by n1) select ((s0.n2).properties -> 'name') as "Group.name", s0.i0 as UserCount from s0 order by s0.i0 desc limit 5; -- case: MATCH (g1:Group) MATCH (g2:Group) WHERE g1.name STARTS WITH 'DOMAIN USERS@' AND g2.name STARTS WITH 'DOMAIN ADMINS@' MATCH p=shortestPath((g1)-[:AddAllowedToAct|AddMember|AdminTo|AllExtendedRights|AllowedToDelegate|CanRDP|Contains|ForceChangePassword|GenericAll|GenericWrite|GetChangesAll|GetChanges|HasSession|MemberOf|Owns|ReadLAPSPassword|SQLAdmin|TrustedBy|WriteAccountRestrictions|WriteOwner*1..]->(g2)) WHERE NONE(r IN relationships(p) WHERE type(r) = 'HasSession' AND startNode(r).name = 'DF-WIN10-DEV01.DUMPSTER.FIRE') RETURN p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s3.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s3.root_id and backward_visited.id = e0.start_id);"} -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'name') like 'DOMAIN ADMINS@%' and ((s0.n0).properties ->> 'name') like 'DOMAIN USERS@%') and n1.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edges_to_path(s2.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s2.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2 where ((not exists (select 1 from edge i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'DF-WIN10-DEV01.DUMPSTER.FIRE') and i0.kind_id = 7) and i0.id = any (s2.ep0)) and s2.ep0 is not null)::bool); +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_root_filter s3_seed_filter) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.start_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and case when (select count(*)::int8 from traversal_terminal_filter where traversal_terminal_filter.id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.end_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s3.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s3.path || e0.id from pg_temp.bsp_forward_front s3 join edge e0 on e0.start_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s3.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s3_seed(root_id) as not materialized (select s3_seed_filter.id as root_id from traversal_terminal_filter s3_seed_filter) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s3_seed join edge e0 on e0.end_id = s3_seed.root_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s3.root_id, e0.start_id, s3.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s3.root_id), false, e0.id || s3.path from pg_temp.bsp_backward_front s3 join edge e0 on e0.end_id = s3.next_id where e0.kind_id = any (array [14, 15, 16, 17, 18, 19, 12, 20, 21, 22, 23, 24, 7, 25, 26, 27, 28, 29, 30, 31]::int2[]) and e0.id != all (s3.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s3.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties ->> 'name') like 'DOMAIN ADMINS@%' and ((s0.n0).properties ->> 'name') like 'DOMAIN USERS@%') and n1.kind_ids operator (pg_catalog.@>) array [13]::int2[]), s2 as (with s3(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct (s1.n0).id, (s1.n1).id from s1 where (s1.n0).id is not null and (s1.n1).id is not null;')::text, false)) select s3.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s1, s3 join node n0 on n0.id = s3.root_id join node n1 on n1.id = s3.next_id where (s1.n0).id = s3.root_id and (s1.n1).id = s3.next_id and case when s3.root_id != s3.next_id then true else shortest_path_self_endpoint_error(s3.root_id, s3.next_id) end) select case when (s2.n0).id is null or s2.ep0 is null or (s2.n1).id is null then null else ordered_edge_ids_to_path(0, s2.n0, s2.ep0, array [s2.n0, s2.n1]::nodecomposite[])::pathcomposite end as p from s2 where ((not exists (select 1 from edge i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'DF-WIN10-DEV01.DUMPSTER.FIRE') and i0.kind_id = 7) and i0.id = any (s2.ep0)) and s2.ep0 is not null)::bool); -- case: match p=shortestPath((s:NodeKind1)-[:EdgeKind1|HasSession*1..]->(d:NodeKind1)) where s.name = 'path-filter-src' and d.name = 'path-filter-dst' with p where none(r in relationships(p) where type(r) = 'HasSession' and startNode(r).name = 'blocked-session-host') return p --- pgsql_params:{"pi0":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -\u003e 'name')) = 'string' and (n0.properties -\u003e\u003e 'name') = 'path-filter-src')) and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.end_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s2.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s2.path || e0.id from forward_front s2 join edge e0 on e0.start_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from forward_visited where forward_visited.root_id = s2.root_id and forward_visited.id = e0.end_id);","pi2":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = 'path-filter-dst')) and n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]);","pi3":"insert into next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.start_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s2.root_id), false, e0.id || s2.path from backward_front s2 join edge e0 on e0.end_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from backward_visited where backward_visited.root_id = s2.root_id and backward_visited.id = e0.start_id);"} -with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into traversal_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((jsonb_typeof((n0.properties -> ''name'')) = ''string'' and (n0.properties ->> ''name'') = ''path-filter-src'')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''path-filter-dst'')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null and n1.id is not null;')::text)) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edges_to_path(s1.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(s1.ep0) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as pc0 from s1) select s0.pc0 as p from s0 where (((select count(*)::int from unnest(((s0.pc0).edges)::edgecomposite[]) as i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'blocked-session-host') and i0.kind_id = 7)) = 0 and ((s0.pc0).edges)::edgecomposite[] is not null)::bool); +-- pgsql_params:{"pi0":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n0.id as root_id from node n0 where ((jsonb_typeof((n0.properties -\u003e 'name')) = 'string' and (n0.properties -\u003e\u003e 'name') = 'path-filter-src')) and n0.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.start_id, e0.end_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.start_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]) and case when (select count(*)::int8 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.start_id) = 0 then true else shortest_path_self_endpoint_error(e0.start_id, e0.start_id) end;","pi1":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.end_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = s2.root_id and traversal_pair_filter.terminal_id = e0.end_id), false, s2.path || e0.id from pg_temp.bsp_forward_front s2 join edge e0 on e0.start_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from pg_temp.bsp_forward_visited where pg_temp.bsp_forward_visited.root_id = s2.root_id and pg_temp.bsp_forward_visited.id = e0.end_id);","pi2":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) with s2_seed(root_id) as not materialized (select n1.id as root_id from node n1 where ((jsonb_typeof((n1.properties -\u003e 'name')) = 'string' and (n1.properties -\u003e\u003e 'name') = 'path-filter-dst')) and n1.kind_ids operator (pg_catalog.@\u003e) array [1]::int2[]) select e0.end_id, e0.start_id, 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = e0.end_id), e0.start_id = e0.end_id, array [e0.id] from s2_seed join edge e0 on e0.end_id = s2_seed.root_id where e0.kind_id = any (array [3, 7]::int2[]);","pi3":"insert into pg_temp.bsp_next_front (root_id, next_id, depth, satisfied, is_cycle, path) select s2.root_id, e0.start_id, s2.depth + 1, exists (select 1 from traversal_pair_filter where traversal_pair_filter.root_id = e0.start_id and traversal_pair_filter.terminal_id = s2.root_id), false, e0.id || s2.path from pg_temp.bsp_backward_front s2 join edge e0 on e0.end_id = s2.next_id where e0.kind_id = any (array [3, 7]::int2[]) and e0.id != all (s2.path) and not exists (select 1 from pg_temp.bsp_backward_visited where pg_temp.bsp_backward_visited.root_id = s2.root_id and pg_temp.bsp_backward_visited.id = e0.start_id);"} +with s0 as (with s1 as (with s2(root_id, next_id, depth, satisfied, is_cycle, path) as (select * from bidirectional_sp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 15, ('')::text, ('')::text, ('insert into pg_temp.bsp_pair_filter (root_id, terminal_id) select distinct n0.id, n1.id from node n0, node n1 where ((jsonb_typeof((n0.properties -> ''name'')) = ''string'' and (n0.properties ->> ''name'') = ''path-filter-src'')) and n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and ((jsonb_typeof((n1.properties -> ''name'')) = ''string'' and (n1.properties ->> ''name'') = ''path-filter-dst'')) and n1.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id is not null and n1.id is not null;')::text, false)) select s2.path as ep0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s2 join node n0 on n0.id = s2.root_id join node n1 on n1.id = s2.next_id where case when s2.root_id != s2.next_id then true else shortest_path_self_endpoint_error(s2.root_id, s2.next_id) end) select case when (s1.n0).id is null or s1.ep0 is null or (s1.n1).id is null then null else ordered_edge_ids_to_path(0, s1.n0, s1.ep0, array [s1.n0, s1.n1]::nodecomposite[])::pathcomposite end as pc0 from s1) select s0.pc0 as p from s0 where (((select count(*)::int from unnest(((s0.pc0).edges)::edgecomposite[]) as i0 where ((jsonb_typeof(((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties -> 'name')) = 'string' and ((start_node((i0.id, i0.start_id, i0.end_id, i0.kind_id, i0.properties)::edgecomposite)::nodecomposite).properties ->> 'name') = 'blocked-session-host') and i0.kind_id = 7)) = 0 and ((s0.pc0).edges)::edgecomposite[] is not null)::bool); diff --git a/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql b/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql index f48a2bcf..741f5895 100644 --- a/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql +++ b/cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql @@ -21,16 +21,16 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (e0.kind_id = 3)) select s0.e0 as r from s0; -- case: match ()-[r]->() return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where type(r) <> 'EdgeKind1' return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (e0.kind_id <> 3)) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (e0.kind_id <> 3)) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where type(r) in ['EdgeKind2'] return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text = any (array ['EdgeKind2']::text[]))) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text = any (array ['EdgeKind2']::text[]))) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where type(r) STARTS WITH 'EdgeKind' return type(r) order by type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text like 'EdgeKind%')) select kind_name((s0.e0).kind_id)::text from s0 order by kind_name((s0.e0).kind_id)::text; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (kind_name(e0.kind_id)::text like 'EdgeKind%')) select kind_name((s0.e0).kind_id)::text as "type(r)" from s0 order by kind_name((s0.e0).kind_id)::text; -- case: match ()-[r]->() where 'EdgeKind1' = type(r) return r with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (3 = e0.kind_id)) select s0.e0 as r from s0; @@ -42,7 +42,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1 from s0, edge e1 join node n2 on n2.id = e1.start_id join node n3 on n3.id = e1.end_id) select s1.e0 as r, s1.e1 as e from s1; -- case: match p = (:NodeKind1)-[:EdgeKind1|EdgeKind2]->(c:NodeKind2) where '123' in c.prop2 or '243' in c.prop2 or size(c.prop2) = 0 return p limit 10 -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('123' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or '243' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or jsonb_array_length((n1.properties -> 'prop2'))::int = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) limit 10) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edges_to_path(s0.n0, (select coalesce(array_agg((_edge.id, _edge.start_id, _edge.end_id, _edge.kind_id, _edge.properties)::edgecomposite order by _path.ordinality), array []::edgecomposite[]) from unnest(array [s0.e0]::int8[]) with ordinality as _path(id, ordinality) join edge _edge on _edge.id = _path.id), array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on ('123' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or '243' = any (jsonb_to_text_array((n1.properties -> 'prop2'))::text[]) or case when jsonb_typeof((n1.properties -> 'prop2')) = 'array' then jsonb_array_length((n1.properties -> 'prop2'))::int else null end = 0) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[]) limit 10) select case when (s0.n0).id is null or s0.e0 is null or (s0.n1).id is null then null else ordered_edge_ids_to_path(0, s0.n0, array [s0.e0]::int8[], array [s0.n0, s0.n1]::nodecomposite[])::pathcomposite end as p from s0 limit 10; -- case: match ()-[r:EdgeKind1]->() return count(r) as the_count select count(*)::int8 as the_count from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[]); @@ -53,10 +53,115 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e -- case: match ()-[r:EdgeKind1]->({name: "123"}) return count(r) as the_count with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0 from edge e0 join node n1 on (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = '123') and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select count(s0.e0)::int8 as the_count from s0; +-- case: match (s)-[r:RegressionKind01]->(e) where id(s) = $start_id return r, e +-- cypher_params: {"start_id":101} +-- pgsql_params:{"pi0":101} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01]->(e) where id(e) = $end_id return r, s +-- cypher_params: {"end_id":202} +-- pgsql_params:{"pi0":202} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = @pi0::float8) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e) where id(s) in $start_ids return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e) where id(e) in $end_ids return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind51]->(e) where id(s) in $start_ids and (e:RegressionKind52 or e:RegressionKind53) return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on ((n1.kind_ids operator (pg_catalog.@>) array [84]::int2[] or n1.kind_ids operator (pg_catalog.@>) array [85]::int2[])) and n1.id = e0.end_id where e0.kind_id = any (array [83]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind54]->(e) where id(s) = $start_id and id(e) in $end_ids return r, e +-- cypher_params: {"end_ids":[202,303],"start_id":101} +-- pgsql_params:{"pi0":101,"pi1":[202,303]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (n1.id = any (@pi1::float8[])) and n1.id = e0.end_id where e0.kind_id = any (array [86]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind55]->(e) where id(s) = $start_id and e.enabled = $enabled and e.score = $score and e.name = $name and e.isassignabletorole = $role_value return r, e +-- cypher_params: {"enabled":true,"name":"target","role_value":"true","score":7,"start_id":101} +-- pgsql_params:{"pi0":101,"pi1":true,"pi2":7,"pi3":"target","pi4":"true"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on (((n1.properties -> 'enabled'))::jsonb = to_jsonb((@pi1::bool)::bool)::jsonb and ((n1.properties -> 'score'))::jsonb = to_jsonb((@pi2::float8)::float8)::jsonb and (jsonb_typeof((n1.properties -> 'name')) = 'string' and (n1.properties ->> 'name') = @pi3::text) and (jsonb_typeof((n1.properties -> 'isassignabletorole')) = 'string' and (n1.properties ->> 'isassignabletorole') = @pi4::text)) and n1.id = e0.end_id join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id where e0.kind_id = any (array [87]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind56]->(e:RegressionKind57) where id(s) = $start_id and ((e.requiresmanagerapproval = false and e.schemaversion > 1 and e.authorizedsignatures = 0 and e.authenticationenabled = true) or (e.requiresmanagerapproval = false and e.schemaversion = 1 and e.authenticationenabled = true)) return r, e +-- cypher_params: {"start_id":101} +-- pgsql_params:{"pi0":101} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on (((((n1.properties -> 'requiresmanagerapproval'))::jsonb = to_jsonb((false)::bool)::jsonb and ((n1.properties ->> 'schemaversion'))::int8 > 1 and ((n1.properties -> 'authorizedsignatures'))::jsonb = to_jsonb((0)::int8)::jsonb and ((n1.properties -> 'authenticationenabled'))::jsonb = to_jsonb((true)::bool)::jsonb) or (((n1.properties -> 'requiresmanagerapproval'))::jsonb = to_jsonb((false)::bool)::jsonb and ((n1.properties -> 'schemaversion'))::jsonb = to_jsonb((1)::int8)::jsonb and ((n1.properties -> 'authenticationenabled'))::jsonb = to_jsonb((true)::bool)::jsonb))) and n1.kind_ids operator (pg_catalog.@>) array [89]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [88]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind58]->(e) where id(s) = $start_id and (e.schannelauthenticationenabled = true or size(e.effectiveekus) = 0 or $eku in e.effectiveekus) return r, e +-- cypher_params: {"eku":"1.3.6.1.5.5.7.3.2","start_id":101} +-- pgsql_params:{"pi0":101,"pi1":"1.3.6.1.5.5.7.3.2"} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::float8) and n0.id = e0.start_id join node n1 on ((((n1.properties -> 'schannelauthenticationenabled'))::jsonb = to_jsonb((true)::bool)::jsonb or case when jsonb_typeof((n1.properties -> 'effectiveekus')) = 'array' then jsonb_array_length((n1.properties -> 'effectiveekus'))::int else null end = 0 or @pi1::text = any (jsonb_to_text_array((n1.properties -> 'effectiveekus'))::text[]))) and n1.id = e0.end_id where e0.kind_id = any (array [90]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind59]->(e) where id(s) in $start_ids and id(e) in $end_ids return r, e +-- cypher_params: {"end_ids":[303,404],"start_ids":[101,202]} +-- pgsql_params:{"pi0":[101,202],"pi1":[303,404]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on (n1.id = any (@pi1::float8[])) and n1.id = e0.end_id where e0.kind_id = any (array [91]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s)-[r:RegressionKind60]->(e:RegressionKind52) where id(s) in $start_ids and e.active = true return r, e +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on (((n1.properties -> 'active'))::jsonb = to_jsonb((true)::bool)::jsonb) and n1.kind_ids operator (pg_catalog.@>) array [84]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [92]::int2[])) select s0.e0 as r, s0.n1 as e from s0; + +-- case: match (s:RegressionKind51)-[r:RegressionKind60]->(e) where id(e) in $end_ids and s.active = true return r, s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on (((n0.properties -> 'active'))::jsonb = to_jsonb((true)::bool)::jsonb) and n0.kind_ids operator (pg_catalog.@>) array [83]::int2[] and n0.id = e0.start_id where e0.kind_id = any (array [92]::int2[])) select s0.e0 as r, s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind60]->(e) where id(e) in $end_ids return s +-- cypher_params: {"end_ids":[202]} +-- pgsql_params:{"pi0":[202]} +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = any (@pi0::float8[])) and n1.id = e0.end_id join node n0 on n0.id = e0.start_id where e0.kind_id = any (array [92]::int2[])) select s0.n0 as s from s0; + +-- case: match (s)-[r:RegressionKind60]->(e) where id(s) in $start_ids return id(e), r +-- cypher_params: {"start_ids":[101]} +-- pgsql_params:{"pi0":[101]} +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n0 on (n0.id = any (@pi0::float8[])) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [92]::int2[])) select s0.n1 as "id(e)", s0.e0 as r from s0; + -- case: match (s)-[r]->(e) where id(e) = $a and not (id(s) = $b) and (r:EdgeKind1 or r:EdgeKind2) and not (s.objectid ends with $c or e.objectid ends with $d) return distinct id(s), id(r), id(e) -- cypher_params: {"a":1,"b":2,"c":"123","d":"456"} -- pgsql_params:{"pi0":1,"pi1":2,"pi2":"123","pi3":"456"} -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on n1.id = e0.end_id join node n0 on (not (n0.id = @pi1::float8)) and n0.id = e0.start_id where ((e0.kind_id = any (array [3]::int2[]) or e0.kind_id = any (array [4]::int2[]))) and (not (cypher_ends_with((n0.properties ->> 'objectid'), (@pi2::text)::text)::bool or cypher_ends_with((n1.properties ->> 'objectid'), (@pi3::text)::text)::bool) and n1.id = @pi0::float8)) select distinct (s0.n0).id, (s0.e0).id, (s0.n1).id from s0; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on n1.id = e0.end_id join node n0 on (not (n0.id = @pi1::float8)) and n0.id = e0.start_id where ((e0.kind_id = any (array [3]::int2[]) or e0.kind_id = any (array [4]::int2[]))) and (not (cypher_ends_with((n0.properties ->> 'objectid'), (@pi2::text)::text)::bool or cypher_ends_with((n1.properties ->> 'objectid'), (@pi3::text)::text)::bool) and n1.id = @pi0::float8)) select distinct (s0.n0).id as "id(s)", (s0.e0).id as "id(r)", (s0.n1).id as "id(e)" from s0; -- case: match (s)-[r]->(e) where s.name = '123' and e:NodeKind1 and not r.property return s, r, e with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '123')) and n0.id = e0.start_id join node n1 on (n1.kind_ids operator (pg_catalog.@>) array [1]::int2[]) and n1.id = e0.end_id where (not ((e0.properties ->> 'property'))::bool)) select s0.n0 as s, s0.e0 as r, s0.n1 as e from s0; @@ -89,22 +194,22 @@ with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::e with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.end_id join node n1 on n1.id = e0.start_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.end_id join node n2 on n2.id = e1.start_id where e1.id != (s0.e0).id) select s1.e0 as e0, s1.n1 as n, s1.e1 as e1 from s1; -- case: match (s)<-[r:EdgeKind1|EdgeKind2]-(e) return s.name, e.name -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.end_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.end_id join node n1 on n1.id = e0.start_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name') as "s.name", ((s0.n1).properties -> 'name') as "e.name" from s0; -- case: match (s)-[:EdgeKind1|EdgeKind2]->(e)-[:EdgeKind1]->() return s.name as s_name, e.name as e_name with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])), s1 as (select s0.e0 as e0, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.kind_id = any (array [3]::int2[]) and e1.id != s0.e0) select ((s1.n0).properties -> 'name') as s_name, ((s1.n1).properties -> 'name') as e_name from s1; -- case: match (s:NodeKind1)-[r:EdgeKind1|EdgeKind2]->(e:NodeKind2) return s.name, e.name -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name') as "s.name", ((s0.n1).properties -> 'name') as "e.name" from s0; -- case: match (s)-[r:EdgeKind1]->() where (s)-[r {prop: 'a'}]->() return s with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (jsonb_typeof((e0.properties -> 'prop')) = 'string' and (e0.properties ->> 'prop') = 'a') and e0.kind_id = any (array [3]::int2[])) select s0.n0 as s from s0 where ((with s1 as (select s0.e0 as e0, s0.n0 as n0 from edge e0 join node n2 on n2.id = (s0.e0).end_id where (s0.n0).id = (s0.e0).start_id) select count(*) > 0 from s1)); -- case: match (s)-[r:EdgeKind1]->(e) where not (s.system_tags contains 'admin_tier_0') and id(e) = 1 return id(s), labels(s), id(r), type(r) -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n1 on (n1.id = 1) and n1.id = e0.end_id join node n0 on (not (coalesce((n0.properties ->> 'system_tags'), '')::text like '%admin\_tier\_0%')) and n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select (s0.n0).id, (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[], (s0.e0).id, kind_name((s0.e0).kind_id)::text from s0; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n1 on (n1.id = 1) and n1.id = e0.end_id join node n0 on (not (coalesce((n0.properties ->> 'system_tags'), '')::text like '%admin\_tier\_0%')) and n0.id = e0.start_id where e0.kind_id = any (array [3]::int2[])) select (s0.n0).id as "id(s)", (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as "labels(s)", (s0.e0).id as "id(r)", kind_name((s0.e0).kind_id)::text as "type(r)" from s0; -- case: match (s)-[r]->(e) where s:NodeKind1 and toLower(s.name) starts with 'test' and r:EdgeKind1 and id(e) in [1, 2] return r limit 1 -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and lower((n0.properties ->> 'name'))::text like 'test%') and n0.id = e0.start_id join node n1 on (n1.id = any (array [1, 2]::int8[])) and n1.id = e0.end_id where (e0.kind_id = any (array [3]::int2[])) limit 1) select s0.e0 as r from s0 limit 1; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[] and lower((n0.properties ->> 'name'))::text like 'test%') and n0.id = e0.start_id join node n1 on (n1.id = any (array [1, 2]::int8[])) and n1.id = e0.end_id where (e0.kind_id = any (array [3]::int2[])) limit 1) select s0.e0 as r from s0 limit 1; -- case: match (n1)-[]->(n2) where n1 <> n2 return n2 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (n0.id <> n1.id)) select s0.n1 as n2 from s0; @@ -113,8 +218,8 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where (n1.id <> n0.id)) select s0.n1 as n2 from s0; -- case: match ()-[r]->()-[e]->(n) where r <> e return n -with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where ((s0.e0).id <> e1.id) and e1.id != (s0.e0).id) select s1.n2 as n from s1; +with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n1.id as n1 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n1 as n1, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n2 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where ((s0.e0).id <> e1.id) and e1.id != (s0.e0).id) select s1.n2 as n from s1; -- case: match (s:NodeKind1:NodeKind2)-[r:EdgeKind1|EdgeKind2]->(e:NodeKind2:NodeKind1) return s.name, e.name -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1, 2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2, 1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name'), ((s0.n1).properties -> 'name') from s0; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on n0.kind_ids operator (pg_catalog.@>) array [1, 2]::int2[] and n0.id = e0.start_id join node n1 on n1.kind_ids operator (pg_catalog.@>) array [2, 1]::int2[] and n1.id = e0.end_id where e0.kind_id = any (array [3, 4]::int2[])) select ((s0.n0).properties -> 'name') as "s.name", ((s0.n1).properties -> 'name') as "e.name" from s0; diff --git a/cypher/models/pgsql/test/translation_cases/unwind.sql b/cypher/models/pgsql/test/translation_cases/unwind.sql index 4c00ab6e..c7cccd4f 100644 --- a/cypher/models/pgsql/test/translation_cases/unwind.sql +++ b/cypher/models/pgsql/test/translation_cases/unwind.sql @@ -33,7 +33,7 @@ with s0 as (select array [1, 2, 3]::int8[] as i0) select i1 as x from s0, unnest with s0 as (select array [1, 2, 3, 1, 2]::int8[] as i0) select distinct i1 as x from s0, unnest(i0) as i1; -- case: with [1, 2, 3] as ids unwind ids as x return count(x) -with s0 as (select array [1, 2, 3]::int8[] as i0) select count(i1)::int8 from s0, unnest(i0) as i1; +with s0 as (select array [1, 2, 3]::int8[] as i0) select count(i1)::int8 as "count(x)" from s0, unnest(i0) as i1; -- case: match (n:NodeKind1) with collect(n.name) as names unwind names as name match (m:NodeKind2) where m.name = name return m with s0 as (with s1 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) select array_remove(coalesce(array_agg(((s1.n0).properties ->> 'name'))::anyarray, array []::text[])::anyarray, null)::anyarray as i0 from s1), s2 as (select s0.i0 as i0, i1 as i1, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, unnest(i0) as i1, node n1 where ((n1.properties ->> 'name') = i1) and n1.kind_ids operator (pg_catalog.@>) array [2]::int2[]) select s2.n1 as m from s2; diff --git a/cypher/models/pgsql/test/translation_cases/update.sql b/cypher/models/pgsql/test/translation_cases/update.sql index 7663e030..28ff09dc 100644 --- a/cypher/models/pgsql/test/translation_cases/update.sql +++ b/cypher/models/pgsql/test/translation_cases/update.sql @@ -39,7 +39,7 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (update node n1 set properties = n1.properties || jsonb_build_object('is_target', true)::jsonb from s0 where (s0.n0).id = n1.id returning (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n0) select 1; -- case: match (n) where n.name = '1234' match (e) where e.tag = n.tag_id set e.is_target = true -with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where ((n1.properties -> 'tag') = ((s0.n0).properties -> 'tag_id'))), s2 as (update node n2 set properties = n2.properties || jsonb_build_object('is_target', true)::jsonb from s1 where (s1.n1).id = n2.id returning s1.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n1) select 1; +with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = '1234'))), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1 where (nullif((n1.properties -> 'tag'), ('null')::jsonb)::jsonb = nullif(((s0.n0).properties -> 'tag_id'), ('null')::jsonb)::jsonb)), s2 as (update node n2 set properties = n2.properties || jsonb_build_object('is_target', true)::jsonb from s1 where (s1.n1).id = n2.id returning s1.n0 as n0, (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n1) select 1; -- case: match (n1), (n3) set n1.target = true set n3.target = true return n1, n3 with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0), s1 as (select s0.n0 as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from s0, node n1), s2 as (update node n2 set properties = n2.properties || jsonb_build_object('target', true)::jsonb from s1 where (s1.n0).id = n2.id returning (n2.id, n2.kind_ids, n2.properties)::nodecomposite as n0, s1.n1 as n1), s3 as (update node n3 set properties = n3.properties || jsonb_build_object('target', true)::jsonb from s2 where (s2.n1).id = n3.id returning s2.n0 as n0, (n3.id, n3.kind_ids, n3.properties)::nodecomposite as n1) select s3.n0 as n1, s3.n1 as n3 from s3; @@ -69,5 +69,5 @@ with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from edge e0 join node n0 on (n0.kind_ids operator (pg_catalog.@>) array [1]::int2[]) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [3]::int2[])), s1 as (update edge e1 set properties = e1.properties || jsonb_build_object('visited', true)::jsonb from s0 where (s0.e0).id = e1.id returning (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e0, s0.n0 as n0) select s1.e0 as r from s1; -- case: match (n)-[]->()-[r]->() where n.name = 'n1' set r.visited = true return r.name -with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on (s0.n1).id = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (update edge e2 set properties = e2.properties || jsonb_build_object('visited', true)::jsonb from s1 where (s1.e1).id = e2.id returning s1.e0 as e0, (e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1) select ((s2.e1).properties -> 'name') from s2; +with s0 as (select e0.id as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, n1.id as n1 from edge e0 join node n0 on ((jsonb_typeof((n0.properties -> 'name')) = 'string' and (n0.properties ->> 'name') = 'n1')) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id), s1 as (select s0.e0 as e0, (e1.id, e1.start_id, e1.end_id, e1.kind_id, e1.properties)::edgecomposite as e1, s0.n0 as n0, s0.n1 as n1 from s0 join edge e1 on s0.n1 = e1.start_id join node n2 on n2.id = e1.end_id where e1.id != s0.e0), s2 as (update edge e2 set properties = e2.properties || jsonb_build_object('visited', true)::jsonb from s1 where (s1.e1).id = e2.id returning s1.e0 as e0, (e2.id, e2.start_id, e2.end_id, e2.kind_id, e2.properties)::edgecomposite as e1, s1.n0 as n0, s1.n1 as n1) select ((s2.e1).properties -> 'name') as "r.name" from s2; diff --git a/cypher/models/pgsql/test/translation_test.go b/cypher/models/pgsql/test/translation_test.go index 12033919..0f7f33c2 100644 --- a/cypher/models/pgsql/test/translation_test.go +++ b/cypher/models/pgsql/test/translation_test.go @@ -12,6 +12,7 @@ import ( "github.com/specterops/dawgs/graph" ) +// translationTestKinds returns the stable kind set and numeric IDs used by translation fixtures. func translationTestKinds() graph.Kinds { // Keep this order stable. Translation case SQL fixtures depend on these IDs. return graph.Kinds{ @@ -48,9 +49,113 @@ func translationTestKinds() graph.Kinds { "WriteAccountRestrictions", "WriteOwner", "AZUser", + // Synthetic reconciliation kinds are append-only. The first 9 and all 30 + // are used by cardinality-sensitive golden cases without renumbering any + // established kind IDs above. + "RegressionKind01", + "RegressionKind02", + "RegressionKind03", + "RegressionKind04", + "RegressionKind05", + "RegressionKind06", + "RegressionKind07", + "RegressionKind08", + "RegressionKind09", + "RegressionKind10", + "RegressionKind11", + "RegressionKind12", + "RegressionKind13", + "RegressionKind14", + "RegressionKind15", + "RegressionKind16", + "RegressionKind17", + "RegressionKind18", + "RegressionKind19", + "RegressionKind20", + "RegressionKind21", + "RegressionKind22", + "RegressionKind23", + "RegressionKind24", + "RegressionKind25", + "RegressionKind26", + "RegressionKind27", + "RegressionKind28", + "RegressionKind29", + "RegressionKind30", + "RegressionKind31", + "RegressionKind32", + "RegressionKind33", + "RegressionKind34", + "RegressionKind35", + "RegressionKind36", + "RegressionKind37", + "RegressionKind38", + "RegressionKind39", + "RegressionKind40", + "RegressionKind41", + "RegressionKind42", + "RegressionKind43", + "RegressionKind44", + "RegressionKind45", + "RegressionKind46", + "RegressionKind47", + "RegressionKind48", + "RegressionKind49", + "RegressionKind50", + "RegressionKind51", + "RegressionKind52", + "RegressionKind53", + "RegressionKind54", + "RegressionKind55", + "RegressionKind56", + "RegressionKind57", + "RegressionKind58", + "RegressionKind59", + "RegressionKind60", + "RegressionKind61", + "RegressionKind62", + "RegressionKind63", + "RegressionKind64", + "RegressionKind65", + "RegressionKind66", + "RegressionKind67", + "RegressionKind68", + "RegressionKind69", + "RegressionKind70", + "RegressionKind71", + "RegressionKind72", + "RegressionKind73", + "RegressionKind74", + "RegressionKind75", + "RegressionKind76", + "RegressionKind77", + "RegressionKind78", + "RegressionKind79", + "RegressionKind80", + "RegressionKind81", + "RegressionKind82", + "RegressionKind83", + "RegressionKind84", + "RegressionKind85", + "RegressionKind86", + "RegressionKind87", + "RegressionKind88", + "RegressionKind89", + "RegressionKind90", + "RegressionKind91", + "RegressionKind92", + "RegressionKind93", + "RegressionKind94", + "RegressionKind95", + "RegressionKind96", + "RegressionKind97", + "RegressionKind98", + "RegressionKind99", + "RegressionKind100", })...) } +// newKindMapper returns a mapper populated with the translation fixture's deterministic kind IDs. func newKindMapper() pgsql.KindMapper { mapper := pgutil.NewInMemoryKindMapper() diff --git a/cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go b/cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go new file mode 100644 index 00000000..e8a7a705 --- /dev/null +++ b/cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go @@ -0,0 +1,170 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package test + +import ( + "testing" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderPostgreSQL_TrustAndPruningForms verifies migrated trust filters and pruning projections retain their SQL contracts. +func TestLegacyBuilderPostgreSQL_TrustAndPruningForms(t *testing.T) { + threshold := time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC) + + testCases := map[string]struct { + // criteria contains the legacy query-builder inputs for the case. + criteria []graph.Criteria + // fragments lists SQL fragments that the translation must contain. + fragments []string + // parameters is the exact parameter map expected from translation. + parameters map[string]any + }{ + "TRUST-01 SameForestTrust ID projection": { + criteria: trustPruningCriteria("RegressionKind40", "RegressionKind41", query.RelationshipID()), + fragments: []string{ + "n0.kind_ids operator (pg_catalog.&&) array [72]::int2[]", + "n1.kind_ids operator (pg_catalog.&&) array [72]::int2[]", + "e0.kind_id = any (array [73]::int2[])", + "e0.properties -> 'lastseen'", + "n0.properties -> 'lastcollected'", + "n1.properties -> 'lastcollected'", + "select (s0.e0).id", + }, + parameters: map[string]any{}, + }, + "TRUST-02 CrossForestTrust full projection": { + criteria: trustPruningCriteria("RegressionKind40", "RegressionKind42", query.Relationship()), + fragments: []string{ + "e0.kind_id = any (array [74]::int2[])", + "select s0.e0 as r", + }, + parameters: map[string]any{}, + }, + "TRUST-03 branch-local derived trust kinds": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("RegressionKind40")), + query.Kind(query.End(), graph.StringKind("RegressionKind40")), + query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind43")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("RegressionKind44")), + ), + ), + )), + query.Returning(query.RelationshipID()), + }, + fragments: []string{ + " or ", + "n0.id = @pi0", + "n1.id = @pi1", + "n0.id = @pi2", + "n1.id = @pi3", + "e0.kind_id = any (array [75]::int2[])", + "e0.kind_id = any (array [76]::int2[])", + }, + parameters: map[string]any{"pi0": uint64(101), "pi1": uint64(202), "pi2": uint64(202), "pi3": uint64(101)}, + }, + "PRUNE-01 relationship TTL": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Relationship(), graph.StringKind("RegressionKind45"), graph.StringKind("RegressionKind46"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + )), + query.Returning(query.RelationshipID()), + }, + fragments: []string{"not (e0.kind_id = any (array [77, 78]::int2[]))", "e0.properties ->> 'lastseen'", "select (s0.e0).id"}, + parameters: map[string]any{"pi0": threshold}, + }, + "PRUNE-02 HasSession TTL": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.KindIn(query.Relationship(), graph.StringKind("HasSession")), + query.Or( + query.Not(query.Exists(query.RelationshipProperty("lastseen"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ), + )), + query.Returning(query.RelationshipID()), + }, + fragments: []string{"not ((e0.properties ? 'lastseen'", " or ", "e0.kind_id = any (array [7]::int2[])"}, + parameters: map[string]any{"pi0": threshold}, + }, + "PRUNE-03 node TTL": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("RegressionKind48"), graph.StringKind("RegressionKind49"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + )), + query.Returning(query.NodeID()), + }, + fragments: []string{"not (n0.kind_ids operator (pg_catalog.&&) array [80, 81]::int2[])", "not ((n0.properties ? 'lastseen'", "select (s0.n0).id"}, + parameters: map[string]any{"pi0": threshold}, + }, + "PRUNE-04 orphan SID prefix": { + criteria: []graph.Criteria{ + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("RegressionKind48"), graph.StringKind("RegressionKind49"))), + query.Not(query.Exists(query.NodeProperty("name"))), + query.StringStartsWith(query.NodeProperty("objectid"), "S-1-5"), + )), + query.Returning(query.NodeID()), + }, + fragments: []string{"not ((n0.properties ? 'name'", "cypher_starts_with", "select (s0.n0).id"}, + parameters: map[string]any{"pi0": "S-1-5"}, + }, + } + + for name, testCase := range testCases { + t.Run(name, func(t *testing.T) { + formatted, translation := translateLegacyQuery(t, testCase.criteria...) + for _, fragment := range testCase.fragments { + require.Contains(t, formatted, fragment) + } + require.Equal(t, testCase.parameters, translation.Parameters) + }) + } +} + +// trustPruningCriteria builds the shared trust-kind and timestamp predicate used by pruning regression cases. +func trustPruningCriteria(domainKind, relationshipKind string, projection graph.Criteria) []graph.Criteria { + return []graph.Criteria{ + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind(domainKind)), + query.Kind(query.End(), graph.StringKind(domainKind)), + query.Kind(query.Relationship(), graph.StringKind(relationshipKind)), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + )), + query.Returning(projection), + } +} diff --git a/cypher/models/pgsql/translate/expansion.go b/cypher/models/pgsql/translate/expansion.go index 12b34f70..78a4c41d 100644 --- a/cypher/models/pgsql/translate/expansion.go +++ b/cypher/models/pgsql/translate/expansion.go @@ -3,6 +3,7 @@ package translate import ( "errors" "fmt" + "strings" "github.com/specterops/dawgs/cypher/models" "github.com/specterops/dawgs/cypher/models/pgsql" @@ -12,18 +13,33 @@ import ( "github.com/specterops/dawgs/graph" ) +// translateDefaultMaxTraversalDepth caps unbounded recursive traversals to prevent runaway expansion. const translateDefaultMaxTraversalDepth int64 = 15 var ( - expansionRootFilter = pgsql.Identifier("traversal_root_filter") - expansionTerminalFilter = pgsql.Identifier("traversal_terminal_filter") - expansionPairFilter = pgsql.Identifier("traversal_pair_filter") - expansionTerminalID = pgsql.Identifier("terminal_id") - expansionVisited = pgsql.Identifier("visited") - expansionForwardVisited = pgsql.Identifier("forward_visited") + // expansionRootFilter names the CTE that materializes admissible traversal roots. + expansionRootFilter = pgsql.Identifier("traversal_root_filter") + + // expansionTerminalFilter names the CTE that materializes admissible traversal terminals. + expansionTerminalFilter = pgsql.Identifier("traversal_terminal_filter") + + // expansionPairFilter names the CTE that materializes admissible root-terminal pairs. + expansionPairFilter = pgsql.Identifier("traversal_pair_filter") + + // expansionTerminalID names the filtered terminal-ID column. + expansionTerminalID = pgsql.Identifier("terminal_id") + + // expansionVisited names the relation that records states visited by shortest-path search. + expansionVisited = pgsql.Identifier("visited") + + // expansionForwardVisited names states visited from the root side of bidirectional search. + expansionForwardVisited = pgsql.Identifier("forward_visited") + + // expansionBackwardVisited names states visited from the terminal side of bidirectional search. expansionBackwardVisited = pgsql.Identifier("backward_visited") ) +// expansionEdgeJoinCondition matches the current node to the start of the next directed edge. func expansionEdgeJoinCondition(traversalStep *TraversalStep) (pgsql.Expression, error) { return pgd.Equals( pgd.EntityID(traversalStep.LeftNode.Identifier), @@ -31,6 +47,7 @@ func expansionEdgeJoinCondition(traversalStep *TraversalStep) (pgsql.Expression, ), nil } +// expansionConstraints limits recursion by maximum depth and rejects cyclic expansion states. func expansionConstraints(traversalStep *TraversalStep) pgsql.Expression { expansionModel := traversalStep.Expansion @@ -45,41 +62,66 @@ func expansionConstraints(traversalStep *TraversalStep) pgsql.Expression { ) } -var ( - ErrUnsupportedExpansionDirection = errors.New("unsupported expansion direction") -) +// ErrUnsupportedExpansionDirection reports a traversal direction that cannot be lowered to SQL. +var ErrUnsupportedExpansionDirection = errors.New("unsupported expansion direction") +// ExpansionBuilder assembles the seed, recursive, and projection statements for one traversal expansion. type ExpansionBuilder struct { - PrimerStatement pgsql.Select - RecursiveStatement pgsql.Select + // PrimerStatement produces the first traversal edge for each root. + PrimerStatement pgsql.Select + + // RecursiveStatement advances each eligible expansion state by one edge. + RecursiveStatement pgsql.Select + + // ProjectionStatement converts internal expansion state into the requested result shape. ProjectionStatement pgsql.Select - ZeroDepthStatement *pgsql.Select - UseUnionAll bool + // ZeroDepthStatement produces empty-path rows when the traversal admits depth zero. + ZeroDepthStatement *pgsql.Select + + // UseUnionAll controls whether recursive branches retain duplicate states. + UseUnionAll bool + + // queryParameters contains literal values lifted while constructing harness calls. queryParameters map[string]any - traversalStep *TraversalStep - model *Expansion - unwindClauses []UnwindClause - unwindSources []pgsql.FromClause + + // graphID identifies the graph partitions referenced by generated traversal SQL. + graphID int32 + + // traversalStep describes the edge, endpoints, direction, and constraints being expanded. + traversalStep *TraversalStep + + // model contains the frame and search options shared by the generated statements. + model *Expansion + + // unwindClauses contains active UNWIND bindings that expansion predicates may reference. + unwindClauses []UnwindClause + + // unwindSources caches the SQL sources corresponding to unwindClauses. + unwindSources []pgsql.FromClause } -func NewExpansionBuilder(queryParameters map[string]any, traversalStep *TraversalStep) (*ExpansionBuilder, error) { +// NewExpansionBuilder validates traversal expansion state and constructs its SQL builder. +func NewExpansionBuilder(queryParameters map[string]any, traversalStep *TraversalStep, graphID int32) (*ExpansionBuilder, error) { if traversalStep.Expansion == nil { return nil, errors.New("traversal step must have expansion set") } return &ExpansionBuilder{ queryParameters: queryParameters, + graphID: graphID, traversalStep: traversalStep, model: traversalStep.Expansion, }, nil } +// SetUnwindClauses registers the active UNWIND bindings and their SQL sources for expansion queries. func (s *ExpansionBuilder) SetUnwindClauses(clauses []UnwindClause) { s.unwindClauses = clauses s.unwindSources = unwindFromClauses(clauses) } +// nextFrontInsert wraps a frontier-producing expression in an insert into the next-front workspace. func nextFrontInsert(body pgsql.SetExpression) pgsql.Insert { return pgsql.Insert{ Table: pgsql.TableReference{ @@ -92,6 +134,7 @@ func nextFrontInsert(body pgsql.SetExpression) pgsql.Insert { } } +// expansionNodeTableReference aliases the graph node table for an expansion binding. func expansionNodeTableReference(binding pgsql.Identifier) pgsql.TableReference { return pgsql.TableReference{ Name: pgsql.TableNode.AsCompoundIdentifier(), @@ -99,6 +142,7 @@ func expansionNodeTableReference(binding pgsql.Identifier) pgsql.TableReference } } +// expansionEdgeTableReference aliases the graph edge table for an expansion binding. func expansionEdgeTableReference(binding pgsql.Identifier) pgsql.TableReference { return pgsql.TableReference{ Name: pgsql.TableEdge.AsCompoundIdentifier(), @@ -106,21 +150,28 @@ func expansionEdgeTableReference(binding pgsql.Identifier) pgsql.TableReference } } +// expansionSeed describes the query and record shape that supply traversal root identifiers. type expansionSeed struct { + // identifier names the seed common table expression. identifier pgsql.Identifier - query pgsql.Select + + // query selects the root identifiers supplied to the expansion. + query pgsql.Select } +// expansionSeedIdentifier derives the CTE name reserved for an expansion's seed rows. func expansionSeedIdentifier(expansionIdentifier pgsql.Identifier) pgsql.Identifier { return pgsql.Identifier(string(expansionIdentifier) + "_seed") } +// expansionSeedColumns returns the single root-identifier column emitted by every seed query. func expansionSeedColumns() *pgsql.RecordShape { return pgsql.NewRecordShape([]pgsql.Identifier{ expansionRootID, }) } +// newExpansionSeed builds a seed query that projects a root expression from the supplied sources and predicate. func newExpansionSeed(identifier pgsql.Identifier, rootExpression pgsql.Expression, from []pgsql.FromClause, where pgsql.Expression) expansionSeed { return expansionSeed{ identifier: identifier, @@ -137,12 +188,14 @@ func newExpansionSeed(identifier pgsql.Identifier, rootExpression pgsql.Expressi } } +// newExpansionNodeSeed builds a seed by scanning candidate root nodes under the supplied constraints. func newExpansionNodeSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { return newExpansionSeed(identifier, pgd.EntityID(nodeIdentifier), []pgsql.FromClause{{ Source: expansionNodeTableReference(nodeIdentifier), }}, constraints) } +// newExpansionNodeFilterSeed reads root identifiers from a materialized filter and joins nodes when constraints require hydration. func newExpansionNodeFilterSeed(identifier, filterIdentifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { var ( filterAlias = pgsql.Identifier(string(identifier) + "_filter") @@ -179,14 +232,9 @@ func newExpansionNodeFilterSeed(identifier, filterIdentifier, nodeIdentifier pgs return seed } -func newExpansionBoundNodeSeed(identifier pgsql.Identifier, previousFrame *Frame, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { - seed := newExpansionSeed(identifier, pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{ - previousFrame.Binding.Identifier, - nodeIdentifier, - }, - Column: pgsql.ColumnID, - }, []pgsql.FromClause{{ +// newExpansionBoundNodeSeed projects distinct bound-node identifiers from the preceding frame. +func newExpansionBoundNodeSeed(identifier pgsql.Identifier, previousFrame *Frame, binding *BoundIdentifier, constraints pgsql.Expression) expansionSeed { + seed := newExpansionSeed(identifier, boundEndpointIDReference(previousFrame, binding), []pgsql.FromClause{{ Source: pgsql.TableReference{ Name: pgsql.CompoundIdentifier{previousFrame.Binding.Identifier}, }, @@ -196,6 +244,7 @@ func newExpansionBoundNodeSeed(identifier pgsql.Identifier, previousFrame *Frame return seed } +// fromClausesContainSource reports whether a FROM list directly names the requested table source. func fromClausesContainSource(fromClauses []pgsql.FromClause, identifier pgsql.Identifier) bool { for _, fromClause := range fromClauses { if tableReference, isTableReference := fromClause.Source.(pgsql.TableReference); isTableReference && @@ -208,6 +257,7 @@ func fromClausesContainSource(fromClauses []pgsql.FromClause, identifier pgsql.I return false } +// prependFrameSourceIfMissing ensures the preceding frame is the first source in a FROM list. func prependFrameSourceIfMissing(fromClauses []pgsql.FromClause, frame *Frame) []pgsql.FromClause { if frame == nil || fromClausesContainSource(fromClauses, frame.Binding.Identifier) { return fromClauses @@ -220,6 +270,7 @@ func prependFrameSourceIfMissing(fromClauses []pgsql.FromClause, frame *Frame) [ }}, fromClauses...) } +// expressionReferencesUnwindBinding reports whether an expression depends on any active UNWIND binding. func expressionReferencesUnwindBinding(expression pgsql.Expression, unwindClauses []UnwindClause) (bool, error) { if expression == nil || len(unwindClauses) == 0 { return false, nil @@ -239,6 +290,7 @@ func expressionReferencesUnwindBinding(expression pgsql.Expression, unwindClause return false, nil } +// seedEndpointConstraintSplit rewrites bound endpoint references for the seed and separates local predicates from deferred ones. func (s *ExpansionBuilder) seedEndpointConstraintSplit(expression pgsql.Expression, nodeIdentifier pgsql.Identifier, previousFrameIdentifier pgsql.Identifier) (pgsql.Expression, pgsql.Expression) { var ( seedExpression = rewriteBoundEndpointSeedReference(expression, previousFrameIdentifier, nodeIdentifier) @@ -254,6 +306,7 @@ func (s *ExpansionBuilder) seedEndpointConstraintSplit(expression pgsql.Expressi return partitionConstraintByLocality(seedExpression, localScope) } +// appendUnwindSourcesIfReferenced adds frame and UNWIND sources only when the supplied expressions use an UNWIND binding. func (s *ExpansionBuilder) appendUnwindSourcesIfReferenced(selectBody *pgsql.Select, expressions ...pgsql.Expression) error { for _, expression := range expressions { if referencesUnwind, err := expressionReferencesUnwindBinding(expression, s.unwindClauses); err != nil { @@ -273,18 +326,47 @@ func (s *ExpansionBuilder) appendUnwindSourcesIfReferenced(selectBody *pgsql.Sel return nil } +// appendUnwindSources appends every active UNWIND source to a select body. func (s *ExpansionBuilder) appendUnwindSources(selectBody *pgsql.Select) { selectBody.From = append(selectBody.From, s.unwindSources...) } +// newExpansionRootIDsParameterSeed builds a root seed from the materialized root-identifier parameter. func newExpansionRootIDsParameterSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { return newExpansionNodeFilterSeed(identifier, expansionRootFilter, nodeIdentifier, constraints) } +// newExpansionTerminalIDsParameterSeed builds a root seed from the materialized terminal-identifier parameter. func newExpansionTerminalIDsParameterSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) expansionSeed { return newExpansionNodeFilterSeed(identifier, expansionTerminalFilter, nodeIdentifier, constraints) } +// newExpansionArrayParameterSeed unnests an identifier-array parameter and filters the corresponding nodes. +func newExpansionArrayParameterSeed(identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression, parameterPosition int) expansionSeed { + parameterAlias := pgsql.Identifier(string(identifier) + "_parameter") + parameterID := pgsql.CompoundIdentifier{parameterAlias, pgsql.ColumnID} + seed := newExpansionSeed(identifier, pgd.EntityID(nodeIdentifier), []pgsql.FromClause{{ + Source: pgsql.FormattingLiteral(fmt.Sprintf( + "unnest($%d::int8[]) as %s(id)", + parameterPosition, + parameterAlias, + )), + Joins: []pgsql.Join{{ + Table: expansionNodeTableReference(nodeIdentifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgd.Equals( + pgd.EntityID(nodeIdentifier), + parameterID, + ), + }, + }}, + }}, constraints) + seed.query.Distinct = true + return seed +} + +// CTE exposes the seed query as a non-materialized common table expression. func (s expansionSeed) CTE() pgsql.CommonTableExpression { return pgsql.CommonTableExpression{ Alias: pgsql.TableAlias{ @@ -298,10 +380,12 @@ func (s expansionSeed) CTE() pgsql.CommonTableExpression { } } +// rootID returns the qualified root-identifier column of the seed CTE. func (s expansionSeed) rootID() pgsql.CompoundIdentifier { return pgsql.CompoundIdentifier{s.identifier, expansionRootID} } +// fromClause references the seed CTE and attaches the supplied joins. func (s expansionSeed) fromClause(joins ...pgsql.Join) pgsql.FromClause { return pgsql.FromClause{ Source: pgsql.TableReference{ @@ -311,6 +395,7 @@ func (s expansionSeed) fromClause(joins ...pgsql.Join) pgsql.FromClause { } } +// edgeJoin joins a seed root identifier to the starting endpoint of an edge binding. func (s expansionSeed) edgeJoin(edgeIdentifier pgsql.Identifier, edgeStartColumn pgsql.CompoundIdentifier) pgsql.Join { return pgsql.Join{ Table: expansionEdgeTableReference(edgeIdentifier), @@ -321,6 +406,7 @@ func (s expansionSeed) edgeJoin(edgeIdentifier pgsql.Identifier, edgeStartColumn } } +// expansionEdgeFromClause references the graph edge table with the joins needed by an expansion query. func expansionEdgeFromClause(edgeIdentifier pgsql.Identifier, joins ...pgsql.Join) pgsql.FromClause { return pgsql.FromClause{ Source: expansionEdgeTableReference(edgeIdentifier), @@ -328,6 +414,7 @@ func expansionEdgeFromClause(edgeIdentifier pgsql.Identifier, joins ...pgsql.Joi } } +// recursiveExpansionEdgeProjection projects every stored column of the recursively selected edge. func recursiveExpansionEdgeProjection(edgeIdentifier pgsql.Identifier) pgsql.Projection { projection := make(pgsql.Projection, len(pgsql.EdgeTableColumns)) @@ -338,6 +425,7 @@ func recursiveExpansionEdgeProjection(edgeIdentifier pgsql.Identifier) pgsql.Pro return projection } +// expansionEdgeNotInPath rejects an edge identifier already present in the accumulated path. func expansionEdgeNotInPath(edgeIdentifier, frameIdentifier pgsql.Identifier) *pgsql.BinaryExpression { return pgsql.NewBinaryExpression( pgd.EntityID(edgeIdentifier), @@ -348,6 +436,7 @@ func expansionEdgeNotInPath(edgeIdentifier, frameIdentifier pgsql.Identifier) *p ) } +// recursiveExpansionEdgeLookupJoin builds the correlated lateral lookup for unused edges leaving the current frontier node. func recursiveExpansionEdgeLookupJoin(traversalStep *TraversalStep) pgsql.Join { var ( expansionModel = traversalStep.Expansion @@ -386,24 +475,30 @@ func recursiveExpansionEdgeLookupJoin(traversalStep *TraversalStep) pgsql.Join { } } -func expansionNodeProjection(nodeIdentifier pgsql.Identifier) pgsql.Projection { +// expansionNodeProjection projects either a node identifier or the complete node record required by its binding. +func expansionNodeProjection(binding *BoundIdentifier) pgsql.Projection { + if binding.IDOnly { + return pgsql.Projection{pgsql.CompoundIdentifier{binding.Identifier, pgsql.ColumnID}} + } + projection := make(pgsql.Projection, len(pgsql.NodeTableColumns)) for idx, column := range pgsql.NodeTableColumns { - projection[idx] = pgsql.CompoundIdentifier{nodeIdentifier, column} + projection[idx] = pgsql.CompoundIdentifier{binding.Identifier, column} } return projection } -func expansionNodeLookupJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expression) pgsql.Join { +// expansionNodeLookupJoin builds a correlated lateral lookup that hydrates a node by identifier. +func expansionNodeLookupJoin(binding *BoundIdentifier, nodeID pgsql.Expression) pgsql.Join { nodeLookup := pgsql.Select{ - Projection: expansionNodeProjection(nodeIdentifier), + Projection: expansionNodeProjection(binding), From: []pgsql.FromClause{{ - Source: expansionNodeTableReference(nodeIdentifier), + Source: expansionNodeTableReference(binding.Identifier), }}, Where: pgd.Equals( - pgsql.CompoundIdentifier{nodeIdentifier, pgsql.ColumnID}, + pgsql.CompoundIdentifier{binding.Identifier, pgsql.ColumnID}, nodeID, ), } @@ -415,7 +510,7 @@ func expansionNodeLookupJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expre // OFFSET 0 keeps PostgreSQL from flattening this correlated lookup into a full-table hash join. Offset: pgsql.NewLiteral(0, pgsql.Int), }, - Binding: models.OptionalValue(nodeIdentifier), + Binding: models.OptionalValue(binding.Identifier), }, JoinOperator: pgsql.JoinOperator{ JoinType: pgsql.JoinTypeInner, @@ -489,6 +584,7 @@ func rewriteBoundEndpointSeedReference(expression pgsql.Expression, previousFram Distinct: typedExpression.Distinct, Function: typedExpression.Function, Parameters: parameters, + OrderBy: typedExpression.OrderBy, Over: typedExpression.Over, CastType: typedExpression.CastType, } @@ -624,6 +720,7 @@ func rewriteBoundEndpointSeedReference(expression pgsql.Expression, previousFram } } +// seededFrontPrimerQuery places a seed CTE in front of the query that initializes a search frontier. func seededFrontPrimerQuery(seed expansionSeed, primer pgsql.Select) pgsql.Query { return pgsql.Query{ CommonTableExpressions: &pgsql.With{ @@ -633,6 +730,7 @@ func seededFrontPrimerQuery(seed expansionSeed, primer pgsql.Select) pgsql.Query } } +// frontPrimerQuery returns a frontier primer with its optional seed CTE attached. func frontPrimerQuery(seed *expansionSeed, primer pgsql.Select) pgsql.Query { if seed == nil { return pgsql.Query{Body: primer} @@ -641,10 +739,12 @@ func frontPrimerQuery(seed *expansionSeed, primer pgsql.Select) pgsql.Query { return seededFrontPrimerQuery(*seed, primer) } +// expansionAllowsZeroDepth reports whether the traversal's lower bound explicitly admits an empty path. func expansionAllowsZeroDepth(expansionModel *Expansion) bool { return expansionModel.Options.MinDepth.Set && expansionModel.Options.MinDepth.Value == 0 } +// zeroDepthNodeJoin joins a node binding to the identifier representing an empty path's endpoint. func zeroDepthNodeJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expression) pgsql.Join { return pgsql.Join{ Table: expansionNodeTableReference(nodeIdentifier), @@ -655,6 +755,7 @@ func zeroDepthNodeJoin(nodeIdentifier pgsql.Identifier, nodeID pgsql.Expression) } } +// zeroDepthTerminalSatisfaction returns the terminal predicate that can be evaluated without traversing an edge. func zeroDepthTerminalSatisfaction(traversalStep *TraversalStep) pgsql.Expression { localSatisfaction, _ := expansionTerminalSatisfactionLocality(traversalStep) if localSatisfaction == nil { @@ -670,6 +771,7 @@ func zeroDepthTerminalSatisfaction(traversalStep *TraversalStep) pgsql.Expressio return localSatisfaction } +// buildZeroDepthExpansionSelect emits the depth-zero expansion state for roots that already satisfy the terminal predicate. func (s *ExpansionBuilder) buildZeroDepthExpansionSelect(seed *expansionSeed) (pgsql.Select, error) { var ( expansionModel = s.traversalStep.Expansion @@ -724,18 +826,22 @@ func (s *ExpansionBuilder) buildZeroDepthExpansionSelect(seed *expansionSeed) (p }, nil } +// usesBoundRootIDs reports whether roots must be read from a binding in the preceding frame. func (s *ExpansionBuilder) usesBoundRootIDs() bool { return s.traversalStep.LeftNodeBound && s.traversalStep.Frame != nil && s.traversalStep.Frame.Previous != nil } +// usesBoundTerminalIDs reports whether terminals must be read from a binding in the preceding frame. func (s *ExpansionBuilder) usesBoundTerminalIDs() bool { return s.traversalStep.RightNodeBound && s.traversalStep.Frame != nil && s.traversalStep.Frame.Previous != nil } +// usesBoundEndpointPairs reports whether both endpoints are paired bindings from the preceding frame. func (s *ExpansionBuilder) usesBoundEndpointPairs() bool { return s.usesBoundRootIDs() && s.usesBoundTerminalIDs() } +// boundNodeIDsFilterStatement inserts distinct non-null bound node identifiers into a filter table. func (s *ExpansionBuilder) boundNodeIDsFilterStatement(filterIdentifier pgsql.Identifier, nodeIdentifier pgsql.Identifier) pgsql.Insert { var ( previousFrameIdentifier = s.traversalStep.Frame.Previous.Binding.Identifier @@ -771,6 +877,7 @@ func (s *ExpansionBuilder) boundNodeIDsFilterStatement(filterIdentifier pgsql.Id } } +// boundRootIDsFilterStatement builds the root-filter insert when the traversal has a bound root. func (s *ExpansionBuilder) boundRootIDsFilterStatement() (pgsql.Insert, bool) { if !s.usesBoundRootIDs() { return pgsql.Insert{}, false @@ -779,6 +886,7 @@ func (s *ExpansionBuilder) boundRootIDsFilterStatement() (pgsql.Insert, bool) { return s.boundNodeIDsFilterStatement(expansionRootFilter, s.traversalStep.LeftNode.Identifier), true } +// boundTerminalIDsFilterStatement builds the terminal-filter insert when the traversal has a bound terminal. func (s *ExpansionBuilder) boundTerminalIDsFilterStatement() (pgsql.Insert, bool) { if !s.usesBoundTerminalIDs() { return pgsql.Insert{}, false @@ -787,6 +895,7 @@ func (s *ExpansionBuilder) boundTerminalIDsFilterStatement() (pgsql.Insert, bool return s.boundNodeIDsFilterStatement(expansionTerminalFilter, s.traversalStep.RightNode.Identifier), true } +// unboundTerminalIDsFilterStatement materializes terminal node identifiers selected by terminal constraints. func (s *ExpansionBuilder) unboundTerminalIDsFilterStatement() (pgsql.Insert, bool) { expansionModel := s.traversalStep.Expansion if !expansionModel.UseMaterializedTerminalFilter { @@ -796,6 +905,7 @@ func (s *ExpansionBuilder) unboundTerminalIDsFilterStatement() (pgsql.Insert, bo return s.nodeIDsFilterStatement(expansionTerminalFilter, s.traversalStep.RightNode.Identifier, expansionModel.TerminalNodeConstraints), true } +// nodeIDsFilterStatement inserts distinct constrained node identifiers into a filter table. func (s *ExpansionBuilder) nodeIDsFilterStatement(filterIdentifier pgsql.Identifier, nodeIdentifier pgsql.Identifier, constraints pgsql.Expression) pgsql.Insert { nodeIDExpression := pgsql.CompoundIdentifier{nodeIdentifier, pgsql.ColumnID} @@ -826,6 +936,7 @@ func (s *ExpansionBuilder) nodeIDsFilterStatement(filterIdentifier pgsql.Identif } } +// boundEndpointPairFilterStatement inserts distinct non-null bound root and terminal pairs from the preceding frame. func (s *ExpansionBuilder) boundEndpointPairFilterStatement() (pgsql.Insert, bool) { if !s.usesBoundEndpointPairs() { return pgsql.Insert{}, false @@ -877,6 +988,7 @@ func (s *ExpansionBuilder) boundEndpointPairFilterStatement() (pgsql.Insert, boo }, true } +// materializedEndpointPairFilterStatement inserts root and terminal pairs selected independently by endpoint constraints. func (s *ExpansionBuilder) materializedEndpointPairFilterStatement() (pgsql.Insert, bool) { expansionModel := s.traversalStep.Expansion if !expansionModel.UseMaterializedEndpointPairFilter { @@ -923,6 +1035,7 @@ func (s *ExpansionBuilder) materializedEndpointPairFilterStatement() (pgsql.Inse }, true } +// boundTerminalFilterSatisfaction tests whether an expansion endpoint occurs in the materialized terminal filter. func boundTerminalFilterSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -947,6 +1060,7 @@ func boundTerminalFilterSatisfaction(expansionModel *Expansion) pgsql.Expression } } +// boundTerminalPairFilterSatisfaction tests whether a root and terminal form a materialized endpoint pair. func boundTerminalPairFilterSatisfaction(rootIDExpression pgsql.Expression, terminalIDExpression pgsql.Expression) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -977,6 +1091,7 @@ func boundTerminalPairFilterSatisfaction(rootIDExpression pgsql.Expression, term } } +// boundRootFilterSatisfaction tests whether an expansion root occurs in the materialized root filter. func boundRootFilterSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1001,6 +1116,7 @@ func boundRootFilterSatisfaction(expansionModel *Expansion) pgsql.Expression { } } +// shortestPathVisitedPruningCondition rejects a root and frontier-node pair already recorded by the search. func shortestPathVisitedPruningCondition(visitedTable pgsql.Identifier, rootIDExpression pgsql.Expression, nextIDExpression pgsql.Expression) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1031,6 +1147,7 @@ func shortestPathVisitedPruningCondition(visitedTable pgsql.Identifier, rootIDEx } } +// forwardContinuationSatisfaction tests whether another eligible edge leaves the forward frontier endpoint. func forwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1055,6 +1172,7 @@ func forwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expression } } +// forwardTerminalSatisfaction selects the cheapest available test that marks a forward frontier row terminal. func (s *ExpansionBuilder) forwardTerminalSatisfaction(expansionModel *Expansion, rootIDExpression pgsql.Expression) pgsql.SelectItem { var satisfied pgsql.Expression @@ -1076,6 +1194,7 @@ func (s *ExpansionBuilder) forwardTerminalSatisfaction(expansionModel *Expansion return satisfiedSelectItem } +// forwardTerminalSatisfactionProjection returns a local terminal predicate when no materialized filter supplies it. func forwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Expression { if expansionModel.TerminalNodeSatisfactionProjection != nil && !expansionModel.UseMaterializedTerminalFilter && @@ -1086,6 +1205,7 @@ func forwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Expr return nil } +// backwardContinuationSatisfaction tests whether another eligible edge enters the backward frontier endpoint. func backwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expression { return pgsql.ExistsExpression{ Subquery: pgsql.Subquery{ @@ -1110,6 +1230,7 @@ func backwardContinuationSatisfaction(expansionModel *Expansion) pgsql.Expressio } } +// backwardTerminalSatisfaction selects the cheapest available test that marks a backward frontier row terminal. func (s *ExpansionBuilder) backwardTerminalSatisfaction(expansionModel *Expansion, terminalIDExpression pgsql.Expression) pgsql.SelectItem { var satisfied pgsql.Expression @@ -1129,6 +1250,7 @@ func (s *ExpansionBuilder) backwardTerminalSatisfaction(expansionModel *Expansio return satisfiedSelectItem } +// backwardTerminalSatisfactionProjection returns a local root predicate when no materialized filter supplies it. func backwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Expression { if expansionModel.PrimerNodeSatisfactionProjection != nil && !expansionModel.UseMaterializedEndpointPairFilter { return pgsql.Expression(expansionModel.PrimerNodeSatisfactionProjection) @@ -1137,6 +1259,7 @@ func backwardTerminalSatisfactionProjection(expansionModel *Expansion) pgsql.Exp return nil } +// prepareForwardFrontPrimerQuery builds the first-edge query and deferred predicate for the forward search frontier. func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expansion) (pgsql.Query, pgsql.Expression, error) { var ( primerSeedConstraints pgsql.Expression @@ -1158,7 +1281,15 @@ func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expans previousFrameIdentifier, ) - if s.usesBoundRootIDs() { + if expansionModel.UsesSingletonEndpointPair() { + rootIDsSeed := newExpansionArrayParameterSeed( + expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), + s.traversalStep.LeftNode.Identifier, + primerSeedConstraints, + 1, + ) + seed = &rootIDsSeed + } else if s.usesBoundRootIDs() { rootIDsSeed := newExpansionRootIDsParameterSeed( expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), s.traversalStep.LeftNode.Identifier, @@ -1226,7 +1357,9 @@ func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expans return pgsql.Query{}, nil, err } - if !expansionModel.HasExplicitEndpointInequality { + if !expansionModel.HasExplicitEndpointInequality && + !expansionModel.UsesSingletonEndpointPair() && + !expansionAllowsZeroDepth(expansionModel) { nextQuery.Where = pgsql.OptionalAnd( nextQuery.Where, shortestPathSeedSelfEndpointGuard(s.model.EdgeStartColumn, expansionModel.UseMaterializedEndpointPairFilter), @@ -1236,6 +1369,7 @@ func (s *ExpansionBuilder) prepareForwardFrontPrimerQuery(expansionModel *Expans return frontPrimerQuery(seed, nextQuery), primerProjectionPredicate, nil } +// prepareForwardFrontRecursiveQuery builds the query that advances the forward frontier by one unused edge. func (s *ExpansionBuilder) prepareForwardFrontRecursiveQuery(expansionModel *Expansion) (pgsql.Select, error) { nextQuery := pgsql.Select{ Where: expansionModel.EdgeConstraints, @@ -1323,6 +1457,7 @@ func (s *ExpansionBuilder) prepareForwardFrontRecursiveQuery(expansionModel *Exp return nextQuery, nil } +// prepareBackwardFrontPrimerQuery builds the first-edge query and deferred predicate for the backward search frontier. func (s *ExpansionBuilder) prepareBackwardFrontPrimerQuery(expansionModel *Expansion) (pgsql.Query, pgsql.Expression, error) { var ( terminalSeedConstraints pgsql.Expression @@ -1344,7 +1479,15 @@ func (s *ExpansionBuilder) prepareBackwardFrontPrimerQuery(expansionModel *Expan previousFrameIdentifier, ) - if s.usesBoundTerminalIDs() { + if expansionModel.UsesSingletonEndpointPair() { + terminalIDsSeed := newExpansionArrayParameterSeed( + expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), + s.traversalStep.RightNode.Identifier, + terminalSeedConstraints, + 2, + ) + seed = &terminalIDsSeed + } else if s.usesBoundTerminalIDs() { terminalIDsSeed := newExpansionTerminalIDsParameterSeed( expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), s.traversalStep.RightNode.Identifier, @@ -1412,6 +1555,7 @@ func (s *ExpansionBuilder) prepareBackwardFrontPrimerQuery(expansionModel *Expan return frontPrimerQuery(seed, nextQuery), terminalProjectionPredicate, nil } +// prepareBackwardFrontRecursiveQuery builds the query that advances the backward frontier by one unused edge. func (s *ExpansionBuilder) prepareBackwardFrontRecursiveQuery(expansionModel *Expansion) (pgsql.Select, error) { nextQuery := pgsql.Select{ Where: expansionModel.EdgeConstraints, @@ -1484,7 +1628,31 @@ func (s *ExpansionBuilder) prepareBackwardFrontRecursiveQuery(expansionModel *Ex return nextQuery, nil } +// shortestPathSearchCTE invokes a shortest-path harness and exposes its rows through the standard search CTE. func shortestPathSearchCTE(functionName pgsql.Identifier, expansionModel *Expansion, harnessParameters []pgsql.Expression) pgsql.CommonTableExpression { + return shortestPathSearchCTEFrom(functionName, expansionModel, harnessParameters, "singleton_endpoints", expansionModel.Frame.Binding.Identifier) +} + +// shortestPathSearchCTEFrom builds the search CTE, substituting validated singleton endpoint identifiers when present. +func shortestPathSearchCTEFrom(functionName pgsql.Identifier, expansionModel *Expansion, harnessParameters []pgsql.Expression, validatedEndpoints, searchAlias pgsql.Identifier) pgsql.CommonTableExpression { + + if expansionModel.UsesSingletonEndpointPair() { + harnessParameters = append([]pgsql.Expression(nil), harnessParameters...) + rootArrayIndex, terminalArrayIndex := bidirectionalHarnessEndpointArrayParameterIndexes(functionName, len(harnessParameters)) + harnessParameters[rootArrayIndex] = pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + }, + CastType: pgsql.Int8Array, + } + harnessParameters[terminalArrayIndex] = pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + }, + CastType: pgsql.Int8Array, + } + } + var ( innerQuery = pgsql.Query{ Body: pgsql.Select{ @@ -1500,27 +1668,77 @@ func shortestPathSearchCTE(functionName pgsql.Identifier, expansionModel *Expans }, } ) + if expansionModel.UsesSingletonEndpointPair() { + selectBody := innerQuery.Body.(pgsql.Select) + selectBody.Projection = []pgsql.SelectItem{ + pgsql.CompoundIdentifier{functionName, pgsql.WildcardIdentifier}, + } + selectBody.From = append([]pgsql.FromClause{{ + Source: pgsql.TableReference{Name: validatedEndpoints.AsCompoundIdentifier()}, + }}, selectBody.From...) + innerQuery.Body = selectBody + } return pgsql.CommonTableExpression{ Alias: pgsql.TableAlias{ - Name: expansionModel.Frame.Binding.Identifier, + Name: searchAlias, Shape: expansionColumns(), }, Query: innerQuery, } } -func boundEndpointProjectionConstraint(prevFrameID, nodeIdentifier, expansionFrameID, expansionColumn pgsql.Identifier) pgsql.Expression { - return pgsql.NewBinaryExpression( - pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{prevFrameID, nodeIdentifier}, - Column: pgsql.ColumnID, +// bidirectionalHarnessEndpointArrayParameterIndexes returns the root and terminal array positions for a harness. +// The SP harness has a trailing allow_zero_depth parameter; the ASP harness does not. +func bidirectionalHarnessEndpointArrayParameterIndexes(functionName pgsql.Identifier, parameterCount int) (int, int) { + switch functionName { + case pgsql.FunctionBidirectionalSPHarness: + return parameterCount - 3, parameterCount - 2 + case pgsql.FunctionBidirectionalASPHarness: + return parameterCount - 2, parameterCount - 1 + default: + panic(fmt.Sprintf("unsupported bidirectional shortest-path harness %q", functionName)) + } +} + +// singletonEndpointValidationCTE validates a single root and terminal pair against both endpoint predicates. +func singletonEndpointValidationCTE(traversalStep *TraversalStep, expansionModel *Expansion) pgsql.CommonTableExpression { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: validatedEndpoints}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{ + &pgsql.AliasedExpression{ + Expression: pgd.EntityID(traversalStep.LeftNode.Identifier), + Alias: models.OptionalValue(expansionRootID), + }, + &pgsql.AliasedExpression{ + Expression: pgd.EntityID(traversalStep.RightNode.Identifier), + Alias: models.OptionalValue(expansionTerminalID), + }, + }, + From: []pgsql.FromClause{ + {Source: expansionNodeTableReference(traversalStep.LeftNode.Identifier)}, + {Source: expansionNodeTableReference(traversalStep.RightNode.Identifier)}, + }, + Where: pgsql.OptionalAnd(expansionModel.PrimerNodeConstraints, expansionModel.TerminalNodeConstraints), + }, }, + } +} + +// boundEndpointProjectionConstraint equates a projected expansion endpoint with its binding in the preceding frame. +func boundEndpointProjectionConstraint(prevFrameID pgsql.Identifier, binding *BoundIdentifier, expansionFrameID, expansionColumn pgsql.Identifier) pgsql.Expression { + return pgsql.NewBinaryExpression( + projectedNodeIDReference(prevFrameID, binding), pgsql.OperatorEquals, pgsql.CompoundIdentifier{expansionFrameID, expansionColumn}, ) } +// applyBoundEndpointProjectionConstraints attaches preceding-frame sources and equalities for bound expansion endpoints. func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQuery *pgsql.Select, expansionModel *Expansion) { if s.traversalStep.Frame == nil || s.traversalStep.Frame.Previous == nil { return @@ -1538,7 +1756,7 @@ func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQue projectionQuery.Where = pgsql.OptionalAnd(projectionQuery.Where, boundEndpointProjectionConstraint( prevFrameID, - s.traversalStep.LeftNode.Identifier, + s.traversalStep.LeftNode, expansionModel.Frame.Binding.Identifier, expansionRootID, ), @@ -1549,7 +1767,7 @@ func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQue projectionQuery.Where = pgsql.OptionalAnd(projectionQuery.Where, boundEndpointProjectionConstraint( prevFrameID, - s.traversalStep.RightNode.Identifier, + s.traversalStep.RightNode, expansionModel.Frame.Binding.Identifier, expansionNextID, ), @@ -1557,6 +1775,7 @@ func (s *ExpansionBuilder) applyBoundEndpointProjectionConstraints(projectionQue } } +// ensureProjectionFrameSource ensures a projection query reads from the requested frame. func ensureProjectionFrameSource(projectionQuery *pgsql.Select, frameIdentifier pgsql.Identifier) { for _, from := range projectionQuery.From { if tableReference, ok := from.Source.(pgsql.TableReference); ok && len(tableReference.Name) == 1 && tableReference.Name[0] == frameIdentifier { @@ -1571,6 +1790,7 @@ func ensureProjectionFrameSource(projectionQuery *pgsql.Select, frameIdentifier }}, projectionQuery.From...) } +// applyShortestPathSeedProjectionConstraints adds deferred seed predicates and any frame source they reference. func (s *ExpansionBuilder) applyShortestPathSeedProjectionConstraints(projectionQuery *pgsql.Select, projectionConstraints pgsql.Expression) { if projectionConstraints == nil { return @@ -1586,6 +1806,7 @@ func (s *ExpansionBuilder) applyShortestPathSeedProjectionConstraints(projection projectionQuery.Where = pgsql.OptionalAnd(projectionQuery.Where, projectionConstraints) } +// shortestPathSelfEndpointGuard rejects a shortest-path request whose root and terminal are identical. // Match Neo4j's shortest-path behavior by surfacing an error for result rows // where the resolved root and terminal endpoints are the same node. func shortestPathSelfEndpointGuard(expansionFrame pgsql.Identifier) pgsql.Expression { @@ -1597,6 +1818,7 @@ func shortestPathSelfEndpointGuard(expansionFrame pgsql.Identifier) pgsql.Expres return shortestPathSelfEndpointGuardCase(rootID, terminalID) } +// shortestPathSelfEndpointGuardCase emits the conditional expression that raises the self-endpoint error. func shortestPathSelfEndpointGuardCase(rootID, terminalID pgsql.Expression) pgsql.Expression { return shortestPathSelfEndpointConditionGuard( pgsql.NewBinaryExpression(rootID, pgsql.OperatorNotEquals, terminalID), @@ -1605,6 +1827,7 @@ func shortestPathSelfEndpointGuardCase(rootID, terminalID pgsql.Expression) pgsq ) } +// shortestPathSelfEndpointConditionGuard applies the self-endpoint check only to rows matching a predicate. func shortestPathSelfEndpointConditionGuard(condition pgsql.Expression, rootID, terminalID pgsql.Expression) pgsql.Expression { return &pgsql.Case{ Conditions: []pgsql.Expression{ @@ -1623,6 +1846,7 @@ func shortestPathSelfEndpointConditionGuard(condition pgsql.Expression, rootID, } } +// shortestPathTerminalFilterSelfEndpointGuard rejects a root present in a singleton terminal filter. // PostgreSQL has no portable expression-level RAISE. Keep the normal path // visible in generated SQL and call the schema helper only for the error path. func shortestPathTerminalFilterSelfEndpointGuard(rootID pgsql.Expression) pgsql.Expression { @@ -1673,6 +1897,7 @@ func shortestPathTerminalFilterSelfEndpointGuard(rootID pgsql.Expression) pgsql. } } +// shortestPathEndpointPairFilterSelfEndpointGuard rejects a self-pair present in the endpoint-pair filter. func shortestPathEndpointPairFilterSelfEndpointGuard(rootID pgsql.Expression) pgsql.Expression { matchingEndpointPairCount := pgsql.Subquery{ Query: pgsql.Query{ @@ -1718,32 +1943,921 @@ func shortestPathEndpointPairFilterSelfEndpointGuard(rootID pgsql.Expression) pg ) } -func shortestPathSeedSelfEndpointGuard(rootID pgsql.Expression, useEndpointPairFilter bool) pgsql.Expression { - if useEndpointPairFilter { - return shortestPathEndpointPairFilterSelfEndpointGuard(rootID) - } +// shortestPathSeedSelfEndpointGuard selects the appropriate self-endpoint check for the active seed filters. +func shortestPathSeedSelfEndpointGuard(rootID pgsql.Expression, useEndpointPairFilter bool) pgsql.Expression { + if useEndpointPairFilter { + return shortestPathEndpointPairFilterSelfEndpointGuard(rootID) + } + + return shortestPathTerminalFilterSelfEndpointGuard(rootID) +} + +// applyShortestPathSelfEndpointGuard adds self-endpoint validation unless an existing inequality already excludes it. +func (s *ExpansionBuilder) applyShortestPathSelfEndpointGuard(projectionQuery *pgsql.Select, expansionModel *Expansion) { + if expansionModel.HasExplicitEndpointInequality || expansionAllowsZeroDepth(expansionModel) { + return + } + + projectionQuery.Where = pgsql.OptionalAnd( + projectionQuery.Where, + shortestPathSelfEndpointGuard(expansionModel.Frame.Binding.Identifier), + ) +} + +// buildShortestPathsHarnessCall assembles the seeded search, harness invocation, and final shortest-path projection. +func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { + var ( + expansionModel = s.traversalStep.Expansion + projectionQuery pgsql.Select + ) + + expansionModel.UseMaterializedTerminalFilter = s.canMaterializeTerminalFilter(expansionModel) + + forwardFrontPrimerQuery, forwardSeedProjectionConstraints, err := s.prepareForwardFrontPrimerQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + forwardFrontRecursiveQuery, err := s.prepareForwardFrontRecursiveQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + projectionQuery.Projection = expansionModel.Projection + + // Select the expansion components for the projection statement + projectionQuery.From = []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier}, + Binding: models.EmptyOptional[pgsql.Identifier](), + }, + Joins: []pgsql.Join{{ + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, + ), + }, + }, { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, + ), + }, + }}, + }} + + s.applyBoundEndpointProjectionConstraints(&projectionQuery, expansionModel) + s.applyShortestPathSeedProjectionConstraints(&projectionQuery, forwardSeedProjectionConstraints) + s.appendUnwindSources(&projectionQuery) + s.applyShortestPathSelfEndpointGuard(&projectionQuery, expansionModel) + + if harnessParameters, err := s.shortestPathsParameters(expansionModel, forwardFrontPrimerQuery, forwardFrontRecursiveQuery); err != nil { + return pgsql.Query{}, err + } else { + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{}, + Body: projectionQuery, + } + + if expansionModel.UsesSingletonEndpointPair() { + query.AddCTE(singletonEndpointValidationCTE(s.traversalStep, expansionModel)) + } + query.AddCTE(shortestPathSearchCTE(harnessFunctionName, expansionModel, harnessParameters)) + return query, nil + } +} + +// BuildShortestPathsRoot builds a unidirectional single-shortest-path harness query. +func (s *ExpansionBuilder) BuildShortestPathsRoot() (pgsql.Query, error) { + return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalSPHarness) +} + +// shortestDistanceColumns returns the harness result shape for identifier-only or rooted distance searches. +func shortestDistanceColumns(idOnly bool) *pgsql.RecordShape { + if idOnly { + return pgsql.NewRecordShape([]pgsql.Identifier{expansionNextID, expansionDepth}) + } + return pgsql.NewRecordShape([]pgsql.Identifier{expansionRootID, expansionNextID, expansionDepth}) +} + +// shortestDistanceEndpointID reads a validated singleton endpoint identifier through a scalar subquery. +func shortestDistanceEndpointID(validatedEndpoints, endpointID pgsql.Identifier) pgsql.Subquery { + return pgsql.Subquery{ + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{validatedEndpoints, endpointID}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + }, + }, + } +} + +// shortestDistanceIDProjection rewrites endpoint identifier projections to use validated search-state columns. +func shortestDistanceIDProjection(projection pgsql.Projection, traversalStep *TraversalStep, stateID, validatedEndpoints pgsql.Identifier) pgsql.Projection { + result := append(pgsql.Projection(nil), projection...) + for idx, item := range result { + aliased, ok := item.(*pgsql.AliasedExpression) + if !ok { + continue + } + identifier, ok := aliased.Expression.(pgsql.CompoundIdentifier) + if !ok || len(identifier) != 2 || identifier[1] != pgsql.ColumnID { + continue + } + var replacement pgsql.Expression + switch identifier[0] { + case traversalStep.LeftNode.Identifier: + replacement = shortestDistanceEndpointID(validatedEndpoints, expansionRootID) + case traversalStep.RightNode.Identifier: + replacement = pgsql.CompoundIdentifier{stateID, expansionNextID} + default: + continue + } + copy := *aliased + copy.Expression = replacement + result[idx] = © + } + return result +} + +// BuildShortestDistanceRoot emits the bounded, distance-only SP-S3-U-D +// recursive search. ID-only endpoint projections use only next ID and depth; +// other projections retain the constant root ID. Neither shape contains path, +// predecessor, visited-edge, cycle, or materialization columns. +func (s *ExpansionBuilder) BuildShortestDistanceRoot() (pgsql.Query, error) { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-S3-U-D requires one validated endpoint pair") + } + if !expansionModel.Options.MaxDepth.Set { + return pgsql.Query{}, errors.New("SP-S3-U-D requires a bounded maximum depth") + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + if expansionModel.Options.MinDepth.GetOr(1) > 0 { + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + } + + stateID := expansionModel.Frame.Binding.Identifier + idOnly := s.traversalStep.LeftNode.IDOnly && s.traversalStep.RightNode.IDOnly + anchorProjection := pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + } + if idOnly { + anchorProjection = pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + } + } + anchor := pgsql.Select{ + Projection: anchorProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + } + + recursiveProjection := pgsql.Projection{ + pgsql.CompoundIdentifier{stateID, expansionRootID}, + expansionModel.EdgeEndColumn, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionDepth}, + pgsql.OperatorAdd, + pgsql.NewLiteral(int64(1), pgsql.Int8), + ), + } + if idOnly { + recursiveProjection = recursiveProjection[1:] + } + recursive := pgsql.Select{ + Projection: recursiveProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + expansionModel.EdgeConstraints, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionDepth}, + pgsql.OperatorLessThan, + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8), + ), + ), + } + + projectionItems := pgsql.Projection(expansionModel.Projection) + var endpointConstraint pgsql.Expression + joins := []pgsql.Join{{ + Table: pgsql.TableReference{Name: validatedEndpoints.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionRootID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + ), + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionNextID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + ), + ), + }, + }} + if idOnly { + projectionItems = shortestDistanceIDProjection(projectionItems, s.traversalStep, stateID, validatedEndpoints) + joins = nil + endpointConstraint = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionNextID}, + pgsql.OperatorEquals, + shortestDistanceEndpointID(validatedEndpoints, expansionTerminalID), + ) + } else { + joins = append(joins, + pgsql.Join{ + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionRootID}, + ), + }, + }, + pgsql.Join{ + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + ) + } + + projection := pgsql.Select{ + Projection: projectionItems, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: joins, + }}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(expansionModel.Options.MinDepth.GetOr(1), pgsql.Int8), + ), + endpointConstraint, + ), + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{Recursive: true}, + Body: projection, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.CompoundIdentifier{stateID, expansionDepth}, + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + } + query.AddCTE(endpointCTE) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: shortestDistanceColumns(idOnly), + }, + Query: pgsql.Query{ + Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + }, + }, + }) + + return query, nil +} + +// shortestPathNodeComposite constructs the stored composite value for a hydrated path node. +func shortestPathNodeComposite(identifier pgsql.Identifier) pgsql.CompositeValue { + value := pgsql.CompositeValue{DataType: pgsql.NodeComposite} + for _, column := range pgsql.NodeTableColumns { + value.Values = append(value.Values, pgsql.CompoundIdentifier{identifier, column}) + } + return value +} + +// shortestPathM0Hydration expands an edge-identifier path into ordered node and edge composites. +func shortestPathM0Hydration(stateID pgsql.Identifier, direction graph.Direction) pgsql.LateralSubquery { + const ( + pathIndex pgsql.Identifier = "m0_path_index" + pathEdge pgsql.Identifier = "m0_edge" + pathTerminal pgsql.Identifier = "m0_terminal" + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + edgeID := &pgsql.ArrayIndex{ + Expression: pgsql.NewParenthetical(pathIDs), + Indexes: []pgsql.Expression{pathIndex}, + CastType: pgsql.Int8, + } + nextNodeColumn := pgsql.ColumnEndID + if direction == graph.DirectionInbound { + nextNodeColumn = pgsql.ColumnStartID + } + joins := []pgsql.Join{{ + Table: expansionEdgeTableReference(pathEdge), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{pathEdge, pgsql.ColumnID}, pgsql.OperatorEquals, edgeID, + ), + }, + }, { + Table: expansionNodeTableReference(pathTerminal), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{pathTerminal, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{pathEdge, nextNodeColumn}, + ), + }, + }} + + return pgsql.LateralSubquery{ + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{shortestPathNodeComposite(pathTerminal)}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pathIndex, + Ascending: true, + }}, + CastType: pgsql.NodeCompositeArray, + }, + Alias: pgsql.AsOptionalIdentifier(hydratedNodes), + }, + &pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{edgeCompositeValue(pathEdge)}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pathIndex, + Ascending: true, + }}, + CastType: pgsql.EdgeCompositeArray, + }, + Alias: pgsql.AsOptionalIdentifier(hydratedEdges), + }, + &pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionCount, + Parameters: []pgsql.Expression{pgsql.Wildcard{}}, + CastType: pgsql.Int8, + }, + Alias: pgsql.AsOptionalIdentifier(hydratedCount), + }, + }, + From: []pgsql.FromClause{{ + Source: pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionGenerateSubscripts, + Parameters: []pgsql.Expression{pathIDs, pgsql.NewLiteral(1, pgsql.Int)}, + }, + Alias: pgsql.AsOptionalIdentifier(pathIndex), + }, + Joins: joins, + }}, + }, + }, + Binding: pgsql.AsOptionalIdentifier(hydrated), + } +} + +// shortestPathM0Projection replaces the raw path state with its hydrated graph-path value. +func shortestPathM0Projection(projection pgsql.Projection, stateID, pathBinding pgsql.Identifier, path pgsql.Expression) pgsql.Projection { + result := append(pgsql.Projection(nil), projection...) + for idx, item := range result { + var aliased pgsql.AliasedExpression + switch typed := item.(type) { + case *pgsql.AliasedExpression: + aliased = *typed + case pgsql.AliasedExpression: + aliased = typed + default: + continue + } + identifier, expressionMatches := aliased.Expression.(pgsql.CompoundIdentifier) + expressionMatches = expressionMatches && len(identifier) == 2 && identifier[0] == stateID && identifier[1] == expansionPath + aliasMatches := aliased.Alias.Set && aliased.Alias.Value == pathBinding + if !expressionMatches && !aliasMatches { + continue + } + aliased.Expression = path + result[idx] = &aliased + } + return result +} + +// BuildShortestPathEdgeM0Root emits the bounded one-path SP-S3-U-E search and +// direction-aware MAT-M0 hydration. Recursive state contains only the current +// node, depth, and ordered edge IDs; node order is derived from edge endpoints. +func (s *ExpansionBuilder) BuildShortestPathEdgeM0Root() (pgsql.Query, error) { + const ( + validatedEndpoints pgsql.Identifier = "singleton_endpoints" + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-S3-U-E+MAT-M0 requires one validated endpoint pair") + } + if !expansionModel.Options.MaxDepth.Set { + return pgsql.Query{}, errors.New("SP-S3-U-E+MAT-M0 requires a bounded maximum depth") + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + if expansionModel.Options.MinDepth.GetOr(1) > 0 { + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + } + + stateID := expansionModel.Frame.Binding.Identifier + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + anchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + pgsql.ArrayLiteral{CastType: pgsql.Int8Array}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + } + recursive := pgsql.Select{ + Projection: pgsql.Projection{ + expansionModel.EdgeEndColumn, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{stateID, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.NewBinaryExpression(pathIDs, pgsql.OperatorConcatenate, pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnID}}, + CastType: pgsql.Int8Array, + }), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + expansionModel.EdgeConstraints, + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{stateID, expansionDepth}, pgsql.OperatorLessThan, pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8)), + relationshipIDNotInPath(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnID}, pathIDs), + ), + ), + } + + hydration := shortestPathM0Hydration(stateID, s.traversalStep.Direction) + rootArray := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{shortestPathNodeComposite(s.traversalStep.LeftNode.Identifier)}, + CastType: pgsql.NodeCompositeArray, + } + nodes := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedNodes}, pgsql.ArrayLiteral{ + CastType: pgsql.NodeCompositeArray, + }, + }, + } + edges := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedEdges}, pgsql.ArrayLiteral{ + CastType: pgsql.EdgeCompositeArray, + }, + }, + } + path := pgsql.CompositeValue{ + DataType: pgsql.PathComposite, + Values: []pgsql.Expression{ + pgsql.NewBinaryExpression(rootArray, pgsql.OperatorConcatenate, nodes), + edges, + }, + } + + projection := pgsql.Select{ + Projection: shortestPathM0Projection(expansionModel.Projection, stateID, expansionModel.PathBinding.Identifier, path), + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{stateID, expansionNextID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + { + Table: hydration, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }, + }, + }}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{stateID, expansionDepth}, pgsql.OperatorGreaterThanOrEqualTo, pgsql.NewLiteral(expansionModel.Options.MinDepth.GetOr(1), pgsql.Int8)), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{hydrated, hydratedCount}, pgsql.OperatorEquals, pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{pathIDs}, + }), + ), + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + }, + Body: projection, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.CompoundIdentifier{stateID, expansionDepth}, + Ascending: true, + }, { + Expression: pathIDs, + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + } + query.AddCTE(endpointCTE) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionNextID, expansionDepth, expansionPath}), + }, + Query: pgsql.Query{ + Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + All: true, + }, + }, + }) + return query, nil +} + +// BuildAllShortestPathsRoot builds a unidirectional all-shortest-paths harness query. +func (s *ExpansionBuilder) BuildAllShortestPathsRoot() (pgsql.Query, error) { + return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalASPHarness) +} + +// compactShortestExecutor reports whether executor emits the compact distance/witness row shape that requires legacy expansion-shape adaptation. +func compactShortestExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorASPA1DAG, + optimize.ShortestPathExecutorASPN1NegativeExhaustion, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + optimize.ShortestPathExecutorS4CanonicalDistance, + optimize.ShortestPathExecutorS4CanonicalWitness, + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness: + return true + default: + return false + } +} + +// buildCompactBoundShortestPathsRoot invokes a typed, static bound-pair +// executor and keeps the legacy expansion row shape at its boundary. That lets +// existing projection and path materialization code consume compact search +// results without carrying entity composites through discovery. +func (s *ExpansionBuilder) buildCompactBoundShortestPathsRoot(functionName pgsql.Identifier, limits ...int64) (pgsql.Query, error) { + const ( + validatedEndpoints pgsql.Identifier = "singleton_endpoints" + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, fmt.Errorf("%s requires one validated endpoint pair", functionName) + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + if expansionModel.Options.MinDepth.GetOr(1) > 0 { + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + } + + maxDepth := expansionModel.Options.MaxDepth.GetOr(translateDefaultMaxTraversalDepth) + parameters := []pgsql.Expression{ + pgsql.NewLiteral(s.graphID, pgsql.Int4), + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(expansionModel.Options.MinDepth.GetOr(1), pgsql.Int4), + pgsql.NewLiteral(maxDepth, pgsql.Int4), + pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array), + pgsql.NewLiteral(s.traversalStep.Direction == graph.DirectionInbound, pgsql.Boolean), + } + for _, limit := range limits { + if limit <= 0 { + return pgsql.Query{}, fmt.Errorf("%s requires positive compact workspace limits", functionName) + } + parameters = append(parameters, pgsql.NewLiteral(limit, pgsql.Int8)) + } + + stateID := expansionModel.Frame.Binding.Identifier + search := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: expansionColumns(), + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{functionName, pgsql.WildcardIdentifier}}, + From: []pgsql.FromClause{ + { + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }, + { + Source: pgsql.FunctionCall{ + Function: functionName, + Parameters: parameters, + }, + }, + }, + }, + }, + } + + projection := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: stateID.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + }, + }}, + } + + // Compact witness and all-path search returns only ordered edge identifiers. Hydrate those + // identifiers at the inline statement boundary, exactly as S3 M0 does, + // instead of invoking the generic ordered_edge_ids_to_path helper. Keeping + // search and hydration as separate SQL operators avoids a second stored + // helper boundary and makes S3/S4 materialization evidence comparable. + if compactExecutorNeedsPathHydration(expansionModel.ShortestPathExecutor) { + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + hydration := shortestPathM0Hydration(stateID, s.traversalStep.Direction) + rootArray := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{shortestPathNodeComposite(s.traversalStep.LeftNode.Identifier)}, + CastType: pgsql.NodeCompositeArray, + } + nodes := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedNodes}, + pgsql.ArrayLiteral{CastType: pgsql.NodeCompositeArray}, + }, + } + edges := pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{hydrated, hydratedEdges}, + pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}, + }, + } + path := pgsql.CompositeValue{ + DataType: pgsql.PathComposite, + Values: []pgsql.Expression{ + pgsql.NewBinaryExpression(rootArray, pgsql.OperatorConcatenate, nodes), + edges, + }, + } + projection.Projection = shortestPathM0Projection(projection.Projection, stateID, expansionModel.PathBinding.Identifier, path) + projection.From[0].Joins = append(projection.From[0].Joins, pgsql.Join{ + Table: hydration, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }) + projection.Where = pgsql.OptionalAnd(projection.Where, pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{hydrated, hydratedCount}, pgsql.OperatorEquals, + pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{pathIDs}, + }, + )) + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{}, + Body: projection, + } + query.AddCTE(endpointCTE) + query.AddCTE(search) + return query, nil +} + +func compactExecutorNeedsPathHydration(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorASPA1DAG, + optimize.ShortestPathExecutorASPN1NegativeExhaustion, + optimize.ShortestPathExecutorS4CanonicalWitness, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return true + default: + return false + } +} + +// BuildAllShortestPathsDAGRoot builds the bound-endpoint query that enumerates all shortest paths from a predecessor DAG. +func (s *ExpansionBuilder) BuildAllShortestPathsDAGRoot() (pgsql.Query, error) { + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionAllShortestPathsDAG) +} + +// BuildAllShortestPathsNoPathProbeRoot builds the exact A1 wrapper that can +// return early only after bounded target-side reachability exhausts. +func (s *ExpansionBuilder) BuildAllShortestPathsNoPathProbeRoot() (pgsql.Query, error) { + return s.buildCompactBoundShortestPathsRoot( + pgsql.FunctionAllShortestPathsNoPathProbe, + s.traversalStep.Expansion.ShortestPathStateLimit, + ) +} + +// BuildB1AllShortestPathsDAGRoot builds strict node-alternating two-sided predecessor-DAG enumeration. +func (s *ExpansionBuilder) BuildB1AllShortestPathsDAGRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionAllShortestPathsB1StrictAlternating, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, + expansion.ShortestPathPredecessorLimit, expansion.ShortestPathEnumerationLimit, + expansion.ShortestPathOutputBytesLimit) +} - return shortestPathTerminalFilterSelfEndpointGuard(rootID) +// BuildB2AllShortestPathsDAGRoot builds smaller-current-level two-sided predecessor-DAG enumeration. +func (s *ExpansionBuilder) BuildB2AllShortestPathsDAGRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionAllShortestPathsB2SmallerCurrentLevel, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, + expansion.ShortestPathPredecessorLimit, expansion.ShortestPathEnumerationLimit, + expansion.ShortestPathOutputBytesLimit) } -func (s *ExpansionBuilder) applyShortestPathSelfEndpointGuard(projectionQuery *pgsql.Select, expansionModel *Expansion) { - if expansionModel.HasExplicitEndpointInequality { - return - } +// BuildCompactShortestPathRoot builds the bound-endpoint query that returns one compact shortest-path witness. +func (s *ExpansionBuilder) BuildCompactShortestPathRoot() (pgsql.Query, error) { + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionShortestPathCompact, s.traversalStep.Expansion.ShortestPathStateLimit) +} - projectionQuery.Where = pgsql.OptionalAnd( - projectionQuery.Where, - shortestPathSelfEndpointGuard(expansionModel.Frame.Binding.Identifier), - ) +// BuildB1CompactShortestPathRoot builds strict node-alternating compact bidirectional search. +func (s *ExpansionBuilder) BuildB1CompactShortestPathRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionShortestPathB1StrictAlternating, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, expansion.ShortestPathPredecessorLimit) } -func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { +// BuildB2CompactShortestPathRoot builds smaller-current-level compact bidirectional search. +func (s *ExpansionBuilder) BuildB2CompactShortestPathRoot() (pgsql.Query, error) { + expansion := s.traversalStep.Expansion + return s.buildCompactBoundShortestPathsRoot(pgsql.FunctionShortestPathB2SmallerCurrentLevel, + expansion.ShortestPathStateLimit, expansion.ShortestPathFrontierLimit, expansion.ShortestPathPredecessorLimit) +} + +// canMaterializeTerminalFilter reports whether terminal constraints can be precomputed as an identifier filter. +func (s *ExpansionBuilder) canMaterializeTerminalFilter(expansionModel *Expansion) bool { + return canMaterializeTerminalFilterForStep(s.traversalStep, expansionModel) +} + +// canMaterializeEndpointPairFilter reports whether root and terminal constraints can be precomputed as endpoint pairs. +func (s *ExpansionBuilder) canMaterializeEndpointPairFilter(expansionModel *Expansion) bool { + return canMaterializeEndpointPairFilterForStep(s.traversalStep, expansionModel) +} + +// buildBiDirectionalShortestPathsHarnessCall assembles both search fronts, the bidirectional harness, and its final projection. +func (s *ExpansionBuilder) buildBiDirectionalShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { var ( expansionModel = s.traversalStep.Expansion projectionQuery pgsql.Select ) - expansionModel.UseMaterializedTerminalFilter = s.canMaterializeTerminalFilter(expansionModel) + if !expansionModel.UsesSingletonEndpointPair() { + expansionModel.UseMaterializedEndpointPairFilter = s.canMaterializeEndpointPairFilter(expansionModel) + } forwardFrontPrimerQuery, forwardSeedProjectionConstraints, err := s.prepareForwardFrontPrimerQuery(expansionModel) if err != nil { @@ -1755,6 +2869,16 @@ func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgs return pgsql.Query{}, err } + backwardFrontPrimerQuery, backwardSeedProjectionConstraints, err := s.prepareBackwardFrontPrimerQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + + backwardFrontRecursiveQuery, err := s.prepareBackwardFrontRecursiveQuery(expansionModel) + if err != nil { + return pgsql.Query{}, err + } + projectionQuery.Projection = expansionModel.Projection // Select the expansion components for the projection statement @@ -1787,11 +2911,18 @@ func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgs }} s.applyBoundEndpointProjectionConstraints(&projectionQuery, expansionModel) - s.applyShortestPathSeedProjectionConstraints(&projectionQuery, forwardSeedProjectionConstraints) + s.applyShortestPathSeedProjectionConstraints(&projectionQuery, pgsql.OptionalAnd(forwardSeedProjectionConstraints, backwardSeedProjectionConstraints)) s.appendUnwindSources(&projectionQuery) s.applyShortestPathSelfEndpointGuard(&projectionQuery, expansionModel) - if harnessParameters, err := s.shortestPathsParameters(expansionModel, forwardFrontPrimerQuery, forwardFrontRecursiveQuery); err != nil { + if harnessParameters, err := s.bidirectionalShortestPathsParameters( + expansionModel, + forwardFrontPrimerQuery, + forwardFrontRecursiveQuery, + backwardFrontPrimerQuery, + backwardFrontRecursiveQuery, + harnessFunctionName == pgsql.FunctionBidirectionalSPHarness, + ); err != nil { return pgsql.Query{}, err } else { query := pgsql.Query{ @@ -1799,112 +2930,225 @@ func (s *ExpansionBuilder) buildShortestPathsHarnessCall(harnessFunctionName pgs Body: projectionQuery, } + if expansionModel.UsesSingletonEndpointPair() { + query.AddCTE(singletonEndpointValidationCTE(s.traversalStep, expansionModel)) + } query.AddCTE(shortestPathSearchCTE(harnessFunctionName, expansionModel, harnessParameters)) return query, nil } } -func (s *ExpansionBuilder) BuildShortestPathsRoot() (pgsql.Query, error) { - return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalSPHarness) -} - -func (s *ExpansionBuilder) BuildAllShortestPathsRoot() (pgsql.Query, error) { - return s.buildShortestPathsHarnessCall(pgsql.FunctionUnidirectionalASPHarness) -} - -func (s *ExpansionBuilder) canMaterializeTerminalFilter(expansionModel *Expansion) bool { - return canMaterializeTerminalFilterForStep(s.traversalStep, expansionModel) -} - -func (s *ExpansionBuilder) canMaterializeEndpointPairFilter(expansionModel *Expansion) bool { - return canMaterializeEndpointPairFilterForStep(s.traversalStep, expansionModel) +// BuildBiDirectionalShortestPathsRoot builds a bidirectional single-shortest-path harness query. +func (s *ExpansionBuilder) BuildBiDirectionalShortestPathsRoot() (pgsql.Query, error) { + return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalSPHarness) } -func (s *ExpansionBuilder) buildBiDirectionalShortestPathsHarnessCall(harnessFunctionName pgsql.Identifier) (pgsql.Query, error) { - var ( - expansionModel = s.traversalStep.Expansion - projectionQuery pgsql.Select +// BuildBiDirectionalShortestPathsRootWithDirectPreflight emits the tool-only +// SP-S0-DIRECT arm. A materialized one-edge probe returns immediately when it +// finds a valid bound-endpoint witness. The workspace-backed incumbent is +// dependent on a zero-or-one-row fallback endpoint CTE, so PostgreSQL cannot +// invoke it on a direct hit. Both branches execute in one statement snapshot. +func (s *ExpansionBuilder) BuildBiDirectionalShortestPathsRootWithDirectPreflight() (pgsql.Query, error) { + const ( + validatedEndpoints pgsql.Identifier = "singleton_endpoints" + directHit pgsql.Identifier = "direct_shortest" + fallbackEndpoints pgsql.Identifier = "fallback_endpoints" + workspaceSearch pgsql.Identifier = "workspace_shortest" ) - expansionModel.UseMaterializedEndpointPairFilter = s.canMaterializeEndpointPairFilter(expansionModel) + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-S0-DIRECT requires one validated endpoint pair") + } + if expansionModel.Options.MinDepth.GetOr(1) != 1 || expansionModel.Options.MaxDepth.GetOr(0) < 1 { + return pgsql.Query{}, errors.New("SP-S0-DIRECT requires minimum depth one and a positive bounded maximum depth") + } forwardFrontPrimerQuery, forwardSeedProjectionConstraints, err := s.prepareForwardFrontPrimerQuery(expansionModel) if err != nil { return pgsql.Query{}, err } - forwardFrontRecursiveQuery, err := s.prepareForwardFrontRecursiveQuery(expansionModel) if err != nil { return pgsql.Query{}, err } - backwardFrontPrimerQuery, backwardSeedProjectionConstraints, err := s.prepareBackwardFrontPrimerQuery(expansionModel) if err != nil { return pgsql.Query{}, err } - backwardFrontRecursiveQuery, err := s.prepareBackwardFrontRecursiveQuery(expansionModel) if err != nil { return pgsql.Query{}, err } - projectionQuery.Projection = expansionModel.Projection + harnessParameters, err := s.bidirectionalShortestPathsParameters( + expansionModel, + forwardFrontPrimerQuery, + forwardFrontRecursiveQuery, + backwardFrontPrimerQuery, + backwardFrontRecursiveQuery, + true, + ) + if err != nil { + return pgsql.Query{}, err + } - // Select the expansion components for the projection statement - projectionQuery.From = []pgsql.FromClause{{ - Source: pgsql.TableReference{ - Name: pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier}, - Binding: models.EmptyOptional[pgsql.Identifier](), - }, - Joins: []pgsql.Join{{ - Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), - JoinOperator: pgsql.JoinOperator{ - JoinType: pgsql.JoinTypeInner, - Constraint: pgsql.NewBinaryExpression( - pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, - pgsql.OperatorEquals, - pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, - ), + projectionQuery := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: expansionModel.Frame.Binding.Identifier.AsCompoundIdentifier(), }, - }, { - Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), - JoinOperator: pgsql.JoinOperator{ - JoinType: pgsql.JoinTypeInner, - Constraint: pgsql.NewBinaryExpression( - pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, - pgsql.OperatorEquals, - pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, - ), + Joins: []pgsql.Join{ + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, + ), + }, + }, }, }}, - }} - - s.applyBoundEndpointProjectionConstraints(&projectionQuery, expansionModel) + } s.applyShortestPathSeedProjectionConstraints(&projectionQuery, pgsql.OptionalAnd(forwardSeedProjectionConstraints, backwardSeedProjectionConstraints)) s.appendUnwindSources(&projectionQuery) s.applyShortestPathSelfEndpointGuard(&projectionQuery, expansionModel) - if harnessParameters, err := s.bidirectionalAllShortestPathsParameters(expansionModel, forwardFrontPrimerQuery, forwardFrontRecursiveQuery, backwardFrontPrimerQuery, backwardFrontRecursiveQuery); err != nil { - return pgsql.Query{}, err - } else { - query := pgsql.Query{ - CommonTableExpressions: &pgsql.With{}, - Body: projectionQuery, - } + directQuery := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(int64(1), pgsql.Int8), + pgsql.NewLiteral(true, pgsql.Boolean), + pgd.Equals(pgd.StartID(s.traversalStep.Edge.Identifier), pgd.EndID(s.traversalStep.Edge.Identifier)), + pgd.ExpressionArrayLiteral(pgd.EntityID(s.traversalStep.Edge.Identifier)), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgd.Equals(expansionModel.EdgeStartColumn, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgd.Equals(expansionModel.EdgeEndColumn, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}), + ), + }, + }}, + }}, + Where: expansionModel.EdgeConstraints, + }, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgd.EntityID(s.traversalStep.Edge.Identifier), + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + } - query.AddCTE(shortestPathSearchCTE(harnessFunctionName, expansionModel, harnessParameters)) - return query, nil + fallbackEndpointQuery := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: validatedEndpoints.AsCompoundIdentifier(), + }, + }}, + Where: pgsql.ExistsExpression{ + Negated: true, + Subquery: pgsql.Subquery{ + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: directHit.AsCompoundIdentifier(), + }, + }}, + }, + }, + }, + }, + }, } -} -func (s *ExpansionBuilder) BuildBiDirectionalShortestPathsRoot() (pgsql.Query, error) { - return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalSPHarness) + stateQuery := pgsql.Query{ + Body: pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: directHit.AsCompoundIdentifier(), + }, + }}, + }, + ROperand: pgsql.Select{ + Projection: pgsql.Projection{pgsql.Wildcard{}}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: workspaceSearch.AsCompoundIdentifier(), + }, + }}, + }, + Operator: pgsql.OperatorUnion, + All: true, + }, + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{}, + Body: projectionQuery, + } + query.AddCTE(singletonEndpointValidationCTE(s.traversalStep, expansionModel)) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: directHit, + Shape: expansionColumns(), + }, + Materialized: &pgsql.Materialized{ + Materialized: true, + }, + Query: directQuery, + }) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: fallbackEndpoints, + }, + Query: fallbackEndpointQuery, + }) + query.AddCTE(shortestPathSearchCTEFrom(pgsql.FunctionBidirectionalSPHarness, expansionModel, harnessParameters, fallbackEndpoints, workspaceSearch)) + query.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: expansionModel.Frame.Binding.Identifier, + Shape: expansionColumns(), + }, + Query: stateQuery, + }) + + return query, nil } +// BuildBiDirectionalAllShortestPathsRoot builds a bidirectional all-shortest-paths harness query. func (s *ExpansionBuilder) BuildBiDirectionalAllShortestPathsRoot() (pgsql.Query, error) { return s.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalASPHarness) } +// boundEndpointFilterParameters renders the available bound endpoint inserts as harness SQL parameters. func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, error) { var ( rootFilterStatement, hasRootFilter = s.boundRootIDsFilterStatement() @@ -1933,13 +3177,13 @@ func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, ) if hasPairFilter { - if formattedFilter, err := format.Statement(pairFilterStatement, format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)); err != nil { + if formattedFilter, err := format.Statement(pairFilterStatement, format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)); err != nil { return nil, err } else { pairFilter = formattedFilter } } else if hasRootFilter { - if formattedFilter, err := format.Statement(rootFilterStatement, format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)); err != nil { + if formattedFilter, err := format.Statement(rootFilterStatement, format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)); err != nil { return nil, err } else { rootFilter = formattedFilter @@ -1947,7 +3191,7 @@ func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, } if !hasPairFilter && hasTerminalFilter { - if formattedFilter, err := format.Statement(terminalFilterStatement, format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)); err != nil { + if formattedFilter, err := format.Statement(terminalFilterStatement, format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)); err != nil { return nil, err } else { terminalFilter = formattedFilter @@ -1966,13 +3210,14 @@ func (s *ExpansionBuilder) boundEndpointFilterParameters() ([]pgsql.Expression, return filterParameters, nil } +// shortestPathsParameters renders a forward search's query fragments, depth limit, and filter inserts as harness parameters. func (s *ExpansionBuilder) shortestPathsParameters(expansionModel *Expansion, forwardFrontPrimerQuery pgsql.SetExpression, forwardFrontRecursiveQuery pgsql.SetExpression) ([]pgsql.Expression, error) { var ( harnessParameters []pgsql.Expression formatFragment = func(query pgsql.SetExpression) (string, error) { return format.Statement( nextFrontInsert(query), - format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)) + format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)) } ) @@ -2010,13 +3255,34 @@ func (s *ExpansionBuilder) shortestPathsParameters(expansionModel *Expansion, fo return harnessParameters, nil } -func (s *ExpansionBuilder) bidirectionalAllShortestPathsParameters(expansionModel *Expansion, forwardFrontPrimerQuery pgsql.SetExpression, forwardFrontRecursiveQuery pgsql.SetExpression, backwardFrontPrimerQuery pgsql.SetExpression, backwardFrontRecursiveQuery pgsql.SetExpression) ([]pgsql.Expression, error) { +// shortestPathWorkspaceFragment rewrites generic workspace identifiers to the reusable bidirectional-search namespace. +func shortestPathWorkspaceFragment(fragment string) string { + return strings.NewReplacer( + "on conflict on constraint forward_visited_pkey", "on conflict on constraint bsp_forward_visited_pkey", + "on conflict on constraint backward_visited_pkey", "on conflict on constraint bsp_backward_visited_pkey", + "forward_visited", "pg_temp.bsp_forward_visited", + "backward_visited", "pg_temp.bsp_backward_visited", + "forward_front", "pg_temp.bsp_forward_front", + "backward_front", "pg_temp.bsp_backward_front", + "next_front", "pg_temp.bsp_next_front", + ).Replace(fragment) +} + +// bidirectionalShortestPathsParameters renders both search fronts and endpoint inputs for the bidirectional harness. +func (s *ExpansionBuilder) bidirectionalShortestPathsParameters(expansionModel *Expansion, forwardFrontPrimerQuery pgsql.SetExpression, forwardFrontRecursiveQuery pgsql.SetExpression, backwardFrontPrimerQuery pgsql.SetExpression, backwardFrontRecursiveQuery pgsql.SetExpression, useReusableWorkspace bool) ([]pgsql.Expression, error) { var ( harnessParameters []pgsql.Expression formatFragment = func(query pgsql.SetExpression) (string, error) { - return format.Statement( + fragment, err := format.Statement( nextFrontInsert(query), - format.NewOutputBuilder().WithMaterializedParameters(s.queryParameters)) + format.NewOutputBuilder().WithTargetGraph(s.graphID).WithMaterializedParameters(s.queryParameters)) + if err != nil { + return "", err + } + if useReusableWorkspace { + fragment = shortestPathWorkspaceFragment(fragment) + } + return fragment, nil } ) @@ -2064,16 +3330,57 @@ func (s *ExpansionBuilder) bidirectionalAllShortestPathsParameters(expansionMode } harnessParameters = append(harnessParameters, pgsql.NewLiteral(expansionModel.Options.MaxDepth.GetOr(translateDefaultMaxTraversalDepth), pgsql.Int)) + if expansionModel.UsesSingletonEndpointPair() { + harnessParameters = append(harnessParameters, + pgsql.ArrayLiteral{ + Values: []pgsql.Expression{expansionModel.SingletonRootID}, + CastType: pgsql.Int8Array, + }, + pgsql.ArrayLiteral{ + Values: []pgsql.Expression{expansionModel.SingletonTerminalID}, + CastType: pgsql.Int8Array, + }, + ) + if useReusableWorkspace { + harnessParameters = append(harnessParameters, pgsql.NewLiteral(expansionAllowsZeroDepth(expansionModel), pgsql.Boolean)) + } + return harnessParameters, nil + } if filterParameters, err := s.boundEndpointFilterParameters(); err != nil { return nil, err } else { + if useReusableWorkspace { + for idx, filterParameter := range filterParameters { + typeCast, isTypeCast := filterParameter.(pgsql.TypeCast) + if !isTypeCast { + continue + } + literal, isLiteral := typeCast.Expression.(pgsql.Literal) + if !isLiteral { + continue + } + if value, isString := literal.Value.(string); isString { + literal.Value = strings.NewReplacer( + "traversal_root_filter", "pg_temp.bsp_root_filter", + "traversal_terminal_filter", "pg_temp.bsp_terminal_filter", + "traversal_pair_filter", "pg_temp.bsp_pair_filter", + ).Replace(value) + typeCast.Expression = literal + filterParameters[idx] = typeCast + } + } + } harnessParameters = append(harnessParameters, filterParameters...) } + if useReusableWorkspace { + harnessParameters = append(harnessParameters, pgsql.NewLiteral(expansionAllowsZeroDepth(expansionModel), pgsql.Boolean)) + } return harnessParameters, nil } +// Build combines the configured expansion stages into a recursive CTE and final projection query. func (s *ExpansionBuilder) Build(expansionIdentifier pgsql.Identifier, commonTableExpressions ...pgsql.CommonTableExpression) pgsql.Query { expansionBody := pgsql.SetExpression(pgsql.SetOperation{ LOperand: s.PrimerStatement, @@ -2130,6 +3437,7 @@ func (s *ExpansionBuilder) Build(expansionIdentifier pgsql.Identifier, commonTab return query } +// projectionAliasExpressions indexes each projected alias or identifier by its underlying expression. func projectionAliasExpressions(projection pgsql.Projection) map[pgsql.Identifier]pgsql.Expression { aliases := make(map[pgsql.Identifier]pgsql.Expression) @@ -2190,6 +3498,23 @@ func rewriteCurrentFrameProjectionWindow(window *pgsql.Window, frameID pgsql.Ide return &rewritten } +func rewriteCurrentFrameProjectionOrderBy(orderBy []*pgsql.OrderBy, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) []*pgsql.OrderBy { + if orderBy == nil { + return nil + } + + rewritten := make([]*pgsql.OrderBy, len(orderBy)) + for idx, item := range orderBy { + if item != nil { + cloned := *item + cloned.Expression = rewriteCurrentFrameProjectionReferences(item.Expression, frameID, aliases) + rewritten[idx] = &cloned + } + } + + return rewritten +} + func rewriteCurrentFrameProjectionItems(projection pgsql.Projection, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Projection { if projection == nil { return nil @@ -2230,6 +3555,7 @@ func rewriteCurrentFrameProjectionFromClauses(fromClauses []pgsql.FromClause, fr return rewritten } +// rewriteCurrentFrameProjectionSetExpression substitutes current-frame aliases throughout a set expression. func rewriteCurrentFrameProjectionSetExpression(setExpression pgsql.SetExpression, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.SetExpression { if setExpression == nil { return nil @@ -2267,6 +3593,7 @@ func rewriteCurrentFrameProjectionSetExpression(setExpression pgsql.SetExpressio } } +// rewriteCurrentFrameProjectionQuery substitutes current-frame aliases throughout a query and its CTEs. func rewriteCurrentFrameProjectionQuery(query pgsql.Query, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Query { query.Body = rewriteCurrentFrameProjectionSetExpression(query.Body, frameID, aliases) @@ -2288,6 +3615,7 @@ func rewriteCurrentFrameProjectionQuery(query pgsql.Query, frameID pgsql.Identif return query } +// rewriteCurrentFrameProjectionSelect substitutes current-frame aliases in every expression-bearing select clause. func rewriteCurrentFrameProjectionSelect(selectBody pgsql.Select, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Select { selectBody.Projection = rewriteCurrentFrameProjectionItems(selectBody.Projection, frameID, aliases) selectBody.From = rewriteCurrentFrameProjectionFromClauses(selectBody.From, frameID, aliases) @@ -2298,6 +3626,7 @@ func rewriteCurrentFrameProjectionSelect(selectBody pgsql.Select, frameID pgsql. return selectBody } +// rewriteCurrentFrameProjectionReferences replaces qualified current-frame references with their projected expressions. func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameID pgsql.Identifier, aliases map[pgsql.Identifier]pgsql.Expression) pgsql.Expression { if expression == nil { return nil @@ -2347,6 +3676,7 @@ func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameI case pgsql.FunctionCall: typedExpression.Parameters = rewriteCurrentFrameProjectionExpressions(typedExpression.Parameters, frameID, aliases) + typedExpression.OrderBy = rewriteCurrentFrameProjectionOrderBy(typedExpression.OrderBy, frameID, aliases) typedExpression.Over = rewriteCurrentFrameProjectionWindow(typedExpression.Over, frameID, aliases) return typedExpression @@ -2357,6 +3687,7 @@ func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameI rewritten := *typedExpression rewritten.Parameters = rewriteCurrentFrameProjectionExpressions(typedExpression.Parameters, frameID, aliases) + rewritten.OrderBy = rewriteCurrentFrameProjectionOrderBy(typedExpression.OrderBy, frameID, aliases) rewritten.Over = rewriteCurrentFrameProjectionWindow(typedExpression.Over, frameID, aliases) return &rewritten @@ -2400,6 +3731,7 @@ func rewriteCurrentFrameProjectionReferences(expression pgsql.Expression, frameI return &pgsql.EdgeArrayFromPathIDs{ PathIDs: rewriteCurrentFrameProjectionReferences(typedExpression.PathIDs, frameID, aliases), + GraphID: rewriteCurrentFrameProjectionReferences(typedExpression.GraphID, frameID, aliases), } case pgsql.ArrayLiteral: @@ -2672,7 +4004,7 @@ func isUnboundSelfLoop(traversalStep *TraversalStep) bool { // otherwise the usual root_id/next_id pair is returned. func expansionProjectionNodeJoins(traversalStep *TraversalStep, frameID pgsql.Identifier) []pgsql.Join { rootJoin := expansionNodeLookupJoin( - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, pgsql.CompoundIdentifier{frameID, expansionRootID}, ) @@ -2681,7 +4013,7 @@ func expansionProjectionNodeJoins(traversalStep *TraversalStep, frameID pgsql.Id } nextJoin := expansionNodeLookupJoin( - traversalStep.RightNode.Identifier, + traversalStep.RightNode, pgsql.CompoundIdentifier{frameID, expansionNextID}, ) @@ -2701,6 +4033,7 @@ func selfLoopIdentityConstraint(traversalStep *TraversalStep, frameID pgsql.Iden ) } +// buildExpansionPatternRoot builds the seed and recursive query for a variable-length traversal that starts a pattern. func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder) (pgsql.Query, error) { var ( traversalStep = traversalStepContext.CurrentStep @@ -2732,7 +4065,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte return pgsql.Query{}, fmt.Errorf("left node is marked as bound but there is no previous frame to reference") } - boundSeed := newExpansionBoundNodeSeed(seedIdentifier, traversalStep.Frame.Previous, traversalStep.LeftNode.Identifier, seedConstraints) + boundSeed := newExpansionBoundNodeSeed(seedIdentifier, traversalStep.Frame.Previous, traversalStep.LeftNode, seedConstraints) seed = &boundSeed expansion.UseUnionAll = true } else if seedConstraints != nil || isUnboundSelfLoop(traversalStep) { @@ -2876,7 +4209,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte projectionConstraints, boundEndpointProjectionConstraint( previousProjectionFrameID, - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, expansionModel.Frame.Binding.Identifier, expansionRootID, ), @@ -2887,7 +4220,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte projectionConstraints, boundEndpointProjectionConstraint( previousProjectionFrameID, - traversalStep.RightNode.Identifier, + traversalStep.RightNode, expansionModel.Frame.Binding.Identifier, expansionNextID, ), @@ -2909,6 +4242,7 @@ func (s *Translator) buildExpansionPatternRoot(traversalStepContext TraversalSte return expansion.Build(expansionModel.Frame.Binding.Identifier), nil } +// buildExpansionPatternStep builds the seed and recursive query for a variable-length traversal after an existing pattern step. func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder) (pgsql.Query, error) { var ( traversalStep = traversalStepContext.CurrentStep @@ -2916,7 +4250,7 @@ func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalSte seed = newExpansionBoundNodeSeed( expansionSeedIdentifier(expansionModel.Frame.Binding.Identifier), traversalStep.Frame.Previous, - traversalStep.LeftNode.Identifier, + traversalStep.LeftNode, expansionModel.PrimerNodeConstraints, ) ) @@ -2998,7 +4332,16 @@ func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalSte Name: pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier}, Binding: models.EmptyOptional[pgsql.Identifier](), }, - Joins: expansionProjectionNodeJoins(traversalStep, expansionModel.Frame.Binding.Identifier), + Joins: []pgsql.Join{ + expansionNodeLookupJoin( + traversalStep.LeftNode, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionRootID}, + ), + expansionNodeLookupJoin( + traversalStep.RightNode, + pgsql.CompoundIdentifier{expansionModel.Frame.Binding.Identifier, expansionNextID}, + ), + }, }) if projectionConstraints, err := s.buildExpansionProjectionConstraints(traversalStepContext); err != nil { @@ -3019,6 +4362,7 @@ func (s *Translator) buildExpansionPatternStep(traversalStepContext TraversalSte return expansion.Build(expansionModel.Frame.Binding.Identifier, seed.CTE()), nil } +// expansionTerminalSatisfactionLocality partitions terminal predicates into traversal-local and deferred expressions. func expansionTerminalSatisfactionLocality(traversalStep *TraversalStep) (pgsql.Expression, pgsql.Expression) { return partitionConstraintByLocality( pgsql.Expression(traversalStep.Expansion.TerminalNodeSatisfactionProjection), @@ -3030,6 +4374,7 @@ func expansionTerminalSatisfactionLocality(traversalStep *TraversalStep) (pgsql. ) } +// applyExpansionSuffixPushdown pushes an eligible fixed-length suffix into the preceding variable expansion's terminal test. func applyExpansionSuffixPushdown(part *PatternPart) (int, error) { var applied int @@ -3040,7 +4385,7 @@ func applyExpansionSuffixPushdown(part *PatternPart) (int, error) { ) if candidateApplied, err := applyExpansionSuffixPushdownCandidate(currentStep, suffixSteps); err != nil { - return applied, err + return 0, err } else if candidateApplied { applied++ } @@ -3049,6 +4394,7 @@ func applyExpansionSuffixPushdown(part *PatternPart) (int, error) { return applied, nil } +// applyExpansionSuffixPushdownCandidate attaches a suffix-existence predicate when all suffix steps can be evaluated locally. func applyExpansionSuffixPushdownCandidate(currentStep *TraversalStep, suffixSteps []*TraversalStep) (bool, error) { if suffixSatisfaction, satisfied := expansionSuffixTerminalSatisfaction(currentStep, suffixSteps); satisfied { currentStep.Expansion.TerminalNodeConstraints = pgsql.OptionalAnd( @@ -3068,6 +4414,7 @@ func applyExpansionSuffixPushdownCandidate(currentStep *TraversalStep, suffixSte return false, nil } +// suffixEdgeLeftEndpoint returns the edge endpoint connected to a suffix step's left node for its direction. func suffixEdgeLeftEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Direction) (pgsql.Expression, bool) { switch direction { case graph.DirectionOutbound: @@ -3079,6 +4426,7 @@ func suffixEdgeLeftEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Dir } } +// suffixEdgeRightEndpoint returns the edge endpoint connected to a suffix step's right node for its direction. func suffixEdgeRightEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Direction) (pgsql.Expression, bool) { switch direction { case graph.DirectionOutbound: @@ -3090,6 +4438,7 @@ func suffixEdgeRightEndpoint(edgeIdentifier pgsql.Identifier, direction graph.Di } } +// suffixBoundNodeIDReference resolves a suffix node to its identifier projection in the preceding frame. func suffixBoundNodeIDReference(currentStep *TraversalStep, node *BoundIdentifier) (pgsql.Expression, bool) { if currentStep == nil || currentStep.Frame == nil || @@ -3100,12 +4449,10 @@ func suffixBoundNodeIDReference(currentStep *TraversalStep, node *BoundIdentifie return nil, false } - return pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{currentStep.Frame.Previous.Binding.Identifier, node.Identifier}, - Column: pgsql.ColumnID, - }, true + return projectedNodeIDReference(currentStep.Frame.Previous.Binding.Identifier, node), true } +// suffixStepEdgeConstraints returns only the predicates local to a suffix step's edge binding. func suffixStepEdgeConstraints(step *TraversalStep) pgsql.Expression { if step == nil || step.EdgeConstraints == nil { return nil @@ -3119,6 +4466,7 @@ func suffixStepEdgeConstraints(step *TraversalStep) pgsql.Expression { return localConstraints } +// expansionSuffixTerminalSatisfaction builds an existence test proving that a fixed suffix continues from an expansion endpoint. func expansionSuffixTerminalSatisfaction(currentStep *TraversalStep, suffixSteps []*TraversalStep) (pgsql.Expression, bool) { if currentStep == nil || currentStep.Expansion == nil || @@ -3218,6 +4566,7 @@ func expansionSuffixTerminalSatisfaction(currentStep *TraversalStep, suffixSteps }, true } +// expansionLocalTerminalSatisfactionProjection projects the local terminal predicate, defaulting to true when none exists. func expansionLocalTerminalSatisfactionProjection(traversalStep *TraversalStep) (pgsql.SelectItem, error) { localSatisfiedConstraint, _ := expansionTerminalSatisfactionLocality(traversalStep) @@ -3228,8 +4577,17 @@ func expansionLocalTerminalSatisfactionProjection(traversalStep *TraversalStep) return pgsql.As[pgsql.SelectItem](localSatisfiedConstraint) } +// buildExpansionPrimerProjection constructs the root, endpoint, depth, satisfaction, cycle, and path columns for the first edge. func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep) ([]pgsql.SelectItem, error) { expansionModel := traversalStep.Expansion + isCycleProjection := pgsql.SelectItem(pgsql.NewLiteral(false, pgsql.Boolean)) + if expansionModel.Options.FindShortestPath || expansionModel.Options.FindAllShortestPaths { + isCycleProjection = pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, + pgsql.OperatorEquals, + expansionModel.EdgeEndColumn, + ) + } if expansionModel.TerminalNodeSatisfactionProjection != nil { satisfiedProjection, err := expansionLocalTerminalSatisfactionProjection(traversalStep) @@ -3242,11 +4600,7 @@ func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep expansionModel.EdgeEndColumn, pgsql.NewLiteral(1, pgsql.Int), satisfiedProjection, - pgsql.NewBinaryExpression( - expansionModel.EdgeStartColumn, - pgsql.OperatorEquals, - expansionModel.EdgeEndColumn, - ), + isCycleProjection, pgsql.ArrayLiteral{ Values: []pgsql.Expression{ pgsql.CompoundIdentifier{traversalStep.Edge.Identifier, pgsql.ColumnID}, @@ -3259,11 +4613,7 @@ func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep expansionModel.EdgeEndColumn, pgsql.NewLiteral(1, pgsql.Int), pgsql.NewLiteral(false, pgsql.Boolean), - pgsql.NewBinaryExpression( - expansionModel.EdgeStartColumn, - pgsql.OperatorEquals, - expansionModel.EdgeEndColumn, - ), + isCycleProjection, pgsql.ArrayLiteral{ Values: []pgsql.Expression{ pgsql.CompoundIdentifier{traversalStep.Edge.Identifier, pgsql.ColumnID}, @@ -3273,6 +4623,7 @@ func (s *Translator) buildExpansionPrimerProjection(traversalStep *TraversalStep } } +// expansionRecursivePathExpression appends or prepends the next edge identifier according to traversal direction. func expansionRecursivePathExpression(traversalStep *TraversalStep) *pgsql.BinaryExpression { var ( expansionModel = traversalStep.Expansion @@ -3287,6 +4638,7 @@ func expansionRecursivePathExpression(traversalStep *TraversalStep) *pgsql.Binar return pgsql.NewBinaryExpression(path, pgsql.OperatorConcatenate, edgeID) } +// buildExpansionRecursiveProjection advances the expansion state and accumulated path by one edge. func (s *Translator) buildExpansionRecursiveProjection(traversalStep *TraversalStep) ([]pgsql.SelectItem, error) { expansionModel := traversalStep.Expansion @@ -3334,6 +4686,7 @@ func (s *Translator) buildExpansionRecursiveProjection(traversalStep *TraversalS } } +// buildExpansionProjectionConstraints combines join, depth, satisfaction, and deferred predicates for projected expansion rows. func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext TraversalStepContext) (pgsql.Expression, error) { var ( currentStep = traversalStepContext.CurrentStep @@ -3347,16 +4700,13 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr if previousStep != nil { joinCondition = pgd.Equals( - pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{previousStep.Frame.Binding.Identifier, currentStep.LeftNode.Identifier}, - Column: pgsql.ColumnID, - }, + projectedNodeIDReference(previousStep.Frame.Binding.Identifier, currentStep.LeftNode), pgd.Column(expansionModel.Frame.Binding.Identifier, expansionRootID), ) } if constraints, err = s.treeTranslator.ConsumeConstraintsFromVisibleSet(expansionModel.Frame.Visible); err != nil { - return projectionConstraints, err + return nil, err } else { // Constraints that target the terminal node may crop up here where it's finally in scope. Additionally, // only accept paths that are marked satisfied from the recursive descent CTE @@ -3368,7 +4718,7 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr } if projectionConstraints, err = ConjoinExpressions(s.kindMapper, expressions); err != nil { - return projectionConstraints, err + return nil, err } // Append any deferred (non-local) constraints onto the projection constraints @@ -3377,7 +4727,7 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr } } else { if projectionConstraints, err = ConjoinExpressions(s.kindMapper, []pgsql.Expression{constraints.Expression, joinCondition}); err != nil { - return projectionConstraints, err + return nil, err } } } @@ -3402,6 +4752,7 @@ func (s *Translator) buildExpansionProjectionConstraints(traversalStepContext Tr return projectionConstraints, nil } +// translateTraversalPatternPartWithExpansion lowers a variable-length pattern step and updates frame bindings for its projected state. func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPart, stepIndex int, isFirstTraversalStep bool, traversalStep *TraversalStep, allowProjectionPruning bool) error { expansionModel := traversalStep.Expansion @@ -3410,6 +4761,31 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar if err := s.translateExpansionConstraints(part, stepIndex, isFirstTraversalStep, traversalStep, expansionModel); err != nil { return err } + if decision, selected := s.shortestPathExecutorDecision(part, stepIndex); selected { + expansionModel.ShortestPathExecutor = decision.SelectedExecutor + expansionModel.ShortestPathTarget = decision.Target + expansionModel.ShortestPathStateLimit = decision.StateLimit + expansionModel.ShortestPathFrontierLimit = decision.FrontierLimit + expansionModel.ShortestPathPredecessorLimit = decision.PredecessorLimit + expansionModel.ShortestPathEnumerationLimit = decision.EnumerationLimit + expansionModel.ShortestPathOutputBytesLimit = decision.OutputBytesLimit + if !expansionModel.Options.MaxDepth.Set && decision.MaximumDepth > 0 { + expansionModel.Options.MaxDepth = models.OptionalValue(decision.MaximumDepth) + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorS3Unidirectional || + decision.SelectedExecutor == optimize.ShortestPathExecutorI1CanonicalDistance || + isGuardedDistanceExecutor(decision.SelectedExecutor) || + decision.SelectedExecutor == optimize.ShortestPathExecutorS4CanonicalDistance || + decision.SelectedExecutor == optimize.ShortestPathExecutorB1AlternatingNodeDistance || + decision.SelectedExecutor == optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance { + expansionModel.PathBinding.DistanceOnly = true + expansionModel.PathBinding.DataType = pgsql.Int + if part.PatternBinding != nil { + part.PatternBinding.DistanceOnly = true + part.PatternBinding.DataType = pgsql.Int + } + } + } // Export the path from the traversal's scope traversalStep.Frame.Export(expansionModel.PathBinding.Identifier) @@ -3455,6 +4831,11 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar // Remove the previous projections of the root and terminal node to reproject them after expansion traversalStep.LeftNode.Dematerialize() traversalStep.RightNode.Dematerialize() + leftNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.LeftNode) + rightNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.RightNode) + if leftNodeIDOnly || rightNodeIDOnly { + s.recordLowering(optimize.LoweringFieldRequirements) + } if boundProjections, err := buildVisibleProjections(s.scope); err != nil { return err @@ -3481,6 +4862,12 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar traversalStep.Projection = boundProjections.Items } + if shortestExecutorEmitsHydratedPath(expansionModel.ShortestPathExecutor) { + expansionModel.PathBinding.DataType = pgsql.PathComposite + if part.PatternBinding != nil { + part.PatternBinding.DataType = pgsql.PathComposite + } + } if expansionModel.Options.FindShortestPath || expansionModel.Options.FindAllShortestPaths { if err := s.translateShortestPathTraversal(part, stepIndex, traversalStep, expansionModel); err != nil { @@ -3491,6 +4878,15 @@ func (s *Translator) translateTraversalPatternPartWithExpansion(part *PatternPar return nil } +func shortestExecutorEmitsHydratedPath(executor optimize.ShortestPathExecutor) bool { + return executor == optimize.ShortestPathExecutorS3EdgeM0 || + executor == optimize.ShortestPathExecutorI1CanonicalWitness || + executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || + executor == optimize.ShortestPathExecutorASPI1DAG || + compactExecutorNeedsPathHydration(executor) +} + +// translateExpansionConstraints consumes applicable constraints and partitions them among expansion bindings and outer frames. func (s *Translator) translateExpansionConstraints(part *PatternPart, stepIndex int, isFirstTraversalStep bool, step *TraversalStep, expansionModel *Expansion) error { if constraints, err := consumePatternConstraints(isFirstTraversalStep, recursivePattern, step, s.treeTranslator); err != nil { return err @@ -3566,6 +4962,7 @@ func (s *Translator) translateExpansionConstraints(part *PatternPart, stepIndex return nil } +// translateShortestPathTraversal selects and parameterizes the physical shortest-path harness for a traversal step. func (s *Translator) translateShortestPathTraversal(part *PatternPart, stepIndex int, traversalStep *TraversalStep, expansionModel *Expansion) error { var ( useBidirectionalSearch bool @@ -3578,12 +4975,50 @@ func (s *Translator) translateShortestPathTraversal(part *PatternPart, stepIndex return err } - expansionModel.UseBidirectionalSearch = useBidirectionalSearch + inlineShortest := expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS3Unidirectional || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalDistance || + isGuardedDistanceExecutor(expansionModel.ShortestPathExecutor) || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || + expansionModel.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG + expansionModel.UseBidirectionalSearch = useBidirectionalSearch && !inlineShortest && !compactShortestExecutor(expansionModel.ShortestPathExecutor) expansionModel.HasExplicitEndpointInequality = s.treeTranslator.HasEndpointInequality( traversalStep.LeftNode.Identifier, traversalStep.RightNode.Identifier, ) s.applyShortestPathFilterMaterialization(part, stepIndex, traversalStep, expansionModel) + if (compactShortestExecutor(expansionModel.ShortestPathExecutor) || expansionModel.UseBidirectionalSearch || inlineShortest) && + !traversalStep.LeftNodeBound && + !traversalStep.RightNodeBound && + (!expansionModel.Options.MinDepth.Set || expansionModel.Options.MinDepth.Value > 0 || inlineShortest || compactShortestExecutor(expansionModel.ShortestPathExecutor)) { + rootAnchor, hasRootAnchor := singletonIDAnchor(expansionModel.PrimerNodeConstraints, traversalStep.LeftNode.Identifier) + terminalAnchor, hasTerminalAnchor := singletonIDAnchor(expansionModel.TerminalNodeConstraints, traversalStep.RightNode.Identifier) + if hasRootAnchor && hasTerminalAnchor { + var err error + if expansionModel.SingletonRootID, err = s.liftSingletonIDAnchor(rootAnchor); err != nil { + return err + } + expansionModel.PrimerNodeConstraints = replaceSingletonIDAnchor( + expansionModel.PrimerNodeConstraints, + traversalStep.LeftNode.Identifier, + expansionModel.SingletonRootID, + ) + if expansionModel.SingletonTerminalID, err = s.liftSingletonIDAnchor(terminalAnchor); err != nil { + return err + } + expansionModel.TerminalNodeConstraints = replaceSingletonIDAnchor( + expansionModel.TerminalNodeConstraints, + traversalStep.RightNode.Identifier, + expansionModel.SingletonTerminalID, + ) + expansionModel.UseMaterializedEndpointPairFilter = false + } + } + + if inlineShortest || compactShortestExecutor(expansionModel.ShortestPathExecutor) { + return nil + } // If this query is a shortest-path look up, the translator will have to use a function harness for // traversal. As such, query fragments for the traversal harness will have to be passed by the parameters @@ -3619,6 +5054,38 @@ func (s *Translator) translateShortestPathTraversal(part *PatternPart, stepIndex return nil } +// liftSingletonIDAnchor converts a literal or parameter singleton identifier into a typed harness parameter. +func (s *Translator) liftSingletonIDAnchor(expression pgsql.Expression) (pgsql.Expression, error) { + switch typedExpression := unwrapParenthetical(expression).(type) { + case pgsql.Literal: + parameterBinding, err := s.scope.DefineNew(pgsql.ParameterIdentifier) + if err != nil { + return nil, err + } + parameter, err := pgsql.AsParameter(parameterBinding.Identifier, typedExpression.Value) + if err != nil { + return nil, err + } + parameter.CastType = pgsql.Int8 + parameterBinding.Parameter = parameter + s.translation.Parameters[parameterBinding.Identifier.String()] = typedExpression.Value + return parameter, nil + + case pgsql.Parameter: + typedExpression.CastType = pgsql.Int8 + return typedExpression, nil + case *pgsql.Parameter: + copy := *typedExpression + copy.CastType = pgsql.Int8 + return ©, nil + case pgsql.TypeCast: + return s.liftSingletonIDAnchor(typedExpression.Expression) + default: + return nil, fmt.Errorf("unsupported singleton endpoint expression: %T", expression) + } +} + +// translateNonTraversalPatternPart lowers a fixed-length pattern part into a new frame and materialized projection. func (s *Translator) translateNonTraversalPatternPart(part *PatternPart) error { if nextFrame, err := s.scope.PushFrame(); err != nil { return err diff --git a/cypher/models/pgsql/translate/expansion_all_shortest_inline.go b/cypher/models/pgsql/translate/expansion_all_shortest_inline.go new file mode 100644 index 00000000..cdb84b71 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_all_shortest_inline.go @@ -0,0 +1,910 @@ +package translate + +import ( + "errors" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" + "github.com/specterops/dawgs/graph" +) + +const ( + // aspI1Distance reserves the stable protocol value used to recognize asp i1 distance across artifacts and executions. + aspI1Distance pgsql.Identifier = "asp_i1_distance" + + // aspI1Direct reserves the stable protocol value used to recognize asp i1 direct across artifacts and executions. + aspI1Direct pgsql.Identifier = "asp_i1_direct" + + // aspI1Preflight reserves the stable protocol value used to recognize asp i1 preflight across artifacts and executions. + aspI1Preflight pgsql.Identifier = "asp_i1_preflight" + + // aspI1PreflightBounded reserves the stable protocol value used to recognize asp i1 preflight bounded across artifacts and executions. + aspI1PreflightBounded pgsql.Identifier = "asp_i1_preflight_bounded" + + // aspI1DistanceBounded reserves the stable protocol value used to recognize asp i1 distance bounded across artifacts and executions. + aspI1DistanceBounded pgsql.Identifier = "asp_i1_distance_bounded" + + // aspI1Target reserves the stable protocol value used to recognize asp i1 target across artifacts and executions. + aspI1Target pgsql.Identifier = "asp_i1_target" + + // aspI1Predecessor reserves the stable protocol value used to recognize asp i1 predecessor across artifacts and executions. + aspI1Predecessor pgsql.Identifier = "asp_i1_predecessor" + + // aspI1PredecessorBounded reserves the stable protocol value used to recognize asp i1 predecessor bounded across artifacts and executions. + aspI1PredecessorBounded pgsql.Identifier = "asp_i1_predecessor_bounded" + + // aspI1Paths reserves the stable protocol value used to recognize asp i1 paths across artifacts and executions. + aspI1Paths pgsql.Identifier = "asp_i1_paths" + + // aspI1PathsBounded reserves the stable protocol value used to recognize asp i1 paths bounded across artifacts and executions. + aspI1PathsBounded pgsql.Identifier = "asp_i1_paths_bounded" + + // aspI1Shortest reserves the stable protocol value used to recognize asp i1 shortest across artifacts and executions. + aspI1Shortest pgsql.Identifier = "asp_i1_shortest" + + // aspI1Admission reserves the stable protocol value used to recognize asp i1 admission across artifacts and executions. + aspI1Admission pgsql.Identifier = "asp_i1_admission" + + // aspI1Decision reserves the stable protocol value used to recognize asp i1 decision across artifacts and executions. + aspI1Decision pgsql.Identifier = "asp_i1_decision" + + // aspI1CandidateMarker reserves the stable protocol value used to recognize asp i1 candidate marker across artifacts and executions. + aspI1CandidateMarker pgsql.Identifier = "asp_i1_candidate_marker" + + // aspI1FallbackMarker reserves the stable protocol value used to recognize asp i1 fallback marker across artifacts and executions. + aspI1FallbackMarker pgsql.Identifier = "asp_i1_fallback_marker" + + // aspI1CandidateBody reserves the stable protocol value used to recognize asp i1 candidate body across artifacts and executions. + aspI1CandidateBody pgsql.Identifier = "asp_i1_candidate_body" + + // aspI1FallbackBody reserves the stable protocol value used to recognize asp i1 fallback body across artifacts and executions. + aspI1FallbackBody pgsql.Identifier = "asp_i1_fallback_body" + + // aspI1CandidateRows reserves the stable protocol value used to recognize asp i1 candidate rows across artifacts and executions. + aspI1CandidateRows pgsql.Identifier = "asp_i1_candidate_rows" + + // aspI1FallbackRows reserves the stable protocol value used to recognize asp i1 fallback rows across artifacts and executions. + aspI1FallbackRows pgsql.Identifier = "asp_i1_fallback_rows" + + // aspI1NodeID reserves the stable protocol value used to recognize asp i1 node id across artifacts and executions. + aspI1NodeID pgsql.Identifier = "node_id" + + // aspI1PredecessorID reserves the stable protocol value used to recognize asp i1 predecessor id across artifacts and executions. + aspI1PredecessorID pgsql.Identifier = "predecessor_id" + + // aspI1EdgeID reserves the stable protocol value used to recognize asp i1 edge id across artifacts and executions. + aspI1EdgeID pgsql.Identifier = "edge_id" + + // aspI1UseCandidate reserves the stable protocol value used to recognize asp i1 use candidate across artifacts and executions. + aspI1UseCandidate pgsql.Identifier = "use_candidate" + + // aspI1UseFallback reserves the stable protocol value used to recognize asp i1 use fallback across artifacts and executions. + aspI1UseFallback pgsql.Identifier = "use_fallback" + + // aspI1Overflow reserves the stable protocol value used to recognize asp i1 overflow across artifacts and executions. + aspI1Overflow pgsql.Identifier = "overflow" + + // aspI1NoPath reserves the stable protocol value used to recognize asp i1 no path across artifacts and executions. + aspI1NoPath pgsql.Identifier = "no_path" + + // aspI1RuntimeReceipt reserves the stable protocol value used to recognize asp i1 runtime receipt across artifacts and executions. + aspI1RuntimeReceipt pgsql.Identifier = "runtime_receipt" + + // aspI1RuntimeAttestationFn reserves the stable protocol value used to recognize asp i1 runtime attestation fn across artifacts and executions. + aspI1RuntimeAttestationFn pgsql.Identifier = "record_requested_traversal_runtime_attestation_v1" + + // aspI1ColumnSizeFn reserves the stable protocol value used to recognize asp i1 column size fn across artifacts and executions. + aspI1ColumnSizeFn pgsql.Identifier = "pg_column_size" +) + +// aspI1Aliased builds the SQL model fragment responsible for asp i1 aliased. +func aspI1Aliased(expression pgsql.Expression, alias pgsql.Identifier) pgsql.SelectItem { + return &pgsql.AliasedExpression{ + Expression: expression, + Alias: models.OptionalValue(alias), + } +} + +// aspI1Table builds the SQL model fragment responsible for asp i1 table. +func aspI1Table(alias, binding pgsql.Identifier) pgsql.TableReference { + return pgsql.TableReference{ + Name: alias.AsCompoundIdentifier(), + Binding: models.OptionalValue(binding), + } +} + +// aspI1CanonicalProjection constructs the SQL model used for asp i1 canonical projection. +func aspI1CanonicalProjection(source pgsql.Identifier) pgsql.Projection { + return pgsql.Projection{ + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionRootID}, expansionRootID), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionNextID}, expansionNextID), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionDepth}, expansionDepth), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionSatisfied}, expansionSatisfied), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionIsCycle}, expansionIsCycle), + aspI1Aliased(pgsql.CompoundIdentifier{source, expansionPath}, expansionPath), + } +} + +// aspI1OverflowAny builds the SQL model fragment responsible for asp i1 overflow any. +func aspI1OverflowAny(overflows ...pgsql.Expression) pgsql.Expression { + var result pgsql.Expression + for _, overflow := range overflows { + if result == nil { + result = overflow + } else { + result = pgsql.NewBinaryExpression(result, pgsql.OperatorOr, overflow) + } + } + return result +} + +// aspI1OutputBytes builds the SQL model fragment responsible for asp i1 output bytes. +func aspI1OutputBytes(source pgsql.Identifier) pgsql.Subquery { + return pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.FunctionCall{ + Function: pgsql.FunctionSum, + Parameters: []pgsql.Expression{pgsql.FunctionCall{ + Function: aspI1ColumnSizeFn, + Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{source, expansionPath}}, + }}, + }, + pgsql.NewLiteral(int64(0), pgsql.Int8), + }, + CastType: pgsql.Int8, + }}, + From: []pgsql.FromClause{tableFrom(source)}, + }}} +} + +// aspI1Marker builds the SQL model fragment responsible for asp i1 marker. +func aspI1Marker(alias pgsql.Identifier, selected pgsql.Identifier) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), orientationArmExecuted)}, + From: []pgsql.FromClause{tableFrom(aspI1Decision)}, + Where: pgsql.CompoundIdentifier{aspI1Decision, selected}, + }}, + } +} + +// inlinePredecessorDAGMode selects the execution behavior used for inline predecessor dag. +type inlinePredecessorDAGMode struct { + // identity retains the identity while inlinePredecessorDAGMode is assembled or evaluated. + identity optimize.ShortestPathExecutor + // fallback retains the fallback while inlinePredecessorDAGMode is assembled or evaluated. + fallback optimize.ShortestPathExecutor + // oneWitness indicates whether one witness applies. + oneWitness bool +} + +// BuildInlineAllShortestPathsDAGRoot emits the guarded ASP-I1 predecessor-DAG statement. +func (s *ExpansionBuilder) BuildInlineAllShortestPathsDAGRoot() (pgsql.Query, error) { + return s.buildInlinePredecessorDAGRoot(inlinePredecessorDAGMode{ + identity: optimize.ShortestPathExecutorASPI1DAG, + fallback: optimize.ShortestPathExecutorASPA1DAG, + }) +} + +// BuildInlineCanonicalShortestPathRoot emits one guarded canonical witness and +// invokes compact S4 exactly once if any candidate resource sentinel overflows. +func (s *ExpansionBuilder) BuildInlineCanonicalShortestPathRoot() (pgsql.Query, error) { + return s.buildInlinePredecessorDAGRoot(inlinePredecessorDAGMode{ + identity: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + fallback: optimize.ShortestPathExecutorS4CanonicalWitness, + oneWitness: true, + }) +} + +// buildInlinePredecessorDAGRoot shares guarded minimum-distance and predecessor +// primitives between the ASP enumerator and the singleton canonical witness. +// Recursive producers are consumed only through materialized cap+1 relations; +// complementary markers prevent candidate/fallback row mixing. +func (s *ExpansionBuilder) buildInlinePredecessorDAGRoot(mode inlinePredecessorDAGMode) (pgsql.Query, error) { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires one validated endpoint pair") + } + if expansionModel.Options.MinDepth.GetOr(1) != 1 || !expansionModel.Options.MaxDepth.Set || expansionModel.Options.MaxDepth.Value < 1 || expansionModel.Options.MaxDepth.Value > 64 { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires min depth 1 and bounded max depth <= 64") + } + if s.traversalStep.Direction != graph.DirectionOutbound && s.traversalStep.Direction != graph.DirectionInbound { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires a directed traversal") + } + for _, limit := range []int64{ + expansionModel.ShortestPathStateLimit, + expansionModel.ShortestPathPredecessorLimit, + expansionModel.ShortestPathEnumerationLimit, + expansionModel.ShortestPathOutputBytesLimit, + } { + if limit <= 0 { + return pgsql.Query{}, errors.New(string(mode.identity) + " requires positive bounded limits") + } + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + + // Exact one/two-hop preflights prevent the recursive distance producer from + // exploring an irrelevant tail when the target is already shallow. The + // preflight itself is consumed through the enumeration cap+1 sentinel so a + // large parallel-edge result falls back before exposing partial rows. + firstEdge := s.traversalStep.Edge.Identifier + secondEdge := pgsql.Identifier("asp_i1_preflight_edge_2") + edgeScope := func(alias pgsql.Identifier) pgsql.Expression { + var scope pgsql.Expression = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{alias, pgsql.ColumnGraphID}, pgsql.OperatorEquals, pgsql.NewLiteral(s.graphID, pgsql.Int4), + ) + if len(expansionModel.RelationshipKindIDs) > 0 { + scope = pgsql.OptionalAnd(scope, pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{alias, pgsql.ColumnKindID}, pgsql.OperatorEquals, + pgsql.NewAnyExpressionHinted(pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array)), + )) + } + return scope + } + startColumn, endColumn := pgsql.ColumnStartID, pgsql.ColumnEndID + if s.traversalStep.Direction == graph.DirectionInbound { + startColumn, endColumn = endColumn, startColumn + } + directWhere := pgsql.OptionalAnd(edgeScope(firstEdge), pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, startColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, endColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}), + )) + direct := pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(pgsql.NewLiteral(int64(1), pgsql.Int8), expansionDepth), + aspI1Aliased(pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{firstEdge, pgsql.ColumnID}}, + CastType: pgsql.Int8Array, + }, expansionPath), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints), {Source: expansionEdgeTableReference(firstEdge)}}, + Where: directWhere, + } + directCTE := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Direct, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth, expansionPath}), + }, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: direct}, + } + directExists := pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(aspI1Direct)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}} + secondJoin := pgsql.OptionalAnd(edgeScope(secondEdge), pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{secondEdge, startColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{firstEdge, endColumn}), + pgsql.NewLiteral(true, pgsql.Boolean), + )) + twoHop := pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(pgsql.NewLiteral(int64(2), pgsql.Int8), expansionDepth), + aspI1Aliased(pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pgsql.CompoundIdentifier{firstEdge, pgsql.ColumnID}, pgsql.CompoundIdentifier{secondEdge, pgsql.ColumnID}, + }, + CastType: pgsql.Int8Array, + }, expansionPath), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints), { + Source: expansionEdgeTableReference(firstEdge), + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(secondEdge), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: secondJoin, + }, + }}, + }}, + Where: pgsql.OptionalAnd( + pgd.Not(directExists), + pgsql.OptionalAnd(edgeScope(firstEdge), pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, startColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{secondEdge, endColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{firstEdge, pgsql.ColumnID}, pgsql.OperatorNotEquals, pgsql.CompoundIdentifier{secondEdge, pgsql.ColumnID}), + ), + )), + ), + } + preflight := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Preflight, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth, expansionPath}), + }, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1Direct, expansionDepth}, + pgsql.CompoundIdentifier{aspI1Direct, expansionPath}, + }, + From: []pgsql.FromClause{tableFrom(aspI1Direct)}, + }, + ROperand: twoHop, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + preflightBounded := boundedTraversalStateProbe( + aspI1PreflightBounded, aspI1Preflight, []pgsql.Identifier{expansionDepth, expansionPath}, expansionModel.ShortestPathEnumerationLimit, + ) + preflightOverflow := boundedProbeOverflow(aspI1PreflightBounded, expansionModel.ShortestPathEnumerationLimit) + preflightExists := pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(aspI1PreflightBounded)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}} + + anchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints)}, + Where: pgd.Not(preflightExists), + } + recursive := pgsql.Select{ + Projection: pgsql.Projection{ + expansionModel.EdgeEndColumn, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1Distance, expansionDepth}, + pgsql.OperatorAdd, + pgsql.NewLiteral(int64(1), pgsql.Int8), + ), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1Distance.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + expansionModel.EdgeStartColumn, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{aspI1Distance, aspI1NodeID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + expansionModel.EdgeConstraints, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1Distance, expansionDepth}, + pgsql.OperatorLessThan, + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8), + ), + ), + } + + distance := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Distance, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{aspI1NodeID, expansionDepth}), + }, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + }}, + } + distanceBounded := boundedTraversalStateProbe( + aspI1DistanceBounded, + aspI1Distance, + []pgsql.Identifier{aspI1NodeID, expansionDepth}, + expansionModel.ShortestPathStateLimit, + ) + stateOverflow := boundedProbeOverflow(aspI1DistanceBounded, expansionModel.ShortestPathStateLimit) + + target := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Target, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth}), + }, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{aspI1DistanceBounded, expansionDepth}}, + From: []pgsql.FromClause{tableFrom(aspI1DistanceBounded)}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1DistanceBounded, aspI1NodeID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + ), + pgd.Not(stateOverflow), + ), + }, + OrderBy: []*pgsql.OrderBy{{ + Expression: pgsql.CompoundIdentifier{aspI1DistanceBounded, expansionDepth}, + Ascending: true, + }}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } + // The endpoint relation is correlated through a scalar subquery so target + // retains a single FROM source and a stable materialization shape. + targetSelect := target.Query.Body.(pgsql.Select) + targetTerminal := pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}}, + From: []pgsql.FromClause{tableFrom(validatedEndpoints)}, + }}} + targetSelect.Where = pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{aspI1DistanceBounded, aspI1NodeID}, + pgsql.OperatorEquals, + targetTerminal, + ), + pgd.Not(stateOverflow), + ) + target.Query.Body = targetSelect + + child, prior := pgsql.Identifier("asp_i1_child"), pgsql.Identifier("asp_i1_prior") + predecessorEdgeConstraint := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{prior, expansionDepth}, + pgsql.OperatorEquals, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{child, expansionDepth}, pgsql.OperatorSubtract, pgsql.NewLiteral(int64(1), pgsql.Int8)), + ), + expansionModel.EdgeConstraints, + ) + if s.traversalStep.Direction == graph.DirectionOutbound { + predecessorEdgeConstraint = pgsql.OptionalAnd(predecessorEdgeConstraint, + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnStartID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{prior, aspI1NodeID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnEndID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{child, aspI1NodeID}), + ), + ) + } else { + predecessorEdgeConstraint = pgsql.OptionalAnd(predecessorEdgeConstraint, + pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnEndID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{prior, aspI1NodeID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnStartID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{child, aspI1NodeID}), + ), + ) + } + + predecessor := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Predecessor, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{ + aspI1NodeID, expansionDepth, aspI1PredecessorID, aspI1EdgeID, + }), + }, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{child, aspI1NodeID}, + pgsql.CompoundIdentifier{child, expansionDepth}, + pgsql.CompoundIdentifier{prior, aspI1NodeID}, + pgsql.CompoundIdentifier{s.traversalStep.Edge.Identifier, pgsql.ColumnID}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1Target.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: aspI1Table(aspI1DistanceBounded, child), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{child, expansionDepth}, pgsql.OperatorGreaterThan, pgsql.NewLiteral(int64(0), pgsql.Int8)), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{child, expansionDepth}, pgsql.OperatorLessThanOrEqualTo, pgsql.CompoundIdentifier{aspI1Target, expansionDepth}), + ), + }, + }, + { + Table: aspI1Table(aspI1DistanceBounded, prior), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }, + { + Table: expansionEdgeTableReference(s.traversalStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: predecessorEdgeConstraint, + }, + }, + }, + }}, + }}, + } + predecessorBounded := boundedTraversalStateProbe( + aspI1PredecessorBounded, + aspI1Predecessor, + []pgsql.Identifier{aspI1NodeID, expansionDepth, aspI1PredecessorID, aspI1EdgeID}, + expansionModel.ShortestPathPredecessorLimit, + ) + predecessorOverflow := boundedProbeOverflow(aspI1PredecessorBounded, expansionModel.ShortestPathPredecessorLimit) + + pathAnchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.CompoundIdentifier{aspI1Target, expansionDepth}, + pgsql.ArrayLiteral{CastType: pgsql.Int8Array}, + }, + From: []pgsql.FromClause{ + tableFrom(aspI1Target), + tableFrom(validatedEndpoints), + }, + Where: pgd.Not(pgsql.NewParenthetical(aspI1OverflowAny(stateOverflow, predecessorOverflow))), + } + pathRecursive := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1PredecessorBounded, aspI1PredecessorID}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1Paths, expansionDepth}, pgsql.OperatorSubtract, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.NewBinaryExpression( + pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{aspI1PredecessorBounded, aspI1EdgeID}}, + CastType: pgsql.Int8Array, + }, + pgsql.OperatorConcatenate, + pgsql.CompoundIdentifier{aspI1Paths, expansionPath}, + ), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1Paths.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.TableReference{Name: aspI1PredecessorBounded.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PredecessorBounded, aspI1NodeID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{aspI1Paths, aspI1NodeID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PredecessorBounded, expansionDepth}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{aspI1Paths, expansionDepth}), + ), + }, + }}, + }}, + } + paths := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Paths, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{aspI1NodeID, expansionDepth, expansionPath}), + }, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pathAnchor, + ROperand: pathRecursive, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + pathsBounded := boundedTraversalStateProbe( + aspI1PathsBounded, + aspI1Paths, + []pgsql.Identifier{aspI1NodeID, expansionDepth, expansionPath}, + expansionModel.ShortestPathEnumerationLimit, + ) + enumerationOverflow := boundedProbeOverflow(aspI1PathsBounded, expansionModel.ShortestPathEnumerationLimit) + + shortest := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1Shortest, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth, expansionPath}), + }, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1Target, expansionDepth}, + pgsql.CompoundIdentifier{aspI1PathsBounded, expansionPath}, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: aspI1PathsBounded.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.TableReference{Name: aspI1Target.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + Where: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PathsBounded, aspI1NodeID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{aspI1PathsBounded, expansionDepth}, pgsql.OperatorEquals, pgsql.NewLiteral(int64(0), pgsql.Int8)), + ), + }}, + } + if mode.oneWitness { + shortestSelect := shortest.Query.Body.(pgsql.Select) + // ORDER BY at a UNION boundary may reference only the set output name, + // not a source relation that belongs to one operand. + shortest.Query.OrderBy = []*pgsql.OrderBy{{ + Expression: pgsql.CompoundIdentifier{expansionPath}, + Ascending: true, + }} + shortest.Query.Limit = pgsql.NewLiteral(int64(1), pgsql.Int8) + shortest.Query.Body = shortestSelect + } + shortestSelect := shortest.Query.Body.(pgsql.Select) + shortestSelect.From = append(shortestSelect.From, tableFrom(validatedEndpoints)) + shortest.Query.Body = shortestSelect + shortest.Query.Body = pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{aspI1PreflightBounded, expansionDepth}, + pgsql.CompoundIdentifier{aspI1PreflightBounded, expansionPath}, + }, + From: []pgsql.FromClause{tableFrom(aspI1PreflightBounded)}, + }, + ROperand: shortest.Query.Body.(pgsql.Select), + Operator: pgsql.OperatorUnion, + All: true, + } + + bytesOverflow := pgsql.NewBinaryExpression( + aspI1OutputBytes(aspI1Shortest), + pgsql.OperatorGreaterThan, + pgsql.NewLiteral(expansionModel.ShortestPathOutputBytesLimit, pgsql.Int8), + ) + overflow := aspI1OverflowAny(preflightOverflow, stateOverflow, predecessorOverflow, enumerationOverflow, bytesOverflow) + useCandidate := pgd.Not(pgsql.NewParenthetical(overflow)) + noPath := pgd.Not(pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(aspI1Shortest)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}) + admission := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Admission}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + aspI1Aliased(overflow, aspI1Overflow), + aspI1Aliased(noPath, aspI1NoPath), + }}}, + } + admissionOverflow := pgsql.CompoundIdentifier{aspI1Admission, aspI1Overflow} + admissionNoPath := pgsql.CompoundIdentifier{aspI1Admission, aspI1NoPath} + useCandidate = pgd.Not(admissionOverflow) + candidateBranch := "inline_predecessor_dag" + noPathBranch := "inline_no_path" + fallbackBranch := "exact_a1_fallback" + if mode.oneWitness { + candidateBranch = "inline_canonical_witness" + noPathBranch = "inline_canonical_no_path" + fallbackBranch = "exact_s4_fallback" + } + branch := pgsql.Case{ + Conditions: []pgsql.Expression{admissionOverflow, admissionNoPath}, + Then: []pgsql.Expression{ + pgsql.NewLiteral(fallbackBranch, pgsql.Text), + pgsql.NewLiteral(noPathBranch, pgsql.Text), + }, + Else: pgsql.NewLiteral(candidateBranch, pgsql.Text), + } + runtimeExecutor := pgsql.Case{ + Conditions: []pgsql.Expression{admissionOverflow}, + Then: []pgsql.Expression{pgsql.NewLiteral(string(mode.fallback), pgsql.Text)}, + Else: pgsql.NewLiteral(string(mode.identity), pgsql.Text), + } + decision := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: aspI1Decision}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(useCandidate, aspI1UseCandidate), + aspI1Aliased(admissionOverflow, aspI1UseFallback), + aspI1Aliased(pgsql.FunctionCall{ + Function: aspI1RuntimeAttestationFn, + Parameters: []pgsql.Expression{ + branch, + admissionOverflow, + runtimeExecutor, + }, + }, aspI1RuntimeReceipt), + }, + From: []pgsql.FromClause{tableFrom(aspI1Admission)}, + }}, + } + + candidateProjection := pgsql.Projection{ + aspI1Aliased(pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, expansionRootID), + aspI1Aliased(pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, expansionNextID), + aspI1Aliased(pgsql.CompoundIdentifier{aspI1Shortest, expansionDepth}, expansionDepth), + aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), expansionSatisfied), + aspI1Aliased(pgsql.NewLiteral(false, pgsql.Boolean), expansionIsCycle), + aspI1Aliased(pgsql.CompoundIdentifier{aspI1Shortest, expansionPath}, expansionPath), + } + candidateQuery := pgsql.Query{Body: pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{ + tableFrom(validatedEndpoints), + tableFrom(aspI1Shortest), + }, + }} + candidateBody, err := gateQueryBehindMarker(aspI1CandidateMarker, aspI1CandidateBody, candidateQuery, candidateProjection) + if err != nil { + return pgsql.Query{}, err + } + candidateRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1CandidateRows, + Shape: expansionColumns(), + }, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: candidateBody}, + } + + fallbackFunction := pgsql.FunctionAllShortestPathsDAG + if mode.oneWitness { + fallbackFunction = pgsql.FunctionShortestPathCompact + } + fallbackProjection := aspI1CanonicalProjection(fallbackFunction) + fallbackParameters := []pgsql.Expression{ + pgsql.NewLiteral(s.graphID, pgsql.Int4), + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(int64(1), pgsql.Int4), + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int4), + pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array), + pgsql.NewLiteral(s.traversalStep.Direction == graph.DirectionInbound, pgsql.Boolean), + } + if mode.oneWitness { + fallbackParameters = append(fallbackParameters, pgsql.NewLiteral(expansionModel.ShortestPathStateLimit, pgsql.Int8)) + } + fallbackQuery := pgsql.Query{Body: pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{ + tableFrom(validatedEndpoints), + {Source: pgsql.FunctionCall{ + Function: fallbackFunction, + Parameters: fallbackParameters, + }}, + }, + }} + fallbackBody, err := gateQueryBehindMarker(aspI1FallbackMarker, aspI1FallbackBody, fallbackQuery, fallbackProjection) + if err != nil { + return pgsql.Query{}, err + } + fallbackRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: aspI1FallbackRows, + Shape: expansionColumns(), + }, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: fallbackBody}, + } + + stateID := expansionModel.Frame.Binding.Identifier + search := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: stateID, + Shape: expansionColumns(), + }, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pgsql.Select{ + Projection: aspI1CanonicalProjection(aspI1CandidateRows), + From: []pgsql.FromClause{tableFrom(aspI1CandidateRows)}, + }, + ROperand: pgsql.Select{ + Projection: aspI1CanonicalProjection(aspI1FallbackRows), + From: []pgsql.FromClause{tableFrom(aspI1FallbackRows)}, + }, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + + projection := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionRootID}, + ), + }, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + ), + }, + }, + }, + }}, + } + // Both one-witness and all-path results carry ordered edge IDs at this + // boundary. Hydrate them inline once per emitted row; routing all-path rows + // through ordered_edge_ids_to_path invokes a separately planned SQL + // function per row and dominates small shortest-path workloads. + { + const ( + hydrated pgsql.Identifier = "m0_hydrated" + hydratedNodes pgsql.Identifier = "nodes" + hydratedEdges pgsql.Identifier = "edges" + hydratedCount pgsql.Identifier = "hydrated_count" + ) + pathIDs := pgsql.CompoundIdentifier{stateID, expansionPath} + hydration := shortestPathM0Hydration(stateID, s.traversalStep.Direction) + path := pgsql.CompositeValue{ + DataType: pgsql.PathComposite, + Values: []pgsql.Expression{ + pgsql.NewBinaryExpression( + pgsql.ArrayLiteral{ + Values: []pgsql.Expression{shortestPathNodeComposite(s.traversalStep.LeftNode.Identifier)}, + CastType: pgsql.NodeCompositeArray, + }, + pgsql.OperatorConcatenate, + pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{hydrated, hydratedNodes}, pgsql.ArrayLiteral{CastType: pgsql.NodeCompositeArray}}, + }, + ), + pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{pgsql.CompoundIdentifier{hydrated, hydratedEdges}, pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}}, + }, + }, + } + projection.Projection = shortestPathM0Projection(projection.Projection, stateID, expansionModel.PathBinding.Identifier, path) + projection.From[0].Joins = append(projection.From[0].Joins, pgsql.Join{ + Table: hydration, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }) + projection.Where = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{hydrated, hydratedCount}, pgsql.OperatorEquals, + pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{pathIDs}, + }, + ) + } + + query := pgsql.Query{ + CommonTableExpressions: &pgsql.With{Recursive: true}, + Body: projection, + } + for _, cte := range []pgsql.CommonTableExpression{ + endpointCTE, + directCTE, + preflight, + preflightBounded, + distance, + distanceBounded, + target, + predecessor, + predecessorBounded, + paths, + pathsBounded, + shortest, + admission, + decision, + aspI1Marker(aspI1CandidateMarker, aspI1UseCandidate), + aspI1Marker(aspI1FallbackMarker, aspI1UseFallback), + candidateRows, + fallbackRows, + search, + } { + query.AddCTE(cte) + } + return query, nil +} diff --git a/cypher/models/pgsql/translate/expansion_endpoint_seeded.go b/cypher/models/pgsql/translate/expansion_endpoint_seeded.go new file mode 100644 index 00000000..3368fd5a --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_endpoint_seeded.go @@ -0,0 +1,334 @@ +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +// endpointSeededIdentifiers names the seed, reverse-state, admitted-state, and fallback CTEs for one rewrite. +type endpointSeededIdentifiers struct { + // endpoints names the materialized terminal-endpoint seed relation. + endpoints pgsql.Identifier + // reverse names the recursive reverse-search relation. + reverse pgsql.Identifier + // states names the deduplicated reverse states admitted for candidate matching. + states pgsql.Identifier + // incumbent names the original forward plan retained as an overflow fallback. + incumbent pgsql.Identifier +} + +// newEndpointSeededIdentifiers derives collision-resistant CTE names from the incumbent final frame. +func newEndpointSeededIdentifiers(finalFrame pgsql.Identifier) endpointSeededIdentifiers { + prefix := string(finalFrame) + "_endpoint_seeded_" + return endpointSeededIdentifiers{ + endpoints: pgsql.Identifier(prefix + "endpoints"), + reverse: pgsql.Identifier(prefix + "reverse"), + states: pgsql.Identifier(prefix + "states"), + incumbent: pgsql.Identifier(prefix + "incumbent"), + } +} + +// selectedEndpointSeededDecision returns the first traversal decision that selected endpoint-seeded reverse search. +func selectedEndpointSeededDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && decision.SelectedStrategy == optimize.ExpansionSearchEndpointSeededReverse { + return decision, true + } + } + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// rewriteTraversalPatternAsEndpointSeededReverse replaces a qualified two-step incumbent chain with guarded reverse search and fallback. +func (s *Translator) rewriteTraversalPatternAsEndpointSeededReverse(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if decision.PrefixLength != 1 || decision.Target.StepIndex != 1 || len(part.TraversalSteps) != 2 { + return fmt.Errorf("endpoint-seeded reverse target requires exactly one fixed prefix step and one terminal expansion") + } + + prefixStep := part.TraversalSteps[0] + expansionStep := part.TraversalSteps[1] + if prefixStep == nil || prefixStep.Edge == nil || prefixStep.Frame == nil || expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.RightNode == nil || expansionStep.LeftNode == nil || expansionStep.Edge == nil { + return fmt.Errorf("endpoint-seeded reverse target has an incomplete traversal step") + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("endpoint-seeded reverse target did not emit an incumbent frame chain") + } + + incumbentFinal := ctes[len(ctes)-1] + if incumbentFinal.Alias.Name != expansionStep.Frame.Binding.Identifier { + return fmt.Errorf("endpoint-seeded reverse final frame mismatch: expected %s but found %s", expansionStep.Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + incumbentSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("endpoint-seeded reverse final frame must be a select") + } + + prefixEdgeIDs := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{prefixStep.Frame.Binding.Identifier, prefixStep.Edge.Identifier}}, + CastType: pgsql.Int8Array, + } + incumbentSelect.Where = pgsql.OptionalAnd(incumbentSelect.Where, pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{expansionStep.Expansion.Frame.Binding.Identifier, expansionPath}, + pgsql.OperatorArrayOverlap, + prefixEdgeIDs, + ))) + incumbentQuery := incumbentFinal.Query + incumbentQuery.Body = incumbentSelect + + ids := newEndpointSeededIdentifiers(incumbentFinal.Alias.Name) + query, err := s.buildGuardedEndpointSeededQuery(decision, prefixStep, expansionStep, ids, incumbentQuery, incumbentSelect.Projection) + if err != nil { + return err + } + + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:len(ctes)-1], pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: query, + }) + s.recordExpansionSearchStrategy(decision.Target, optimize.ExpansionSearchEndpointSeededReverse) + return nil +} + +// buildGuardedEndpointSeededQuery unions bounded endpoint-seeded candidates with the incumbent overflow fallback. +func (s *Translator) buildGuardedEndpointSeededQuery( + decision optimize.ExpansionSearchStrategyDecision, + prefixStep *TraversalStep, + expansionStep *TraversalStep, + ids endpointSeededIdentifiers, + incumbent pgsql.Query, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + endpointCTE, err := buildEndpointSeedCTE(decision, expansionStep, ids) + if err != nil { + return pgsql.Query{}, err + } + + reverseCTE, err := buildEndpointReverseCTE(decision, expansionStep, ids) + if err != nil { + return pgsql.Query{}, err + } + + statesCTE := buildEndpointStateProbeCTE(decision, ids) + incumbentCTE := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.incumbent}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: incumbent, + } + + candidateProjection, fallbackProjection, err := endpointSeededProjections(prefixStep, expansionStep, ids, incumbentProjection) + if err != nil { + return pgsql.Query{}, err + } + + prefixFrame := prefixStep.Frame.Binding.Identifier + admitted, fallbackGate := boundedAdmissionGates( + boundedProbeLimit{ + source: ids.endpoints, + limit: decision.EndpointLimit, + }, + boundedProbeLimit{ + source: ids.states, + limit: decision.StateLimit, + }, + ) + + prefixEdgeIDs := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{prefixFrame, prefixStep.Edge.Identifier}}, + CastType: pgsql.Int8Array, + } + candidateWhere := pgsql.OptionalAnd( + admitted, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.states, expansionDepth}, pgsql.OperatorGreaterThanOrEqualTo, pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8)), + ) + candidateWhere = pgsql.OptionalAnd(candidateWhere, pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionPath}, pgsql.OperatorArrayOverlap, prefixEdgeIDs, + ))) + + candidate := pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: prefixFrame.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{Name: ids.states.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(prefixFrame, expansionStep.LeftNode), pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.states, expansionNextID}, + ), + }, + }, + { + Table: pgsql.TableReference{Name: ids.endpoints.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.endpoints, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.states, expansionRootID}, + ), + }, + }, + }, + }}, + Where: candidateWhere, + } + + fallback := pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{tableFrom(ids.incumbent)}, + Where: fallbackGate, + } + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: []pgsql.CommonTableExpression{endpointCTE, reverseCTE, statesCTE, incumbentCTE}, + }, + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: candidate, + ROperand: fallback, + }, + }, nil +} + +// buildEndpointSeedCTE materializes locally constrained terminal IDs up to the endpoint guard limit. +func buildEndpointSeedCTE(decision optimize.ExpansionSearchStrategyDecision, expansionStep *TraversalStep, ids endpointSeededIdentifiers) (pgsql.CommonTableExpression, error) { + local, external := partitionConstraintByLocality(expansionStep.Expansion.TerminalNodeConstraints, pgsql.AsIdentifierSet(expansionStep.RightNode.Identifier)) + if external != nil { + return pgsql.CommonTableExpression{}, fmt.Errorf("endpoint-seeded reverse terminal predicate is not local") + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.endpoints}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{ + &pgsql.AliasedExpression{ + Expression: pgd.EntityID(expansionStep.RightNode.Identifier), + Alias: models.OptionalValue(pgsql.ColumnID), + }, + &pgsql.AliasedExpression{ + Expression: suffixSeededNodeValue(expansionStep.RightNode), + Alias: models.OptionalValue(expansionStep.RightNode.Identifier), + }, + }, + From: []pgsql.FromClause{{Source: expansionNodeTableReference(expansionStep.RightNode.Identifier)}}, + Where: local, + }, + Limit: pgsql.NewLiteral(decision.EndpointLimit+1, pgsql.Int8), + }, + }, nil +} + +// buildEndpointReverseCTE builds recursive reverse traversal from terminal seeds while preserving edge uniqueness. +func buildEndpointReverseCTE(decision optimize.ExpansionSearchStrategyDecision, expansionStep *TraversalStep, ids endpointSeededIdentifiers) (pgsql.CommonTableExpression, error) { + localEdgeConstraint, external := partitionConstraintByLocality(expansionStep.Expansion.EdgeConstraints, pgsql.AsIdentifierSet(expansionStep.Edge.Identifier)) + if external != nil { + return pgsql.CommonTableExpression{}, fmt.Errorf("endpoint-seeded reverse relationship predicate is not local") + } + emptyPath := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + seed := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.endpoints, pgsql.ColumnID}, + pgsql.CompoundIdentifier{ids.endpoints, pgsql.ColumnID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + emptyPath, + }, + From: []pgsql.FromClause{tableFrom(ids.endpoints)}, + } + path := pgsql.CompoundIdentifier{ids.reverse, expansionPath} + recursiveWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, pgsql.OperatorLessThan, pgsql.NewLiteral(decision.MaximumDepth, pgsql.Int8)), + pgsql.NewBinaryExpression(pgd.EntityID(expansionStep.Edge.Identifier), pgsql.OperatorNotEquals, pgsql.NewAllExpression(path)), + ) + recursiveWhere = pgsql.OptionalAnd(recursiveWhere, localEdgeConstraint) + recursive := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.reverse, expansionRootID}, + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, expansionStep.Expansion.EdgeStartIdentifier}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.FunctionCall{ + Function: pgsql.Identifier("array_prepend"), + Parameters: []pgsql.Expression{pgd.EntityID(expansionStep.Edge.Identifier), path}, + CastType: pgsql.Int8Array, + }, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: ids.reverse.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(expansionStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, expansionStep.Expansion.EdgeEndIdentifier}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.reverse, expansionNextID}, + ), + }, + }}, + }}, + Where: recursiveWhere, + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.reverse, + Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionRootID, expansionNextID, expansionDepth, expansionPath}), + }, + Query: pgsql.Query{Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: seed, + ROperand: recursive, + }}, + }, nil +} + +// buildEndpointStateProbeCTE materializes at most the guarded number of reverse states for candidate matching. +func buildEndpointStateProbeCTE(decision optimize.ExpansionSearchStrategyDecision, ids endpointSeededIdentifiers) pgsql.CommonTableExpression { + return boundedTraversalStateProbe(ids.states, ids.reverse, []pgsql.Identifier{ + expansionRootID, + expansionNextID, + expansionDepth, + expansionPath, + }, decision.StateLimit) +} + +// endpointSeededProjections aligns reverse-search results and incumbent rows to the original projection shape. +func endpointSeededProjections(prefixStep, expansionStep *TraversalStep, ids endpointSeededIdentifiers, incumbent pgsql.Projection) (pgsql.Projection, pgsql.Projection, error) { + prefixFrame := prefixStep.Frame.Binding.Identifier + candidate := make(pgsql.Projection, 0, len(incumbent)) + fallback := make(pgsql.Projection, 0, len(incumbent)) + for _, item := range incumbent { + alias, ok := selectItemAlias(item) + if !ok { + return nil, nil, fmt.Errorf("endpoint-seeded reverse final projection contains an unaliased item %T", item) + } + var expression pgsql.Expression + switch { + case expansionStep.Expansion.PathBinding != nil && alias == expansionStep.Expansion.PathBinding.Identifier: + expression = pgsql.CompoundIdentifier{ids.states, expansionPath} + case alias == expansionStep.LeftNode.Identifier: + expression = pgsql.CompoundIdentifier{prefixFrame, alias} + case alias == expansionStep.RightNode.Identifier: + expression = pgsql.CompoundIdentifier{ids.endpoints, alias} + default: + expression = pgsql.CompoundIdentifier{prefixFrame, alias} + } + candidate = append(candidate, &pgsql.AliasedExpression{ + Expression: expression, + Alias: models.OptionalValue(alias), + }) + fallback = append(fallback, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.incumbent, alias}, + Alias: models.OptionalValue(alias), + }) + } + return candidate, fallback, nil +} diff --git a/cypher/models/pgsql/translate/expansion_orientation.go b/cypher/models/pgsql/translate/expansion_orientation.go new file mode 100644 index 00000000..0bbabcde --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_orientation.go @@ -0,0 +1,765 @@ +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +const ( + // orientationRootID reserves the stable protocol value used to recognize orientation root id across artifacts and executions. + orientationRootID pgsql.Identifier = "root_id" + + // orientationDegreeSample reserves the stable protocol value used to recognize orientation degree sample across artifacts and executions. + orientationDegreeSample pgsql.Identifier = "sampled" + + // orientationRootRows reserves the stable protocol value used to recognize orientation root rows across artifacts and executions. + orientationRootRows pgsql.Identifier = "root_rows" + + // orientationSuffixRows reserves the stable protocol value used to recognize orientation suffix rows across artifacts and executions. + orientationSuffixRows pgsql.Identifier = "suffix_rows" + + // orientationBoundaryRows reserves the stable protocol value used to recognize orientation boundary rows across artifacts and executions. + orientationBoundaryRows pgsql.Identifier = "boundary_rows" + + // orientationForwardDegreeRows reserves the stable protocol value used to recognize orientation forward degree rows across artifacts and executions. + orientationForwardDegreeRows pgsql.Identifier = "forward_degree_rows" + + // orientationReverseDegreeRows reserves the stable protocol value used to recognize orientation reverse degree rows across artifacts and executions. + orientationReverseDegreeRows pgsql.Identifier = "reverse_degree_rows" + + // orientationProbesComplete reserves the stable protocol value used to recognize orientation probes complete across artifacts and executions. + orientationProbesComplete pgsql.Identifier = "probes_complete" + + // orientationForwardScore reserves the stable protocol value used to recognize orientation forward score across artifacts and executions. + orientationForwardScore pgsql.Identifier = "forward_score" + + // orientationReverseScore reserves the stable protocol value used to recognize orientation reverse score across artifacts and executions. + orientationReverseScore pgsql.Identifier = "reverse_score" + + // orientationUseReverse reserves the stable protocol value used to recognize orientation use reverse across artifacts and executions. + orientationUseReverse pgsql.Identifier = "use_reverse" + + // orientationWouldSelectReverse reserves the stable protocol value used to recognize orientation would select reverse across artifacts and executions. + orientationWouldSelectReverse pgsql.Identifier = "would_select_reverse" + + // orientationShadowSelected reserves the stable protocol value used to recognize orientation shadow selected across artifacts and executions. + orientationShadowSelected pgsql.Identifier = "selected" + + // orientationArmExecuted reserves the stable protocol value used to recognize orientation arm executed across artifacts and executions. + orientationArmExecuted pgsql.Identifier = "executed" +) + +// expansionOrientationIdentifiers gives every probe, decision, candidate, +// and fallback relation a stable suffix suitable for plan and telemetry +// attribution. +type expansionOrientationIdentifiers struct { + // rootProbe retains the root probe while expansionOrientationIdentifiers is assembled or evaluated. + rootProbe pgsql.Identifier + // rootPresence retains the root presence while expansionOrientationIdentifiers is assembled or evaluated. + rootPresence pgsql.Identifier + // suffixProbe retains the suffix probe while expansionOrientationIdentifiers is assembled or evaluated. + suffixProbe pgsql.Identifier + // boundaries retains the boundaries while expansionOrientationIdentifiers is assembled or evaluated. + boundaries pgsql.Identifier + // forwardDegreeProbe retains the forward degree probe while expansionOrientationIdentifiers is assembled or evaluated. + forwardDegreeProbe pgsql.Identifier + // reverseDegreeProbe retains the reverse degree probe while expansionOrientationIdentifiers is assembled or evaluated. + reverseDegreeProbe pgsql.Identifier + // metrics retains the metrics while expansionOrientationIdentifiers is assembled or evaluated. + metrics pgsql.Identifier + // decision retains the decision while expansionOrientationIdentifiers is assembled or evaluated. + decision pgsql.Identifier + // admission retains the admission while expansionOrientationIdentifiers is assembled or evaluated. + admission pgsql.Identifier + // shadowForward retains the shadow forward while expansionOrientationIdentifiers is assembled or evaluated. + shadowForward pgsql.Identifier + // shadowReverse retains the shadow reverse while expansionOrientationIdentifiers is assembled or evaluated. + shadowReverse pgsql.Identifier + // shadowSelection retains the shadow selection while expansionOrientationIdentifiers is assembled or evaluated. + shadowSelection pgsql.Identifier + // reverseGate retains the reverse gate while expansionOrientationIdentifiers is assembled or evaluated. + reverseGate pgsql.Identifier + // reverseSeed retains the reverse seed while expansionOrientationIdentifiers is assembled or evaluated. + reverseSeed pgsql.Identifier + // reverseSeedRows records the number of reverse seed rows. + reverseSeedRows pgsql.Identifier + // executedCandidate retains the executed candidate while expansionOrientationIdentifiers is assembled or evaluated. + executedCandidate pgsql.Identifier + // executedIncumbent retains the executed incumbent while expansionOrientationIdentifiers is assembled or evaluated. + executedIncumbent pgsql.Identifier + // candidateBody retains the candidate body while expansionOrientationIdentifiers is assembled or evaluated. + candidateBody pgsql.Identifier + // incumbentBody retains the incumbent body while expansionOrientationIdentifiers is assembled or evaluated. + incumbentBody pgsql.Identifier + // reverse retains the reverse while expansionOrientationIdentifiers is assembled or evaluated. + reverse pgsql.Identifier + // states retains the states while expansionOrientationIdentifiers is assembled or evaluated. + states pgsql.Identifier + // incumbent retains the incumbent while expansionOrientationIdentifiers is assembled or evaluated. + incumbent pgsql.Identifier +} + +// newExpansionOrientationIdentifiers constructs expansion orientation identifiers. +func newExpansionOrientationIdentifiers(finalFrame pgsql.Identifier) expansionOrientationIdentifiers { + prefix := string(finalFrame) + "_orientation_" + return expansionOrientationIdentifiers{ + rootProbe: pgsql.Identifier(prefix + "root_probe"), + rootPresence: pgsql.Identifier(prefix + "root_presence"), + suffixProbe: pgsql.Identifier(prefix + "suffix_probe"), + boundaries: pgsql.Identifier(prefix + "boundaries"), + forwardDegreeProbe: pgsql.Identifier(prefix + "forward_degree_probe"), + reverseDegreeProbe: pgsql.Identifier(prefix + "reverse_degree_probe"), + metrics: pgsql.Identifier(prefix + "metrics"), + decision: pgsql.Identifier(prefix + "decision"), + admission: pgsql.Identifier(prefix + "admission"), + shadowForward: pgsql.Identifier(prefix + "shadow_forward"), + shadowReverse: pgsql.Identifier(prefix + "shadow_reverse"), + shadowSelection: pgsql.Identifier(prefix + "shadow_selection"), + reverseGate: pgsql.Identifier(prefix + "reverse_gate"), + reverseSeed: pgsql.Identifier(prefix + "reverse_seed"), + reverseSeedRows: pgsql.Identifier(prefix + "reverse_seed_rows"), + executedCandidate: pgsql.Identifier(prefix + "executed_candidate"), + executedIncumbent: pgsql.Identifier(prefix + "executed_incumbent"), + candidateBody: pgsql.Identifier(prefix + "candidate_body"), + incumbentBody: pgsql.Identifier(prefix + "incumbent_body"), + reverse: pgsql.Identifier(prefix + "reverse"), + states: pgsql.Identifier(prefix + "states"), + incumbent: pgsql.Identifier(prefix + "incumbent"), + } +} + +// pairwiseRelationshipIDUniqueness excludes every repeated relationship in a +// fixed orientation region. This is intentionally explicit: constraints +// attached while translating the incumbent traversal may be partitioned away +// when the region is rebuilt as an independent seed relation. +func pairwiseRelationshipIDUniqueness(relationships []pgsql.Identifier) pgsql.Expression { + var constraint pgsql.Expression + for right := 1; right < len(relationships); right++ { + for left := 0; left < right; left++ { + constraint = pgsql.OptionalAnd( + constraint, + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{relationships[right], pgsql.ColumnID}, + pgsql.OperatorNotEquals, + pgsql.CompoundIdentifier{relationships[left], pgsql.ColumnID}, + ), + ) + } + } + return constraint +} + +// expansionOrientationReverseDominates mirrors orientation-probe-v1's SQL +// hysteresis rule: reverse evidence must be strictly below 75 percent of +// forward evidence, so equality and ties keep the incumbent. +func expansionOrientationReverseDominates(forwardScore, reverseScore int64) bool { + return reverseScore*optimize.ExpansionSearchOrientationReverseScoreMultiplier < forwardScore*optimize.ExpansionSearchOrientationForwardScoreMultiplier +} + +// boundedProbeOverflow detects the cap+1 sentinel row of a bounded relation. +func boundedProbeOverflow(source pgsql.Identifier, limit int64) pgsql.ExistsExpression { + return pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(source)}, + }, + Offset: pgsql.NewLiteral(limit, pgsql.Int8), + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}} +} + +// boundedTraversalStateProbe materializes a cap+1 view over recursive state. +// Orientation families provide their own state columns and retain their +// existing candidate/fallback semantics around this common admission boundary. +func boundedTraversalStateProbe( + alias, source pgsql.Identifier, + columns []pgsql.Identifier, + limit int64, + executionMarker ...pgsql.Identifier, +) pgsql.CommonTableExpression { + projection := make(pgsql.Projection, 0, len(columns)) + for _, column := range columns { + projection = append(projection, pgsql.CompoundIdentifier{source, column}) + } + query := pgsql.Query{ + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{tableFrom(source)}, + }, + Limit: pgsql.NewLiteral(limit+1, pgsql.Int8), + } + if len(executionMarker) > 0 && executionMarker[0] != "" { + marker := executionMarker[0] + bodyAlias := pgsql.Identifier(string(alias) + "_body") + body := query.Body.(pgsql.Select) + body.Where = pgsql.CompoundIdentifier{marker, orientationArmExecuted} + query.Body = body + query.Offset = pgsql.NewLiteral(int64(0), pgsql.Int8) + + outerProjection := make(pgsql.Projection, 0, len(columns)) + for _, column := range columns { + outerProjection = append(outerProjection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{bodyAlias, column}, + Alias: models.OptionalValue(column), + }) + } + query = pgsql.Query{Body: pgsql.Select{ + Projection: outerProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: marker.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.LateralSubquery{ + Query: query, + Binding: models.OptionalValue(bodyAlias), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + }} + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: query, + } +} + +// boundedAdmissionGates returns exact complementary candidate and incumbent +// gates for independent bounded probes. Empty input admits the candidate and +// suppresses fallback; every ordinary orientation supplies at least one gate. +type boundedProbeLimit struct { + // source retains the source while boundedProbeLimit is assembled or evaluated. + source pgsql.Identifier + // limit retains the limit while boundedProbeLimit is assembled or evaluated. + limit int64 +} + +// boundedAdmissionGates builds the SQL model fragment responsible for bounded admission gates. +func boundedAdmissionGates(probes ...boundedProbeLimit) (candidate, fallback pgsql.Expression) { + for _, probe := range probes { + overflow := boundedProbeOverflow(probe.source, probe.limit) + candidate = pgsql.OptionalAnd(candidate, pgd.Not(overflow)) + if fallback == nil { + fallback = overflow + } else { + fallback = pgsql.NewBinaryExpression(fallback, pgsql.OperatorOr, overflow) + } + } + if candidate == nil { + candidate = pgsql.NewLiteral(true, pgsql.Boolean) + } + if fallback == nil { + fallback = pgsql.NewLiteral(false, pgsql.Boolean) + } + return candidate, fallback +} + +// orientationCount builds the SQL model fragment responsible for orientation count. +func orientationCount(source pgsql.Identifier) pgsql.Subquery { + return pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.FunctionCall{ + Function: pgsql.FunctionCount, + Parameters: []pgsql.Expression{pgsql.Wildcard{}}, + CastType: pgsql.Int8, + }}, + From: []pgsql.FromClause{tableFrom(source)}, + }}} +} + +// buildExpansionOrientationRootProbe materializes duplicate-preserving root +// evidence. It is evidence only; candidate and fallback continue to read the +// exact root relation. +func buildExpansionOrientationRootProbe(rootFrame pgsql.Identifier, root *BoundIdentifier, ids expansionOrientationIdentifiers, cap int64) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.rootProbe}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: projectedNodeIDReference(rootFrame, root), + Alias: models.OptionalValue(orientationRootID), + }}, + From: []pgsql.FromClause{tableFrom(rootFrame)}, + }, + Limit: pgsql.NewLiteral(cap+1, pgsql.Int8), + }, + } +} + +// buildExpansionOrientationRootPresence builds expansion orientation root presence. +func buildExpansionOrientationRootPresence(ids expansionOrientationIdentifiers) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.rootPresence}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(ids.rootProbe)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } +} + +// buildExpansionOrientationDegreeProbe counts a bounded stream of typed +// adjacencies. The inner limit preserves the cap+1 sentinel and duplicate seed +// work, while the scalar outer result avoids materializing and rescanning one +// boolean tuple per adjacency. +func buildExpansionOrientationDegreeProbe( + alias, seedSource, seedColumn pgsql.Identifier, + edgeAlias, edgeSeedColumn pgsql.Identifier, + edgeConstraint pgsql.Expression, + cap int64, +) pgsql.CommonTableExpression { + sampleRows := pgsql.Identifier(string(alias) + "_samples") + samples := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(true, pgsql.Boolean)}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: seedSource.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(edgeAlias), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edgeAlias, edgeSeedColumn}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{seedSource, seedColumn}, + ), + }, + }}, + }}, + Where: edgeConstraint, + }, + Limit: pgsql.NewLiteral(cap+1, pgsql.Int8), + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionCount, + Parameters: []pgsql.Expression{pgsql.Wildcard{}}, + CastType: pgsql.Int8, + }, + Alias: models.OptionalValue(orientationDegreeSample), + }}, + From: []pgsql.FromClause{{Source: pgsql.LateralSubquery{ + Query: samples, + Binding: models.OptionalValue(sampleRows), + }}}, + }}, + } +} + +// orientationDegreeCount reads the scalar count emitted by a bounded degree +// probe. +func orientationDegreeCount(source pgsql.Identifier) pgsql.Expression { + return pgsql.CompoundIdentifier{source, orientationDegreeSample} +} + +// buildExpansionOrientationMetrics builds expansion orientation metrics. +func buildExpansionOrientationMetrics(ids expansionOrientationIdentifiers, caps optimize.ExpansionSearchProbeCaps) pgsql.CommonTableExpression { + complete := pgsql.OptionalAnd( + pgd.Not(boundedProbeOverflow(ids.rootProbe, caps.RootRowLimit)), + pgd.Not(boundedProbeOverflow(ids.suffixProbe, caps.ReverseSeedRowLimit)), + ) + complete = pgsql.OptionalAnd(complete, pgsql.NewBinaryExpression( + orientationDegreeCount(ids.forwardDegreeProbe), + pgsql.OperatorLessThanOrEqualTo, + pgsql.NewLiteral(caps.DirectionalDegreeRowLimit, pgsql.Int8), + )) + complete = pgsql.OptionalAnd(complete, pgsql.NewBinaryExpression( + orientationDegreeCount(ids.reverseDegreeProbe), + pgsql.OperatorLessThanOrEqualTo, + pgsql.NewLiteral(caps.DirectionalDegreeRowLimit, pgsql.Int8), + )) + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.metrics}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{ + Expression: orientationCount(ids.rootProbe), + Alias: models.OptionalValue(orientationRootRows), + }, + &pgsql.AliasedExpression{ + Expression: orientationCount(ids.suffixProbe), + Alias: models.OptionalValue(orientationSuffixRows), + }, + &pgsql.AliasedExpression{ + Expression: orientationCount(ids.boundaries), + Alias: models.OptionalValue(orientationBoundaryRows), + }, + &pgsql.AliasedExpression{ + Expression: orientationDegreeCount(ids.forwardDegreeProbe), + Alias: models.OptionalValue(orientationForwardDegreeRows), + }, + &pgsql.AliasedExpression{ + Expression: orientationDegreeCount(ids.reverseDegreeProbe), + Alias: models.OptionalValue(orientationReverseDegreeRows), + }, + &pgsql.AliasedExpression{ + Expression: complete, + Alias: models.OptionalValue(orientationProbesComplete), + }, + }, + From: []pgsql.FromClause{ + tableFrom(ids.forwardDegreeProbe), + tableFrom(ids.reverseDegreeProbe), + }, + }}, + } +} + +// buildExpansionOrientationDecision renders the immutable score formula for +// the requested policy identity. V1 counts one forward-degree sample per root; +// v2 weights those samples by the traversal's inclusive maximum depth. +func buildExpansionOrientationDecision(ids expansionOrientationIdentifiers, policy optimize.ExpansionSearchPolicy, maximumDepth int64) (pgsql.CommonTableExpression, error) { + var ( + forwardWork pgsql.Expression = pgsql.CompoundIdentifier{ids.metrics, orientationForwardDegreeRows} + reverseMultiplier = optimize.ExpansionSearchOrientationReverseScoreMultiplier + forwardMultiplier = optimize.ExpansionSearchOrientationForwardScoreMultiplier + ) + switch policy { + case optimize.ExpansionSearchPolicyOrientationProbeV1: + case optimize.ExpansionSearchPolicyOrientationProbeV2: + if maximumDepth <= 0 { + return pgsql.CommonTableExpression{}, fmt.Errorf("%s requires a positive maximum depth", policy) + } + forwardWork = pgsql.NewBinaryExpression( + pgsql.NewLiteral(maximumDepth, pgsql.Int8), + pgsql.OperatorMultiply, + forwardWork, + ) + reverseMultiplier = optimize.ExpansionSearchOrientationV2ReverseScoreMultiplier + forwardMultiplier = optimize.ExpansionSearchOrientationV2ForwardScoreMultiplier + default: + return pgsql.CommonTableExpression{}, fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + forwardScore := pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.metrics, orientationRootRows}, + pgsql.OperatorAdd, + forwardWork, + ) + reverseScore := pgsql.NewBinaryExpression( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.metrics, orientationSuffixRows}, + pgsql.OperatorAdd, + pgsql.CompoundIdentifier{ids.metrics, orientationBoundaryRows}, + ), + pgsql.OperatorAdd, + pgsql.CompoundIdentifier{ids.metrics, orientationReverseDegreeRows}, + ) + dominates := pgsql.NewBinaryExpression( + pgsql.NewBinaryExpression(pgsql.NewParenthetical(reverseScore), pgsql.OperatorMultiply, pgsql.NewLiteral(reverseMultiplier, pgsql.Int8)), + pgsql.OperatorLessThan, + pgsql.NewBinaryExpression(pgsql.NewParenthetical(forwardScore), pgsql.OperatorMultiply, pgsql.NewLiteral(forwardMultiplier, pgsql.Int8)), + ) + useReverse := pgsql.OptionalAnd(pgsql.CompoundIdentifier{ids.metrics, orientationProbesComplete}, dominates) + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.decision}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{ + Expression: forwardScore, + Alias: models.OptionalValue(orientationForwardScore), + }, + &pgsql.AliasedExpression{ + Expression: reverseScore, + Alias: models.OptionalValue(orientationReverseScore), + }, + &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.metrics, orientationProbesComplete}, + Alias: models.OptionalValue(orientationProbesComplete), + }, + &pgsql.AliasedExpression{ + Expression: useReverse, + Alias: models.OptionalValue(orientationUseReverse), + }, + &pgsql.AliasedExpression{ + Expression: useReverse, + Alias: models.OptionalValue(orientationWouldSelectReverse), + }, + }, + From: []pgsql.FromClause{tableFrom(ids.metrics)}, + }}, + }, nil +} + +// buildExpansionOrientationShadowMarkers turns the SQL-visible policy result +// into two mutually exclusive, named plan branches. The final one-row relation +// preserves would_select_reverse without adding a column to the public query +// result. JSON EXPLAIN can therefore attribute the shadow choice while the +// incumbent remains the only executable traversal arm. +func buildExpansionOrientationShadowMarkers(ids expansionOrientationIdentifiers) []pgsql.CommonTableExpression { + shadowMarker := func(alias pgsql.Identifier, selected bool, predicate pgsql.Expression) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.NewLiteral(selected, pgsql.Boolean), + Alias: models.OptionalValue(orientationShadowSelected), + }}, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + Where: predicate, + }}, + } + } + + forward := shadowMarker( + ids.shadowForward, + false, + pgd.Not(pgsql.CompoundIdentifier{ids.decision, orientationWouldSelectReverse}), + ) + reverse := shadowMarker( + ids.shadowReverse, + true, + pgsql.CompoundIdentifier{ids.decision, orientationWouldSelectReverse}, + ) + selection := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.shadowSelection}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.shadowForward, orientationShadowSelected}, + Alias: models.OptionalValue(orientationWouldSelectReverse), + }}, + From: []pgsql.FromClause{tableFrom(ids.shadowForward)}, + }, + ROperand: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.shadowReverse, orientationShadowSelected}, + Alias: models.OptionalValue(orientationWouldSelectReverse), + }}, + From: []pgsql.FromClause{tableFrom(ids.shadowReverse)}, + }, + }}, + } + incumbent := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.executedIncumbent}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.Identifier("record_traversal_runtime_attestation_v1"), + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(string(optimize.ExpansionSearchStepwiseForward), pgsql.Text), + pgsql.NewLiteral("shadow_incumbent", pgsql.Text), + pgsql.NewLiteral(false, pgsql.Boolean), + }, + CastType: pgsql.Boolean, + }, + Alias: models.OptionalValue(orientationArmExecuted), + }}, + From: []pgsql.FromClause{tableFrom(ids.shadowSelection)}, + }}, + } + + return []pgsql.CommonTableExpression{forward, reverse, selection, incumbent} +} + +// buildExpansionOrientationAdmission materializes the recursive-state +// sentinel once. Both execution markers consume this one decision row so the +// cap+1 state relation is not rescanned independently by each gate and receipt. +func buildExpansionOrientationAdmission(ids expansionOrientationIdentifiers, stateLimit int64) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.admission}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.decision, orientationUseReverse}, + Alias: models.OptionalValue(orientationUseReverse), + }, + &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.decision, orientationProbesComplete}, + Alias: models.OptionalValue(orientationProbesComplete), + }, + &pgsql.AliasedExpression{ + Expression: boundedProbeOverflow(ids.states, stateLimit), + Alias: models.OptionalValue[pgsql.Identifier]("state_overflow"), + }, + }, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + }}, + } +} + +// buildExpansionOrientationExecutionMarkers materializes exactly one named +// marker for the arm admitted by the tournament. Unlike recursive-loop row +// counts, these relations remain unambiguous when a selected arm legitimately +// produces no traversal rows. Candidate admission requires both the policy +// choice and a complete state probe; state overflow selects the incumbent. +func buildExpansionOrientationExecutionMarkers(ids expansionOrientationIdentifiers) []pgsql.CommonTableExpression { + stateOverflow := pgsql.CompoundIdentifier{ids.admission, pgsql.Identifier("state_overflow")} + stateAdmitted := pgd.Not(stateOverflow) + useReverse := pgsql.CompoundIdentifier{ids.admission, orientationUseReverse} + probeOverflow := pgd.Not(pgsql.CompoundIdentifier{ids.admission, orientationProbesComplete}) + candidateGate := pgsql.OptionalAnd(useReverse, stateAdmitted) + incumbentGate := pgsql.NewBinaryExpression(pgd.Not(useReverse), pgsql.OperatorOr, stateOverflow) + fallbackExecuted := pgsql.NewBinaryExpression(probeOverflow, pgsql.OperatorOr, stateOverflow) + + marker := func(alias pgsql.Identifier, gate pgsql.Expression, runtimeIdentity, runtimeBranch string, fallback pgsql.Expression) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.Identifier("record_traversal_runtime_attestation_v1"), + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(runtimeIdentity, pgsql.Text), + pgsql.NewLiteral(runtimeBranch, pgsql.Text), + fallback, + }, + CastType: pgsql.Boolean, + }, + Alias: models.OptionalValue(orientationArmExecuted), + }}, + From: []pgsql.FromClause{tableFrom(ids.admission)}, + Where: gate, + }}, + } + } + + return []pgsql.CommonTableExpression{ + marker(ids.executedCandidate, candidateGate, string(optimize.ExpansionSearchSuffixSeededReverse), "suffix_seeded_reverse", pgsql.NewLiteral(false, pgsql.Boolean)), + marker(ids.executedIncumbent, incumbentGate, string(optimize.ExpansionSearchStepwiseForward), "exact_forward_incumbent", fallbackExecuted), + } +} + +// buildExpansionOrientationReverseSeed puts the policy marker on the outer +// side of a correlated LATERAL boundary scan. PostgreSQL therefore cannot +// initialize the reverse recursion's seed scan when the policy keeps the +// incumbent; the lateral subquery has no invocation row in that case. +func buildExpansionOrientationReverseSeed(ids expansionOrientationIdentifiers) []pgsql.CommonTableExpression { + gate := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.reverseGate}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.NewLiteral(true, pgsql.Boolean), + Alias: models.OptionalValue(orientationArmExecuted), + }}, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + Where: pgsql.CompoundIdentifier{ids.decision, orientationUseReverse}, + }}, + } + seedRows := pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.boundaries)}, + Where: pgsql.CompoundIdentifier{ids.reverseGate, orientationArmExecuted}, + }, + // OFFSET 0 is a deliberate planner boundary for this correlated gate. + Offset: pgsql.NewLiteral(int64(0), pgsql.Int8), + } + + seed := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.reverseSeed}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.reverseSeedRows, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: ids.reverseGate.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.LateralSubquery{ + Query: seedRows, + Binding: models.OptionalValue(ids.reverseSeedRows), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + }}, + } + return []pgsql.CommonTableExpression{gate, seed} +} + +// gateQueryBehindMarker makes an execution marker the outer relation of a +// correlated LATERAL query. Merely listing a marker after a materialized CTE +// does not prove PostgreSQL avoids initializing that CTE; this dependency does. +func gateQueryBehindMarker( + marker, bodyAlias pgsql.Identifier, + query pgsql.Query, + exposedProjection pgsql.Projection, +) (pgsql.Select, error) { + body, ok := query.Body.(pgsql.Select) + if !ok { + return pgsql.Select{}, fmt.Errorf("gated orientation body must be a select, found %T", query.Body) + } + body.Where = pgsql.OptionalAnd( + body.Where, + pgsql.CompoundIdentifier{marker, orientationArmExecuted}, + ) + query.Body = body + // The correlated reference and OFFSET 0 keep the expensive inner query + // below the marker-driven LATERAL invocation boundary. + query.Offset = pgsql.NewLiteral(int64(0), pgsql.Int8) + + projection := make(pgsql.Projection, 0, len(exposedProjection)) + for _, item := range exposedProjection { + alias, ok := selectItemAlias(item) + if !ok { + return pgsql.Select{}, fmt.Errorf("gated orientation projection contains an unaliased item %T", item) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{bodyAlias, alias}, + Alias: models.OptionalValue(alias), + }) + } + + return pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: marker.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: pgsql.LateralSubquery{ + Query: query, + Binding: models.OptionalValue(bodyAlias), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }}, + }}, + }, nil +} + +// expansionOrientationStateProbe builds the SQL model fragment responsible for expansion orientation state probe. +func expansionOrientationStateProbe(decision optimize.ExpansionSearchStrategyDecision, ids expansionOrientationIdentifiers, carryNodePath bool) pgsql.CommonTableExpression { + columns := []pgsql.Identifier{ + fixedSuffixBoundaryID, + expansionNextID, + expansionDepth, + expansionPath, + } + if carryNodePath { + columns = append(columns, expansionNodePath) + } + return boundedTraversalStateProbe(ids.states, ids.reverse, columns, decision.Admission.StateLimit, ids.reverseGate) +} diff --git a/cypher/models/pgsql/translate/expansion_orientation_test.go b/cypher/models/pgsql/translate/expansion_orientation_test.go new file mode 100644 index 00000000..1b6b3413 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_orientation_test.go @@ -0,0 +1,662 @@ +package translate + +import ( + "context" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/format" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// guardedSuffixOrientationQuery reserves the stable protocol value used to recognize guarded suffix orientation query across artifacts and executions. +const guardedSuffixOrientationQuery = ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path +` + +// TestExpansionOrientationReverseDominanceHasStrictHysteresis verifies expansion orientation reverse dominance has strict hysteresis behavior. +func TestExpansionOrientationReverseDominanceHasStrictHysteresis(t *testing.T) { + require.False(t, expansionOrientationReverseDominates(0, 0)) + require.False(t, expansionOrientationReverseDominates(100, 75)) + require.False(t, expansionOrientationReverseDominates(4, 3)) + require.True(t, expansionOrientationReverseDominates(100, 74)) + require.True(t, expansionOrientationReverseDominates(4, 2)) +} + +// TestExpansionOrientationBooleanModesRemainV1ByDefault verifies expansion orientation boolean modes remain v1 by default behavior. +func TestExpansionOrientationBooleanModesRemainV1ByDefault(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // options retains the options while anonymous record is assembled or evaluated. + options ToolOptions + }{ + { + name: "guarded", + options: ToolOptions{EnableExpansionOrientationTournament: true}, + }, + { + name: "shadow", + options: ToolOptions{EnableExpansionOrientationShadow: true}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + translate := func(options ToolOptions) (Result, string) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "v1-default-root", + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return translation, formatted + } + + implicit, implicitSQL := translate(testCase.options) + explicitOptions := testCase.options + explicitOptions.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicyOrientationProbeV1 + explicit, explicitSQL := translate(explicitOptions) + + require.Equal(t, implicitSQL, explicitSQL) + require.Contains(t, implicitSQL, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + require.NotContains(t, implicitSQL, "16 * s5_orientation_metrics.forward_degree_rows") + require.Equal(t, implicit.Optimization.LoweringPlan.ExpansionSearchStrategy, explicit.Optimization.LoweringPlan.ExpansionSearchStrategy) + decision := implicit.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), decision.SelectorVersion) + outcome := requireTraversalTargetOutcome(t, implicit.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.SelectorVersion) + }) + } +} + +// TestExpansionOrientationProbeV2IsExplicitAndDepthWeighted verifies expansion orientation probe v2 is explicit and depth weighted behavior. +func TestExpansionOrientationProbeV2IsExplicitAndDepthWeighted(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // options retains the options while anonymous record is assembled or evaluated. + options ToolOptions + // expectedMode identifies the expected mode. + expectedMode string + // expectedBoundary retains the expected boundary while anonymous record is assembled or evaluated. + expectedBoundary string + // expectedCandidates retains the expected candidates while anonymous record is assembled or evaluated. + expectedCandidates []optimize.ExpansionSearchStrategy + }{ + { + name: "guarded", + options: ToolOptions{EnableExpansionOrientationTournament: true}, + expectedMode: "guarded_tool", + expectedBoundary: optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, + expectedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward, optimize.ExpansionSearchSuffixSeededReverse}, + }, + { + name: "shadow", + options: ToolOptions{EnableExpansionOrientationShadow: true}, + expectedMode: "shadow_tool", + expectedBoundary: optimize.ExpansionSearchExecutionBoundaryInlineStatement, + expectedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + options := testCase.options + options.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicyOrientationProbeV2 + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "v2-depth-root", + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + 16 * s5_orientation_metrics.forward_degree_rows) * 3") + require.Contains(t, formatted, "s5_orientation_metrics.probes_complete and (s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 <") + require.NotContains(t, formatted, "(s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, int64(16), decision.MaximumDepth) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV2, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV2, decision.EmittedPolicy) + require.Equal(t, testCase.expectedCandidates, decision.EmittedCandidates) + require.Equal(t, testCase.expectedMode, decision.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), decision.SelectorVersion) + require.Equal(t, testCase.expectedBoundary, decision.ExecutionBoundary) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.SelectorVersion) + require.Equal(t, testCase.expectedBoundary, outcome.ExecutionBoundary) + }) + } +} + +// TestBoundedAdmissionGatesAreStrictComplements verifies bounded admission gates are strict complements behavior. +func TestBoundedAdmissionGatesAreStrictComplements(t *testing.T) { + admitted, fallback := boundedAdmissionGates( + boundedProbeLimit{ + source: "endpoint_probe", + limit: 32, + }, + boundedProbeLimit{ + source: "state_probe", + limit: 4096, + }, + ) + require.NotNil(t, admitted) + require.NotNil(t, fallback) + + query := pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + &pgsql.AliasedExpression{ + Expression: admitted, + Alias: models.OptionalValue[pgsql.Identifier]("admitted"), + }, + &pgsql.AliasedExpression{ + Expression: fallback, + Alias: models.OptionalValue[pgsql.Identifier]("fallback"), + }, + }}} + rendered, err := format.Statement(query, format.NewOutputBuilder()) + require.NoError(t, err) + require.Contains(t, rendered, "not exists") + require.Contains(t, rendered, "offset 32 limit 1") + require.Contains(t, rendered, "offset 4096 limit 1") + require.Contains(t, rendered, "or exists") +} + +// TestGuardedSuffixOrientationTournamentEmitsBoundedDisjointBranches verifies guarded suffix orientation tournament emits bounded disjoint branches behavior. +func TestGuardedSuffixOrientationTournamentEmitsBoundedDisjointBranches(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "guarded-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "s5_orientation_root_probe as materialized") + require.Contains(t, formatted, "s5_orientation_suffix_probe as materialized") + require.Contains(t, formatted, "s5_orientation_boundaries as materialized") + require.Contains(t, formatted, "s5_orientation_forward_degree_probe as materialized") + require.Contains(t, formatted, "s5_orientation_reverse_degree_probe as materialized") + require.Contains(t, formatted, "select count(*)::int8 as sampled from lateral (select true from s5_orientation_root_probe") + require.Contains(t, formatted, "select count(*)::int8 as sampled from lateral (select true from s5_orientation_boundaries") + require.Contains(t, formatted, "s5_orientation_metrics as materialized") + require.Contains(t, formatted, "s5_orientation_decision as materialized") + require.Contains(t, formatted, "s5_orientation_states as materialized") + require.Contains(t, formatted, "s5_orientation_admission as materialized") + require.Contains(t, formatted, "s5_orientation_executed_candidate as materialized") + require.Contains(t, formatted, "s5_orientation_executed_incumbent as materialized") + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1('EXPANSION-SUFFIX-SEEDED-REVERSE', 'suffix_seeded_reverse', false)") + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1('EXPANSION-STEPWISE-FORWARD', 'exact_forward_incumbent'") + require.Contains(t, formatted, "s5_orientation_incumbent as materialized") + require.Contains(t, formatted, "limit 513") + require.Contains(t, formatted, "limit 16385") + require.Contains(t, formatted, "limit 4097") + require.Contains(t, formatted, "select (s0.n0).id as root_id from s0 limit 513") + require.NotContains(t, formatted, "select distinct (s0.n0).id as root_id from s0 limit 513") + require.Contains(t, formatted, "select distinct s5_orientation_suffix_probe.boundary_id as boundary_id") + require.Contains(t, formatted, "e3.id != e2.id limit 513") + require.Contains(t, formatted, "offset 512 limit 1") + require.Contains(t, formatted, "s5_orientation_forward_degree_probe.sampled <= 16384") + require.Contains(t, formatted, "s5_orientation_reverse_degree_probe.sampled <= 16384") + require.Contains(t, formatted, "from s5_orientation_forward_degree_probe, s5_orientation_reverse_degree_probe") + require.Contains(t, formatted, "offset 4096 limit 1") + require.Contains(t, formatted, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + require.Contains(t, formatted, "s5_orientation_admission.use_reverse and not s5_orientation_admission.state_overflow") + require.Contains(t, formatted, "not s5_orientation_admission.use_reverse or s5_orientation_admission.state_overflow") + require.Contains(t, formatted, "not s5_orientation_admission.probes_complete or s5_orientation_admission.state_overflow") + require.Equal(t, 1, strings.Count(formatted, "offset 4096 limit 1")) + require.Contains(t, formatted, "from s5_orientation_executed_candidate join lateral") + require.Contains(t, formatted, "s5_orientation_executed_candidate.executed offset 0") + require.Contains(t, formatted, "s5_orientation_incumbent as materialized (with") + require.Contains(t, formatted, "from s5_orientation_executed_incumbent join lateral") + require.Contains(t, formatted, "s5_orientation_executed_incumbent.executed offset 0") + require.Contains(t, formatted, "s5_orientation_reverse_gate as materialized") + require.Contains(t, formatted, "from s5_orientation_reverse_gate join lateral") + require.Contains(t, formatted, "s5_orientation_reverse_gate.executed offset 0") + require.Contains(t, formatted, "s5_orientation_states as materialized (select") + require.Contains(t, formatted, "from s5_orientation_reverse_gate join lateral (select s5_orientation_reverse.boundary_id") + require.Contains(t, formatted, "s5_orientation_reverse(boundary_id, next_id, depth, path, node_path)") + require.Contains(t, formatted, "generate_subscripts(s5_orientation_states.node_path") + require.Contains(t, formatted, "generate_subscripts(s5_orientation_states.path") + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path("), "only the exact incumbent fallback should retain generic path hydration") + require.Contains(t, formatted, "select s5.pc0 as path from s5") + guardedSuffixProjection := regexp.MustCompile(`(?s)s5_orientation_suffix_probe as materialized \(select (.*?) from s5_orientation_root_presence`).FindStringSubmatch(formatted) + require.Len(t, guardedSuffixProjection, 2) + require.Contains(t, guardedSuffixProjection[1], "n1.id as boundary_id") + require.Contains(t, guardedSuffixProjection[1], "e1.id as e1") + require.Contains(t, guardedSuffixProjection[1], "e2.id as e2") + require.Contains(t, guardedSuffixProjection[1], "e3.id as e3") + require.Contains(t, guardedSuffixProjection[1], "::nodecomposite") + require.Contains(t, formatted, "e3.id != e1.id") + require.Contains(t, formatted, "e3.id != e2.id") + require.Contains(t, formatted, "union all") + require.NotContains(t, formatted, "_orientation_shadow_") + require.NotContains(t, formatted, "s5_orientation_incumbent as materialized (with s1 as (with recursive s2_seed(root_id) as not materialized (select distinct (s0.n0).id as root_id from s0 limit") + + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchStepwiseForward, + optimize.ExpansionSearchSuffixSeededReverse, + }, decision.EmittedCandidates) + require.Equal(t, "guarded_tool", decision.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + require.Equal(t, optimize.ExpansionSearchProbeCaps{ + RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: optimize.ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, decision.ProbeCaps) + require.Equal(t, optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, decision.Admission) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Empty(t, outcome.Applied) + require.Empty(t, outcome.SkipReason) + requireOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) +} + +// TestProductionCanaryExpansionOrientationUsesVersionedGuardedPolicy verifies production canary expansion orientation uses versioned guarded policy behavior. +func TestProductionCanaryExpansionOrientationUsesVersionedGuardedPolicy(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "guarded-fixed-suffix-root", + }, DefaultGraphID, ProductionOptions{ + EnableExpansionOrientation: true, + ExpansionOrientationPolicy: optimize.ExpansionSearchPolicyOrientationProbeV1, + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV1), + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1") + require.Contains(t, formatted, "(s5_orientation_metrics.suffix_rows + s5_orientation_metrics.boundary_rows + s5_orientation_metrics.reverse_degree_rows) * 4 < (s5_orientation_metrics.root_rows + s5_orientation_metrics.forward_degree_rows) * 3") + require.NotContains(t, formatted, "16 * s5_orientation_metrics.forward_degree_rows") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, "production_canary", outcome.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.SelectorVersion) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) +} + +// TestProductionCanaryExpansionOrientationV2PreservesDepthWeightedFormula verifies +// manifest-authorized v2 reaches SQL generation without being rewritten to v1. +func TestProductionCanaryExpansionOrientationV2PreservesDepthWeightedFormula(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "guarded-fixed-suffix-root", + }, DefaultGraphID, ProductionOptions{ + EnableExpansionOrientation: true, + ExpansionOrientationPolicy: optimize.ExpansionSearchPolicyOrientationProbeV2, + SelectorVersion: string(optimize.ExpansionSearchPolicyOrientationProbeV2), + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "16 * s5_orientation_metrics.forward_degree_rows") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 1, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.EmittedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), outcome.SelectorVersion) + require.Equal(t, "production_canary", outcome.SelectionMode) +} + +// TestSuffixOrientationShadowEmitsWouldSelectMetadataAndOnlyIncumbent verifies suffix orientation shadow emits would select metadata and only incumbent behavior. +func TestSuffixOrientationShadowEmitsWouldSelectMetadataAndOnlyIncumbent(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "shadow-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationShadow: true}) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "s5_orientation_root_probe as materialized") + require.Contains(t, formatted, "s5_orientation_suffix_probe as materialized") + require.Contains(t, formatted, "s5_orientation_forward_degree_probe as materialized") + require.Contains(t, formatted, "s5_orientation_reverse_degree_probe as materialized") + require.Contains(t, formatted, "select count(*)::int8 as sampled from lateral (select true from s5_orientation_root_probe") + require.Contains(t, formatted, "select count(*)::int8 as sampled from lateral (select true from s5_orientation_boundaries") + require.Contains(t, formatted, "s5_orientation_metrics as materialized") + require.Contains(t, formatted, "s5_orientation_decision as materialized") + require.Contains(t, formatted, "as would_select_reverse") + require.Contains(t, formatted, "s5_orientation_shadow_forward as materialized") + require.Contains(t, formatted, "s5_orientation_shadow_reverse as materialized") + require.Contains(t, formatted, "s5_orientation_shadow_selection as materialized") + require.Contains(t, formatted, "s5_orientation_executed_incumbent as materialized") + require.Contains(t, formatted, "record_traversal_runtime_attestation_v1('EXPANSION-STEPWISE-FORWARD', 'shadow_incumbent', false)") + require.Contains(t, formatted, "from s5_orientation_executed_incumbent join lateral") + require.Contains(t, formatted, "s5_orientation_executed_incumbent.executed offset 0") + require.Contains(t, formatted, "limit 513") + require.Contains(t, formatted, "limit 16385") + require.Contains(t, formatted, "offset 512 limit 1") + require.Contains(t, formatted, "s5_orientation_forward_degree_probe.sampled <= 16384") + require.Contains(t, formatted, "s5_orientation_reverse_degree_probe.sampled <= 16384") + require.NotContains(t, formatted, "s5_orientation_states") + require.NotContains(t, formatted, "s5_orientation_reverse(boundary_id") + require.NotContains(t, formatted, "limit 4097") + forwardDegreeProjection := regexp.MustCompile(`(?s)s5_orientation_forward_degree_probe as materialized \(select (.*?) from lateral \(select true from s5_orientation_root_probe`).FindStringSubmatch(formatted) + require.Len(t, forwardDegreeProjection, 2) + require.Equal(t, "count(*)::int8 as sampled", forwardDegreeProjection[1]) + reverseDegreeProjection := regexp.MustCompile(`(?s)s5_orientation_reverse_degree_probe as materialized \(select (.*?) from lateral \(select true from s5_orientation_boundaries`).FindStringSubmatch(formatted) + require.Len(t, reverseDegreeProjection, 2) + require.Equal(t, "count(*)::int8 as sampled", reverseDegreeProjection[1]) + suffixProjection := regexp.MustCompile(`(?s)s5_orientation_suffix_probe as materialized \(select (.*?) from s5_orientation_root_presence`).FindStringSubmatch(formatted) + require.Len(t, suffixProjection, 2) + require.Equal(t, "n1.id as boundary_id", suffixProjection[1]) + + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, decision.EmittedCandidates) + require.Equal(t, "shadow_tool", decision.SelectionMode) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward)}, outcome.EmittedCandidates) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Selected) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) + require.Empty(t, outcome.Applied) + require.Empty(t, outcome.SkipReason) +} + +// TestSuffixOrientationShadowIsParameterStable verifies suffix orientation shadow is parameter stable behavior. +func TestSuffixOrientationShadowIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translate := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationShadow: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + first := translate("shadow-root-a") + second := translate("shadow-root-b") + require.Equal(t, first, second) + require.Contains(t, first, "@pi0::text") +} + +// TestGuardedSuffixOrientationSQLIsParameterStable verifies guarded suffix orientation sql is parameter stable behavior. +func TestGuardedSuffixOrientationSQLIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translate := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + first := translate("root-a") + second := translate("root-b") + require.Equal(t, first, second) + require.Contains(t, first, "@pi0::text") +} + +// TestGuardedSuffixOrientationAlignsSupportedOutputShapes verifies guarded suffix orientation aligns supported output shapes behavior. +func TestGuardedSuffixOrientationAlignsSupportedOutputShapes(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // projection retains the projection while anonymous record is assembled or evaluated. + projection string + // expected retains the expected while anonymous record is assembled or evaluated. + expected string + }{ + { + name: "endpoint IDs", + projection: "id(head), id(terminal)", + expected: `select s5.n2 as "id(head)", s5.n4 as "id(terminal)"`, + }, + { + name: "ordered path IDs", + projection: "length(path)", + expected: `as "length(path)"`, + }, + { + name: "full path", + projection: "path", + expected: "ordered_edge_ids_to_path", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN `+testCase.projection) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "output-root", + }, DefaultGraphID, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, testCase.expected) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV1, translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0].EmittedPolicy) + }) + } +} + +// TestProductionFixedSuffixTranslationRemainsIncumbent verifies production fixed suffix translation remains incumbent behavior. +func TestProductionFixedSuffixTranslationRemainsIncumbent(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "production-root", + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.NotContains(t, formatted, "_orientation_") + require.Contains(t, formatted, "s2(root_id, next_id, depth, satisfied, is_cycle, path)") + + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.SelectedStrategy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, decision.EmittedCandidates) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) +} + +// TestGuardedSuffixOrientationUsesOnlyTargetGraphRelations verifies guarded suffix orientation uses only target graph relations behavior. +func TestGuardedSuffixOrientationUsesOnlyTargetGraphRelations(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "graph-scoped-root", + }, 42, ToolOptions{EnableExpansionOrientationTournament: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "node_42") + require.Contains(t, formatted, "edge_42") + require.Contains(t, formatted, "ordered_edge_ids_to_path(42,") + require.NotRegexp(t, regexp.MustCompile(`(?i)(from|join) (node|edge)(?:\s|;)`), formatted) +} + +// TestExpansionOrientationTournamentRejectsConflictingForceWithoutMutation verifies expansion orientation tournament rejects conflicting force without mutation behavior. +func TestExpansionOrientationTournamentRejectsConflictingForceWithoutMutation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err = applyToolOptions(&plan, ToolOptions{ + EnableExpansionOrientationTournament: true, + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }) + require.ErrorContains(t, err, "mutually exclusive") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +// TestExpansionOrientationShadowRejectsConflictingModesWithoutMutation verifies expansion orientation shadow rejects conflicting modes without mutation behavior. +func TestExpansionOrientationShadowRejectsConflictingModesWithoutMutation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + for _, options := range []ToolOptions{ + { + EnableExpansionOrientationTournament: true, + EnableExpansionOrientationShadow: true, + }, + { + EnableExpansionOrientationShadow: true, + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }, + } { + err := applyToolOptions(&plan, options) + require.ErrorContains(t, err, "mutually exclusive") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) + } +} + +// TestExpansionOrientationPolicyRequiresSupportedEnabledMode verifies expansion orientation policy requires supported enabled mode behavior. +func TestExpansionOrientationPolicyRequiresSupportedEnabledMode(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err = applyToolOptions(&plan, ToolOptions{ExpansionOrientationPolicy: optimize.ExpansionSearchPolicyOrientationProbeV2}) + require.ErrorContains(t, err, "requires tournament or shadow mode") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) + + err = applyToolOptions(&plan, ToolOptions{ + ExpansionOrientationPolicy: optimize.ExpansionSearchPolicy("orientation-probe-v3"), + EnableExpansionOrientationTournament: true, + }) + require.ErrorContains(t, err, "unsupported expansion orientation policy") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +// TestExpansionOrientationShadowRequiresExactlyOneEligibleTarget verifies expansion orientation shadow requires exactly one eligible target behavior. +func TestExpansionOrientationShadowRequiresExactlyOneEligibleTarget(t *testing.T) { + plan := optimize.Plan{LoweringPlan: optimize.LoweringPlan{ + ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + }, + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + }, + }, + }} + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err := applyExpansionOrientationShadow(&plan) + require.ErrorContains(t, err, "matched 2 structurally eligible fixed-suffix targets; expected exactly one") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +// TestExpansionOrientationTournamentRequiresExactlyOneEligibleTarget verifies expansion orientation tournament requires exactly one eligible target behavior. +func TestExpansionOrientationTournamentRequiresExactlyOneEligibleTarget(t *testing.T) { + plan := optimize.Plan{LoweringPlan: optimize.LoweringPlan{ + ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + FallbackReason: optimize.ExpansionSearchFallbackTournamentUnqualified, + }, + { + Family: "fixed_suffix_expansion", + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + StaticallyEligible: true, + EmittedCandidates: []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward}, + FallbackReason: optimize.ExpansionSearchFallbackTournamentUnqualified, + }, + }, + }} + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err := applyExpansionOrientationTournament(&plan) + require.ErrorContains(t, err, "matched 2 structurally eligible fixed-suffix targets; expected exactly one") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +// TestExpansionOrientationTournamentRejectsNonInitialVariableRegion verifies expansion orientation tournament rejects non initial variable region behavior. +func TestExpansionOrientationTournamentRejectsNonInitialVariableRegion(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH ()-[:Prefix]->(root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) + RETURN id(root) + `) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + require.False(t, plan.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) + require.NotEqual(t, "fixed_suffix_expansion", plan.LoweringPlan.ExpansionSearchStrategy[0].Family) + require.ErrorContains(t, applyExpansionOrientationTournament(&plan), "has no structurally eligible fixed-suffix target") +} diff --git a/cypher/models/pgsql/translate/expansion_shortest_distance_inline.go b/cypher/models/pgsql/translate/expansion_shortest_distance_inline.go new file mode 100644 index 00000000..917861de --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_shortest_distance_inline.go @@ -0,0 +1,494 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package translate + +import ( + "errors" + + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" + "github.com/specterops/dawgs/graph" +) + +const ( + spI2Distance pgsql.Identifier = "sp_i2_distance" + spI2Direct pgsql.Identifier = "sp_i2_v2_direct" + spI2DistanceBounded pgsql.Identifier = "sp_i2_distance_bounded" + spI2Target pgsql.Identifier = "sp_i2_target" + spI2SelectedDistance pgsql.Identifier = "sp_i2_selected_distance" + spI2Admission pgsql.Identifier = "sp_i2_admission" + spI2Decision pgsql.Identifier = "sp_i2_decision" + spI2CandidateMarker pgsql.Identifier = "sp_i2_candidate_marker" + spI2FallbackMarker pgsql.Identifier = "sp_i2_fallback_marker" + spI2CandidateBody pgsql.Identifier = "sp_i2_candidate_body" + spI2FallbackBody pgsql.Identifier = "sp_i2_fallback_body" + spI2CandidateRows pgsql.Identifier = "sp_i2_candidate_rows" + spI2FallbackRows pgsql.Identifier = "sp_i2_fallback_rows" + spI2NodeID pgsql.Identifier = "node_id" + spI2Overflow pgsql.Identifier = "overflow" + spI2UseCandidate pgsql.Identifier = "use_candidate" + spI2UseFallback pgsql.Identifier = "use_fallback" + spI2RuntimeReceipt pgsql.Identifier = "runtime_receipt" + spI2RuntimeAttestationFn pgsql.Identifier = "record_requested_traversal_runtime_attestation_v1" +) + +type spI2Architecture struct { + consolidatedAdmission bool + directFloor bool + scalarProjection bool +} + +// BuildInlineGuardedShortestDistanceRoot emits reverse-physical, ID-only +// minimum-distance discovery. The bounded candidate remains invisible until +// independent total-state and per-level frontier gates pass; overflow invokes +// exact compact S4 in the same top-level statement. +func (s *ExpansionBuilder) BuildInlineGuardedShortestDistanceRoot() (pgsql.Query, error) { + return s.buildInlineGuardedShortestDistanceRoot(optimize.ShortestPathExecutorI2GuardedDistance, spI2Architecture{}) +} + +// buildInlineGuardedShortestDistanceRoot retains the byte-stable V1 rendering +// when consolidatedAdmission is false. V2 uses the same proven recursive and +// fallback semantics while replacing only admission/target orchestration. +func (s *ExpansionBuilder) buildInlineGuardedShortestDistanceRoot(runtimeIdentity optimize.ShortestPathExecutor, architecture spI2Architecture) (pgsql.Query, error) { + const validatedEndpoints pgsql.Identifier = "singleton_endpoints" + + expansionModel := s.traversalStep.Expansion + if !expansionModel.UsesSingletonEndpointPair() { + return pgsql.Query{}, errors.New("SP-I2-C-D requires one validated endpoint pair") + } + if expansionModel.Options.MinDepth.GetOr(1) != 1 || !expansionModel.Options.MaxDepth.Set || expansionModel.Options.MaxDepth.Value < 1 || expansionModel.Options.MaxDepth.Value > 64 { + return pgsql.Query{}, errors.New("SP-I2-C-D requires min depth 1 and bounded max depth <= 64") + } + if s.traversalStep.Direction != graph.DirectionOutbound && s.traversalStep.Direction != graph.DirectionInbound { + return pgsql.Query{}, errors.New("SP-I2-C-D requires a directed traversal") + } + if expansionModel.ShortestPathStateLimit <= 0 || expansionModel.ShortestPathFrontierLimit <= 0 { + return pgsql.Query{}, errors.New("SP-I2-C-D requires positive state and frontier limits") + } + + endpointCTE := singletonEndpointValidationCTE(s.traversalStep, expansionModel) + endpointSelect := endpointCTE.Query.Body.(pgsql.Select) + endpointSelect.Where = pgsql.OptionalAnd(endpointSelect.Where, shortestPathSelfEndpointGuardCase( + pgd.EntityID(s.traversalStep.LeftNode.Identifier), + pgd.EntityID(s.traversalStep.RightNode.Identifier), + )) + endpointCTE.Query.Body = endpointSelect + + // Search begins at the public terminal and follows the opposite physical + // adjacency direction toward the public root. This is the canonical escape + // from logical-inbound hidden fan-in while remaining correct for either + // directed pattern orientation. + edge := s.traversalStep.Edge.Identifier + joinColumn, nextColumn := pgsql.ColumnEndID, pgsql.ColumnStartID + if s.traversalStep.Direction == graph.DirectionInbound { + joinColumn, nextColumn = pgsql.ColumnStartID, pgsql.ColumnEndID + } + var edgeScope pgsql.Expression = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edge, pgsql.ColumnGraphID}, pgsql.OperatorEquals, pgsql.NewLiteral(s.graphID, pgsql.Int4), + ) + if len(expansionModel.RelationshipKindIDs) > 0 { + edgeScope = pgsql.OptionalAnd(edgeScope, pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edge, pgsql.ColumnKindID}, pgsql.OperatorEquals, + pgsql.NewAnyExpressionHinted(pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array)), + )) + } + var direct pgsql.CommonTableExpression + if architecture.directFloor { + directWhere := pgsql.OptionalAnd(edgeScope, pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edge, joinColumn}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + ), + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edge, nextColumn}, pgsql.OperatorEquals, + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + ), + )) + direct = pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2Direct, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth})}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{aspI1Aliased(pgsql.NewLiteral(int64(1), pgsql.Int8), expansionDepth)}, + From: []pgsql.FromClause{tableFrom(validatedEndpoints), {Source: expansionEdgeTableReference(edge)}}, + Where: directWhere, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } + } + anchor := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + }, + From: []pgsql.FromClause{tableFrom(validatedEndpoints)}, + } + if architecture.directFloor { + anchor.Where = pgd.Not(pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(spI2Direct)}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}) + } + targetRoot := shortestDistanceEndpointID(validatedEndpoints, expansionRootID) + recursive := pgsql.Select{ + Projection: pgsql.Projection{ + pgsql.CompoundIdentifier{edge, nextColumn}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{spI2Distance, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: spI2Distance.AsCompoundIdentifier()}, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(edge), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{edge, joinColumn}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{spI2Distance, spI2NodeID}, + ), + }, + }}, + }}, + Where: pgsql.OptionalAnd(edgeScope, pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{spI2Distance, expansionDepth}, pgsql.OperatorLessThan, + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int8), + ), + // A walk that has reached the requested root already provides a + // complete distance candidate. Expanding out of it can only create + // longer walks, and on cycles it needlessly repeats work through the + // maximum depth. + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{spI2Distance, spI2NodeID}, pgsql.OperatorNotEquals, targetRoot, + ), + )), + } + distance := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2Distance, Shape: pgsql.NewRecordShape([]pgsql.Identifier{spI2NodeID, expansionDepth})}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: anchor, + ROperand: recursive, + Operator: pgsql.OperatorUnion, + }}, + } + distanceBounded := boundedTraversalStateProbe( + spI2DistanceBounded, spI2Distance, []pgsql.Identifier{spI2NodeID, expansionDepth}, expansionModel.ShortestPathStateLimit, + ) + stateOverflow := boundedProbeOverflow(spI2DistanceBounded, expansionModel.ShortestPathStateLimit) + var frontierOverflow pgsql.Expression = pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(spI2DistanceBounded)}, + GroupBy: []pgsql.Expression{pgsql.CompoundIdentifier{spI2DistanceBounded, expansionDepth}}, + Having: pgsql.NewBinaryExpression( + pgsql.FunctionCall{Function: pgsql.FunctionCount, Parameters: []pgsql.Expression{pgsql.Wildcard{}}, CastType: pgsql.Int8}, + pgsql.OperatorGreaterThan, + pgsql.NewLiteral(expansionModel.ShortestPathFrontierLimit, pgsql.Int8), + ), + }, Limit: pgsql.NewLiteral(int64(1), pgsql.Int8)}}} + frontierGuardDominated := architecture.consolidatedAdmission && expansionModel.ShortestPathFrontierLimit >= expansionModel.ShortestPathStateLimit + if frontierGuardDominated { + // Every frontier row is also a state row. When the frontier cap is at + // least the state cap, state admission strictly dominates the frontier + // check and the depth aggregate is redundant. + frontierOverflow = pgsql.NewLiteral(false, pgsql.Boolean) + } + overflow := aspI1OverflowAny(stateOverflow, frontierOverflow) + + admission := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2Admission}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + aspI1Aliased(overflow, spI2Overflow), + }}}, + } + if architecture.consolidatedAdmission { + admission.Query.Body = pgsql.Select{Projection: pgsql.Projection{ + aspI1Aliased(overflow, spI2Overflow), + aspI1Aliased(pgsql.NewLiteral(frontierGuardDominated, pgsql.Boolean), pgsql.Identifier("frontier_guard_dominated")), + aspI1Aliased(pgsql.NewLiteral(expansionModel.ShortestPathStateLimit, pgsql.Int8), pgsql.Identifier("state_limit")), + aspI1Aliased(pgsql.NewLiteral(expansionModel.ShortestPathFrontierLimit, pgsql.Int8), pgsql.Identifier("frontier_limit")), + }} + } + if architecture.directFloor { + admissionSelect := admission.Query.Body.(pgsql.Select) + admissionSelect.Where = pgd.Not(pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(spI2Direct)}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}) + admission.Query.Body = admissionSelect + } + admissionOverflow := pgsql.CompoundIdentifier{spI2Admission, spI2Overflow} + + targetWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{spI2DistanceBounded, spI2NodeID}, pgsql.OperatorEquals, targetRoot), + pgd.Not(pgsql.NewParenthetical(overflow)), + ) + targetFrom := []pgsql.FromClause{tableFrom(spI2DistanceBounded)} + if architecture.consolidatedAdmission { + targetWhere = pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{spI2DistanceBounded, spI2NodeID}, pgsql.OperatorEquals, targetRoot), + pgd.Not(admissionOverflow), + ) + targetFrom = append(targetFrom, tableFrom(spI2Admission)) + } + if architecture.directFloor { + targetWhere = pgsql.OptionalAnd( + pgd.Not(pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(spI2Direct)}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}), + targetWhere, + ) + } + target := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2Target, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth})}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.CompoundIdentifier{spI2DistanceBounded, expansionDepth}}, + From: targetFrom, + Where: targetWhere, + }, + OrderBy: []*pgsql.OrderBy{{Expression: pgsql.CompoundIdentifier{spI2DistanceBounded, expansionDepth}, Ascending: true}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } + var selectedDistance pgsql.CommonTableExpression + selectedDistanceSource := spI2Target + if architecture.directFloor { + selectedDistanceSource = spI2SelectedDistance + selectedDistance = pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2SelectedDistance, Shape: pgsql.NewRecordShape([]pgsql.Identifier{expansionDepth})}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pgsql.Select{Projection: pgsql.Projection{pgsql.CompoundIdentifier{spI2Direct, expansionDepth}}, From: []pgsql.FromClause{tableFrom(spI2Direct)}}, + ROperand: pgsql.Select{Projection: pgsql.Projection{pgsql.CompoundIdentifier{spI2Target, expansionDepth}}, From: []pgsql.FromClause{tableFrom(spI2Target)}}, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + } + + noPath := pgd.Not(pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(selectedDistanceSource)}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}) + branch := pgsql.Case{ + Conditions: []pgsql.Expression{admissionOverflow, noPath}, + Then: []pgsql.Expression{ + pgsql.NewLiteral("exact_s4_distance_fallback", pgsql.Text), + pgsql.NewLiteral("inline_canonical_distance_no_path", pgsql.Text), + }, + Else: pgsql.NewLiteral("inline_canonical_distance", pgsql.Text), + } + runtimeExecutor := pgsql.Case{ + Conditions: []pgsql.Expression{admissionOverflow}, + Then: []pgsql.Expression{pgsql.NewLiteral(string(optimize.ShortestPathExecutorS4CanonicalDistance), pgsql.Text)}, + Else: pgsql.NewLiteral(string(runtimeIdentity), pgsql.Text), + } + decision := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2Decision}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(pgd.Not(admissionOverflow), spI2UseCandidate), + aspI1Aliased(admissionOverflow, spI2UseFallback), + aspI1Aliased(pgsql.FunctionCall{Function: spI2RuntimeAttestationFn, Parameters: []pgsql.Expression{branch, admissionOverflow, runtimeExecutor}}, spI2RuntimeReceipt), + }, + From: []pgsql.FromClause{tableFrom(spI2Admission)}, + }}, + } + if architecture.directFloor { + directDecision := pgsql.Select{ + Projection: pgsql.Projection{ + aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), spI2UseCandidate), + aspI1Aliased(pgsql.NewLiteral(false, pgsql.Boolean), spI2UseFallback), + aspI1Aliased(pgsql.FunctionCall{Function: spI2RuntimeAttestationFn, Parameters: []pgsql.Expression{ + pgsql.NewLiteral("inline_direct_distance", pgsql.Text), + pgsql.NewLiteral(false, pgsql.Boolean), + pgsql.NewLiteral(string(runtimeIdentity), pgsql.Text), + }}, spI2RuntimeReceipt), + }, + From: []pgsql.FromClause{tableFrom(spI2Direct)}, + } + recursiveDecision := decision.Query.Body.(pgsql.Select) + decision.Query.Body = pgsql.SetOperation{ + LOperand: directDecision, + ROperand: recursiveDecision, + Operator: pgsql.OperatorUnion, + All: true, + } + } + marker := func(alias, selected pgsql.Identifier) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), orientationArmExecuted)}, + From: []pgsql.FromClause{tableFrom(spI2Decision)}, + Where: pgsql.CompoundIdentifier{spI2Decision, selected}, + }}, + } + } + + candidateProjection := pgsql.Projection{ + aspI1Aliased(pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, expansionRootID), + aspI1Aliased(pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, expansionNextID), + aspI1Aliased(pgsql.CompoundIdentifier{selectedDistanceSource, expansionDepth}, expansionDepth), + aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), expansionSatisfied), + aspI1Aliased(pgsql.NewLiteral(false, pgsql.Boolean), expansionIsCycle), + aspI1Aliased(pgsql.ArrayLiteral{CastType: pgsql.Int8Array}, expansionPath), + } + candidateQuery := pgsql.Query{Body: pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{tableFrom(validatedEndpoints), tableFrom(selectedDistanceSource)}, + }} + candidateBody, err := gateQueryBehindMarker(spI2CandidateMarker, spI2CandidateBody, candidateQuery, candidateProjection) + if err != nil { + return pgsql.Query{}, err + } + candidateRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2CandidateRows, Shape: expansionColumns()}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: candidateBody}, + } + + fallbackProjection := aspI1CanonicalProjection(pgsql.FunctionShortestPathCompact) + fallbackQuery := pgsql.Query{Body: pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{ + tableFrom(validatedEndpoints), + {Source: pgsql.FunctionCall{ + Function: pgsql.FunctionShortestPathCompact, + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(s.graphID, pgsql.Int4), + pgsql.CompoundIdentifier{validatedEndpoints, expansionRootID}, + pgsql.CompoundIdentifier{validatedEndpoints, expansionTerminalID}, + pgsql.NewLiteral(int64(1), pgsql.Int4), + pgsql.NewLiteral(expansionModel.Options.MaxDepth.Value, pgsql.Int4), + pgsql.NewLiteral(append([]int16(nil), expansionModel.RelationshipKindIDs...), pgsql.Int2Array), + pgsql.NewLiteral(s.traversalStep.Direction == graph.DirectionInbound, pgsql.Boolean), + pgsql.NewLiteral(expansionModel.ShortestPathStateLimit, pgsql.Int8), + }, + }}, + }, + }} + fallbackBody, err := gateQueryBehindMarker(spI2FallbackMarker, spI2FallbackBody, fallbackQuery, fallbackProjection) + if err != nil { + return pgsql.Query{}, err + } + fallbackRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: spI2FallbackRows, Shape: expansionColumns()}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: fallbackBody}, + } + + stateID := expansionModel.Frame.Binding.Identifier + search := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: stateID, Shape: expansionColumns()}, + Query: pgsql.Query{Body: pgsql.SetOperation{ + LOperand: pgsql.Select{Projection: aspI1CanonicalProjection(spI2CandidateRows), From: []pgsql.FromClause{tableFrom(spI2CandidateRows)}}, + ROperand: pgsql.Select{Projection: aspI1CanonicalProjection(spI2FallbackRows), From: []pgsql.FromClause{tableFrom(spI2FallbackRows)}}, + Operator: pgsql.OperatorUnion, + All: true, + }}, + } + projection := pgsql.Select{ + Projection: expansionModel.Projection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: stateID.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: expansionNodeTableReference(s.traversalStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.LeftNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionRootID}, + )}, + }, + { + Table: expansionNodeTableReference(s.traversalStep.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{s.traversalStep.RightNode.Identifier, pgsql.ColumnID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{stateID, expansionNextID}, + )}, + }, + }, + }}, + } + if architecture.scalarProjection { + if scalarProjection, ok := spI2ScalarProjection(expansionModel.Projection, stateID, s.traversalStep.LeftNode.Identifier, s.traversalStep.RightNode.Identifier); !ok { + return pgsql.Query{}, errors.New("SP-I2 V2 scalar projection requires state-local distance-only output") + } else { + projection.Projection = scalarProjection + } + projection.From = []pgsql.FromClause{tableFrom(stateID)} + } + + query := pgsql.Query{CommonTableExpressions: &pgsql.With{Recursive: true}, Body: projection} + query.AddCTE(endpointCTE) + if architecture.directFloor { + query.AddCTE(direct) + } + query.AddCTE(distance) + query.AddCTE(distanceBounded) + if architecture.consolidatedAdmission { + query.AddCTE(admission) + query.AddCTE(target) + } else { + query.AddCTE(target) + query.AddCTE(admission) + } + if architecture.directFloor { + query.AddCTE(selectedDistance) + } + query.AddCTE(decision) + query.AddCTE(marker(spI2CandidateMarker, spI2UseCandidate)) + query.AddCTE(marker(spI2FallbackMarker, spI2UseFallback)) + query.AddCTE(candidateRows) + query.AddCTE(fallbackRows) + query.AddCTE(search) + return query, nil +} + +func spI2ScalarProjection(projection []pgsql.SelectItem, stateID, leftNode, rightNode pgsql.Identifier) ([]pgsql.SelectItem, bool) { + localScope := pgsql.AsIdentifierSet(stateID) + scalar := make([]pgsql.SelectItem, 0, len(projection)) + for _, item := range projection { + var ( + expression pgsql.Expression + alias pgsql.Identifier + aliased bool + ) + switch typed := item.(type) { + case pgsql.AliasedExpression: + expression = typed.Expression + alias, aliased = typed.Alias.Value, typed.Alias.Set + case *pgsql.AliasedExpression: + if typed == nil { + return nil, false + } + expression = typed.Expression + alias, aliased = typed.Alias.Value, typed.Alias.Set + case pgsql.Expression: + expression = typed + default: + return nil, false + } + if expression != nil && optimize.ExpressionReferencesOnlyLocalIdentifiers(expression, localScope) { + scalar = append(scalar, item) + continue + } + identifier, ok := expression.(pgsql.CompoundIdentifier) + if !ok || len(identifier) != 2 || identifier.Field() != pgsql.ColumnID || !aliased { + return nil, false + } + switch identifier.Root() { + case leftNode: + scalar = append(scalar, aspI1Aliased(pgsql.CompoundIdentifier{stateID, expansionRootID}, alias)) + case rightNode: + scalar = append(scalar, aspI1Aliased(pgsql.CompoundIdentifier{stateID, expansionNextID}, alias)) + default: + return nil, false + } + } + return scalar, len(scalar) > 0 +} diff --git a/cypher/models/pgsql/translate/expansion_shortest_distance_inline_test.go b/cypher/models/pgsql/translate/expansion_shortest_distance_inline_test.go new file mode 100644 index 00000000..f8339690 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_shortest_distance_inline_test.go @@ -0,0 +1,163 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package translate + +import ( + "context" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +const guardedDistanceToolQuery = ` + MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) +` + +// TestGuardedDistanceToolCapsDriveBothAdmissionSentinels verifies reduced-cap +// fallback can be exercised diagnostically without relaxing production caps. +func TestGuardedDistanceToolCapsDriveBothAdmissionSentinels(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), guardedDistanceToolQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), query, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistance, + GuardedDistanceStateLimit: 10, + GuardedDistanceFrontierLimit: 10, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + decision := translation.Optimization.LoweringPlan.ShortestPathExecutor[0] + require.Equal(t, int64(10), decision.StateLimit) + require.Equal(t, int64(10), decision.FrontierLimit) + require.Contains(t, formatted, "limit 11") + require.Contains(t, formatted, "offset 10 limit 1") + require.Contains(t, formatted, "having count(*)::int8 > 10") + require.Contains(t, formatted, "shortest_path_compact(") +} + +func TestGuardedDistanceV2ConsolidatesAdmissionAndDominatesEqualFrontierCap(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), guardedDistanceToolQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), query, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistanceV2, + GuardedDistanceStateLimit: 10, + GuardedDistanceFrontierLimit: 10, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.NotContains(t, formatted, "group by sp_i2_distance_bounded.depth") + require.Contains(t, formatted, "true as frontier_guard_dominated") + require.Less(t, strings.Index(formatted, "sp_i2_admission as materialized"), strings.Index(formatted, "sp_i2_target(depth) as materialized")) + require.Contains(t, formatted, "else 'SP-I2-C-D-V2' end") +} + +func TestGuardedDistanceV2RetainsOneIndependentFrontierCheckForUnequalCaps(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), guardedDistanceToolQuery) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), query, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1, + GuardedDistanceStateLimit: 20, + GuardedDistanceFrontierLimit: 10, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Equal(t, 1, strings.Count(formatted, "group by sp_i2_distance_bounded.depth")) + require.Contains(t, formatted, "false as frontier_guard_dominated") + require.Contains(t, formatted, "limit 21") + require.Contains(t, formatted, "having count(*)::int8 > 10") + require.Contains(t, formatted, "else 'SP-I2-C-D-V2-E1' end") +} + +func TestGuardedDistanceV2DevelopmentComponentArms(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), guardedDistanceToolQuery) + require.NoError(t, err) + + for _, testCase := range []struct { + name string + executor optimize.ShortestPathExecutor + directFloor bool + scalarProjection bool + expectedIdentity string + expectedDirectName string + }{ + {name: "direct", executor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, directFloor: true, expectedIdentity: "SP-I2-C-D-V2-E1D", expectedDirectName: "sp_i2_v2_direct(depth) as materialized"}, + {name: "projection", executor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, scalarProjection: true, expectedIdentity: "SP-I2-C-D-V2-E1P"}, + {name: "combined", executor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP, directFloor: true, scalarProjection: true, expectedIdentity: "SP-I2-C-D-V2-E1DP", expectedDirectName: "sp_i2_v2_direct(depth) as materialized"}, + } { + t.Run(testCase.name, func(t *testing.T) { + translation, err := TranslateForTool(context.Background(), query, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: testCase.executor, + GuardedDistanceStateLimit: 10, + GuardedDistanceFrontierLimit: 10, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "else '"+testCase.expectedIdentity+"' end") + if testCase.directFloor { + require.Contains(t, formatted, testCase.expectedDirectName) + require.Contains(t, formatted, "'inline_direct_distance'") + require.Contains(t, formatted, "where not exists (select 1 from sp_i2_v2_direct limit 1)") + require.Contains(t, formatted, "sp_i2_selected_distance(depth) as materialized") + } else { + require.NotContains(t, formatted, "sp_i2_v2_direct") + require.NotContains(t, formatted, "inline_direct_distance") + } + if testCase.scalarProjection { + require.NotContains(t, formatted, "join node as") + } + }) + } +} + +func TestGuardedDistanceV2ScalarProjectionRejectsEntityHydration(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p), s + `) + require.NoError(t, err) + _, err = TranslateForTool(context.Background(), query, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + GuardedDistanceStateLimit: 10, + GuardedDistanceFrontierLimit: 10, + }) + require.ErrorContains(t, err, "no structurally eligible distance-only target") +} + +// TestGuardedDistanceToolCapsAreIsolated rejects partial, negative, and +// unrelated overrides before they can mutate an optimized plan. +func TestGuardedDistanceToolCapsAreIsolated(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), guardedDistanceToolQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(query) + require.NoError(t, err) + + for _, options := range []ToolOptions{ + {GuardedDistanceStateLimit: 10, GuardedDistanceFrontierLimit: 10}, + {ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistance, GuardedDistanceStateLimit: 10}, + {ForceShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistance, GuardedDistanceStateLimit: -1, GuardedDistanceFrontierLimit: 10}, + } { + planCopy := plan + require.Error(t, applyToolOptions(&planCopy, options)) + } +} diff --git a/cypher/models/pgsql/translate/expansion_shortest_distance_inline_v2.go b/cypher/models/pgsql/translate/expansion_shortest_distance_inline_v2.go new file mode 100644 index 00000000..084b06d2 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_shortest_distance_inline_v2.go @@ -0,0 +1,33 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package translate + +import ( + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// BuildInlineGuardedShortestDistanceRootV2 emits the selected E1 V2 +// architecture. Admission is materialized once before target selection. With +// production's equal caps, total-state admission dominates frontier admission +// and no GROUP BY depth aggregate is rendered. Unequal diagnostic caps retain +// exactly one independent frontier check. +func (s *ExpansionBuilder) BuildInlineGuardedShortestDistanceRootV2() (pgsql.Query, error) { + return s.buildInlineGuardedShortestDistanceRoot(optimize.ShortestPathExecutorI2GuardedDistanceV2, spI2Architecture{consolidatedAdmission: true}) +} + +func spI2DevelopmentArchitecture(executor optimize.ShortestPathExecutor) spI2Architecture { + switch executor { + case optimize.ShortestPathExecutorI2GuardedDistanceV2E0: + return spI2Architecture{} + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1D: + return spI2Architecture{consolidatedAdmission: true, directFloor: true} + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1P: + return spI2Architecture{consolidatedAdmission: true, scalarProjection: true} + case optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP: + return spI2Architecture{consolidatedAdmission: true, directFloor: true, scalarProjection: true} + default: + return spI2Architecture{consolidatedAdmission: true} + } +} diff --git a/cypher/models/pgsql/translate/expansion_suffix_guarded.go b/cypher/models/pgsql/translate/expansion_suffix_guarded.go new file mode 100644 index 00000000..f7f14226 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_suffix_guarded.go @@ -0,0 +1,537 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +const ( + suffixGuardSuffixOverflow pgsql.Identifier = "suffix_overflow" + suffixGuardStateOverflow pgsql.Identifier = "state_overflow" + suffixGuardUseCandidate pgsql.Identifier = "use_candidate" + suffixGuardUseFallback pgsql.Identifier = "use_fallback" + suffixGuardRuntimeReceipt pgsql.Identifier = "runtime_receipt" + suffixGuardAttestationFn pgsql.Identifier = "record_requested_traversal_runtime_attestation_v1" + suffixRetryStatusSetting = "dawgs.suffix_reverse_retry_status" +) + +// suffixReverseGuardIdentifiers assigns stable, policy-specific names to the +// bounded inputs, admission decision, and mutually exclusive execution arms. +// Keeping this namespace separate from orientation makes plan evidence unable +// to attribute this static guard to the rejected topology selector family. +type suffixReverseGuardIdentifiers struct { + rootPresence pgsql.Identifier + suffixProbe pgsql.Identifier + boundaries pgsql.Identifier + reverse pgsql.Identifier + states pgsql.Identifier + admission pgsql.Identifier + decision pgsql.Identifier + candidateMarker pgsql.Identifier + fallbackMarker pgsql.Identifier + candidateBody pgsql.Identifier + fallbackBody pgsql.Identifier + fallbackRows pgsql.Identifier +} + +func newSuffixReverseGuardIdentifiers(finalFrame pgsql.Identifier) suffixReverseGuardIdentifiers { + prefix := string(finalFrame) + "_suffix_guard_" + return suffixReverseGuardIdentifiers{ + rootPresence: pgsql.Identifier(prefix + "root_presence"), + suffixProbe: pgsql.Identifier(prefix + "suffix_probe"), + boundaries: pgsql.Identifier(prefix + "boundaries"), + reverse: pgsql.Identifier(prefix + "reverse"), + states: pgsql.Identifier(prefix + "states"), + admission: pgsql.Identifier(prefix + "admission"), + decision: pgsql.Identifier(prefix + "decision"), + candidateMarker: pgsql.Identifier(prefix + "candidate_marker"), + fallbackMarker: pgsql.Identifier(prefix + "fallback_marker"), + candidateBody: pgsql.Identifier(prefix + "candidate_body"), + fallbackBody: pgsql.Identifier(prefix + "fallback_body"), + fallbackRows: pgsql.Identifier(prefix + "fallback_rows"), + } +} + +// rewriteTraversalPatternAsSuffixReverseGuard replaces the incumbent frame +// chain with a static full-path reverse candidate and the unchanged incumbent +// behind a bounded, same-statement fallback boundary. +func (s *Translator) rewriteTraversalPatternAsSuffixReverseGuard(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if (decision.EmittedPolicy != optimize.ExpansionSearchPolicySuffixReverseGuardV1 && + decision.EmittedPolicy != optimize.ExpansionSearchPolicySuffixReverseRetryV1 && + decision.EmittedPolicy != optimize.ExpansionSearchPolicyTopologyFixedSuffixV1 && + decision.EmittedPolicy != optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) || + decision.ObservationMode != optimize.ExpansionSearchObservationFullPath || part.PatternBinding == nil { + return fmt.Errorf("suffix reverse guard requires the full-path policy envelope") + } + if len(part.TraversalSteps) != decision.SuffixEndStep+1 || decision.SuffixLength != 3 || decision.Target.StepIndex != 0 { + return fmt.Errorf("suffix reverse guard requires one expansion followed by exactly three terminal suffix steps") + } + if decision.ProbeCaps.ReverseSeedRowLimit <= 0 || decision.Admission.StateLimit <= 0 || + decision.Admission.FallbackStrategy != optimize.ExpansionSearchStepwiseForward { + return fmt.Errorf("suffix reverse guard requires positive immutable suffix/state caps and exact stepwise-forward fallback") + } + + expansionStep := part.TraversalSteps[decision.Target.StepIndex] + if expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.Frame.Previous == nil || + !expansionStep.LeftNodeBound || expansionStep.Edge == nil || expansionStep.LeftNode == nil { + return fmt.Errorf("suffix reverse guard requires a complete expansion and bound root") + } + suffix := part.TraversalSteps[decision.SuffixStartStep : decision.SuffixEndStep+1] + for _, step := range suffix { + if step == nil || step.Frame == nil || step.Edge == nil || step.LeftNode == nil || step.RightNode == nil { + return fmt.Errorf("suffix reverse guard has an incomplete fixed suffix step") + } + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("suffix reverse guard did not emit an incumbent frame chain") + } + incumbentChain := append([]pgsql.CommonTableExpression(nil), ctes[firstCTE:]...) + incumbentFinal := incumbentChain[len(incumbentChain)-1] + if incumbentFinal.Alias.Name != suffix[len(suffix)-1].Frame.Binding.Identifier { + return fmt.Errorf("suffix reverse guard final frame mismatch: expected %s but found %s", suffix[len(suffix)-1].Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + incumbentSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("suffix reverse guard final frame must be a select") + } + + query, err := s.buildSuffixReverseGuardQuery( + part, + decision, + expansionStep, + suffix, + expansionStep.Frame.Previous.Binding.Identifier, + newSuffixReverseGuardIdentifiers(incumbentFinal.Alias.Name), + incumbentChain, + incumbentFinal.Alias.Name, + incumbentSelect.Projection, + ) + if err != nil { + return err + } + + part.PatternBinding.DataType = pgsql.PathComposite + part.PatternBinding.Dependencies = nil + part.PatternBinding.MaterializedBy(suffix[len(suffix)-1].Frame) + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:firstCTE], pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: query, + }) + s.recordExpansionSearchPolicy(decision.Target, decision.EmittedPolicy) + return nil +} + +// buildSuffixReverseGuardQuery performs no topology probes. A bounded suffix +// payload gates reverse seeds, bounded reverse state gates visibility, and one +// materialized decision records the runtime receipt before complementary +// marker-driven candidate/fallback branches execute. +func (s *Translator) buildSuffixReverseGuardQuery( + part *PatternPart, + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids suffixReverseGuardIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + _, externalEdgeConstraint := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + if externalEdgeConstraint != nil { + return pgsql.Query{}, fmt.Errorf("suffix reverse guard relationship predicate is not local") + } + suffixIDs := suffixSeededIdentifiers{ + rootPresence: ids.rootPresence, + suffix: ids.suffixProbe, + boundaries: ids.boundaries, + reverse: ids.reverse, + } + rootPresence := buildSuffixReverseGuardRootPresence(rootFrame, ids) + suffixProbe, err := s.buildFixedSuffixProbeCTE(expansionStep, suffix, suffixIDs, decision.ProbeCaps.ReverseSeedRowLimit) + if err != nil { + return pgsql.Query{}, err + } + boundaries := buildSuffixReverseGuardBoundaries(ids, decision.ProbeCaps.ReverseSeedRowLimit) + reverse, err := buildSuffixSeededReverseCTE(expansionStep, decision, suffixIDs, "", "", true) + if err != nil { + return pgsql.Query{}, err + } + states := boundedTraversalStateProbe( + ids.states, + ids.reverse, + []pgsql.Identifier{fixedSuffixBoundaryID, expansionNextID, expansionDepth, expansionPath, expansionNodePath}, + decision.Admission.StateLimit, + ) + admission := buildSuffixReverseGuardAdmission(ids, decision.ProbeCaps.ReverseSeedRowLimit, decision.Admission.StateLimit) + retryOnly := decision.EmittedPolicy == optimize.ExpansionSearchPolicySuffixReverseRetryV1 || + decision.EmittedPolicy == optimize.ExpansionSearchPolicyTopologyFixedSuffixV1 || + decision.EmittedPolicy == optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1 + decisionCTE := buildSuffixReverseGuardDecision(ids, retryOnly) + markers := buildSuffixReverseGuardMarkers(ids) + + candidateProjection, err := suffixSeededFinalProjection(part, expansionStep, suffix, rootFrame, suffixIDs, ids.states, incumbentProjection, nil) + if err != nil { + return pgsql.Query{}, err + } + candidateProjection = append(candidateProjection, &pgsql.AliasedExpression{ + Expression: suffixSeededOrderedPathComposite(s.graphID, expansionStep, suffix, suffixIDs, ids.states), + Alias: models.OptionalValue(part.PatternBinding.Identifier), + }) + + suffixEdgeIDs := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + for _, step := range suffix { + suffixEdgeIDs.Values = append(suffixEdgeIDs.Values, pgsql.CompoundIdentifier{ids.suffixProbe, step.Edge.Identifier}) + } + candidateWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8), + ), + pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionPath}, + pgsql.OperatorArrayOverlap, + suffixEdgeIDs, + )), + ) + candidateQuery := pgsql.Query{Body: pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{Name: rootFrame.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{Name: ids.states.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(rootFrame, expansionStep.LeftNode), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.states, expansionNextID}, + )}, + }, + { + Table: pgsql.TableReference{Name: ids.suffixProbe.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{JoinType: pgsql.JoinTypeInner, Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.suffixProbe, fixedSuffixBoundaryID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.states, fixedSuffixBoundaryID}, + )}, + }, + }, + }}, + Where: candidateWhere, + }} + candidateExecutor := pgsql.Identifier(string(ids.candidateBody) + "_executor") + candidate, err := gateQueryBehindMarker(ids.candidateMarker, candidateExecutor, candidateQuery, candidateProjection) + if err != nil { + return pgsql.Query{}, err + } + + candidateRows := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.candidateBody}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: candidate}, + } + candidateOutput, err := suffixReverseGuardOutputSelect(ids.candidateBody, candidateProjection) + if err != nil { + return pgsql.Query{}, err + } + if retryOnly { + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: []pgsql.CommonTableExpression{ + rootPresence, + suffixProbe, + boundaries, + reverse, + states, + admission, + decisionCTE, + markers[0], + candidateRows, + }, + }, + Body: candidateOutput, + Limit: pgsql.NewLiteral(decision.Admission.OutputRowLimit+1, pgsql.Int8), + }, nil + } + + incumbentPath, err := expressionForPathComposite(part.PatternBinding, s.scope) + if err != nil { + return pgsql.Query{}, err + } + fallbackRows, fallbackProjection, err := buildSuffixReverseGuardFallbackCTE( + ids.fallbackRows, + incumbentChain, + incumbentFinal, + incumbentProjection, + pgsql.Projection{&pgsql.AliasedExpression{ + Expression: incumbentPath, + Alias: models.OptionalValue(part.PatternBinding.Identifier), + }}, + ) + if err != nil { + return pgsql.Query{}, err + } + fallbackQuery := pgsql.Query{Body: pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{tableFrom(ids.fallbackRows)}, + }} + fallbackExecutor := pgsql.Identifier(string(ids.fallbackBody) + "_executor") + fallback, err := gateQueryBehindMarker(ids.fallbackMarker, fallbackExecutor, fallbackQuery, fallbackProjection) + if err != nil { + return pgsql.Query{}, err + } + fallbackOutput := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.fallbackBody}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: fallback}, + } + fallbackOutputSelect, err := suffixReverseGuardOutputSelect(ids.fallbackBody, fallbackProjection) + if err != nil { + return pgsql.Query{}, err + } + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: []pgsql.CommonTableExpression{ + rootPresence, + suffixProbe, + boundaries, + reverse, + states, + admission, + decisionCTE, + markers[0], + markers[1], + fallbackRows, + candidateRows, + fallbackOutput, + }, + }, + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: candidateOutput, + ROperand: fallbackOutputSelect, + }, + }, nil +} + +func suffixReverseGuardOutputSelect(alias pgsql.Identifier, prototype pgsql.Projection) (pgsql.Select, error) { + projection := make(pgsql.Projection, 0, len(prototype)) + for _, item := range prototype { + itemAlias, ok := selectItemAlias(item) + if !ok { + return pgsql.Select{}, fmt.Errorf("suffix reverse guard output projection contains an unaliased item %T", item) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{alias, itemAlias}, + Alias: models.OptionalValue(itemAlias), + }) + } + return pgsql.Select{Projection: projection, From: []pgsql.FromClause{tableFrom(alias)}}, nil +} + +func buildSuffixReverseGuardRootPresence(rootFrame pgsql.Identifier, ids suffixReverseGuardIdentifiers) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.rootPresence}, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(rootFrame)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } +} + +// buildSuffixReverseGuardBoundaries produces no reverse seed when the suffix +// cap+1 sentinel exists. Duplicate suffix paths remain in suffixProbe for final +// bag semantics; only recursive starting nodes are deduplicated. +func buildSuffixReverseGuardBoundaries(ids suffixReverseGuardIdentifiers, suffixRowLimit int64) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.boundaries}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Distinct: true, + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.suffixProbe, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.suffixProbe)}, + Where: pgsql.OptionalAnd( + pgd.Not(boundedProbeOverflow(ids.suffixProbe, suffixRowLimit)), + pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, From: []pgsql.FromClause{tableFrom(ids.rootPresence)}}, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }}}, + ), + }}, + } +} + +func buildSuffixReverseGuardAdmission(ids suffixReverseGuardIdentifiers, suffixRowLimit, stateLimit int64) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.admission}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{ + aspI1Aliased(boundedProbeOverflow(ids.suffixProbe, suffixRowLimit), suffixGuardSuffixOverflow), + aspI1Aliased(boundedProbeOverflow(ids.states, stateLimit), suffixGuardStateOverflow), + }}}, + } +} + +// buildSuffixReverseGuardDecision records exactly one requested-runtime +// attestation and exposes complementary booleans consumed by execution markers. +func buildSuffixReverseGuardDecision(ids suffixReverseGuardIdentifiers, retryOnly bool) pgsql.CommonTableExpression { + suffixOverflow := pgsql.CompoundIdentifier{ids.admission, suffixGuardSuffixOverflow} + stateOverflow := pgsql.CompoundIdentifier{ids.admission, suffixGuardStateOverflow} + overflow := pgsql.NewBinaryExpression(suffixOverflow, pgsql.OperatorOr, stateOverflow) + runtimeExecutor := pgsql.Case{ + Conditions: []pgsql.Expression{overflow}, + Then: []pgsql.Expression{pgsql.NewLiteral(string(optimize.ExpansionSearchStepwiseForward), pgsql.Text)}, + Else: pgsql.NewLiteral(string(optimize.ExpansionSearchSuffixSeededReverse), pgsql.Text), + } + branch := pgsql.Case{ + Conditions: []pgsql.Expression{suffixOverflow, stateOverflow}, + Then: []pgsql.Expression{ + pgsql.NewLiteral("exact_forward_suffix_overflow", pgsql.Text), + pgsql.NewLiteral("exact_forward_state_overflow", pgsql.Text), + }, + Else: pgsql.NewLiteral("suffix_seeded_reverse", pgsql.Text), + } + if retryOnly { + branch = pgsql.Case{ + Conditions: []pgsql.Expression{suffixOverflow, stateOverflow}, + Then: []pgsql.Expression{ + pgsql.NewLiteral("forward_retry_suffix_overflow", pgsql.Text), + pgsql.NewLiteral("forward_retry_state_overflow", pgsql.Text), + }, + Else: pgsql.NewLiteral("reverse_complete", pgsql.Text), + } + } + fallbackExecuted := pgsql.Expression(overflow) + recordedExecutor := pgsql.Expression(runtimeExecutor) + if retryOnly { + fallbackExecuted = pgsql.NewLiteral(false, pgsql.Boolean) + recordedExecutor = pgsql.NewLiteral(string(optimize.ExpansionSearchStepwiseForward), pgsql.Text) + } + projection := pgsql.Projection{ + aspI1Aliased(pgd.Not(pgsql.NewParenthetical(overflow)), suffixGuardUseCandidate), + aspI1Aliased(overflow, suffixGuardUseFallback), + aspI1Aliased(pgsql.FunctionCall{ + Function: suffixGuardAttestationFn, + Parameters: []pgsql.Expression{ + branch, + fallbackExecuted, + recordedExecutor, + }, + }, suffixGuardRuntimeReceipt), + } + if retryOnly { + projection = append(projection, aspI1Aliased(pgsql.FunctionCall{ + Function: pgsql.Identifier("set_config"), + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(suffixRetryStatusSetting, pgsql.Text), + branch, + pgsql.NewLiteral(true, pgsql.Boolean), + }, + }, pgsql.Identifier("retry_status"))) + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.decision}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{tableFrom(ids.admission)}, + }}, + } +} + +func buildSuffixReverseGuardMarkers(ids suffixReverseGuardIdentifiers) []pgsql.CommonTableExpression { + marker := func(alias, selected pgsql.Identifier) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{aspI1Aliased(pgsql.NewLiteral(true, pgsql.Boolean), orientationArmExecuted)}, + From: []pgsql.FromClause{tableFrom(ids.decision)}, + Where: pgsql.CompoundIdentifier{ids.decision, selected}, + }}, + } + } + return []pgsql.CommonTableExpression{ + marker(ids.candidateMarker, suffixGuardUseCandidate), + marker(ids.fallbackMarker, suffixGuardUseFallback), + } +} + +// buildSuffixReverseGuardFallbackCTE nests the original frame chain unchanged. +// The materialized CTE sits under the fallback marker's correlated lateral +// boundary, so the incumbent executor is not initialized on admitted runs. +func buildSuffixReverseGuardFallbackCTE( + alias pgsql.Identifier, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, + extraProjection pgsql.Projection, +) (pgsql.CommonTableExpression, pgsql.Projection, error) { + projection := make(pgsql.Projection, 0, len(incumbentProjection)+len(extraProjection)) + fallback := make(pgsql.Projection, 0, len(incumbentProjection)+len(extraProjection)) + appendItem := func(item pgsql.SelectItem, from pgsql.Identifier) error { + itemAlias, ok := selectItemAlias(item) + if !ok { + return fmt.Errorf("suffix reverse guard incumbent projection contains an unaliased item %T", item) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{from, itemAlias}, + Alias: models.OptionalValue(itemAlias), + }) + fallback = append(fallback, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{alias, itemAlias}, + Alias: models.OptionalValue(itemAlias), + }) + return nil + } + for _, item := range incumbentProjection { + if err := appendItem(item, incumbentFinal); err != nil { + return pgsql.CommonTableExpression{}, nil, err + } + } + for _, item := range extraProjection { + itemAlias, ok := selectItemAlias(item) + if !ok { + return pgsql.CommonTableExpression{}, nil, fmt.Errorf("suffix reverse guard extra incumbent projection contains an unaliased item %T", item) + } + projection = append(projection, item) + fallback = append(fallback, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{alias, itemAlias}, + Alias: models.OptionalValue(itemAlias), + }) + } + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: alias}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{ + CommonTableExpressions: &pgsql.With{Expressions: incumbentChain}, + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{tableFrom(incumbentFinal)}, + }, + }, + }, fallback, nil +} diff --git a/cypher/models/pgsql/translate/expansion_suffix_guarded_test.go b/cypher/models/pgsql/translate/expansion_suffix_guarded_test.go new file mode 100644 index 00000000..8af8bb70 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_suffix_guarded_test.go @@ -0,0 +1,363 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package translate + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +func translateSuffixReverseGuard(t *testing.T, query string, options ToolOptions) (Result, string) { + t.Helper() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-root", + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return translation, formatted +} + +// TestSuffixReverseGuardEmitsStaticFullPathDualArm verifies that the tool-only +// policy has a distinct identity, emits no orientation probes, and retains an +// exact marker-gated fallback in the same statement. +func TestSuffixReverseGuardEmitsStaticFullPathDualArm(t *testing.T) { + translation, formatted := translateSuffixReverseGuard(t, guardedSuffixOrientationQuery, ToolOptions{ + EnableExpansionSuffixReverseGuard: true, + }) + + require.NotNil(t, translation.Optimization.LoweringPlan) + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchPolicySuffixReverseGuardV1, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicySuffixReverseGuardV1, decision.EmittedPolicy) + require.Equal(t, optimize.ExpansionSearchSelectorFixedSuffixPathV1, decision.SelectorVersion) + require.Equal(t, optimize.ExpansionSearchObservationFullPath, decision.ObservationMode) + require.Equal(t, "guarded_tool", decision.SelectionMode) + require.Equal(t, optimize.ExpansionSearchExecutionBoundaryGuardedDualArm, decision.ExecutionBoundary) + require.Equal(t, []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchSuffixSeededReverse, + optimize.ExpansionSearchStepwiseForward, + }, decision.EmittedCandidates) + require.Equal(t, optimize.ExpansionSearchProbeCaps{ + ReverseSeedRowLimit: optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, + }, decision.ProbeCaps) + require.Equal(t, optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchSuffixReverseGuardStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, decision.Admission) + require.Equal(t, optimize.ExpansionSearchSuffixReverseGuardStateLimit, decision.StateLimit) + require.Equal(t, optimize.ExpansionSearchStepwiseForward, decision.FallbackStrategy) + require.Contains(t, decision.EligibilityFacts, optimize.ExpansionSearchEligibilityFact{ + Name: "full_path_observation", + Eligible: true, + }) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicySuffixReverseGuardV1), outcome.EmittedPolicy) + require.Equal(t, optimize.ExpansionSearchSelectorFixedSuffixPathV1, outcome.SelectorVersion) + + for _, cte := range []string{ + "_suffix_guard_root_presence", + "_suffix_guard_suffix_probe", + "_suffix_guard_boundaries", + "_suffix_guard_states", + "_suffix_guard_admission", + "_suffix_guard_decision", + "_suffix_guard_candidate_marker", + "_suffix_guard_fallback_marker", + "_suffix_guard_candidate_body", + "_suffix_guard_fallback_body", + } { + require.Contains(t, formatted, cte) + } + require.Equal(t, 1, strings.Count(formatted, "record_requested_traversal_runtime_attestation_v1(")) + require.Contains(t, formatted, "EXPANSION-STEPWISE-FORWARD") + require.Contains(t, formatted, "EXPANSION-SUFFIX-SEEDED-REVERSE") + require.Contains(t, formatted, "suffix_seeded_reverse") + require.Contains(t, formatted, "exists (select 1") + require.Contains(t, formatted, "generate_subscripts") + require.Contains(t, formatted, "union all") + require.NotContains(t, formatted, "_orientation_") + require.NotContains(t, formatted, "forward_degree") + require.NotContains(t, formatted, "reverse_degree") + require.NotContains(t, formatted, "would_select") +} + +// TestSuffixReverseGuardToolCapsAreExplicit verifies that diagnostic cap +// overrides are copied into both lowering metadata and cap+1 SQL sentinels. +func TestSuffixReverseGuardToolCapsAreExplicit(t *testing.T) { + translation, formatted := translateSuffixReverseGuard(t, guardedSuffixOrientationQuery, ToolOptions{ + EnableExpansionSuffixReverseGuard: true, + SuffixReverseGuardSuffixRowLimit: 7, + SuffixReverseGuardStateLimit: 11, + }) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, int64(7), decision.ProbeCaps.ReverseSeedRowLimit) + require.Equal(t, int64(11), decision.Admission.StateLimit) + require.Contains(t, formatted, "limit 8") + require.Contains(t, formatted, "offset 7") + require.Contains(t, formatted, "limit 12") + require.Contains(t, formatted, "offset 11") +} + +// TestSuffixReverseRetryEmitsOnlyBoundedCandidate verifies the P1 development +// statement contains no incumbent body and reports a transaction-local status +// before any buffered row can be published. +func TestSuffixReverseRetryEmitsOnlyBoundedCandidate(t *testing.T) { + translation, formatted := translateSuffixReverseGuard(t, guardedSuffixOrientationQuery, ToolOptions{ + EnableExpansionSuffixReverseRetry: true, + }) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchPolicySuffixReverseRetryV1, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicySuffixReverseRetryV1, decision.EmittedPolicy) + require.Equal(t, "transaction_retry_tool", decision.SelectionMode) + require.Equal(t, optimize.ExpansionSearchExecutionBoundaryTransactionRetry, decision.ExecutionBoundary) + require.Equal(t, optimize.ExpansionSearchSuffixSeededReverse, decision.SelectedStrategy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse}, decision.EmittedCandidates) + require.Equal(t, optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchSuffixReverseGuardStateLimit, + OutputRowLimit: optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, + OutputBytesLimit: optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, decision.Admission) + require.Contains(t, formatted, "set_config('dawgs.suffix_reverse_retry_status'") + require.Contains(t, formatted, "forward_retry_suffix_overflow") + require.Contains(t, formatted, "forward_retry_state_overflow") + require.Contains(t, formatted, "reverse_complete") + require.Contains(t, formatted, "limit 4097") + require.Contains(t, formatted, "_suffix_guard_candidate_body") + require.NotContains(t, formatted, "_suffix_guard_fallback_body") + require.NotContains(t, formatted, "_suffix_guard_fallback_rows") +} + +func TestProductionTopologyFixedSuffixEmitsOneGuardedCandidate(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-root", + }, DefaultGraphID, ProductionOptions{ + EnableTopologyFixedSuffix: true, + TopologyFixedSuffixCaps: &ProductionFixedSuffixCaps{ + SuffixRowLimit: 7, StateLimit: 11, OutputRowLimit: 13, OutputBytesLimit: 17, + }, + SelectorVersion: string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1), + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchPolicyTopologyFixedSuffixV1, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicyTopologyFixedSuffixV1, decision.EmittedPolicy) + require.Equal(t, "production_canary", decision.SelectionMode) + require.Equal(t, optimize.ExpansionSearchExecutionBoundaryTransactionRetry, decision.ExecutionBoundary) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse}, decision.EmittedCandidates) + require.Contains(t, formatted, "set_config('dawgs.suffix_reverse_retry_status'") + require.NotContains(t, formatted, "_suffix_guard_fallback_body") +} + +func TestProductionTopologyFixedSuffixFirstUseEmitsRetryCandidate(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-root", + }, DefaultGraphID, ProductionOptions{ + EnableTopologyFixedSuffix: true, + TopologyFixedSuffixCaps: &ProductionFixedSuffixCaps{ + SuffixRowLimit: 7, StateLimit: 11, OutputRowLimit: 13, OutputBytesLimit: 17, + }, + SelectorVersion: string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1), + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1, decision.PlannedPolicy) + require.Equal(t, optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1, decision.EmittedPolicy) + require.Equal(t, optimize.ExpansionSearchExecutionBoundaryTransactionRetry, decision.ExecutionBoundary) + require.Contains(t, formatted, "set_config('dawgs.suffix_reverse_retry_status'") + require.NotContains(t, formatted, "_suffix_guard_fallback_body") +} + +func TestSuffixReverseRetryLowersEveryIndependentFixedSuffixTarget(t *testing.T) { + plan := &optimize.Plan{LoweringPlan: optimize.LoweringPlan{ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + {Family: "fixed_suffix_expansion", CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, StructurallyEligible: true, StaticallyEligible: true, ObservationMode: optimize.ExpansionSearchObservationFullPath}, + {Family: "fixed_suffix_expansion", CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, StructurallyEligible: true, StaticallyEligible: true, ObservationMode: optimize.ExpansionSearchObservationFullPath}, + }}} + require.NoError(t, applyExpansionSuffixReverseRetryPolicy(plan, 0, 0, 0, 0)) + for _, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + require.Equal(t, optimize.ExpansionSearchSuffixSeededReverse, decision.SelectedStrategy) + require.Equal(t, "transaction_retry_tool", decision.SelectionMode) + require.Equal(t, optimize.ExpansionSearchPolicySuffixReverseRetryV1, decision.EmittedPolicy) + } +} + +// TestSuffixRouteComponentEmitsOneExactReverseStatement verifies the new +// default-off preflight arm has no probe, fallback, retry, or production-policy +// identity while retaining one runtime receipt for diagnostic attestation. +func TestSuffixRouteComponentEmitsOneExactReverseStatement(t *testing.T) { + translation, formatted := translateSuffixReverseGuard(t, guardedSuffixOrientationQuery, ToolOptions{ + EnableExpansionSuffixRouteComponent: true, + }) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Empty(t, decision.PlannedPolicy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, "component_tool", decision.SelectionMode) + require.Equal(t, optimize.ExpansionSearchSelectorSuffixRouteComponentV1, decision.SelectorVersion) + require.Equal(t, optimize.ExpansionSearchExecutionBoundaryInlineStatement, decision.ExecutionBoundary) + require.Equal(t, optimize.ExpansionSearchSuffixSeededReverse, decision.SelectedStrategy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse}, decision.EmittedCandidates) + require.Empty(t, decision.ProbeCaps) + require.Empty(t, decision.Admission) + require.Empty(t, decision.FallbackStrategy) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, decision.Target) + require.Equal(t, "component_tool", outcome.SelectionMode) + require.Equal(t, optimize.ExpansionSearchSelectorSuffixRouteComponentV1, outcome.SelectorVersion) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), outcome.Applied) + require.Empty(t, outcome.EmittedPolicy) + + require.Contains(t, formatted, "_suffix_seeded_component_receipt") + require.Contains(t, formatted, "record_requested_traversal_runtime_attestation_v1('suffix_route_component', false, 'EXPANSION-SUFFIX-SEEDED-REVERSE')") + require.Contains(t, formatted, "EXPANSION-SUFFIX-SEEDED-REVERSE") + require.NotContains(t, formatted, "_suffix_guard_") + require.NotContains(t, formatted, "_orientation_") + require.NotContains(t, formatted, "EXPANSION-STEPWISE-FORWARD") + require.NotContains(t, formatted, "forward_retry_") +} + +// TestSuffixRouteComponentAdmitsEndpointObservation verifies the direct +// component measures both output shapes while retry/guard remain path-only. +func TestSuffixRouteComponentAdmitsEndpointObservation(t *testing.T) { + query := strings.Replace(guardedSuffixOrientationQuery, "RETURN path", "RETURN id(terminal)", 1) + translation, formatted := translateSuffixReverseGuard(t, query, ToolOptions{ + EnableExpansionSuffixRouteComponent: true, + }) + decision := translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchObservationEndpointIDs, decision.ObservationMode) + require.Equal(t, optimize.ExpansionSearchSuffixSeededReverse, decision.SelectedStrategy) + require.Contains(t, formatted, "_suffix_seeded_component_receipt") +} + +// TestSuffixReverseGuardRejectsEndpointOnlyObservation verifies that endpoint +// cases remain on the incumbent and cannot be silently enrolled by tooling. +func TestSuffixReverseGuardRejectsEndpointOnlyObservation(t *testing.T) { + query := strings.Replace(guardedSuffixOrientationQuery, "RETURN path", "RETURN id(terminal)", 1) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + for name, options := range map[string]ToolOptions{ + "guard": {EnableExpansionSuffixReverseGuard: true}, + "retry": {EnableExpansionSuffixReverseRetry: true}, + } { + t.Run(name, func(t *testing.T) { + _, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-endpoint", + }, DefaultGraphID, options) + require.ErrorContains(t, err, "statically eligible full-path fixed-suffix target") + }) + } + + incumbent, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-endpoint", + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(incumbent) + require.NoError(t, err) + require.NotContains(t, formatted, "_suffix_guard_") +} + +// TestSuffixReverseGuardRejectsMutation verifies that the static full-path +// envelope does not override the optimizer's read-only qualification. +func TestSuffixReverseGuardRejectsMutation(t *testing.T) { + query := strings.Replace(guardedSuffixOrientationQuery, "RETURN path", "CREATE (created) RETURN path", 1) + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + for name, options := range map[string]ToolOptions{ + "guard": {EnableExpansionSuffixReverseGuard: true}, + "retry": {EnableExpansionSuffixReverseRetry: true}, + } { + t.Run(name, func(t *testing.T) { + _, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-mutation", + }, DefaultGraphID, options) + require.ErrorContains(t, err, "statically eligible full-path fixed-suffix target") + }) + } +} + +// TestSuffixReverseGuardTemplateIsParameterStable verifies that tool policy +// selection and cap literals do not specialize SQL to runtime parameter data. +func TestSuffixReverseGuardTemplateIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translate := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{EnableExpansionSuffixReverseGuard: true}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + require.Equal(t, translate("root-a"), translate("root-b")) +} + +// TestSuffixReverseGuardOptionsAreIsolated verifies the new policy cannot be +// combined with unrelated experimental selectors and rejects invalid caps. +func TestSuffixReverseGuardOptionsAreIsolated(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + + for _, options := range []ToolOptions{ + { + EnableExpansionSuffixReverseGuard: true, + EnableExpansionOrientationTournament: true, + }, + { + EnableExpansionSuffixReverseGuard: true, + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }, + { + EnableExpansionSuffixReverseGuard: true, + EnableExpansionSuffixReverseRetry: true, + }, + {SuffixReverseGuardSuffixRowLimit: 1}, + {EnableExpansionSuffixReverseGuard: true, SuffixReverseGuardSuffixRowLimit: -1}, + {EnableExpansionSuffixReverseGuard: true, SuffixReverseGuardStateLimit: -1}, + {EnableExpansionSuffixReverseRetry: true, SuffixReverseRetryOutputRowLimit: -1}, + {EnableExpansionSuffixReverseRetry: true, SuffixReverseRetryOutputBytesLimit: -1}, + } { + planCopy := plan + require.Error(t, applyToolOptions(&planCopy, options)) + } +} + +// TestProductionTranslationDoesNotEmitSuffixReverseGuard verifies that the +// zero-value production path remains unchanged by this tool-only feature. +func TestProductionTranslationDoesNotEmitSuffixReverseGuard(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedSuffixOrientationQuery) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "suffix-guard-production-default", + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.NotContains(t, formatted, "_suffix_guard_") + require.Empty(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0].EmittedPolicy) +} diff --git a/cypher/models/pgsql/translate/expansion_suffix_seeded.go b/cypher/models/pgsql/translate/expansion_suffix_seeded.go new file mode 100644 index 00000000..4e9e5e90 --- /dev/null +++ b/cypher/models/pgsql/translate/expansion_suffix_seeded.go @@ -0,0 +1,1239 @@ +package translate + +import ( + "fmt" + + "github.com/specterops/dawgs/cypher/models" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/pgd" +) + +const ( + // fixedSuffixBoundaryID names the column containing the node where reverse search enters the fixed suffix. + fixedSuffixBoundaryID pgsql.Identifier = "boundary_id" +) + +// suffixSeededIdentifiers names the root-presence, suffix, boundary, and reverse-search CTEs for one rewrite. +type suffixSeededIdentifiers struct { + // rootPresence names the relation that records whether the bound root produced rows. + rootPresence pgsql.Identifier + // suffix names the materialized matches for the fixed terminal suffix. + suffix pgsql.Identifier + // boundaries names the distinct suffix-boundary nodes used to seed reverse search. + boundaries pgsql.Identifier + // reverse names the recursive relation that searches from each boundary toward the root. + reverse pgsql.Identifier + // componentReceipt names the forced direct-component runtime receipt. + componentReceipt pgsql.Identifier +} + +// newSuffixSeededIdentifiers derives collision-resistant CTE names from the incumbent final frame. +func newSuffixSeededIdentifiers(finalFrame pgsql.Identifier) suffixSeededIdentifiers { + prefix := string(finalFrame) + "_suffix_seeded_" + return suffixSeededIdentifiers{ + rootPresence: pgsql.Identifier(prefix + "root_presence"), + suffix: pgsql.Identifier(prefix + "suffix"), + boundaries: pgsql.Identifier(prefix + "boundaries"), + reverse: pgsql.Identifier(prefix + "reverse"), + componentReceipt: pgsql.Identifier(prefix + "component_receipt"), + } +} + +// selectedFixedSuffixDecision returns the first traversal decision that selected suffix-seeded reverse search. +func selectedFixedSuffixDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && decision.SelectedStrategy == optimize.ExpansionSearchSuffixSeededReverse { + return decision, true + } + } + + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// selectedGuardedFixedSuffixDecision returns a tool-enabled suffix policy +// without treating its runtime decision as a compile-time selected arm. The +// decision's selection mode distinguishes guarded execution from true shadow. +func selectedGuardedFixedSuffixDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && + decision.Family == "fixed_suffix_expansion" && + decision.CandidateStrategy == optimize.ExpansionSearchSuffixSeededReverse && + supportedExpansionOrientationPolicy(decision.EmittedPolicy) { + return decision, true + } + } + + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// selectedSuffixReverseGuardDecision returns the statically selected, +// full-path-only suffix reverse guard without conflating it with an orientation +// policy. Runtime admission still chooses between exact reverse and exact +// stepwise-forward execution. +func selectedSuffixReverseGuardDecision(part *PatternPart, decisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision) (optimize.ExpansionSearchStrategyDecision, bool) { + for _, step := range part.TraversalSteps { + if step == nil || !step.HasSourceTarget { + continue + } + if decision, found := decisions[step.SourceTarget]; found && + (decision.EmittedPolicy == optimize.ExpansionSearchPolicySuffixReverseGuardV1 || + decision.EmittedPolicy == optimize.ExpansionSearchPolicySuffixReverseRetryV1 || + decision.EmittedPolicy == optimize.ExpansionSearchPolicyTopologyFixedSuffixV1 || + decision.EmittedPolicy == optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) && + decision.ObservationMode == optimize.ExpansionSearchObservationFullPath { + return decision, true + } + } + + return optimize.ExpansionSearchStrategyDecision{}, false +} + +// rewriteTraversalPatternAsSuffixSeededReverse replaces a qualified incumbent frame chain with fixed-suffix reverse search. +func (s *Translator) rewriteTraversalPatternAsSuffixSeededReverse(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if len(part.TraversalSteps) != decision.SuffixEndStep+1 || decision.SuffixLength != 3 || decision.Target.StepIndex != 0 { + return fmt.Errorf("forced suffix-seeded reverse target requires one expansion followed by exactly three terminal suffix steps") + } + + expansionStep := part.TraversalSteps[decision.Target.StepIndex] + if expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.Frame.Previous == nil || !expansionStep.LeftNodeBound { + return fmt.Errorf("forced suffix-seeded reverse target requires a bound root materialized by a previous frame") + } + + suffix := part.TraversalSteps[decision.SuffixStartStep : decision.SuffixEndStep+1] + for _, step := range suffix { + if step == nil || step.Frame == nil || step.Edge == nil || step.LeftNode == nil || step.RightNode == nil { + return fmt.Errorf("forced suffix-seeded reverse target has an incomplete fixed suffix step") + } + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("forced suffix-seeded reverse target did not emit an incumbent frame chain") + } + incumbentFinal := ctes[len(ctes)-1] + if incumbentFinal.Alias.Name != suffix[len(suffix)-1].Frame.Binding.Identifier { + return fmt.Errorf("forced suffix-seeded reverse final frame mismatch: expected %s but found %s", suffix[len(suffix)-1].Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + + finalSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("forced suffix-seeded reverse final frame must be a select") + } + + ids := newSuffixSeededIdentifiers(incumbentFinal.Alias.Name) + rootFrame := expansionStep.Frame.Previous.Binding.Identifier + suffixSeededQuery, err := s.buildSuffixSeededReverseQuery(part, decision, expansionStep, suffix, rootFrame, ids, finalSelect.Projection) + if err != nil { + return err + } + if part.PatternBinding != nil { + part.PatternBinding.DataType = pgsql.PathComposite + part.PatternBinding.Dependencies = nil + part.PatternBinding.MaterializedBy(suffix[len(suffix)-1].Frame) + } + + replacement := pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: suffixSeededQuery, + } + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:firstCTE], replacement) + s.recordExpansionSearchStrategy(decision.Target, optimize.ExpansionSearchSuffixSeededReverse) + return nil +} + +// rewriteTraversalPatternAsGuardedSuffixOrientation emits a tool-selected, +// versioned orientation policy. Guarded mode wraps the incumbent and reverse +// arm in disjoint runtime gates; shadow mode executes the same bounded probes +// but leaves the incumbent as the only traversal arm. +func (s *Translator) rewriteTraversalPatternAsGuardedSuffixOrientation(part *PatternPart, decision optimize.ExpansionSearchStrategyDecision, firstCTE int) error { + if len(part.TraversalSteps) != decision.SuffixEndStep+1 || decision.SuffixLength != 3 || decision.Target.StepIndex != 0 { + return fmt.Errorf("guarded suffix orientation requires one expansion followed by exactly three terminal suffix steps") + } + + expansionStep := part.TraversalSteps[decision.Target.StepIndex] + if expansionStep == nil || expansionStep.Expansion == nil || expansionStep.Frame == nil || expansionStep.Frame.Previous == nil || !expansionStep.LeftNodeBound || expansionStep.Edge == nil || expansionStep.LeftNode == nil { + return fmt.Errorf("guarded suffix orientation requires a complete expansion and bound root") + } + + suffix := part.TraversalSteps[decision.SuffixStartStep : decision.SuffixEndStep+1] + for _, step := range suffix { + if step == nil || step.Frame == nil || step.Edge == nil || step.LeftNode == nil || step.RightNode == nil { + return fmt.Errorf("guarded suffix orientation has an incomplete fixed suffix step") + } + } + + ctes := s.query.CurrentPart().Model.CommonTableExpressions.Expressions + if firstCTE < 0 || firstCTE >= len(ctes) { + return fmt.Errorf("guarded suffix orientation did not emit an incumbent frame chain") + } + incumbentChain := append([]pgsql.CommonTableExpression(nil), ctes[firstCTE:]...) + incumbentFinal := incumbentChain[len(incumbentChain)-1] + if incumbentFinal.Alias.Name != suffix[len(suffix)-1].Frame.Binding.Identifier { + return fmt.Errorf("guarded suffix orientation final frame mismatch: expected %s but found %s", suffix[len(suffix)-1].Frame.Binding.Identifier, incumbentFinal.Alias.Name) + } + incumbentSelect, ok := incumbentFinal.Query.Body.(pgsql.Select) + if !ok { + return fmt.Errorf("guarded suffix orientation final frame must be a select") + } + + ids := newExpansionOrientationIdentifiers(incumbentFinal.Alias.Name) + rootFrame := expansionStep.Frame.Previous.Binding.Identifier + var ( + query pgsql.Query + err error + ) + if decision.SelectionMode == "shadow_tool" { + query, err = s.buildShadowSuffixOrientationQuery( + decision, + expansionStep, + suffix, + rootFrame, + ids, + incumbentChain, + incumbentFinal.Alias.Name, + incumbentSelect.Projection, + ) + } else { + query, err = s.buildGuardedSuffixOrientationQuery( + part, + decision, + expansionStep, + suffix, + rootFrame, + ids, + incumbentChain, + incumbentFinal.Alias.Name, + incumbentSelect.Projection, + ) + } + if err != nil { + return err + } + materializedCandidatePath := decision.SelectionMode != "shadow_tool" && part.PatternBinding != nil + if materializedCandidatePath { + part.PatternBinding.DataType = pgsql.PathComposite + part.PatternBinding.Dependencies = nil + part.PatternBinding.MaterializedBy(suffix[len(suffix)-1].Frame) + } + + s.query.CurrentPart().Model.CommonTableExpressions.Expressions = append(ctes[:firstCTE], pgsql.CommonTableExpression{ + Alias: incumbentFinal.Alias, + Query: query, + }) + s.recordExpansionSearchPolicy(decision.Target, decision.EmittedPolicy) + return nil +} + +// buildShadowSuffixOrientationQuery executes only bounded policy probes and +// the exact incumbent. Named, mutually exclusive marker CTEs preserve the +// policy's would_select_reverse result for plan-derived diagnostic metadata; +// they never dispatch the reverse traversal candidate. +func (s *Translator) buildShadowSuffixOrientationQuery( + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids expansionOrientationIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + if decision.ProbeCaps.RootRowLimit <= 0 || decision.ProbeCaps.ReverseSeedRowLimit <= 0 || decision.ProbeCaps.DirectionalDegreeRowLimit <= 0 { + return pgsql.Query{}, fmt.Errorf("shadow suffix orientation requires positive immutable probe caps") + } + + localEdgeConstraint, externalEdgeConstraint := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + if externalEdgeConstraint != nil { + return pgsql.Query{}, fmt.Errorf("shadow suffix orientation relationship predicate is not local") + } + + suffixIDs := suffixSeededIdentifiers{ + rootPresence: ids.rootPresence, + suffix: ids.suffixProbe, + boundaries: ids.boundaries, + } + rootProbe := buildExpansionOrientationRootProbe(rootFrame, expansionStep.LeftNode, ids, decision.ProbeCaps.RootRowLimit) + rootPresence := buildExpansionOrientationRootPresence(ids) + suffixProbe, err := s.buildFixedSuffixEvidenceProbeCTE(expansionStep, suffix, suffixIDs, decision.ProbeCaps.ReverseSeedRowLimit) + if err != nil { + return pgsql.Query{}, err + } + boundaries := buildFixedSuffixBoundariesCTE(suffixIDs) + forwardDegree := buildExpansionOrientationDegreeProbe( + ids.forwardDegreeProbe, + ids.rootProbe, + orientationRootID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeStartIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + reverseDegree := buildExpansionOrientationDegreeProbe( + ids.reverseDegreeProbe, + ids.boundaries, + fixedSuffixBoundaryID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeEndIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + metrics := buildExpansionOrientationMetrics(ids, decision.ProbeCaps) + policyDecision, err := buildExpansionOrientationDecision(ids, decision.EmittedPolicy, decision.MaximumDepth) + if err != nil { + return pgsql.Query{}, err + } + shadowMarkers := buildExpansionOrientationShadowMarkers(ids) + incumbent, incumbentOutput, err := buildExpansionOrientationIncumbentCTE(ids, incumbentChain, incumbentFinal, incumbentProjection, nil) + if err != nil { + return pgsql.Query{}, err + } + gatedIncumbent, err := gateQueryBehindMarker( + ids.executedIncumbent, + ids.incumbentBody, + pgsql.Query{Body: pgsql.Select{ + Projection: incumbentOutput, + From: []pgsql.FromClause{tableFrom(ids.incumbent)}, + }}, + incumbentOutput, + ) + if err != nil { + return pgsql.Query{}, err + } + + expressions := []pgsql.CommonTableExpression{ + rootProbe, + rootPresence, + suffixProbe, + boundaries, + forwardDegree, + reverseDegree, + metrics, + policyDecision, + } + expressions = append(expressions, shadowMarkers...) + expressions = append(expressions, incumbent) + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: expressions, + }, + Body: gatedIncumbent, + }, nil +} + +// buildGuardedSuffixOrientationQuery emits bounded evidence, a versioned +// decision, reverse-state admission, and strictly complementary candidate and +// incumbent branches. No candidate row can pass until every evidence and +// state sentinel proves completeness. +func (s *Translator) buildGuardedSuffixOrientationQuery( + part *PatternPart, + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids expansionOrientationIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + if decision.ProbeCaps.RootRowLimit <= 0 || decision.ProbeCaps.ReverseSeedRowLimit <= 0 || decision.ProbeCaps.DirectionalDegreeRowLimit <= 0 || decision.Admission.StateLimit <= 0 { + return pgsql.Query{}, fmt.Errorf("guarded suffix orientation requires positive immutable probe and admission caps") + } + + localEdgeConstraint, externalEdgeConstraint := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + if externalEdgeConstraint != nil { + return pgsql.Query{}, fmt.Errorf("guarded suffix orientation relationship predicate is not local") + } + + suffixIDs := suffixSeededIdentifiers{ + rootPresence: ids.rootPresence, + suffix: ids.suffixProbe, + boundaries: ids.boundaries, + reverse: ids.reverse, + } + rootProbe := buildExpansionOrientationRootProbe(rootFrame, expansionStep.LeftNode, ids, decision.ProbeCaps.RootRowLimit) + rootPresence := buildExpansionOrientationRootPresence(ids) + suffixProbe, err := s.buildFixedSuffixProbeCTE(expansionStep, suffix, suffixIDs, decision.ProbeCaps.ReverseSeedRowLimit) + if err != nil { + return pgsql.Query{}, err + } + boundaries := buildFixedSuffixBoundariesCTE(suffixIDs) + forwardDegree := buildExpansionOrientationDegreeProbe( + ids.forwardDegreeProbe, + ids.rootProbe, + orientationRootID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeStartIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + reverseDegree := buildExpansionOrientationDegreeProbe( + ids.reverseDegreeProbe, + ids.boundaries, + fixedSuffixBoundaryID, + expansionStep.Edge.Identifier, + expansionStep.Expansion.EdgeEndIdentifier, + localEdgeConstraint, + decision.ProbeCaps.DirectionalDegreeRowLimit, + ) + metrics := buildExpansionOrientationMetrics(ids, decision.ProbeCaps) + policyDecision, err := buildExpansionOrientationDecision(ids, decision.EmittedPolicy, decision.MaximumDepth) + if err != nil { + return pgsql.Query{}, err + } + reverseSeed := buildExpansionOrientationReverseSeed(ids) + reverseIDs := suffixIDs + reverseIDs.boundaries = ids.reverseSeed + materializeOrderedPath := part.PatternBinding != nil + reverse, err := buildSuffixSeededReverseCTE(expansionStep, decision, reverseIDs, "", "", materializeOrderedPath) + if err != nil { + return pgsql.Query{}, err + } + states := expansionOrientationStateProbe(decision, ids, materializeOrderedPath) + admission := buildExpansionOrientationAdmission(ids, decision.Admission.StateLimit) + executionMarkers := buildExpansionOrientationExecutionMarkers(ids) + var incumbentExtras pgsql.Projection + if materializeOrderedPath { + incumbentPath, pathErr := expressionForPathComposite(part.PatternBinding, s.scope) + if pathErr != nil { + return pgsql.Query{}, pathErr + } + incumbentExtras = append(incumbentExtras, &pgsql.AliasedExpression{ + Expression: incumbentPath, + Alias: models.OptionalValue(part.PatternBinding.Identifier), + }) + } + incumbent, fallbackProjection, err := buildExpansionOrientationIncumbentCTE(ids, incumbentChain, incumbentFinal, incumbentProjection, incumbentExtras) + if err != nil { + return pgsql.Query{}, err + } + candidateProjection, err := suffixSeededFinalProjection(part, expansionStep, suffix, rootFrame, suffixIDs, ids.states, incumbentProjection, nil) + if err != nil { + return pgsql.Query{}, err + } + if materializeOrderedPath { + candidateProjection = append(candidateProjection, &pgsql.AliasedExpression{ + Expression: suffixSeededOrderedPathComposite(s.graphID, expansionStep, suffix, suffixIDs, ids.states), + Alias: models.OptionalValue(part.PatternBinding.Identifier), + }) + } + + suffixEdgeIDs := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + for _, step := range suffix { + suffixEdgeIDs.Values = append(suffixEdgeIDs.Values, pgsql.CompoundIdentifier{ids.suffixProbe, step.Edge.Identifier}) + } + var candidateWhere pgsql.Expression = pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8), + ) + candidateWhere = pgsql.OptionalAnd(candidateWhere, pgd.Not(pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.states, expansionPath}, + pgsql.OperatorArrayOverlap, + suffixEdgeIDs, + ))) + + candidate := pgsql.Select{ + Projection: candidateProjection, + From: []pgsql.FromClause{ + { + Source: pgsql.TableReference{Name: rootFrame.AsCompoundIdentifier()}, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{Name: ids.states.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(rootFrame, expansionStep.LeftNode), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.states, expansionNextID}, + ), + }, + }, + { + Table: pgsql.TableReference{Name: ids.suffixProbe.AsCompoundIdentifier()}, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.suffixProbe, fixedSuffixBoundaryID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.states, fixedSuffixBoundaryID}, + ), + }, + }, + }, + }, + }, + Where: candidateWhere, + } + candidate, err = gateQueryBehindMarker( + ids.executedCandidate, + ids.candidateBody, + pgsql.Query{Body: candidate}, + candidateProjection, + ) + if err != nil { + return pgsql.Query{}, err + } + + fallback := pgsql.Select{ + Projection: fallbackProjection, + From: []pgsql.FromClause{tableFrom(ids.incumbent)}, + } + fallback, err = gateQueryBehindMarker( + ids.executedIncumbent, + ids.incumbentBody, + pgsql.Query{Body: fallback}, + fallbackProjection, + ) + if err != nil { + return pgsql.Query{}, err + } + expressions := []pgsql.CommonTableExpression{ + rootProbe, + rootPresence, + suffixProbe, + boundaries, + forwardDegree, + reverseDegree, + metrics, + policyDecision, + } + expressions = append(expressions, reverseSeed...) + expressions = append(expressions, reverse, states, admission) + expressions = append(expressions, executionMarkers...) + expressions = append(expressions, incumbent) + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: expressions, + }, + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: candidate, + ROperand: fallback, + }, + }, nil +} + +// buildFixedSuffixBoundariesCTE builds fixed suffix boundaries cte. +func buildFixedSuffixBoundariesCTE(ids suffixSeededIdentifiers) pgsql.CommonTableExpression { + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.boundaries}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{ + Distinct: true, + Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.suffix, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.suffix)}, + }}, + } +} + +// buildExpansionOrientationIncumbentCTE nests the original unmodified frame +// chain as the exact fallback. It has no tournament cap and preserves the +// incumbent's projection and bag semantics. +func buildExpansionOrientationIncumbentCTE( + ids expansionOrientationIdentifiers, + incumbentChain []pgsql.CommonTableExpression, + incumbentFinal pgsql.Identifier, + incumbentProjection pgsql.Projection, + extraProjection pgsql.Projection, +) (pgsql.CommonTableExpression, pgsql.Projection, error) { + projection := make(pgsql.Projection, 0, len(incumbentProjection)+len(extraProjection)) + fallback := make(pgsql.Projection, 0, len(incumbentProjection)+len(extraProjection)) + for _, item := range incumbentProjection { + alias, ok := selectItemAlias(item) + if !ok { + return pgsql.CommonTableExpression{}, nil, fmt.Errorf("guarded suffix orientation incumbent projection contains an unaliased item %T", item) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{incumbentFinal, alias}, + Alias: models.OptionalValue(alias), + }) + fallback = append(fallback, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.incumbent, alias}, + Alias: models.OptionalValue(alias), + }) + } + for _, item := range extraProjection { + alias, ok := selectItemAlias(item) + if !ok { + return pgsql.CommonTableExpression{}, nil, fmt.Errorf("guarded suffix orientation extra incumbent projection contains an unaliased item %T", item) + } + projection = append(projection, item) + fallback = append(fallback, &pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.incumbent, alias}, + Alias: models.OptionalValue(alias), + }) + } + + incumbentQuery := pgsql.Query{ + CommonTableExpressions: &pgsql.With{Expressions: incumbentChain}, + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{tableFrom(incumbentFinal)}, + }, + } + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.incumbent}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: incumbentQuery, + }, fallback, nil +} + +// buildSuffixSeededReverseQuery joins bound roots to reverse states seeded by materialized fixed-suffix matches. +func (s *Translator) buildSuffixSeededReverseQuery( + part *PatternPart, + decision optimize.ExpansionSearchStrategyDecision, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids suffixSeededIdentifiers, + incumbentProjection pgsql.Projection, +) (pgsql.Query, error) { + rootPresence := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.rootPresence, + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: []pgsql.SelectItem{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(rootFrame)}, + }, + Limit: pgsql.NewLiteral(int64(1), pgsql.Int8), + }, + } + + suffixCTE, err := s.buildFixedSuffixCTE(expansionStep, suffix, ids) + if err != nil { + return pgsql.Query{}, err + } + + boundaries := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.boundaries, + }, + Materialized: &pgsql.Materialized{ + Materialized: true, + }, + Query: pgsql.Query{ + Body: pgsql.Select{ + Distinct: true, + Projection: []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{ids.suffix, fixedSuffixBoundaryID}, + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }}, + From: []pgsql.FromClause{tableFrom(ids.suffix)}, + }, + }, + } + materializeOrderedPath := part.PatternBinding != nil + reverse, err := buildSuffixSeededReverseCTE(expansionStep, decision, ids, "", "", materializeOrderedPath) + if err != nil { + return pgsql.Query{}, err + } + + projection, err := suffixSeededFinalProjection(part, expansionStep, suffix, rootFrame, ids, ids.reverse, incumbentProjection, nil) + if err != nil { + return pgsql.Query{}, err + } + if materializeOrderedPath { + projection = append(projection, &pgsql.AliasedExpression{ + Expression: suffixSeededOrderedPathComposite(s.graphID, expansionStep, suffix, ids, ids.reverse), + Alias: models.OptionalValue(part.PatternBinding.Identifier), + }) + } + + var componentReceipt *pgsql.CommonTableExpression + if decision.SelectionMode == "component_tool" { + receipt := pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{Name: ids.componentReceipt}, + Materialized: &pgsql.Materialized{Materialized: true}, + Query: pgsql.Query{Body: pgsql.Select{Projection: pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: suffixGuardAttestationFn, + Parameters: []pgsql.Expression{ + pgsql.NewLiteral("suffix_route_component", pgsql.Text), + pgsql.NewLiteral(false, pgsql.Boolean), + pgsql.NewLiteral(string(optimize.ExpansionSearchSuffixSeededReverse), pgsql.Text), + }, + }, + Alias: models.OptionalValue(pgsql.Identifier("runtime_receipt")), + }}}}, + } + componentReceipt = &receipt + } + + suffixEdgeIDs := pgsql.ArrayLiteral{ + CastType: pgsql.Int8Array, + } + for _, step := range suffix { + suffixEdgeIDs.Values = append(suffixEdgeIDs.Values, pgsql.CompoundIdentifier{ids.suffix, step.Edge.Identifier}) + } + + reversePath := pgsql.CompoundIdentifier{ids.reverse, expansionPath} + finalWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, + pgsql.OperatorGreaterThanOrEqualTo, + pgsql.NewLiteral(decision.MinimumDepth, pgsql.Int8), + ), + pgd.Not(pgsql.NewBinaryExpression(reversePath, pgsql.OperatorArrayOverlap, suffixEdgeIDs)), + ) + + expressions := []pgsql.CommonTableExpression{ + rootPresence, + suffixCTE, + boundaries, + reverse, + } + if componentReceipt != nil { + expressions = append(expressions, *componentReceipt) + } + from := []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: rootFrame.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: pgsql.TableReference{ + Name: ids.reverse.AsCompoundIdentifier(), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + projectedNodeIDReference(rootFrame, expansionStep.LeftNode), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.reverse, expansionNextID}, + ), + }, + }, + { + Table: pgsql.TableReference{ + Name: ids.suffix.AsCompoundIdentifier(), + }, + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.suffix, fixedSuffixBoundaryID}, + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{ids.reverse, fixedSuffixBoundaryID}, + ), + }, + }, + }, + }} + if componentReceipt != nil { + finalWhere = pgsql.OptionalAnd(finalWhere, pgsql.ExistsExpression{Subquery: pgsql.Subquery{Query: pgsql.Query{ + Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.NewLiteral(int64(1), pgsql.Int8)}, + From: []pgsql.FromClause{tableFrom(ids.componentReceipt)}, + }, + }}}) + } + + return pgsql.Query{ + CommonTableExpressions: &pgsql.With{ + Recursive: true, + Expressions: expressions, + }, + Body: pgsql.Select{ + Projection: projection, + From: from, + Where: finalWhere, + }, + }, nil +} + +// buildFixedSuffixCTE materializes every locally valid fixed-suffix path and its boundary node. +func (s *Translator) buildFixedSuffixCTE(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers) (pgsql.CommonTableExpression, error) { + return s.buildFixedSuffixCTEWithOptions(expansionStep, suffix, ids, false, false, 0) +} + +// buildFixedSuffixProbeCTE builds a bounded suffix probe used to guard the specialized branch. +func (s *Translator) buildFixedSuffixProbeCTE(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, rowLimit int64) (pgsql.CommonTableExpression, error) { + return s.buildFixedSuffixCTEWithOptions(expansionStep, suffix, ids, false, false, rowLimit) +} + +// buildFixedSuffixEvidenceProbeCTE preserves the suffix join and row +// multiplicity used by orientation scoring while projecting only the boundary +// ID needed by the shadow policy. Candidate execution is impossible in shadow +// mode, so materializing edge IDs and node composites would be pure overhead. +func (s *Translator) buildFixedSuffixEvidenceProbeCTE(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, rowLimit int64) (pgsql.CommonTableExpression, error) { + return s.buildFixedSuffixCTEWithOptions(expansionStep, suffix, ids, false, true, rowLimit) +} + +// buildFixedSuffixCTEWithOptions builds the fixed-suffix join chain with an +// optional evidence-only projection and row limit. +func (s *Translator) buildFixedSuffixCTEWithOptions(expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, projectNodeIDs, evidenceOnly bool, rowLimit int64) (pgsql.CommonTableExpression, error) { + localScope := pgsql.NewIdentifierSet() + for _, step := range suffix { + localScope.Add(step.Edge.Identifier) + localScope.Add(step.LeftNode.Identifier) + localScope.Add(step.RightNode.Identifier) + } + + projection := pgsql.Projection{&pgsql.AliasedExpression{ + Expression: pgd.EntityID(suffix[0].LeftNode.Identifier), + Alias: models.OptionalValue(fixedSuffixBoundaryID), + }} + if !evidenceOnly { + for _, step := range suffix { + projection = append(projection, &pgsql.AliasedExpression{ + Expression: pgd.EntityID(step.Edge.Identifier), + Alias: models.OptionalValue(step.Edge.Identifier), + }) + } + for idx, step := range suffix { + binding := step.RightNode + expression := suffixSeededNodeValue(binding) + if projectNodeIDs { + expression = pgd.EntityID(binding.Identifier) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: expression, + Alias: models.OptionalValue(binding.Identifier), + }) + if idx == 0 { + leftExpression := suffixSeededNodeValue(step.LeftNode) + if projectNodeIDs { + leftExpression = pgd.EntityID(step.LeftNode.Identifier) + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: leftExpression, + Alias: models.OptionalValue(step.LeftNode.Identifier), + }) + } + } + } + + first := suffix[0] + from := pgsql.FromClause{ + Source: pgsql.TableReference{ + Name: ids.rootPresence.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionEdgeTableReference(first.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewLiteral(true, pgsql.Boolean), + }, + }, + { + Table: expansionNodeTableReference(first.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(first.LeftNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{first.Edge.Identifier, pgsql.ColumnStartID}, + ), + }, + }, + { + Table: expansionNodeTableReference(first.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(first.RightNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{first.Edge.Identifier, pgsql.ColumnEndID}, + ), + }, + }, + }, + } + for _, step := range suffix[1:] { + from.Joins = append(from.Joins, + pgsql.Join{ + Table: expansionEdgeTableReference(step.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{step.Edge.Identifier, pgsql.ColumnStartID}, pgsql.OperatorEquals, pgd.EntityID(step.LeftNode.Identifier), + ), + }, + }, + pgsql.Join{ + Table: expansionNodeTableReference(step.RightNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(step.RightNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{step.Edge.Identifier, pgsql.ColumnEndID}, + ), + }, + }, + ) + } + + var boundaryConstraint pgsql.Expression + if expansionStep.Expansion != nil { + boundaryConstraint = expansionStep.Expansion.TerminalNodeConstraints + } + localBoundaryConstraint, _ := partitionConstraintByLocality(boundaryConstraint, localScope) + where := localBoundaryConstraint + suffixRelationships := make([]pgsql.Identifier, 0, len(suffix)) + for _, step := range suffix { + suffixRelationships = append(suffixRelationships, step.Edge.Identifier) + localLeftConstraint, _ := partitionConstraintByLocality(step.LeftNodeConstraints, localScope) + localEdgeConstraint, _ := partitionConstraintByLocality(step.EdgeConstraints.Expression, localScope) + localRightConstraint, _ := partitionConstraintByLocality(step.RightNodeConstraints, localScope) + where = pgsql.OptionalAnd(where, localLeftConstraint) + where = pgsql.OptionalAnd(where, localEdgeConstraint) + where = pgsql.OptionalAnd(where, localRightConstraint) + } + where = pgsql.OptionalAnd(where, pairwiseRelationshipIDUniqueness(suffixRelationships)) + + query := pgsql.Query{ + Body: pgsql.Select{ + Projection: projection, + From: []pgsql.FromClause{from}, + Where: where, + }, + } + if rowLimit > 0 { + query.Limit = pgsql.NewLiteral(rowLimit+1, pgsql.Int8) + } + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.suffix, + }, + Materialized: &pgsql.Materialized{ + Materialized: true, + }, + Query: query, + }, nil +} + +// buildSuffixSeededReverseCTE recursively walks from suffix boundaries back toward bound roots without reusing edges. +func buildSuffixSeededReverseCTE(expansionStep *TraversalStep, decision optimize.ExpansionSearchStrategyDecision, ids suffixSeededIdentifiers, gateSource, gateColumn pgsql.Identifier, carryNodePath bool) (pgsql.CommonTableExpression, error) { + if expansionStep.Edge == nil || expansionStep.RightNode == nil { + return pgsql.CommonTableExpression{}, fmt.Errorf("forced suffix-seeded reverse expansion step is incomplete") + } + + emptyPath := pgsql.ArrayLiteral{ + CastType: pgsql.Int8Array, + } + seed := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}, + pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}, + pgsql.NewLiteral(int64(0), pgsql.Int8), + emptyPath, + }, + From: []pgsql.FromClause{tableFrom(ids.boundaries)}, + } + if carryNodePath { + seed.Projection = append(seed.Projection, pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pgsql.CompoundIdentifier{ids.boundaries, fixedSuffixBoundaryID}}, + CastType: pgsql.Int8Array, + }) + } + if gateSource != "" && gateColumn != "" { + seed.From = append(seed.From, tableFrom(gateSource)) + seed.Where = pgsql.CompoundIdentifier{gateSource, gateColumn} + } + + path := pgsql.CompoundIdentifier{ids.reverse, expansionPath} + localEdgeConstraint, _ := partitionConstraintByLocality( + expansionStep.Expansion.EdgeConstraints, + pgsql.AsIdentifierSet(expansionStep.Edge.Identifier), + ) + recursiveWhere := pgsql.OptionalAnd( + pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, + pgsql.OperatorLessThan, + pgsql.NewLiteral(decision.MaximumDepth, pgsql.Int8), + ), + pgsql.NewBinaryExpression( + pgd.EntityID(expansionStep.Edge.Identifier), + pgsql.OperatorNotEquals, + pgsql.NewAllExpression(path), + ), + ) + recursiveWhere = pgsql.OptionalAnd(recursiveWhere, localEdgeConstraint) + + recursive := pgsql.Select{ + Projection: []pgsql.SelectItem{ + pgsql.CompoundIdentifier{ids.reverse, fixedSuffixBoundaryID}, + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnStartID}, + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{ids.reverse, expansionDepth}, pgsql.OperatorAdd, pgsql.NewLiteral(int64(1), pgsql.Int8)), + pgsql.FunctionCall{ + Function: pgsql.Identifier("array_prepend"), + Parameters: []pgsql.Expression{ + pgd.EntityID(expansionStep.Edge.Identifier), path, + }, + CastType: pgsql.Int8Array, + }, + }, + From: []pgsql.FromClause{{ + Source: pgsql.TableReference{ + Name: ids.reverse.AsCompoundIdentifier(), + }, + Joins: []pgsql.Join{ + { + Table: expansionEdgeTableReference(expansionStep.Edge.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnEndID}, pgsql.OperatorEquals, pgsql.CompoundIdentifier{ids.reverse, expansionNextID}, + ), + }, + }, + { + Table: expansionNodeTableReference(expansionStep.LeftNode.Identifier), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.NewBinaryExpression( + pgd.EntityID(expansionStep.LeftNode.Identifier), pgsql.OperatorEquals, pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnStartID}, + ), + }, + }, + }, + }}, + Where: recursiveWhere, + } + if carryNodePath { + recursive.Projection = append(recursive.Projection, pgsql.FunctionCall{ + Function: pgsql.Identifier("array_prepend"), + Parameters: []pgsql.Expression{ + pgsql.CompoundIdentifier{expansionStep.Edge.Identifier, pgsql.ColumnStartID}, + pgsql.CompoundIdentifier{ids.reverse, expansionNodePath}, + }, + CastType: pgsql.Int8Array, + }) + } + + shape := []pgsql.Identifier{fixedSuffixBoundaryID, expansionNextID, expansionDepth, expansionPath} + if carryNodePath { + shape = append(shape, expansionNodePath) + } + + return pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: ids.reverse, + Shape: pgsql.NewRecordShape(shape), + }, + Query: pgsql.Query{ + Body: pgsql.SetOperation{ + Operator: pgsql.OperatorUnion, + All: true, + LOperand: seed, + ROperand: recursive, + }, + }, + }, nil +} + +// suffixSeededOrderedPathComposite hydrates the node and edge arrays already +// carried in path order by the reverse traversal. Keeping hydration in the +// translated statement lets PostgreSQL prune directly to the selected graph +// partitions and avoids recursively reconstructing node order from edge IDs. +func suffixSeededOrderedPathComposite(graphID int32, expansionStep *TraversalStep, suffix []*TraversalStep, ids suffixSeededIdentifiers, reverseSource pgsql.Identifier) pgsql.Expression { + const ( + pathIndex pgsql.Identifier = "_ordered_path_index" + pathNode pgsql.Identifier = "_ordered_path_node" + edgeIndex pgsql.Identifier = "_ordered_edge_index" + pathEdge pgsql.Identifier = "_ordered_path_edge" + ) + + nodeIDs := pgsql.Expression(pgsql.CompoundIdentifier{reverseSource, expansionNodePath}) + suffixNodeIDs := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + for _, step := range suffix { + suffixNodeIDs.Values = append(suffixNodeIDs.Values, projectedNodeIDReference(ids.suffix, step.RightNode)) + } + nodeIDs = pgsql.NewBinaryExpression(nodeIDs, pgsql.OperatorConcatenate, suffixNodeIDs) + + edgeIDs := pgsql.Expression(pgsql.CompoundIdentifier{reverseSource, expansionPath}) + suffixEdgeIDs := pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + for _, step := range suffix { + suffixEdgeIDs.Values = append(suffixEdgeIDs.Values, pgsql.CompoundIdentifier{ids.suffix, step.Edge.Identifier}) + } + edgeIDs = pgsql.NewBinaryExpression(edgeIDs, pgsql.OperatorConcatenate, suffixEdgeIDs) + + nodeID := &pgsql.ArrayIndex{ + Expression: pgsql.NewParenthetical(nodeIDs), + Indexes: []pgsql.Expression{pathIndex}, + CastType: pgsql.Int8, + } + nodes := pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{shortestPathNodeComposite(pathNode)}, + OrderBy: []*pgsql.OrderBy{{ + Expression: pathIndex, + Ascending: true, + }}, + CastType: pgsql.NodeCompositeArray, + }, + pgsql.ArrayLiteral{CastType: pgsql.NodeCompositeArray}, + }, + }}, + From: []pgsql.FromClause{{ + Source: pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionGenerateSubscripts, + Parameters: []pgsql.Expression{nodeIDs, pgsql.NewLiteral(1, pgsql.Int)}, + }, + Alias: models.OptionalValue(pathIndex), + }, + Joins: []pgsql.Join{{ + Table: expansionNodeTableReference(pathNode), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{pathNode, pgsql.ColumnID}, pgsql.OperatorEquals, nodeID), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{pathNode, pgsql.ColumnGraphID}, pgsql.OperatorEquals, pgsql.NewLiteral(graphID, pgsql.Int4)), + ), + }, + }}, + }}, + }}} + edgeID := &pgsql.ArrayIndex{ + Expression: pgsql.NewParenthetical(edgeIDs), + Indexes: []pgsql.Expression{edgeIndex}, + CastType: pgsql.Int8, + } + edges := pgsql.Subquery{Query: pgsql.Query{Body: pgsql.Select{ + Projection: pgsql.Projection{pgsql.FunctionCall{ + Function: pgsql.FunctionCoalesce, + Parameters: []pgsql.Expression{ + pgsql.FunctionCall{ + Function: pgsql.FunctionArrayAggregate, + Parameters: []pgsql.Expression{edgeCompositeValue(pathEdge)}, + OrderBy: []*pgsql.OrderBy{{ + Expression: edgeIndex, + Ascending: true, + }}, + CastType: pgsql.EdgeCompositeArray, + }, + pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}, + }, + }}, + From: []pgsql.FromClause{{ + Source: pgsql.AliasedExpression{ + Expression: pgsql.FunctionCall{ + Function: pgsql.FunctionGenerateSubscripts, + Parameters: []pgsql.Expression{edgeIDs, pgsql.NewLiteral(1, pgsql.Int)}, + }, + Alias: models.OptionalValue(edgeIndex), + }, + Joins: []pgsql.Join{{ + Table: expansionEdgeTableReference(pathEdge), + JoinOperator: pgsql.JoinOperator{ + JoinType: pgsql.JoinTypeInner, + Constraint: pgsql.OptionalAnd( + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{pathEdge, pgsql.ColumnID}, pgsql.OperatorEquals, edgeID), + pgsql.NewBinaryExpression(pgsql.CompoundIdentifier{pathEdge, pgsql.ColumnGraphID}, pgsql.OperatorEquals, pgsql.NewLiteral(graphID, pgsql.Int4)), + ), + }, + }}, + }}, + }}} + + return pgsql.CompositeValue{ + DataType: pgsql.PathComposite, + Values: []pgsql.Expression{ + nodes, + edges, + }, + } +} + +// suffixSeededFinalProjection reconstructs the incumbent projection from root, reverse-state, and suffix columns. +func suffixSeededFinalProjection( + part *PatternPart, + expansionStep *TraversalStep, + suffix []*TraversalStep, + rootFrame pgsql.Identifier, + ids suffixSeededIdentifiers, + reverseStateSource pgsql.Identifier, + incumbent pgsql.Projection, + suffixOverrides map[pgsql.Identifier]pgsql.Expression, +) (pgsql.Projection, error) { + suffixBindings := map[pgsql.Identifier]struct{}{} + for _, step := range suffix { + suffixBindings[step.Edge.Identifier] = struct{}{} + suffixBindings[step.LeftNode.Identifier] = struct{}{} + suffixBindings[step.RightNode.Identifier] = struct{}{} + } + + projection := make(pgsql.Projection, 0, len(incumbent)) + for _, item := range incumbent { + alias, ok := selectItemAlias(item) + if !ok { + return nil, fmt.Errorf("forced suffix-seeded reverse final projection contains an unaliased item %T", item) + } + + var expression pgsql.Expression + switch { + case expansionStep.Expansion != nil && expansionStep.Expansion.PathBinding != nil && alias == expansionStep.Expansion.PathBinding.Identifier: + expression = pgsql.CompoundIdentifier{reverseStateSource, expansionPath} + case alias == expansionStep.LeftNode.Identifier: + expression = pgsql.CompoundIdentifier{rootFrame, alias} + case suffixOverrides[alias] != nil: + expression = suffixOverrides[alias] + default: + if _, found := suffixBindings[alias]; found { + expression = pgsql.CompoundIdentifier{ids.suffix, alias} + } else { + expression = pgsql.CompoundIdentifier{rootFrame, alias} + } + } + projection = append(projection, &pgsql.AliasedExpression{ + Expression: expression, + Alias: models.OptionalValue(alias), + }) + } + + return projection, nil +} + +// selectItemAlias returns an explicit alias or the identifier naturally exposed by a select item. +func selectItemAlias(item pgsql.SelectItem) (pgsql.Identifier, bool) { + switch typed := item.(type) { + case *pgsql.AliasedExpression: + return typed.Alias.Value, typed.Alias.Set + case pgsql.AliasedExpression: + return typed.Alias.Value, typed.Alias.Set + default: + return "", false + } +} + +// suffixSeededNodeValue returns a node's scalar ID or composite value according to its projection representation. +func suffixSeededNodeValue(binding *BoundIdentifier) pgsql.Expression { + if binding.IDOnly { + return pgd.EntityID(binding.Identifier) + } + return aggregateNodeComposite(binding.Identifier) +} + +// tableFrom wraps a relation name as a single PostgreSQL FROM clause. +func tableFrom(identifier pgsql.Identifier) pgsql.FromClause { + return pgsql.FromClause{ + Source: pgsql.TableReference{ + Name: identifier.AsCompoundIdentifier(), + }, + } +} diff --git a/cypher/models/pgsql/translate/expansion_test.go b/cypher/models/pgsql/translate/expansion_test.go index 3eee1caa..a81627a5 100644 --- a/cypher/models/pgsql/translate/expansion_test.go +++ b/cypher/models/pgsql/translate/expansion_test.go @@ -22,6 +22,8 @@ func translateCypher(t *testing.T, cypher string) string { kindMapper := pgutil.NewInMemoryKindMapper() kindMapper.Put(graph.StringKind("NodeKind1")) + kindMapper.Put(graph.StringKind("EdgeKind1")) + kindMapper.Put(graph.StringKind("EdgeKind2")) query, err := frontend.ParseCypher(frontend.NewContext(), cypher) require.NoError(t, err) @@ -48,6 +50,8 @@ func TestSelfLoopExpansionInLaterFrameSeedsIndependently(t *testing.T) { require.NotContains(t, formatted, "(s0.n1)") // The self-loop identity constraint still ties the endpoints. require.Contains(t, formatted, "s2.root_id = s2.next_id") + // The projection hydrates the shared endpoint alias only once. + require.Equal(t, 2, strings.Count(formatted, "node n1"), formatted) // The carried x binding is still projected. require.Contains(t, formatted, "s1.n0 as x") } @@ -64,15 +68,45 @@ func TestSelfLoopExpansionCarriedNodeStaysBound(t *testing.T) { require.Contains(t, formatted, "(s0.n0).id = s3.root_id") } +func TestReversedExpansionPathRestoresLogicalSegmentOrder(t *testing.T) { + formatted := translateCypher(t, `MATCH p = (s:NodeKind1)-[:EdgeKind1*0..]->(g)-[:EdgeKind2]->(d:NodeKind1) WHERE d.name = 'terminal' RETURN p`) + + require.Contains(t, formatted, ".ep0 || array [", formatted) +} + const ( + // shortestPathSeedTestPreviousFrame identifies the frame that supplies bound endpoint values in seed tests. shortestPathSeedTestPreviousFrame pgsql.Identifier = "s0" - shortestPathSeedTestFrame pgsql.Identifier = "s1" - shortestPathSeedTestRoot pgsql.Identifier = "n0" - shortestPathSeedTestTerminal pgsql.Identifier = "n1" - shortestPathSeedTestOther pgsql.Identifier = "x" - shortestPathSeedTestEdge pgsql.Identifier = "e0" + + // shortestPathSeedTestFrame identifies the generated shortest-path frame in seed tests. + shortestPathSeedTestFrame pgsql.Identifier = "s1" + + // shortestPathSeedTestRoot identifies the root-node binding in seed tests. + shortestPathSeedTestRoot pgsql.Identifier = "n0" + + // shortestPathSeedTestTerminal identifies the terminal-node binding in seed tests. + shortestPathSeedTestTerminal pgsql.Identifier = "n1" + + // shortestPathSeedTestOther identifies an unrelated binding used to test locality rejection. + shortestPathSeedTestOther pgsql.Identifier = "x" + + // shortestPathSeedTestEdge identifies the relationship binding in seed tests. + shortestPathSeedTestEdge pgsql.Identifier = "e0" ) +// TestShortestDistanceColumnsCompactsOnlyIDOnlyState verifies that compact state omits root ID only when endpoint identity is already carried. +func TestShortestDistanceColumnsCompactsOnlyIDOnlyState(t *testing.T) { + require.Equal(t, + []pgsql.Identifier{expansionNextID, expansionDepth}, + shortestDistanceColumns(true).Columns, + ) + require.Equal(t, + []pgsql.Identifier{expansionRootID, expansionNextID, expansionDepth}, + shortestDistanceColumns(false).Columns, + ) +} + +// shortestPathSeedTestBoundColumn references a composite field from the fixture's preceding frame. func shortestPathSeedTestBoundColumn(nodeIdentifier pgsql.Identifier, column pgsql.Identifier) pgsql.RowColumnReference { return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{shortestPathSeedTestPreviousFrame, nodeIdentifier}, @@ -80,6 +114,7 @@ func shortestPathSeedTestBoundColumn(nodeIdentifier pgsql.Identifier, column pgs } } +// shortestPathSeedTestLocalFunctionPredicate builds a deterministic predicate that depends only on the selected node. func shortestPathSeedTestLocalFunctionPredicate(nodeIdentifier pgsql.Identifier, value string) pgsql.Expression { return pgsql.NewBinaryExpression( pgsql.FunctionCall{ @@ -94,6 +129,7 @@ func shortestPathSeedTestLocalFunctionPredicate(nodeIdentifier pgsql.Identifier, ) } +// shortestPathSeedTestExternalPredicate builds a predicate that deliberately depends on an unrelated binding. func shortestPathSeedTestExternalPredicate(nodeIdentifier pgsql.Identifier) pgsql.Expression { return pgsql.NewBinaryExpression( shortestPathSeedTestBoundColumn(nodeIdentifier, pgsql.ColumnID), @@ -102,6 +138,7 @@ func shortestPathSeedTestExternalPredicate(nodeIdentifier pgsql.Identifier) pgsq ) } +// newShortestPathSeedTestBuilder creates a shortest-path builder with deterministic fixture bindings and parameters. func newShortestPathSeedTestBuilder(leftBound, rightBound bool) (*ExpansionBuilder, *Expansion) { previousFrame := &Frame{ Binding: &BoundIdentifier{Identifier: shortestPathSeedTestPreviousFrame}, @@ -169,6 +206,26 @@ func TestShortestPathSelfEndpointGuardsUseCaseErrorHelper(t *testing.T) { require.NotContains(t, endpointPairFilterGuard, " / ") } +// TestForwardPrimerSkipsSelfEndpointGuardWhenZeroDepthIsAllowed verifies that a zero-length path may use the same root and terminal. +func TestForwardPrimerSkipsSelfEndpointGuardWhenZeroDepthIsAllowed(t *testing.T) { + builder, expansionModel := newShortestPathSeedTestBuilder(false, false) + expansionModel.UseMaterializedEndpointPairFilter = true + expansionModel.Options.MinDepth = models.OptionalValue[int64](0) + + query, _, err := builder.prepareForwardFrontPrimerQuery(expansionModel) + require.NoError(t, err) + formatted, err := format.SyntaxNode(query) + require.NoError(t, err) + require.NotContains(t, formatted, "shortest_path_self_endpoint_error") + + expansionModel.Options.MinDepth = models.OptionalValue[int64](1) + query, _, err = builder.prepareForwardFrontPrimerQuery(expansionModel) + require.NoError(t, err) + formatted, err = format.SyntaxNode(query) + require.NoError(t, err) + require.Contains(t, formatted, "shortest_path_self_endpoint_error") +} + func TestBoundRootShortestPathPrimerKeepsOnlySeedLocalConstraints(t *testing.T) { builder, expansionModel := newShortestPathSeedTestBuilder(true, false) expansionModel.PrimerNodeConstraints = pgsql.NewBinaryExpression( @@ -217,6 +274,25 @@ func TestBoundTerminalShortestPathPrimerKeepsOnlySeedLocalConstraints(t *testing require.Contains(t, formattedQuery, "(s0.n1).id = s1.next_id") } +// TestBidirectionalAllShortestPathsSingletonEndpointsPreserveMaxDepth verifies the ASP harness receives its +// depth argument before the endpoint arrays. Unlike the SP harness, ASP has no trailing allow-zero-depth flag. +func TestBidirectionalAllShortestPathsSingletonEndpointsPreserveMaxDepth(t *testing.T) { + builder, expansionModel := newShortestPathSeedTestBuilder(true, true) + expansionModel.BackwardPrimerQueryParameter = &BoundIdentifier{Identifier: "pi2"} + expansionModel.BackwardRecursiveQueryParameter = &BoundIdentifier{Identifier: "pi3"} + expansionModel.Options.MaxDepth = models.OptionalValue[int64](7) + expansionModel.SingletonRootID = pgd.IntLiteral(101) + expansionModel.SingletonTerminalID = pgd.IntLiteral(202) + + query, err := builder.buildBiDirectionalShortestPathsHarnessCall(pgsql.FunctionBidirectionalASPHarness) + require.NoError(t, err) + + formattedQuery, err := format.Statement(query, format.NewOutputBuilder()) + require.NoError(t, err) + require.Contains(t, formattedQuery, "bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, 7, array [singleton_endpoints.root_id]::int8[], array [singleton_endpoints.terminal_id]::int8[])") + require.NotContains(t, formattedQuery, "bidirectional_asp_harness(@pi0::text, @pi1::text, @pi2::text, @pi3::text, array") +} + func TestZeroDepthExpansionRejectsEdgeDependentTerminalSatisfaction(t *testing.T) { builder, expansionModel := newShortestPathSeedTestBuilder(false, false) seed := newExpansionNodeSeed(expansionSeedIdentifier(shortestPathSeedTestFrame), shortestPathSeedTestRoot, nil) diff --git a/cypher/models/pgsql/translate/expression.go b/cypher/models/pgsql/translate/expression.go index c8c5ff70..bd1ae197 100644 --- a/cypher/models/pgsql/translate/expression.go +++ b/cypher/models/pgsql/translate/expression.go @@ -10,6 +10,7 @@ import ( "github.com/specterops/dawgs/cypher/models/walk" ) +// unwrapParenthetical removes every enclosing parenthetical expression and returns the innermost operand. func unwrapParenthetical(parenthetical pgsql.Expression) pgsql.Expression { next := parenthetical @@ -26,6 +27,7 @@ func unwrapParenthetical(parenthetical pgsql.Expression) pgsql.Expression { return parenthetical } +// expressionHasCompositeProperties reports whether a data type exposes an entity properties field. func expressionHasCompositeProperties(expressionType pgsql.DataType) bool { switch expressionType { case pgsql.NodeComposite, pgsql.EdgeComposite, pgsql.ExpansionRootNode, pgsql.ExpansionEdge, pgsql.ExpansionTerminalNode: @@ -36,10 +38,12 @@ func expressionHasCompositeProperties(expressionType pgsql.DataType) bool { } } +// isCompositePropertyLookupTarget reports whether a type-hinted expression exposes composite properties. func isCompositePropertyLookupTarget(expression pgsql.TypeHinted) bool { return expressionHasCompositeProperties(expression.TypeHint()) } +// translateCompositePropertyLookup pushes a lookup of the properties field from a composite expression. func (s *Translator) translateCompositePropertyLookup(target pgsql.Expression, lookup *cypher.PropertyLookup) error { if fieldIdentifierLiteral, err := pgsql.AsLiteral(lookup.Symbol); err != nil { return err @@ -53,6 +57,8 @@ func (s *Translator) translateCompositePropertyLookup(target pgsql.Expression, l return s.treeTranslator.CompleteBinaryExpression(s.scope, pgsql.OperatorPropertyLookup) } } + +// translatePropertyLookup lowers a validated Cypher property access according to its translated atom type. func (s *Translator) translatePropertyLookup(lookup *cypher.PropertyLookup) error { if err := cypher.ValidatePropertyKeyName(lookup.Symbol); err != nil { return err @@ -157,6 +163,7 @@ func (s *Translator) translatePropertyLookup(lookup *cypher.PropertyLookup) erro return nil } +// translateCypherAssignmentOperator maps supported Cypher assignment operators to their PostgreSQL AST equivalents. func translateCypherAssignmentOperator(operator cypher.AssignmentOperator) (pgsql.Operator, error) { switch operator { case cypher.OperatorAssignment: @@ -191,6 +198,7 @@ func ExtractSyntaxNodeReferences(root pgsql.SyntaxNode) (*pgsql.IdentifierSet, e )) } +// rewriteStringWildCardLiteral escapes LIKE metacharacters in a literal string operand. func rewriteStringWildCardLiteral(expression pgsql.Expression) (pgsql.Expression, error) { switch typedExpression := expression.(type) { case pgsql.Literal: @@ -210,6 +218,7 @@ func rewriteStringWildCardLiteral(expression pgsql.Expression) (pgsql.Expression } } +// rewritePropertyLookupOperator selects JSON text extraction, JSON extraction, and casts for the requested result type. func rewritePropertyLookupOperator(propertyLookup *pgsql.BinaryExpression, dataType pgsql.DataType) pgsql.Expression { if dataType.IsArrayType() { // Ensure that array conversions use JSONB @@ -241,6 +250,7 @@ func rewritePropertyLookupOperator(propertyLookup *pgsql.BinaryExpression, dataT } } +// isJSONScalarEqualityType reports whether a scalar can be normalized to JSONB for Cypher equality. func isJSONScalarEqualityType(dataType pgsql.DataType) bool { switch dataType { case pgsql.Boolean, pgsql.Float4, pgsql.Float8, pgsql.Int, pgsql.Int2, pgsql.Int4, pgsql.Int8, pgsql.Numeric: @@ -251,6 +261,7 @@ func isJSONScalarEqualityType(dataType pgsql.DataType) bool { } } +// rewriteJSONScalarEqualityOperand converts a non-null supported scalar to comparable JSONB. func rewriteJSONScalarEqualityOperand(expression pgsql.Expression) (pgsql.Expression, bool) { if literal, isLiteral := expression.(pgsql.Literal); isLiteral && literal.Null { return nil, false @@ -271,6 +282,7 @@ func rewriteJSONScalarEqualityOperand(expression pgsql.Expression) (pgsql.Expres } } +// rewriteStringEqualityOperand accepts a non-null text expression for string-specific equality handling. func rewriteStringEqualityOperand(expression pgsql.Expression) (pgsql.Expression, bool) { if literal, isLiteral := expression.(pgsql.Literal); isLiteral && literal.Null { return nil, false @@ -285,6 +297,7 @@ func rewriteStringEqualityOperand(expression pgsql.Expression) (pgsql.Expression return expression, true } +// lookupRequiresElementType reports whether an array comparison expects a property's element type rather than its array type. func lookupRequiresElementType(typeHint pgsql.DataType, operator pgsql.Operator, otherOperand pgsql.SyntaxNode) bool { if typeHint.IsArrayType() { switch operator { @@ -301,6 +314,7 @@ func lookupRequiresElementType(typeHint pgsql.DataType, operator pgsql.Operator, return false } +// TypeCastExpression applies a type hint, rewriting property comparisons when the operator requires element typing. func TypeCastExpression(expression pgsql.Expression, dataType pgsql.DataType) (pgsql.Expression, error) { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression); isPropertyLookup { lookupTypeHint := dataType @@ -316,6 +330,24 @@ func TypeCastExpression(expression pgsql.Expression, dataType pgsql.DataType) (p return pgsql.NewTypeCast(expression, dataType), nil } +// jsonNullLiteral returns the JSONB representation of a JSON null value. +func jsonNullLiteral() pgsql.Expression { + return pgsql.NewTypeCast(pgsql.NewLiteral(pgsql.StringLiteralNull, pgsql.Text), pgsql.JSONB) +} + +// nullifyJSONPropertyLookup converts a JSON null property value to SQL NULL with NULLIF. +func nullifyJSONPropertyLookup(propertyLookup *pgsql.BinaryExpression) pgsql.Expression { + return pgsql.FunctionCall{ + Function: pgsql.FunctionNullIf, + Parameters: []pgsql.Expression{ + propertyLookup, + jsonNullLiteral(), + }, + CastType: pgsql.JSONB, + } +} + +// rewritePropertyLookupOperands assigns extraction operators and casts using the comparison's opposite operand. func rewritePropertyLookupOperands(kindMapper *contextAwareKindMapper, expression *pgsql.BinaryExpression) error { var ( leftPropertyLookup, hasLeftPropertyLookup = expressionToPropertyLookupBinaryExpression(expression.LOperand) @@ -328,6 +360,8 @@ func rewritePropertyLookupOperands(kindMapper *contextAwareKindMapper, expressio (pgsql.OperatorIsComparator(expression.Operator) || expression.Operator == pgsql.OperatorCypherNotEquals) { leftPropertyLookup.Operator = pgsql.OperatorJSONField rightPropertyLookup.Operator = pgsql.OperatorJSONField + expression.LOperand = nullifyJSONPropertyLookup(leftPropertyLookup) + expression.ROperand = nullifyJSONPropertyLookup(rightPropertyLookup) return nil } @@ -414,6 +448,7 @@ func rewritePropertyLookupOperands(kindMapper *contextAwareKindMapper, expressio return nil } +// newFunctionCallComparatorError returns a focused type-mismatch error for function comparisons with special Cypher semantics. func newFunctionCallComparatorError(functionCall pgsql.FunctionCall, operator pgsql.Operator, comparisonType pgsql.DataType) error { switch functionCall.Function { case pgsql.FunctionCoalesce: @@ -516,6 +551,7 @@ func NewExpressionTreeTranslator(kindMapper *contextAwareKindMapper) *Expression } } +// mergeUserAndTranslationConstraints combines user predicates with translator-added safety constraints. func mergeUserAndTranslationConstraints(userConstraints, translationConstraints *Constraint) *Constraint { if userConstraints.Expression != nil { // Fold the user constraints into the translation constraints wrapped in a parenthetical @@ -528,10 +564,14 @@ func mergeUserAndTranslationConstraints(userConstraints, translationConstraints return translationConstraints } +// HasAnyConstraints reports whether the supplied scope can evaluate any satisfiable user or translator constraint. func (s *ExpressionTreeTranslator) HasAnyConstraints(scope *pgsql.IdentifierSet) (bool, error) { - if hasUser, err := s.UserConstraints.HasConstraints(scope); err != nil || hasUser { - return hasUser, err + if hasUser, err := s.UserConstraints.HasConstraints(scope); err != nil { + return false, err + } else if hasUser { + return true, nil } + return s.TranslationConstraints.HasConstraints(scope) } @@ -580,6 +620,7 @@ func (s *ExpressionTreeTranslator) PopOperand() (pgsql.Expression, error) { return s.treeBuilder.PopOperand(s.kindMapper) } +// popOperandAsUserConstraint removes the next operand, normalizes bare property truth tests, and records its dependencies. func (s *ExpressionTreeTranslator) popOperandAsUserConstraint() error { if nextExpression, err := s.PopOperand(); err != nil { return err @@ -662,6 +703,7 @@ func (s *ExpressionTreeTranslator) PopBinaryExpression(operator pgsql.Operator) } } +// rewriteIdentityOperands replaces entity comparisons with comparisons of their scalar identity fields. func rewriteIdentityOperands(scope *Scope, newExpression *pgsql.BinaryExpression) error { switch typedLOperand := newExpression.LOperand.(type) { case pgsql.Identifier: @@ -759,6 +801,7 @@ func rewriteIdentityOperands(scope *Scope, newExpression *pgsql.BinaryExpression return nil } +// isPropertyLookup reports whether expression is a property-lookup binary expression, including wrapped forms. func isPropertyLookup(expression pgsql.Expression) bool { _, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression) return isPropertyLookup @@ -793,6 +836,7 @@ func isConcatenationOperation(lOperand, rOperand pgsql.Expression, lOperandType, return false } +// isEmptyArrayLiteralPropertyComparison finds a property lookup paired with an untyped empty array literal. func isEmptyArrayLiteralPropertyComparison(expression *pgsql.BinaryExpression) (*pgsql.BinaryExpression, bool) { var ( hasPropertyLookup bool @@ -821,11 +865,13 @@ func isEmptyArrayLiteralPropertyComparison(expression *pgsql.BinaryExpression) ( return propertyLookup, hasPropertyLookup && hasEmptyArrayLiteral } +// isEmptyAnyArrayLiteral reports whether expression is an empty array with no inferred element type. func isEmptyAnyArrayLiteral(expression pgsql.Expression) bool { arrayLiteral, isArrayLiteral := expression.(pgsql.ArrayLiteral) return isArrayLiteral && arrayLiteral.CastType == pgsql.AnyArray && len(arrayLiteral.Values) == 0 } +// isKnownEmptyArrayExpression reports whether expression is an untyped empty array or a parameter statically typed as NULL. func isKnownEmptyArrayExpression(expression pgsql.Expression) bool { if isEmptyAnyArrayLiteral(expression) { return true @@ -841,14 +887,12 @@ func isKnownEmptyArrayExpression(expression pgsql.Expression) bool { } } -func jsonNullLiteral() pgsql.Expression { - return pgsql.NewTypeCast(pgsql.NewLiteral(pgsql.StringLiteralNull, pgsql.Text), pgsql.JSONB) -} - +// jsonEmptyArrayLiteral returns the JSONB representation of an empty array. func jsonEmptyArrayLiteral() pgsql.Expression { return pgsql.NewTypeCast(pgsql.NewLiteral(pgsql.StringLiteralEmptyArray, pgsql.Text), pgsql.JSONB) } +// rewritePropertyLookupNullCheck preserves Cypher null semantics for missing keys and explicit JSON null values. func rewritePropertyLookupNullCheck(propertyLookup *pgsql.BinaryExpression, isNotNull bool) pgsql.Expression { propertyLookup.Operator = pgsql.OperatorJSONField @@ -880,14 +924,17 @@ func rewritePropertyLookupNullCheck(propertyLookup *pgsql.BinaryExpression, isNo )) } +// jsonFieldPropertyLookup copies a property lookup using JSONB field extraction. func jsonFieldPropertyLookup(propertyLookup *pgsql.BinaryExpression) *pgsql.BinaryExpression { return pgsql.NewBinaryExpression(propertyLookup.LOperand, pgsql.OperatorJSONField, propertyLookup.ROperand) } +// jsonTextPropertyLookup copies a property lookup using text field extraction. func jsonTextPropertyLookup(propertyLookup *pgsql.BinaryExpression) *pgsql.BinaryExpression { return pgsql.NewBinaryExpression(propertyLookup.LOperand, pgsql.OperatorJSONTextField, propertyLookup.ROperand) } +// jsonbTypeof returns a call that inspects an expression's JSONB value type. func jsonbTypeof(expression pgsql.Expression) pgsql.Expression { return pgsql.FunctionCall{ Function: pgsql.FunctionJSONBTypeof, @@ -895,6 +942,7 @@ func jsonbTypeof(expression pgsql.Expression) pgsql.Expression { } } +// jsonbStringTypeCheck reports at SQL runtime whether a property contains a JSON string. func jsonbStringTypeCheck(propertyLookup *pgsql.BinaryExpression) pgsql.Expression { return pgsql.NewBinaryExpression( jsonbTypeof(jsonFieldPropertyLookup(propertyLookup)), @@ -903,6 +951,7 @@ func jsonbStringTypeCheck(propertyLookup *pgsql.BinaryExpression) pgsql.Expressi ) } +// toJSONBTextOperand converts expression through text to a JSONB scalar for type-safe comparison. func toJSONBTextOperand(expression pgsql.Expression) pgsql.Expression { return pgsql.FunctionCall{ Function: pgsql.FunctionToJSONB, @@ -913,6 +962,7 @@ func toJSONBTextOperand(expression pgsql.Expression) pgsql.Expression { } } +// buildStringPropertyEqualityComparison compares a property's text extraction with a text operand in the original operand order. func buildStringPropertyEqualityComparison(propertyLookup *pgsql.BinaryExpression, textOperand pgsql.Expression, propertyOnLeft bool, operator pgsql.Operator) pgsql.Expression { textPropertyLookup := jsonTextPropertyLookup(propertyLookup) @@ -923,6 +973,7 @@ func buildStringPropertyEqualityComparison(propertyLookup *pgsql.BinaryExpressio return pgsql.NewBinaryExpression(textOperand, operator, textPropertyLookup) } +// buildStringPropertyEqualityPredicate recognizes string/property equality and builds its type-aware predicate. func buildStringPropertyEqualityPredicate(expression *pgsql.BinaryExpression) (pgsql.Expression, bool) { if !expression.Operator.IsIn(pgsql.OperatorEquals, pgsql.OperatorCypherNotEquals) { return nil, false @@ -948,6 +999,7 @@ func buildStringPropertyEqualityPredicate(expression *pgsql.BinaryExpression) (p return nil, false } +// buildStringPropertyComparisonPredicate guards text comparison by JSON type while preserving inequality for non-string values. func buildStringPropertyComparisonPredicate(propertyLookup *pgsql.BinaryExpression, textOperand pgsql.Expression, propertyOnLeft bool, operator pgsql.Operator) pgsql.Expression { stringComparison := buildStringPropertyEqualityComparison(propertyLookup, textOperand, propertyOnLeft, operator) @@ -983,6 +1035,7 @@ func buildStringPropertyComparisonPredicate(propertyLookup *pgsql.BinaryExpressi )) } +// buildEmptyArrayPropertyComparison compares a property with [] while retaining null taint and optional negation. func buildEmptyArrayPropertyComparison(propertyLookup *pgsql.BinaryExpression, negated bool) *pgsql.BinaryExpression { var ( emptyArrayExpression = pgsql.NewBinaryExpression( @@ -1029,6 +1082,7 @@ func buildEmptyArrayPropertyComparison(propertyLookup *pgsql.BinaryExpression, n ) } +// cypherStringPredicateTextOperand converts a predicate operand to text while retaining null propagation. func cypherStringPredicateTextOperand(operand pgsql.Expression) (pgsql.Expression, error) { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(operand); isPropertyLookup { propertyLookup.Operator = pgsql.OperatorJSONTextField @@ -1042,6 +1096,7 @@ func cypherStringPredicateTextOperand(operand pgsql.Expression) (pgsql.Expressio return pgsql.NewTypeCast(operand, pgsql.Text), nil } +// cypherStringPredicateFunction maps a Cypher string predicate operator to its PostgreSQL helper function. func cypherStringPredicateFunction(function pgsql.Identifier, lOperand, rOperand pgsql.Expression) (pgsql.Expression, error) { leftText, err := cypherStringPredicateTextOperand(lOperand) if err != nil { @@ -1063,6 +1118,7 @@ func cypherStringPredicateFunction(function pgsql.Identifier, lOperand, rOperand }, nil } +// rewriteBinaryExpression applies operator-specific casts, wildcard escaping, and Cypher null semantics before pushing the result. func (s *ExpressionTreeTranslator) rewriteBinaryExpression(newExpression *pgsql.BinaryExpression) error { switch newExpression.Operator { case pgsql.OperatorAdd: diff --git a/cypher/models/pgsql/translate/expression_test.go b/cypher/models/pgsql/translate/expression_test.go index 9c9df618..393b94af 100644 --- a/cypher/models/pgsql/translate/expression_test.go +++ b/cypher/models/pgsql/translate/expression_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" ) +// mustAsLiteral converts value to a PostgreSQL literal and panics when the value type is unsupported. func mustAsLiteral(value any) pgsql.Literal { if literal, err := pgsql.AsLiteral(value); err != nil { panic(fmt.Sprintf("%v", err)) @@ -225,6 +226,7 @@ func TestInferUnaryExpressionType(t *testing.T) { } } +// TestInferWrappedExpressionType verifies that wrappers preserve or derive the data type of their enclosed expressions. func TestInferWrappedExpressionType(t *testing.T) { testCases := []struct { Name string @@ -284,6 +286,17 @@ func TestInferWrappedExpressionType(t *testing.T) { Name: "all expression over scalar", ExpectedType: pgsql.UnknownDataType, Expression: pgsql.NewAllExpression(mustAsLiteral(int64(1))), + }, { + Name: "case expression ignores null branch during inference", + ExpectedType: pgsql.Int, + Expression: pgsql.Case{ + Conditions: []pgsql.Expression{mustAsLiteral(true)}, + Then: []pgsql.Expression{pgsql.FunctionCall{ + Function: pgsql.FunctionJSONBArrayLength, + CastType: pgsql.Int, + }}, + Else: pgsql.NullLiteral(), + }, }} for _, nextCase := range testCases { @@ -296,6 +309,7 @@ func TestInferWrappedExpressionType(t *testing.T) { } } +// TestPropertyLookupEqualityScalarRewrites verifies scalar equality operators receive type-aware property extraction. func TestPropertyLookupEqualityScalarRewrites(t *testing.T) { var ( propertyLookup = func(property string) *pgsql.BinaryExpression { @@ -390,7 +404,13 @@ func TestPropertyLookupEqualityScalarRewrites(t *testing.T) { LOperand: propertyLookup("left"), Operator: pgsql.OperatorEquals, ROperand: propertyLookup("right"), - Expected: "(n.properties -> 'left') = (n.properties -> 'right')", + Expected: "nullif((n.properties -> 'left'), ('null')::jsonb)::jsonb = nullif((n.properties -> 'right'), ('null')::jsonb)::jsonb", + }, { + Name: "property ordering treats JSON null as SQL null", + LOperand: propertyLookup("left"), + Operator: pgsql.OperatorLessThan, + ROperand: propertyLookup("right"), + Expected: "nullif((n.properties -> 'left'), ('null')::jsonb)::jsonb < nullif((n.properties -> 'right'), ('null')::jsonb)::jsonb", }} ) @@ -500,6 +520,7 @@ func TestExpressionTreeTranslator(t *testing.T) { validateConstraints(t, treeTranslator, idents, expectedTranslation) } +// validateConstraints requires the generated constraint collection to contain exactly the expected SQL expressions. func validateConstraints(t *testing.T, constraintTracker *translate.ExpressionTreeTranslator, idents *pgsql.IdentifierSet, expectedTranslation string) { constraint, err := constraintTracker.ConsumeConstraintsFromVisibleSet(idents) diff --git a/cypher/models/pgsql/translate/format.go b/cypher/models/pgsql/translate/format.go index f750b8c1..3d0cfdda 100644 --- a/cypher/models/pgsql/translate/format.go +++ b/cypher/models/pgsql/translate/format.go @@ -11,11 +11,12 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/format" ) +// Translated renders a translation result as PostgreSQL for its target graph. func Translated(translation Result) (string, error) { - return format.Statement(translation.Statement, format.NewOutputBuilder()) + return format.Statement(translation.Statement, format.NewOutputBuilder().WithTargetGraph(translation.GraphID)) } -// postgres comments can be terminated by \r, \n, or both per the source: +// newlineToCommentReplacer prefixes every PostgreSQL line-comment continuation after \r, \n, or both, per the scanner source: // https://github.com/postgres/postgres/blob/824d5f6241ea7a0a85c9d2b3d27beb78e42a36ab/src/backend/parser/scan.l#L186-L211 var newlineToCommentReplacer = strings.NewReplacer( "\r\n", "\n-- ", @@ -23,6 +24,7 @@ var newlineToCommentReplacer = strings.NewReplacer( "\n", "\n-- ", ) +// FromCypher renders a Cypher query as a SQL comment followed by its PostgreSQL translation. func FromCypher(ctx context.Context, regularQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, stripLiterals bool, graphID int32) (format.Formatted, error) { var ( output = &bytes.Buffer{} @@ -53,7 +55,7 @@ func FromCypher(ctx context.Context, regularQuery *cypher.RegularQuery, kindMapp if translation, err := Translate(ctx, regularQuery, kindMapper, nil, graphID); err != nil { return format.Formatted{}, err - } else if sqlQuery, err := format.Statement(translation.Statement, format.NewOutputBuilder()); err != nil { + } else if sqlQuery, err := format.Statement(translation.Statement, format.NewOutputBuilder().WithTargetGraph(translation.GraphID)); err != nil { return format.Formatted{}, err } else { output.WriteString(sqlQuery) diff --git a/cypher/models/pgsql/translate/format_test.go b/cypher/models/pgsql/translate/format_test.go index 9958bab9..f163fc60 100644 --- a/cypher/models/pgsql/translate/format_test.go +++ b/cypher/models/pgsql/translate/format_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestFromCypherProperlyEscapesDebugComment verifies that every source-query line remains inside the emitted PostgreSQL comment. func TestFromCypherProperlyEscapesDebugComment(t *testing.T) { t.Parallel() diff --git a/cypher/models/pgsql/translate/function.go b/cypher/models/pgsql/translate/function.go index 319ac897..1f8290a5 100644 --- a/cypher/models/pgsql/translate/function.go +++ b/cypher/models/pgsql/translate/function.go @@ -11,6 +11,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// legacyToIntegerFunction identifies the legacy spelling normalized to Cypher's toInteger function. const legacyToIntegerFunction = "toint" func SymbolsFor(node pgsql.SyntaxNode) (*pgsql.SymbolTable, error) { @@ -27,6 +28,7 @@ func SymbolsFor(node pgsql.SyntaxNode) (*pgsql.SymbolTable, error) { })) } +// asFunctionCall unwraps parentheses and returns a PostgreSQL function call when expression contains one. func asFunctionCall(node pgsql.SyntaxNode) (pgsql.FunctionCall, bool) { switch typedNode := node.(type) { case pgsql.FunctionCall: @@ -110,6 +112,7 @@ func ContainsAggregateFunction(node pgsql.SyntaxNode) (bool, error) { })) } +// appendIfReferencedGroupByExpression appends expression to GROUP BY only when it contains a binding reference. func appendIfReferencedGroupByExpression(groupByExpressions []pgsql.Expression, expression pgsql.Expression) ([]pgsql.Expression, error) { if references, err := ExtractSyntaxNodeReferences(expression); err != nil { return nil, err @@ -120,6 +123,7 @@ func appendIfReferencedGroupByExpression(groupByExpressions []pgsql.Expression, } } +// appendNonAggregateGroupByExpressions adds non-aggregate projection expressions required by PostgreSQL grouping rules. func appendNonAggregateGroupByExpressions(groupByExpressions []pgsql.Expression, expressions ...pgsql.Expression) ([]pgsql.Expression, error) { for _, expression := range expressions { nextGroupByExpressions, err := NonAggregateGroupByExpressions(expression) @@ -272,6 +276,7 @@ func NonAggregateGroupByExpressions(expression pgsql.Expression) ([]pgsql.Expres } } +// bindingExpressionType returns the effective data type of a bound identifier expression. func bindingExpressionType(binding *BoundIdentifier) pgsql.DataType { switch binding.DataType { case pgsql.ExpansionEdge: @@ -288,6 +293,7 @@ func bindingExpressionType(binding *BoundIdentifier) pgsql.DataType { } } +// inferRowColumnReferenceType infers a composite field's data type from the referenced binding and column. func inferRowColumnReferenceType(expression pgsql.RowColumnReference) pgsql.DataType { switch expression.Column { case pgsql.ColumnGraphID, pgsql.ColumnID, pgsql.ColumnStartID, pgsql.ColumnEndID: @@ -313,6 +319,7 @@ func inferRowColumnReferenceType(expression pgsql.RowColumnReference) pgsql.Data } } +// inferExpressionType resolves an expression's SQL data type using scope information when needed. func (s *Translator) inferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { switch typedExpression := unwrapParenthetical(expression).(type) { case pgsql.Identifier: @@ -339,6 +346,7 @@ func (s *Translator) inferExpressionType(expression pgsql.Expression) (pgsql.Dat return InferExpressionType(expression) } +// inferArrayExpressionType resolves an array expression's element-derived PostgreSQL array type. func (s *Translator) inferArrayExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { if expressionType, err := s.inferExpressionType(expression); err != nil { return pgsql.UnsetDataType, err @@ -351,6 +359,7 @@ func (s *Translator) inferArrayExpressionType(expression pgsql.Expression) (pgsq } } +// expressionForPath returns the path representation carried by binding or reports that it cannot satisfy the requested use. func (s *Translator) expressionForPath(expression pgsql.Expression) (pgsql.Expression, error) { switch typedExpression := unwrapParenthetical(expression).(type) { case pgsql.Identifier: @@ -378,6 +387,7 @@ func (s *Translator) expressionForPath(expression pgsql.Expression) (pgsql.Expre } } +// translateHeadFunction lowers head(list) to safe PostgreSQL array indexing. func (s *Translator) translateHeadFunction(functionInvocation *cypher.FunctionInvocation) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -400,6 +410,7 @@ func (s *Translator) translateHeadFunction(functionInvocation *cypher.FunctionIn return nil } +// translateTailFunction lowers tail(list) to a PostgreSQL array slice that excludes the first element. func (s *Translator) translateTailFunction(functionInvocation *cypher.FunctionInvocation) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -429,6 +440,7 @@ func (s *Translator) translateTailFunction(functionInvocation *cypher.FunctionIn return nil } +// cypherMinMaxFunction selects the Cypher-aware minimum or maximum SQL aggregate for the invocation name. func cypherMinMaxFunction(function pgsql.Identifier, argument pgsql.Expression) pgsql.FunctionCall { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(argument); isPropertyLookup { propertyLookup.Operator = pgsql.OperatorJSONField @@ -456,6 +468,7 @@ func cypherMinMaxFunction(function pgsql.Identifier, argument pgsql.Expression) } } +// translatePathComponentFunction lowers nodes(path) or relationships(path) from the binding's carried path representation. func (s *Translator) translatePathComponentFunction(functionInvocation *cypher.FunctionInvocation, column pgsql.Identifier, castType pgsql.DataType) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -498,6 +511,86 @@ func (s *Translator) translatePathComponentFunction(functionInvocation *cypher.F return nil } +// translatePathLengthFunction lowers length(path) to the cardinality of carried ordered edge IDs when possible. +func (s *Translator) translatePathLengthFunction(functionInvocation *cypher.FunctionInvocation) error { + if functionInvocation.NumArguments() != 1 { + return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) + } + + argument, err := s.treeTranslator.PopOperand() + if err != nil { + return err + } + + if literal, isLiteral := argument.(pgsql.Literal); isLiteral && literal.Null { + s.treeTranslator.PushOperand(pgsql.NewTypeCast(literal, pgsql.Int)) + return nil + } + + if identifier, isIdentifier := unwrapParenthetical(argument).(pgsql.Identifier); isIdentifier { + binding, bound := s.scope.Lookup(identifier) + if !bound { + binding, bound = s.scope.AliasedLookup(identifier) + } + if !bound { + return fmt.Errorf("unable to resolve path identifier %s", identifier) + } + if binding.DistanceOnly { + var distance pgsql.Expression = binding.Identifier + if binding.LastProjection != nil { + distance = pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier} + } else { + for _, dependency := range binding.Dependencies { + if dependency.DistanceOnly && dependency.LastProjection != nil { + distance = pgsql.CompoundIdentifier{dependency.LastProjection.Binding.Identifier, dependency.Identifier} + break + } + } + } + s.treeTranslator.PushOperand(pgsql.NewTypeCast(distance, pgsql.Int)) + return nil + } + if binding.DataType != pgsql.PathComposite { + return fmt.Errorf("expected path expression but received %s", binding.DataType) + } + + var edges pgsql.Expression + if binding.LastProjection == nil { + edges, err = pathCompositeEdgeIDArrayExpression(s.scope, binding) + } else { + edges = pgsql.RowColumnReference{ + Identifier: pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier}, + Column: pgsql.ColumnEdges, + } + } + if err != nil { + return err + } + + s.treeTranslator.PushOperand(pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{edges}, + CastType: pgsql.Int, + }) + return nil + } + + pathExpression, err := s.expressionForPath(argument) + if err != nil { + return err + } + s.treeTranslator.PushOperand(pgsql.FunctionCall{ + Function: pgsql.FunctionCardinality, + Parameters: []pgsql.Expression{pgsql.RowColumnReference{ + Identifier: pathExpression, + Column: pgsql.ColumnEdges, + }}, + CastType: pgsql.Int, + }) + return nil +} + +// prepareCollectExpression prepares a value and result type for PostgreSQL array aggregation. func prepareCollectExpression(scope *Scope, collectedExpression pgsql.Expression, functionName string) (pgsql.Expression, pgsql.DataType, error) { castType := pgsql.AnyArray @@ -529,6 +622,7 @@ func prepareCollectExpression(scope *Scope, collectedExpression pgsql.Expression return collectedExpression, castType, nil } +// prepareCollectIDExpression extracts a scalar entity ID before collection and records the ID-only alias. func prepareCollectIDExpression(scope *Scope, collectedExpression pgsql.Expression) (pgsql.Expression, bool) { identifier, isIdentifier := unwrapParenthetical(collectedExpression).(pgsql.Identifier) if !isIdentifier { @@ -551,6 +645,7 @@ func prepareCollectIDExpression(scope *Scope, collectedExpression pgsql.Expressi } } +// translateNodeLabelsExpression lowers labels(node) to kind-name lookup over the node's kind IDs. func translateNodeLabelsExpression(identifier pgsql.Identifier) pgsql.TypeHinted { const ( kindAlias pgsql.Identifier = "_kind" @@ -602,6 +697,7 @@ func translateNodeLabelsExpression(identifier pgsql.Identifier) pgsql.TypeHinted }, pgsql.TextArray) } +// relationshipEndpointFunctionArgument expands a bound edge identifier to the composite accepted by startNode or endNode. func (s *Translator) relationshipEndpointFunctionArgument(argument pgsql.Expression) pgsql.Expression { identifier, isIdentifier := unwrapParenthetical(argument).(pgsql.Identifier) if !isIdentifier { @@ -619,6 +715,7 @@ func (s *Translator) relationshipEndpointFunctionArgument(argument pgsql.Express return argument } +// translateRelationshipEndpointFunction lowers startNode or endNode with graph-scoped entity hydration. func (s *Translator) translateRelationshipEndpointFunction(function pgsql.Identifier, functionInvocation *cypher.FunctionInvocation) error { if functionInvocation.NumArguments() != 1 { return fmt.Errorf("expected only one argument for cypher function: %s", functionInvocation.Name) @@ -637,6 +734,7 @@ func (s *Translator) translateRelationshipEndpointFunction(function pgsql.Identi return nil } +// translateFunction dispatches a Cypher function invocation to its function-specific PostgreSQL lowering. func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocation) { switch formattedName := strings.ToLower(typedExpression.Name); formattedName { case cypher.DurationFunction: @@ -662,6 +760,8 @@ func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocatio s.SetError(err) } else if referenceArgument, typeOK := argument.(pgsql.Identifier); !typeOK { s.SetErrorf("expected an identifier for the cypher function: %s but received %T", typedExpression.Name, argument) + } else if binding, bound := s.scope.Lookup(referenceArgument); bound && binding.IDOnly && binding.LastProjection != nil { + s.treeTranslator.PushOperand(referenceArgument) } else { s.treeTranslator.PushOperand(pgsql.CompoundIdentifier{referenceArgument, pgsql.ColumnID}) } @@ -790,29 +890,37 @@ func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocatio } else if argument, err := s.treeTranslator.PopOperand(); err != nil { s.SetError(err) } else { - var functionCall pgsql.FunctionCall + var sizeExpression pgsql.Expression if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(argument); isPropertyLookup { // Ensure that the JSONB array length function receives the JSONB type propertyLookup.Operator = pgsql.OperatorJSONField - functionCall = pgsql.FunctionCall{ - Function: pgsql.FunctionJSONBArrayLength, - Parameters: []pgsql.Expression{argument}, - CastType: pgsql.Int, + sizeExpression = pgsql.Case{ + Conditions: []pgsql.Expression{pgsql.NewBinaryExpression( + jsonbTypeof(argument), + pgsql.OperatorEquals, + pgsql.NewLiteral("array", pgsql.Text), + )}, + Then: []pgsql.Expression{pgsql.FunctionCall{ + Function: pgsql.FunctionJSONBArrayLength, + Parameters: []pgsql.Expression{argument}, + CastType: pgsql.Int, + }}, + Else: pgsql.NullLiteral(), } } else if isKnownEmptyArrayExpression(argument) { s.treeTranslator.PushOperand(pgsql.NewLiteral(0, pgsql.Int)) return } else { - functionCall = pgsql.FunctionCall{ + sizeExpression = pgsql.FunctionCall{ Function: pgsql.FunctionCardinality, Parameters: []pgsql.Expression{argument}, CastType: pgsql.Int, } } - s.treeTranslator.PushOperand(functionCall) + s.treeTranslator.PushOperand(sizeExpression) } case cypher.HeadFunction: @@ -835,6 +943,11 @@ func (s *Translator) translateFunction(typedExpression *cypher.FunctionInvocatio s.SetError(err) } + case cypher.PathLengthFunction: + if err := s.translatePathLengthFunction(typedExpression); err != nil { + s.SetError(err) + } + case cypher.ToUpperFunction: if typedExpression.NumArguments() != 1 { s.SetError(fmt.Errorf("expected only one argument for cypher function: %s", typedExpression.Name)) @@ -1001,6 +1114,7 @@ func functionWrapCollectToArray(distinct bool, collectedExpression pgsql.Express } } +// translateDateTimeFunctionCall lowers Cypher temporal constructors and validates supported argument forms. func (s *Translator) translateDateTimeFunctionCall(cypherFunc *cypher.FunctionInvocation, dataType pgsql.DataType) error { // Ensure the local date time function uses the default precision const defaultTimestampPrecision = 6 @@ -1068,6 +1182,7 @@ func (s *Translator) translateDateTimeFunctionCall(cypherFunc *cypher.FunctionIn return nil } +// translateCoalesceFunction lowers coalesce after reconciling every argument to one compatible result type. func (s *Translator) translateCoalesceFunction(functionInvocation *cypher.FunctionInvocation) error { if numArgs := functionInvocation.NumArguments(); numArgs == 0 { s.SetError(fmt.Errorf("expected at least one argument for cypher function: %s", functionInvocation.Name)) diff --git a/cypher/models/pgsql/translate/function_test.go b/cypher/models/pgsql/translate/function_test.go index 066f0cf0..e8d84acc 100644 --- a/cypher/models/pgsql/translate/function_test.go +++ b/cypher/models/pgsql/translate/function_test.go @@ -9,6 +9,7 @@ import ( "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql" "github.com/specterops/dawgs/drivers/pg/pgutil" + "github.com/specterops/dawgs/graph" "github.com/stretchr/testify/require" ) @@ -57,6 +58,25 @@ func TestPathComponentFunctionsTranslateNullArguments(t *testing.T) { require.Contains(t, formatted, "(null)::edgecomposite[]") } +// TestListSizeGuardsDynamicJSONPropertiesByType verifies that size() distinguishes JSON strings and arrays at runtime. +func TestListSizeGuardsDynamicJSONPropertiesByType(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + kindMapper.Put(graph.StringKind("TestNode")) + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (n:TestNode) RETURN size(n.values)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "case when jsonb_typeof") + require.Contains(t, formatted, "= 'array' then jsonb_array_length") + require.Contains(t, formatted, "else null end") +} + +// TestTailFunctionDoesNotDuplicatePathComponentExpression verifies nested tail calls hydrate path components only once. func TestTailFunctionDoesNotDuplicatePathComponentExpression(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -68,10 +88,12 @@ func TestTailFunctionDoesNotDuplicatePathComponentExpression(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path"), formatted) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path"), formatted) + require.NotContains(t, formatted, "ordered_edges_to_path") require.NotContains(t, formatted, "cardinality(((case when") } +// TestTailPredicateStagesPathComponentExpression verifies predicates reuse a staged path-component projection. func TestTailPredicateStagesPathComponentExpression(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -83,11 +105,13 @@ func TestTailPredicateStagesPathComponentExpression(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path")) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path")) + require.NotContains(t, formatted, "ordered_edges_to_path") require.Contains(t, formatted, "lateral (select") require.Contains(t, formatted, ".nodes") } +// TestProjectionStagesPathBeforeReadingComponents verifies path hydration is staged before node and edge access. func TestProjectionStagesPathBeforeReadingComponents(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -100,11 +124,13 @@ func TestProjectionStagesPathBeforeReadingComponents(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) require.Contains(t, formatted, "lateral (select") - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path"), formatted) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path"), formatted) + require.NotContains(t, formatted, "ordered_edges_to_path") require.Contains(t, formatted, ".nodes") require.Contains(t, formatted, ".edges") } +// TestProjectionStagesRepeatedPathComponents verifies repeated component access shares one staged path hydration. func TestProjectionStagesRepeatedPathComponents(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() @@ -117,12 +143,230 @@ func TestProjectionStagesRepeatedPathComponents(t *testing.T) { formatted, err := Translated(translation) require.NoError(t, err) require.Contains(t, formatted, "lateral (select") - require.Equal(t, 1, strings.Count(formatted, "ordered_edges_to_path"), formatted) - require.Equal(t, 1, strings.Count(formatted, "from unnest"), formatted) + require.Equal(t, 1, strings.Count(formatted, "ordered_edge_ids_to_path"), formatted) + require.NotContains(t, formatted, "ordered_edges_to_path") + require.NotContains(t, formatted, "from unnest") require.Contains(t, formatted, ".nodes") require.Contains(t, formatted, ".edges") } +// TestPathLengthUsesScalarDistanceWithoutHydration verifies that length(path) +// consumes the selected scalar-distance result without carrying edge IDs. +func TestPathLengthUsesOrderedEdgeIDsWithoutHydration(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = 1 AND id(e) = 2 RETURN length(p)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "(s0.ep0)::int") + require.NotContains(t, formatted, "cardinality(") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.NotContains(t, formatted, "ordered_edges_to_path") + require.NotContains(t, formatted, "from unnest") +} + +// TestIDOnlyTerminalProjectionCarriesScalarID verifies that an ID-only terminal consumer receives scalar state. +func TestIDOnlyTerminalProjectionCarriesScalarID(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + kindMapper.Put(graph.StringKind("TestNode")) + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH ()-[]->(e:TestNode) RETURN id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "n1.id as n1") + require.Contains(t, formatted, "select s0.n1 as \"id(e)\"") + require.NotContains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "n1.kind_ids operator") +} + +// TestIDOnlyTerminalProjectionRetainsCompositeForMixedUse verifies that mixed ID and property consumers retain the terminal composite. +func TestIDOnlyTerminalProjectionRetainsCompositeForMixedUse(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH ()-[]->(e) RETURN id(e), e.name`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id") + require.Contains(t, formatted, "(s0.n1).properties") +} + +// TestIDOnlyTerminalProjectionRetainsCompositeForLaterPatternReuse verifies that a reused terminal remains a complete entity binding. +func TestIDOnlyTerminalProjectionRetainsCompositeForLaterPatternReuse(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH ()-[]->(e) MATCH (e)-[]->() RETURN id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.NotContains(t, formatted, "n1.id as n1") +} + +// TestIDOnlyTerminalProjectionRetainsCompositeForObservedPath verifies that observing the path retains complete terminal state. +func TestIDOnlyTerminalProjectionRetainsCompositeForObservedPath(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH p = ()-[*1..]->(e) WHERE id(e) = 2 RETURN p`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "ordered_edge_ids_to_path") +} + +// TestIDOnlyExpansionContinuationCarriesScalarID verifies that an ID-only intermediate binding continues as scalar state. +func TestIDOnlyExpansionContinuationCarriesScalarID(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[]->(e) RETURN id(mid), id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "join lateral (select n1.id from node n1 where n1.id = s1.next_id offset 0) n1 on true") + require.Contains(t, formatted, "s0.n1 = e1.start_id") + require.Contains(t, formatted, "s0.n1 as n1") + require.NotContains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.NotContains(t, formatted, "(s0.n1).id = e1.start_id") +} + +// TestIDOnlyExpansionContinuationRetainsCompositeForPropertyUse verifies that a property consumer prevents scalar-only continuation state. +func TestIDOnlyExpansionContinuationRetainsCompositeForPropertyUse(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[]->(e) RETURN mid.name`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id = e1.start_id") +} + +// TestIDOnlyExpansionContinuationSeedsFollowingExpansionFromScalarID verifies that a following traversal can join from a scalar intermediate ID. +func TestIDOnlyExpansionContinuationSeedsFollowingExpansionFromScalarID(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[*1..]->(e) RETURN id(mid), id(e)`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "select distinct s0.n1 as root_id from s0") + require.Contains(t, formatted, "s0.n1 = s3.root_id") + require.NotContains(t, formatted, "select distinct (s0.n1).id as root_id from s0") +} + +// TestIDOnlyExpansionContinuationRetainsCompositeForObservedPath verifies that observing the path prevents scalar-only intermediate state. +func TestIDOnlyExpansionContinuationRetainsCompositeForObservedPath(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH p = (s)-[*1..]->(mid)-[]->(e) RETURN p`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id = e1.start_id") + require.Contains(t, formatted, "ordered_edge_ids_to_path") +} + +// TestIDOnlyExpansionContinuationRetainsCompositeForMutation verifies that mutating an intermediate node retains its complete entity value. +func TestIDOnlyExpansionContinuationRetainsCompositeForMutation(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + + query, err := frontend.ParseCypher(frontend.NewContext(), `MATCH (s)-[*1..]->(mid)-[]->(e) DELETE mid`) + require.NoError(t, err) + + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "(n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1") + require.Contains(t, formatted, "(s0.n1).id = e1.start_id") + require.Contains(t, formatted, "delete from node") +} + +// TestBoundPairShortestPathUsesStableSingletonEndpoints verifies deterministic +// scalar endpoint construction for the contained compact executor. +func TestBoundPairShortestPathUsesStableSingletonArrays(t *testing.T) { + kindMapper := pgutil.NewInMemoryKindMapper() + translateQuery := func(cypherQuery string) (Result, string) { + query, err := frontend.ParseCypher(frontend.NewContext(), cypherQuery) + require.NoError(t, err) + translation, err := Translate(context.Background(), query, kindMapper, nil, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return translation, formatted + } + + first, firstSQL := translateQuery(`MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = 1 AND id(e) = 2 RETURN p LIMIT 1`) + second, secondSQL := translateQuery(`MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = 41 AND id(e) = 42 RETURN p LIMIT 1`) + + require.Equal(t, firstSQL, secondSQL) + require.Contains(t, firstSQL, "shortest_path_compact(") + require.NotContains(t, firstSQL, "insert into pg_temp.bsp_pair_filter") + require.NotContains(t, firstSQL, "traversal_pair_filter") + require.Contains(t, firstSQL, "limit 1") + require.Contains(t, firstSQL, "with singleton_endpoints as") + require.Contains(t, firstSQL, "singleton_endpoints.root_id") + require.Contains(t, firstSQL, "singleton_endpoints.terminal_id") + require.NotContains(t, firstSQL, "n0.id = 1") + require.NotContains(t, secondSQL, "n0.id = 41") + var firstEndpointValues, secondEndpointValues []any + for _, value := range first.Parameters { + if _, isString := value.(string); !isString { + firstEndpointValues = append(firstEndpointValues, value) + } + } + for _, value := range second.Parameters { + if _, isString := value.(string); !isString { + secondEndpointValues = append(secondEndpointValues, value) + } + } + require.ElementsMatch(t, []any{int64(1), int64(2)}, firstEndpointValues) + require.ElementsMatch(t, []any{int64(41), int64(42)}, secondEndpointValues) + +} + func TestRelationshipEndpointFunctionsUseEdgeCompositeArguments(t *testing.T) { t.Parallel() diff --git a/cypher/models/pgsql/translate/graph_scope_test.go b/cypher/models/pgsql/translate/graph_scope_test.go new file mode 100644 index 00000000..fe2a97a7 --- /dev/null +++ b/cypher/models/pgsql/translate/graph_scope_test.go @@ -0,0 +1,63 @@ +package translate + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/stretchr/testify/require" +) + +// TestTargetGraphUsesConcreteRelationsInOuterAndHarnessSQL verifies graph partitioning in both the outer query and shortest-path harness. +func TestTargetGraphUsesConcreteRelationsInOuterAndHarnessSQL(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s:Group)-[:MemberOf*1..]->(e:Domain)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + LIMIT 1 + `) + require.NoError(t, err) + + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), + "end_id": int64(2), + }, 42) + require.NoError(t, err) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "node_42") + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.Contains(t, formatted, "join edge_42") + require.NotRegexp(t, `(?i)(from|join) (node|edge)(?:\s|;)`, formatted) + + var fragments []string + for _, value := range translation.Parameters { + if fragment, ok := value.(string); ok && strings.Contains(fragment, "pg_temp.bsp_") { + fragments = append(fragments, fragment) + } + } + require.Empty(t, fragments, "contained inline execution should not emit workspace harness fragments") + for _, fragment := range fragments { + require.Contains(t, fragment, "edge_42", fmt.Sprintf("unscoped harness fragment: %s", fragment)) + require.NotRegexp(t, `(?i)(from|join) (node|edge)(?:\s|;)`, fragment) + } +} + +// TestFixedSuffixTargetGraphUsesOnlyConcreteRelations verifies that suffix-seeded translation never falls back to unpartitioned graph tables. +func TestFixedSuffixTargetGraphUsesOnlyConcreteRelations(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), optimizerFixedSuffixQuery) + require.NoError(t, err) + + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), nil, 42) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "node_42") + require.Contains(t, formatted, "edge_42") + require.NotRegexp(t, `(?i)(from|join) (node|edge)(?:\s|;)`, formatted) + require.Equal(t, 2, strings.Count(formatted, "ordered_edge_ids_to_path(42,")) +} diff --git a/cypher/models/pgsql/translate/hinting.go b/cypher/models/pgsql/translate/hinting.go index 5d837c4e..afd0944b 100644 --- a/cypher/models/pgsql/translate/hinting.go +++ b/cypher/models/pgsql/translate/hinting.go @@ -18,6 +18,7 @@ func GetTypeHint(expression pgsql.Expression) (pgsql.DataType, bool) { return pgsql.UnsetDataType, false } +// applyUnaryExpressionTypeHints casts a unary operand to the type required by its operator. func applyUnaryExpressionTypeHints(expression *pgsql.UnaryExpression) error { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression.Operand); isPropertyLookup { expression.Operand = rewritePropertyLookupOperator(propertyLookup, pgsql.Boolean) @@ -26,6 +27,7 @@ func applyUnaryExpressionTypeHints(expression *pgsql.UnaryExpression) error { return nil } +// inferBinaryExpressionType returns the result type implied by a binary operator and its operand hints. func inferBinaryExpressionType(expression *pgsql.BinaryExpression) (pgsql.DataType, error) { var ( leftHint, isLeftHinted = GetTypeHint(expression.LOperand) @@ -93,6 +95,7 @@ func inferBinaryExpressionType(expression *pgsql.BinaryExpression) (pgsql.DataTy } } +// inferUnaryExpressionType returns the result type implied by a unary operator and operand hint. func inferUnaryExpressionType(expression pgsql.UnaryExpression) (pgsql.DataType, error) { switch expression.Operator { case pgsql.OperatorNot, pgsql.OperatorIs, pgsql.OperatorIsNot: @@ -112,6 +115,7 @@ func inferUnaryExpressionType(expression pgsql.UnaryExpression) (pgsql.DataType, } } +// inferAllExpressionType returns the boolean type of a valid ALL predicate after checking its operands. func inferAllExpressionType(expression pgsql.AllExpression) (pgsql.DataType, error) { if expressionType, err := InferExpressionType(expression.Expression); err != nil { return pgsql.UnsetDataType, err @@ -122,6 +126,46 @@ func inferAllExpressionType(expression pgsql.AllExpression) (pgsql.DataType, err } } +// inferCaseExpressionType finds the common result type of a CASE expression's branches. +func inferCaseExpressionType(expression pgsql.Case) (pgsql.DataType, error) { + var ( + resultType = pgsql.UnknownDataType + branches = append(append([]pgsql.Expression(nil), expression.Then...), expression.Else) + ) + + for _, branch := range branches { + if branch == nil { + continue + } + + branchType, err := InferExpressionType(branch) + if err != nil { + return pgsql.UnsetDataType, err + } + if branchType == pgsql.Null || !branchType.IsKnown() { + continue + } + + if !resultType.IsKnown() { + resultType = branchType + continue + } + + if resultType == branchType { + continue + } + + if supertype, valid := resultType.CoerceToSupertype(branchType); valid { + resultType = supertype + } else { + return pgsql.UnknownDataType, nil + } + } + + return resultType, nil +} + +// InferExpressionType derives the PostgreSQL data type produced by an expression when it can be determined statically. func InferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { switch typedExpression := expression.(type) { case pgsql.Identifier, pgsql.RowColumnReference: @@ -193,6 +237,16 @@ func InferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { case pgsql.AllExpression: return inferAllExpressionType(typedExpression) + case *pgsql.Case: + if typedExpression == nil { + return pgsql.UnknownDataType, nil + } + + return inferCaseExpressionType(*typedExpression) + + case pgsql.Case: + return inferCaseExpressionType(typedExpression) + case *pgsql.AliasedExpression: if typedExpression == nil { return pgsql.UnknownDataType, nil @@ -218,12 +272,17 @@ func InferExpressionType(expression pgsql.Expression) (pgsql.DataType, error) { } } +// contextAwareKindMapper adapts request-scoped kind resolution to translation helpers that do not accept a context. type contextAwareKindMapper struct { - ctx context.Context + // ctx carries cancellation and deadlines into kind-name lookups. + ctx context.Context + // kindMapper performs the underlying graph kind-name resolution. kindMapper pgsql.KindMapper + // parameters is the translation parameter map shared with the owning translator. parameters map[string]any } +// newContextAwareKindMapper wraps a mapper with request context and retains the associated translation parameter map. func newContextAwareKindMapper(ctx context.Context, kindMapper pgsql.KindMapper, parameters map[string]any) *contextAwareKindMapper { return &contextAwareKindMapper{ ctx: ctx, @@ -240,6 +299,7 @@ func (s *contextAwareKindMapper) AssertKinds(kinds graph.Kinds) ([]int16, error) return s.kindMapper.AssertKinds(s.ctx, kinds) } +// relationshipTypeKindIDExpression returns the scalar kind-ID field used to implement type(relationship). func relationshipTypeKindIDExpression(expression pgsql.Expression) (pgsql.Expression, bool) { functionCall, isFunctionCall := unwrapParenthetical(expression).(pgsql.FunctionCall) if !isFunctionCall || functionCall.Function != pgsql.FunctionKindName || len(functionCall.Parameters) != 1 { @@ -249,6 +309,7 @@ func relationshipTypeKindIDExpression(expression pgsql.Expression) (pgsql.Expres return functionCall.Parameters[0], true } +// literalKindID resolves a string kind name through the context-bound mapper and returns its numeric PostgreSQL literal. func literalKindID(kindMapper *contextAwareKindMapper, literal pgsql.Literal) (pgsql.Literal, bool, error) { if literal.CastType != pgsql.Text { return pgsql.Literal{}, false, nil @@ -270,10 +331,12 @@ func literalKindID(kindMapper *contextAwareKindMapper, literal pgsql.Literal) (p return pgsql.NewLiteral(kindIDs[0], pgsql.Int2), true, nil } +// mapsRelationshipTypeLiteralToKindID reports whether operator permits mapping a relationship type name to its kind ID. func mapsRelationshipTypeLiteralToKindID(operator pgsql.Operator) bool { return operator.IsIn(pgsql.OperatorEquals, pgsql.OperatorNotEquals, pgsql.OperatorCypherNotEquals) } +// applyTypeFunctionLikeTypeHints normalizes operands for equality involving type(relationship). func applyTypeFunctionLikeTypeHints(kindMapper *contextAwareKindMapper, expression *pgsql.BinaryExpression) error { mapTypeLiteralToKindID := mapsRelationshipTypeLiteralToKindID(expression.Operator) @@ -431,6 +494,7 @@ func applyTypeFunctionLikeTypeHints(kindMapper *contextAwareKindMapper, expressi return nil } +// applyBinaryExpressionTypeHints rewrites property lookups and casts operands to types compatible with the binary operator. func applyBinaryExpressionTypeHints(kindMapper *contextAwareKindMapper, expression *pgsql.BinaryExpression) error { switch expression.Operator { case pgsql.OperatorPropertyLookup: diff --git a/cypher/models/pgsql/translate/limit_pushdown_test.go b/cypher/models/pgsql/translate/limit_pushdown_test.go index e7edcb5f..bc87c832 100644 --- a/cypher/models/pgsql/translate/limit_pushdown_test.go +++ b/cypher/models/pgsql/translate/limit_pushdown_test.go @@ -9,13 +9,23 @@ import ( ) const ( - limitPushdownTestSourceFrame pgsql.Identifier = "s0" - limitPushdownTestHarnessFrame pgsql.Identifier = "s1" + // limitPushdownTestSourceFrame identifies the source frame referenced by limit-pushdown fixtures. + limitPushdownTestSourceFrame pgsql.Identifier = "s0" + + // limitPushdownTestHarnessFrame identifies the shortest-path harness frame in limit-pushdown fixtures. + limitPushdownTestHarnessFrame pgsql.Identifier = "s1" + + // limitPushdownTestPreviousFrame identifies the frame that supplies bound endpoints in fixtures. limitPushdownTestPreviousFrame pgsql.Identifier = "s2" - limitPushdownTestRootAlias pgsql.Identifier = "n0" + + // limitPushdownTestRootAlias identifies the root-node binding in limit-pushdown fixtures. + limitPushdownTestRootAlias pgsql.Identifier = "n0" + + // limitPushdownTestTerminalAlias identifies the terminal-node binding in limit-pushdown fixtures. limitPushdownTestTerminalAlias pgsql.Identifier = "n1" ) +// limitPushdownTestEndpointRef references an endpoint ID projected by the fixture source frame. func limitPushdownTestEndpointRef(alias pgsql.Identifier) pgsql.RowColumnReference { return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{limitPushdownTestSourceFrame, alias}, @@ -23,6 +33,7 @@ func limitPushdownTestEndpointRef(alias pgsql.Identifier) pgsql.RowColumnReferen } } +// limitPushdownTestEndpointInequality builds the Cypher inequality used to exclude identical endpoints. func limitPushdownTestEndpointInequality(leftAlias, rightAlias pgsql.Identifier) pgsql.Expression { return pgsql.NewBinaryExpression( limitPushdownTestEndpointRef(leftAlias), @@ -31,6 +42,7 @@ func limitPushdownTestEndpointInequality(leftAlias, rightAlias pgsql.Identifier) ) } +// limitPushdownTestBoundEndpointConstraint equates a previous-frame endpoint ID with a harness expansion column. func limitPushdownTestBoundEndpointConstraint(endpointAlias, expansionColumn pgsql.Identifier) pgsql.Expression { return pgsql.NewBinaryExpression( pgsql.RowColumnReference{ @@ -42,6 +54,7 @@ func limitPushdownTestBoundEndpointConstraint(endpointAlias, expansionColumn pgs ) } +// limitPushdownTestSourceWhere combines the fixture's root, terminal, and endpoint-pair constraints. func limitPushdownTestSourceWhere(t *testing.T, part *QueryPart, where pgsql.Expression) { t.Helper() @@ -55,6 +68,7 @@ func limitPushdownTestSourceWhere(t *testing.T, part *QueryPart, where pgsql.Exp sourceCTE.Query.Body = selectBody } +// limitPushdownTestJoin joins one bound endpoint from the previous frame to the shortest-path harness. func limitPushdownTestJoin(nodeAlias, expansionColumn pgsql.Identifier) pgsql.Join { return pgsql.Join{ Table: pgsql.TableReference{ @@ -72,6 +86,7 @@ func limitPushdownTestJoin(nodeAlias, expansionColumn pgsql.Identifier) pgsql.Jo } } +// limitPushdownTestPart constructs a query part containing a bounded shortest-path harness and final projection. func limitPushdownTestPart(harnessFunction pgsql.Identifier) *QueryPart { part := NewQueryPart(1, 0) part.Limit = pgsql.NewLiteral(10, pgsql.Int) @@ -100,6 +115,7 @@ func limitPushdownTestPart(harnessFunction pgsql.Identifier) *QueryPart { return part } +// limitPushdownTestTail returns the terminal query part used to determine whether a limit may be pushed down. func limitPushdownTestTail(where pgsql.Expression) pgsql.Select { return pgsql.Select{ From: []pgsql.FromClause{{ @@ -228,6 +244,7 @@ func TestLimitPushdownTailSourceAllowsBidirectionalShortestPathEndpointInequalit require.Equal(t, limitPushdownTestSourceFrame, sourceFrame) } +// TestPushDownShortestPathLimitAppendsHarnessLimitWithEndpointInequality verifies endpoint filtering does not displace the harness limit. func TestPushDownShortestPathLimitAppendsHarnessLimitWithEndpointInequality(t *testing.T) { var ( part = limitPushdownTestPart(pgsql.FunctionUnidirectionalSPHarness) @@ -245,6 +262,7 @@ func TestPushDownShortestPathLimitAppendsHarnessLimitWithEndpointInequality(t *t require.Len(t, sourceCTE.Query.CommonTableExpressions.Expressions, 1) harnessCTE := sourceCTE.Query.CommonTableExpressions.Expressions[0] + require.Equal(t, part.Limit, harnessCTE.Query.Limit) selectBody, isSelect := harnessCTE.Query.Body.(pgsql.Select) require.True(t, isSelect) require.Len(t, selectBody.From, 1) diff --git a/cypher/models/pgsql/translate/model.go b/cypher/models/pgsql/translate/model.go index 9315b327..293c24ef 100644 --- a/cypher/models/pgsql/translate/model.go +++ b/cypher/models/pgsql/translate/model.go @@ -12,17 +12,40 @@ import ( ) const ( - expansionRootID pgsql.Identifier = "root_id" - expansionNextID pgsql.Identifier = "next_id" - expansionDepth pgsql.Identifier = "depth" - expansionSatisfied pgsql.Identifier = "satisfied" - expansionIsCycle pgsql.Identifier = "is_cycle" - expansionPath pgsql.Identifier = "path" - expansionForwardFront pgsql.Identifier = "forward_front" + // expansionRootID names the recursive-state column containing the traversal's initial node ID. + expansionRootID pgsql.Identifier = "root_id" + + // expansionNextID names the recursive-state column containing the current frontier node ID. + expansionNextID pgsql.Identifier = "next_id" + + // expansionDepth names the recursive-state column containing the number of traversed edges. + expansionDepth pgsql.Identifier = "depth" + + // expansionSatisfied names the recursive-state column that marks a satisfied terminal predicate. + expansionSatisfied pgsql.Identifier = "satisfied" + + // expansionIsCycle names the recursive-state column that marks an edge-reusing path. + expansionIsCycle pgsql.Identifier = "is_cycle" + + // expansionPath names the recursive-state column containing ordered traversed edge IDs. + expansionPath pgsql.Identifier = "path" + + // expansionNodePath names an optional ordered node-ID trail carried by + // specialized traversals that can hydrate a complete path without walking + // the edge stream again. + expansionNodePath pgsql.Identifier = "node_path" + + // expansionForwardFront names the current forward frontier in bidirectional search. + expansionForwardFront pgsql.Identifier = "forward_front" + + // expansionBackwardFront names the current backward frontier in bidirectional search. expansionBackwardFront pgsql.Identifier = "backward_front" - expansionNextFront pgsql.Identifier = "next_front" + + // expansionNextFront names the staging relation for the next bidirectional-search frontier. + expansionNextFront pgsql.Identifier = "next_front" ) +// expansionColumns returns the canonical root, frontier, depth, satisfaction, cycle, and path state shape. func expansionColumns() *pgsql.RecordShape { return pgsql.NewRecordShape([]pgsql.Identifier{ expansionRootID, @@ -34,20 +57,31 @@ func expansionColumns() *pgsql.RecordShape { }) } +// NodeSelect groups SQL model state that must remain consistent while translating node select. type NodeSelect struct { - Frame *Frame - Binding *BoundIdentifier - Select pgsql.Select + // Frame supplies the frame input to the NodeSelect contract. + Frame *Frame + // Binding supplies the binding input to the NodeSelect contract. + Binding *BoundIdentifier + // Select supplies the select input to the NodeSelect contract. + Select pgsql.Select + // Constraints supplies the constraints input to the NodeSelect contract. Constraints pgsql.Expression } +// ExpansionOptions configures expansion. type ExpansionOptions struct { - FindShortestPath bool + // FindShortestPath identifies the filesystem find shortest path. + FindShortestPath bool + // FindAllShortestPaths identifies the filesystem find all shortest paths. FindAllShortestPaths bool - MinDepth models.Optional[int64] - MaxDepth models.Optional[int64] + // MinDepth supplies the min depth input to the ExpansionOptions contract. + MinDepth models.Optional[int64] + // MaxDepth supplies the max depth input to the ExpansionOptions contract. + MaxDepth models.Optional[int64] } +// newExpansionOptions derives shortest-path and depth options from a pattern part and relationship range. func newExpansionOptions(part *PatternPart, relationshipPattern *cypher.RelationshipPattern) ExpansionOptions { return ExpansionOptions{ FindShortestPath: part.ShortestPath, @@ -57,47 +91,102 @@ func newExpansionOptions(part *PatternPart, relationshipPattern *cypher.Relation } } +// Expansion contains the bindings, constraints, and execution choices for one variable-length traversal. type Expansion struct { - Frame *Frame + // Frame is the scope frame that materializes the expansion result. + Frame *Frame + // PathBinding is the optional Cypher path variable backed by recursive path state. PathBinding *BoundIdentifier - Options ExpansionOptions - - PrimerNodeConstraints pgsql.Expression - PrimerNodeSatisfactionProjection pgsql.SelectItem - PrimerNodeJoinCondition pgsql.Expression - EdgeConstraints pgsql.Expression - PreviousRelationshipUniqueness pgsql.Expression - EdgeJoinCondition pgsql.Expression - RecursiveConstraints pgsql.Expression - ExpansionNodeJoinCondition pgsql.Expression - TerminalNodeConstraints pgsql.Expression + // Options records shortest-path mode and traversal depth bounds. + Options ExpansionOptions + + // PrimerNodeConstraints restricts root nodes used to seed recursive traversal. + PrimerNodeConstraints pgsql.Expression + // PrimerNodeSatisfactionProjection evaluates terminal satisfaction at the seed node. + PrimerNodeSatisfactionProjection pgsql.SelectItem + // PrimerNodeJoinCondition joins the expansion seed to its root node. + PrimerNodeJoinCondition pgsql.Expression + // EdgeConstraints restricts relationships admitted into the expansion. + EdgeConstraints pgsql.Expression + // PreviousRelationshipUniqueness rejects relationships already traversed by preceding fixed steps. + PreviousRelationshipUniqueness pgsql.Expression + // EdgeJoinCondition joins a relationship to the current traversal frontier. + EdgeJoinCondition pgsql.Expression + // RecursiveConstraints restricts recursive states independently of edge and node predicates. + RecursiveConstraints pgsql.Expression + // ExpansionNodeJoinCondition joins the traversed relationship to its next node. + ExpansionNodeJoinCondition pgsql.Expression + // TerminalNodeConstraints restricts nodes considered valid expansion terminals. + TerminalNodeConstraints pgsql.Expression + // TerminalNodeSatisfactionProjection computes whether a recursive state satisfies terminal predicates. TerminalNodeSatisfactionProjection pgsql.SelectItem + // DeferredNodeSatisfactionConstraint retains terminal predicates that require outer bindings. DeferredNodeSatisfactionConstraint pgsql.Expression - UseMaterializedTerminalFilter bool - UseMaterializedEndpointPairFilter bool - HasExplicitEndpointInequality bool - - PrimerQueryParameter *BoundIdentifier - BackwardPrimerQueryParameter *BoundIdentifier - RecursiveQueryParameter *BoundIdentifier + // UseMaterializedTerminalFilter enables lookup against precomputed terminal node IDs. + UseMaterializedTerminalFilter bool + // UseMaterializedEndpointPairFilter enables lookup against precomputed root-terminal ID pairs. + UseMaterializedEndpointPairFilter bool + // HasExplicitEndpointInequality reports whether the source query already excludes identical endpoints. + HasExplicitEndpointInequality bool + + // PrimerQueryParameter identifies the harness parameter containing the forward primer query. + PrimerQueryParameter *BoundIdentifier + // BackwardPrimerQueryParameter identifies the harness parameter containing the backward primer query. + BackwardPrimerQueryParameter *BoundIdentifier + // RecursiveQueryParameter identifies the harness parameter containing the forward recursive query. + RecursiveQueryParameter *BoundIdentifier + // BackwardRecursiveQueryParameter identifies the harness parameter containing the backward recursive query. BackwardRecursiveQueryParameter *BoundIdentifier + // UseBidirectionalSearch reports whether shortest-path traversal expands from both endpoints. UseBidirectionalSearch bool - + // ShortestPathExecutor selects the physical implementation for this shortest-path expansion. + ShortestPathExecutor optimize.ShortestPathExecutor + // ShortestPathTarget locates this expansion in the optimizer's lowering plan. + ShortestPathTarget optimize.TraversalStepTarget + // ShortestPathStateLimit caps distinct seen state for compact executors. + ShortestPathStateLimit int64 + // ShortestPathFrontierLimit caps current and queued frontier state. + ShortestPathFrontierLimit int64 + // ShortestPathPredecessorLimit caps retained witness predecessors. + ShortestPathPredecessorLimit int64 + // ShortestPathEnumerationLimit caps staged all-shortest-path arrays. + ShortestPathEnumerationLimit int64 + // ShortestPathOutputBytesLimit caps staged all-shortest-path array bytes. + ShortestPathOutputBytesLimit int64 + // SingletonRootID holds the statically resolved root ID when exactly one root is known. + SingletonRootID pgsql.Expression + // SingletonTerminalID holds the statically resolved terminal ID when exactly one terminal is known. + SingletonTerminalID pgsql.Expression + // RelationshipKindIDs contains the statically resolved relationship kinds admitted by the expansion. + RelationshipKindIDs []int16 + + // EdgeStartIdentifier is the unqualified edge endpoint column from which the chosen direction advances. EdgeStartIdentifier pgsql.Identifier - EdgeStartColumn pgsql.CompoundIdentifier - EdgeEndIdentifier pgsql.Identifier - EdgeEndColumn pgsql.CompoundIdentifier - + // EdgeStartColumn is the qualified edge endpoint expression from which the chosen direction advances. + EdgeStartColumn pgsql.CompoundIdentifier + // EdgeEndIdentifier is the unqualified edge endpoint column reached by the chosen direction. + EdgeEndIdentifier pgsql.Identifier + // EdgeEndColumn is the qualified edge endpoint expression reached by the chosen direction. + EdgeEndColumn pgsql.CompoundIdentifier + + // Projection contains the select items exposed by the completed expansion frame. Projection []pgsql.SelectItem } +// UsesSingletonEndpointPair reports whether both expansion endpoints are statically singleton IDs. +func (s *Expansion) UsesSingletonEndpointPair() bool { + return s != nil && s.SingletonRootID != nil && s.SingletonTerminalID != nil +} + +// NewExpansionModel builds the SQL model fragment responsible for new expansion model. func NewExpansionModel(part *PatternPart, relationshipPattern *cypher.RelationshipPattern) *Expansion { return &Expansion{ Options: newExpansionOptions(part, relationshipPattern), } } +// CompletePattern builds the SQL model fragment responsible for complete pattern. func (s *Expansion) CompletePattern(traversalStep *TraversalStep) error { // This determines which side of the expansion is treated as the root (where the traversal begins) switch traversalStep.Direction { @@ -119,16 +208,19 @@ func (s *Expansion) CompletePattern(traversalStep *TraversalStep) error { return nil } +// FlipDirection builds the SQL model fragment responsible for flip direction. func (s *Expansion) FlipDirection() { oldEdgeStartColumn := s.EdgeStartColumn s.EdgeStartColumn = s.EdgeEndColumn s.EdgeEndColumn = oldEdgeStartColumn } +// CanExecuteBidirectionalSearch builds the SQL model fragment responsible for can execute bidirectional search. func (s *Expansion) CanExecuteBidirectionalSearch() bool { return s.PrimerNodeConstraints != nil && s.TerminalNodeConstraints != nil } +// CanExecuteBidirectionalSearch builds the SQL model fragment responsible for can execute bidirectional search. func (s *TraversalStep) CanExecuteBidirectionalSearch() bool { if s.Expansion == nil { return false @@ -138,18 +230,22 @@ func (s *TraversalStep) CanExecuteBidirectionalSearch() bool { (s.LeftNodeBound && s.RightNodeBound && s.Frame != nil && s.Frame.Previous != nil) } +// hasPreviousFrameBinding reports whether the step can reference bindings materialized by a prior frame. func (s *TraversalStep) hasPreviousFrameBinding() bool { return s.Frame != nil && s.Frame.Previous != nil } +// usesBoundEndpointPairs reports whether both endpoints come from a previous frame. func (s *TraversalStep) usesBoundEndpointPairs() bool { return s.LeftNodeBound && s.RightNodeBound && s.hasPreviousFrameBinding() } +// usesBoundTerminalIDs reports whether the terminal endpoint comes from a previous frame. func (s *TraversalStep) usesBoundTerminalIDs() bool { return s.RightNodeBound && s.hasPreviousFrameBinding() } +// canMaterializeTerminalFilterForStep reports whether terminal constraints are local and useful as an independent filter. func canMaterializeTerminalFilterForStep(traversalStep *TraversalStep, expansionModel *Expansion) bool { if traversalStep == nil || expansionModel == nil || traversalStep.RightNode == nil || expansionModel.TerminalNodeConstraints == nil || @@ -168,6 +264,7 @@ func canMaterializeTerminalFilterForStep(traversalStep *TraversalStep, expansion return externalConstraints == nil } +// canMaterializeEndpointPairFilterForStep reports whether both local endpoint constraints restrict harness search columns. func canMaterializeEndpointPairFilterForStep(traversalStep *TraversalStep, expansionModel *Expansion) bool { // Pair filters enumerate the exact root/terminal combinations the // bidirectional harness must resolve. Kind-only endpoint predicates are not @@ -186,14 +283,17 @@ func canMaterializeEndpointPairFilterForStep(traversalStep *TraversalStep, expan return true } +// endpointSelectivity scores an endpoint expression using binding and previous-frame context. func (s *TraversalStep) endpointSelectivity(scope *Scope, expression pgsql.Expression, bound bool) (int, error) { return optimize.NewSelectivityModel(scope).EndpointSelectivity(expression, bound, s.hasPreviousFrameBinding()) } +// isBidirectionalSearchAnchor reports whether a selectivity score is strong enough to seed bidirectional search. func isBidirectionalSearchAnchor(selectivity int) bool { return optimize.IsBidirectionalSearchAnchor(selectivity) } +// hasIDEqualityConstraint reports whether identifier's ID equals a row-independent value in a conjunction. func hasIDEqualityConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { for _, term := range flattenConjunction(expression) { binaryExpression, isBinaryExpression := unwrapParenthetical(term).(*pgsql.BinaryExpression) @@ -218,6 +318,7 @@ func hasIDEqualityConstraint(expression pgsql.Expression, identifier pgsql.Ident return false } +// hasLocalIDEqualityConstraint reports whether an ID equality depends only on identifier and static values. func hasLocalIDEqualityConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { if !hasIDEqualityConstraint(expression, identifier) { return false @@ -226,6 +327,7 @@ func hasLocalIDEqualityConstraint(expression pgsql.Expression, identifier pgsql. return hasLocalEndpointConstraint(expression, identifier) } +// hasLocalEndpointConstraint reports whether expression references identifier without any external binding. func hasLocalEndpointConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { if expression == nil || !referencesIdentifier(expression, identifier) { return false @@ -235,6 +337,7 @@ func hasLocalEndpointConstraint(expression pgsql.Expression, identifier pgsql.Id return externalConstraints == nil } +// referencesIdentifier reports whether expression contains a direct, compound, or row-column reference rooted at identifier. func referencesIdentifier(expression pgsql.Expression, identifier pgsql.Identifier) bool { references := false @@ -267,11 +370,13 @@ func referencesIdentifier(expression pgsql.Expression, identifier pgsql.Identifi return references } +// hasPairAwareEndpointConstraint reports whether a local constraint restricts endpoint values beyond node kinds. func hasPairAwareEndpointConstraint(expression pgsql.Expression, identifier pgsql.Identifier) bool { return hasLocalEndpointConstraint(expression, identifier) && referencesEndpointSearchColumn(expression, identifier) } +// referencesEndpointSearchColumn reports whether expression reads a non-kind field used to restrict endpoint search. func referencesEndpointSearchColumn(expression pgsql.Expression, identifier pgsql.Identifier) bool { references := false @@ -292,6 +397,7 @@ func referencesEndpointSearchColumn(expression pgsql.Expression, identifier pgsq return references } +// isStaticIDEqualityOperand reports whether expression contains no row or identifier references. func isStaticIDEqualityOperand(expression pgsql.Expression) bool { if expression == nil { return false @@ -312,6 +418,7 @@ func isStaticIDEqualityOperand(expression pgsql.Expression) bool { return isStatic } +// isIdentifierIDReference reports whether expression is exactly identifier.id. func isIdentifierIDReference(expression pgsql.Expression, identifier pgsql.Identifier) bool { compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(expression).(pgsql.CompoundIdentifier) return isCompoundIdentifier && len(compoundIdentifier) == 2 && @@ -319,6 +426,85 @@ func isIdentifierIDReference(expression pgsql.Expression, identifier pgsql.Ident compoundIdentifier[1] == pgsql.ColumnID } +// isSingletonIDOperand reports whether expression denotes one non-null integer ID literal or parameter. +func isSingletonIDOperand(expression pgsql.Expression) bool { + switch typedExpression := unwrapParenthetical(expression).(type) { + case pgsql.Literal: + return !typedExpression.Null + case pgsql.Parameter, *pgsql.Parameter: + return true + case pgsql.TypeCast: + switch typedExpression.CastType { + case pgsql.Int, pgsql.Int2, pgsql.Int4, pgsql.Int8: + return isSingletonIDOperand(typedExpression.Expression) + default: + return false + } + default: + return false + } +} + +// singletonIDAnchor returns the sole static value equated with identifier.id, rejecting ambiguous multiple equalities. +func singletonIDAnchor(expression pgsql.Expression, identifier pgsql.Identifier) (pgsql.Expression, bool) { + var anchor pgsql.Expression + + for _, term := range flattenConjunction(expression) { + binaryExpression, isBinaryExpression := unwrapParenthetical(term).(*pgsql.BinaryExpression) + if !isBinaryExpression || binaryExpression.Operator != pgsql.OperatorEquals { + continue + } + + var candidate pgsql.Expression + switch { + case isIdentifierIDReference(binaryExpression.LOperand, identifier) && isSingletonIDOperand(binaryExpression.ROperand): + candidate = binaryExpression.ROperand + case isIdentifierIDReference(binaryExpression.ROperand, identifier) && isSingletonIDOperand(binaryExpression.LOperand): + candidate = binaryExpression.LOperand + default: + continue + } + + if anchor != nil { + // Multiple ID equalities may be contradictory and require the generic + // validation path until the singleton validator can retain every term. + return nil, false + } + anchor = candidate + } + + return anchor, anchor != nil +} + +// replaceSingletonIDAnchor substitutes replacement for the static side of identifier's singleton ID equality. +func replaceSingletonIDAnchor(expression pgsql.Expression, identifier pgsql.Identifier, replacement pgsql.Expression) pgsql.Expression { + switch typedExpression := expression.(type) { + case *pgsql.Parenthetical: + typedExpression.Expression = replaceSingletonIDAnchor(typedExpression.Expression, identifier, replacement) + return typedExpression + + case *pgsql.BinaryExpression: + if typedExpression.Operator == pgsql.OperatorEquals { + switch { + case isIdentifierIDReference(typedExpression.LOperand, identifier) && isSingletonIDOperand(typedExpression.ROperand): + typedExpression.ROperand = replacement + return typedExpression + case isIdentifierIDReference(typedExpression.ROperand, identifier) && isSingletonIDOperand(typedExpression.LOperand): + typedExpression.LOperand = replacement + return typedExpression + } + } + + typedExpression.LOperand = replaceSingletonIDAnchor(typedExpression.LOperand, identifier, replacement) + typedExpression.ROperand = replaceSingletonIDAnchor(typedExpression.ROperand, identifier, replacement) + return typedExpression + + default: + return expression + } +} + +// CanExecuteSelectiveBidirectionalSearch builds the SQL model fragment responsible for can execute selective bidirectional search. func (s *TraversalStep) CanExecuteSelectiveBidirectionalSearch(scope *Scope) (bool, error) { if s.Expansion == nil { return false, nil @@ -348,6 +534,7 @@ func (s *TraversalStep) CanExecuteSelectiveBidirectionalSearch(scope *Scope) (bo return false, nil } +// CanExecutePairAwareBidirectionalSearch builds the SQL model fragment responsible for can execute pair aware bidirectional search. func (s *TraversalStep) CanExecutePairAwareBidirectionalSearch(scope *Scope) (bool, error) { if canExecute, err := s.CanExecuteSelectiveBidirectionalSearch(scope); canExecute || err != nil { return canExecute, err @@ -376,80 +563,116 @@ func (s *TraversalStep) CanExecutePairAwareBidirectionalSearch(scope *Scope) (bo } } +// flattenConjunction returns the independent terms of a nested PostgreSQL AND expression. func flattenConjunction(expr pgsql.Expression) []pgsql.Expression { return optimize.FlattenConjunction(expr) } +// expressionReferencesOnlyLocalIdentifiers reports whether every binding referenced by expression belongs to localScope. func expressionReferencesOnlyLocalIdentifiers(expression pgsql.Expression, localScope *pgsql.IdentifierSet) bool { return optimize.ExpressionReferencesOnlyLocalIdentifiers(expression, localScope) } +// subqueryReferencesOnlyLocalIdentifiers reports whether a subquery has no dependencies outside localScope. func subqueryReferencesOnlyLocalIdentifiers(subquery pgsql.Subquery, localScope *pgsql.IdentifierSet) bool { return optimize.SubqueryReferencesOnlyLocalIdentifiers(subquery, localScope) } +// queryReferencesOnlyLocalIdentifiers reports whether a query has no dependencies outside localScope. func queryReferencesOnlyLocalIdentifiers(query pgsql.Query, localScope *pgsql.IdentifierSet) bool { return optimize.QueryReferencesOnlyLocalIdentifiers(query, localScope) } +// addFromClauseBindings adds every alias introduced by fromClauses to localScope. func addFromClauseBindings(localScope *pgsql.IdentifierSet, fromClauses []pgsql.FromClause) { optimize.AddFromClauseBindings(localScope, fromClauses) } +// addFromExpressionBinding adds the alias introduced by a FROM expression to localScope. func addFromExpressionBinding(localScope *pgsql.IdentifierSet, expression pgsql.Expression) { optimize.AddFromExpressionBinding(localScope, expression) } +// selectReferencesOnlyLocalIdentifiers reports whether a SELECT body has no dependencies outside localScope. func selectReferencesOnlyLocalIdentifiers(selectBody pgsql.Select, localScope *pgsql.IdentifierSet) bool { return optimize.SelectReferencesOnlyLocalIdentifiers(selectBody, localScope) } +// fromExpressionReferencesOnlyLocalIdentifiers reports whether a FROM expression has no dependencies outside localScope. func fromExpressionReferencesOnlyLocalIdentifiers(expression pgsql.Expression, localScope *pgsql.IdentifierSet) bool { return optimize.FromExpressionReferencesOnlyLocalIdentifiers(expression, localScope) } +// isLocalToScope reports whether expression can be evaluated using only identifiers in localScope. func isLocalToScope(expression pgsql.Expression, localScope *pgsql.IdentifierSet) bool { return optimize.IsLocalToScope(expression, localScope) } +// partitionConstraintByLocality separates conjuncts evaluable in localScope from those requiring outer bindings. func partitionConstraintByLocality(expression pgsql.Expression, localScope *pgsql.IdentifierSet) (pgsql.Expression, pgsql.Expression) { return optimize.PartitionConstraintByLocality(expression, localScope) } +// ProjectionPruningApplication groups SQL model state that must remain consistent while translating projection pruning application. type ProjectionPruningApplication struct { - LeftNode *BoundIdentifier + // LeftNode supplies the left node input to the ProjectionPruningApplication contract. + LeftNode *BoundIdentifier + // Relationship supplies the relationship input to the ProjectionPruningApplication contract. Relationship *BoundIdentifier - RightNode *BoundIdentifier - PathBinding *BoundIdentifier + // RightNode supplies the right node input to the ProjectionPruningApplication contract. + RightNode *BoundIdentifier + // PathBinding supplies the path binding input to the ProjectionPruningApplication contract. + PathBinding *BoundIdentifier } +// TraversalStep groups SQL model state that must remain consistent while translating traversal step. type TraversalStep struct { - Frame *Frame - SourceTarget optimize.TraversalStepTarget + // Frame supplies the frame input to the TraversalStep contract. + Frame *Frame + // SourceTarget supplies the source target input to the TraversalStep contract. + SourceTarget optimize.TraversalStepTarget + // HasSourceTarget indicates whether has source target applies. HasSourceTarget bool - Direction graph.Direction - Expansion *Expansion - PathReversed bool + // Direction selects the traversal orientation covered by the contract. + Direction graph.Direction + // Expansion supplies the expansion input to the TraversalStep contract. + Expansion *Expansion + // PathReversed indicates whether path reversed applies. + PathReversed bool // OmitPreviousFrameSource suppresses comma-joining the previous frame as a FROM source. Pattern // predicate roots require this so that references to the enclosing query part's frame remain // correlated to the outer row instead of re-scanning the outer CTE. OmitPreviousFrameSource bool - ProjectionPruning ProjectionPruningApplication - LeftNode *BoundIdentifier - LeftNodeBound bool - UseExpandInto bool - LeftNodeConstraints pgsql.Expression - LeftNodeJoinCondition pgsql.Expression - Edge *BoundIdentifier - EdgeConstraints *Constraint - EdgeJoinCondition pgsql.Expression - RightNode *BoundIdentifier - RightNodeBound bool - RightNodeConstraints pgsql.Expression + // ProjectionPruning supplies the projection pruning input to the TraversalStep contract. + ProjectionPruning ProjectionPruningApplication + // LeftNode supplies the left node input to the TraversalStep contract. + LeftNode *BoundIdentifier + // LeftNodeBound indicates whether left node bound applies. + LeftNodeBound bool + // UseExpandInto indicates whether use expand into applies. + UseExpandInto bool + // LeftNodeConstraints supplies the left node constraints input to the TraversalStep contract. + LeftNodeConstraints pgsql.Expression + // LeftNodeJoinCondition supplies the left node join condition input to the TraversalStep contract. + LeftNodeJoinCondition pgsql.Expression + // Edge supplies the edge input to the TraversalStep contract. + Edge *BoundIdentifier + // EdgeConstraints supplies the edge constraints input to the TraversalStep contract. + EdgeConstraints *Constraint + // EdgeJoinCondition supplies the edge join condition input to the TraversalStep contract. + EdgeJoinCondition pgsql.Expression + // RightNode supplies the right node input to the TraversalStep contract. + RightNode *BoundIdentifier + // RightNodeBound indicates whether right node bound applies. + RightNodeBound bool + // RightNodeConstraints supplies the right node constraints input to the TraversalStep contract. + RightNodeConstraints pgsql.Expression + // RightNodeJoinCondition supplies the right node join condition input to the TraversalStep contract. RightNodeJoinCondition pgsql.Expression - Projection []pgsql.SelectItem + // Projection supplies the projection input to the TraversalStep contract. + Projection []pgsql.SelectItem } // StartNode will find the root node of this pattern segment based on the segment's direction @@ -476,6 +699,7 @@ func (s *TraversalStep) EndNode() (*BoundIdentifier, error) { } } +// FlipNodes builds the SQL model fragment responsible for flip nodes. func (s *TraversalStep) FlipNodes() { if s.Expansion != nil { // If the expansion is set then column identifiers must also be swapped @@ -502,23 +726,35 @@ func (s *TraversalStep) FlipNodes() { s.PathReversed = !s.PathReversed } +// PatternPart groups SQL model state that must remain consistent while translating pattern part. type PatternPart struct { - IsTraversal bool - ShortestPath bool + // IsTraversal indicates whether is traversal applies. + IsTraversal bool + // ShortestPath identifies the filesystem shortest path. + ShortestPath bool + // AllShortestPaths identifies the filesystem all shortest paths. AllShortestPaths bool // PathDirectionReversed is set when the optimizer reversed the originating cypher pattern's // element order and relationship directions. Path materialization uses it to restore the // original left-to-right logical order for a bound path. PathDirectionReversed bool - PatternBinding *BoundIdentifier - Target optimize.PatternTarget - HasTarget bool - TraversalSteps []*TraversalStep - NodeSelect NodeSelect - Constraints *ConstraintTracker - nextSourceStep int -} - + // PatternBinding supplies the pattern binding input to the PatternPart contract. + PatternBinding *BoundIdentifier + // Target supplies the target input to the PatternPart contract. + Target optimize.PatternTarget + // HasTarget indicates whether has target applies. + HasTarget bool + // TraversalSteps supplies the traversal steps input to the PatternPart contract. + TraversalSteps []*TraversalStep + // NodeSelect supplies the node select input to the PatternPart contract. + NodeSelect NodeSelect + // Constraints supplies the constraints input to the PatternPart contract. + Constraints *ConstraintTracker + // nextSourceStep retains the next source step while PatternPart is assembled or evaluated. + nextSourceStep int +} + +// nextSourceTarget returns the optimizer coordinates for the next traversal step and advances the step cursor. func (s *PatternPart) nextSourceTarget() (optimize.TraversalStepTarget, bool) { if s == nil { return optimize.TraversalStepTarget{}, false @@ -534,10 +770,12 @@ func (s *PatternPart) nextSourceTarget() (optimize.TraversalStepTarget, bool) { return s.Target.TraversalStep(stepIndex), true } +// LastStep builds the SQL model fragment responsible for last step. func (s *PatternPart) LastStep() *TraversalStep { return s.TraversalSteps[len(s.TraversalSteps)-1] } +// ContainsExpansions builds the SQL model fragment responsible for contains expansions. func (s *PatternPart) ContainsExpansions() bool { for _, traversalStep := range s.TraversalSteps { if traversalStep.Expansion != nil { @@ -548,14 +786,18 @@ func (s *PatternPart) ContainsExpansions() bool { return false } +// Pattern groups SQL model state that must remain consistent while translating pattern. type Pattern struct { + // Parts supplies the parts input to the Pattern contract. Parts []*PatternPart } +// Reset builds the SQL model fragment responsible for reset. func (s *Pattern) Reset() { s.Parts = s.Parts[:0] } +// NewPart builds the SQL model fragment responsible for new part. func (s *Pattern) NewPart() *PatternPart { newPatternPart := &PatternPart{ Constraints: NewConstraintTracker(), @@ -565,79 +807,119 @@ func (s *Pattern) NewPart() *PatternPart { return newPatternPart } +// CurrentPart builds the SQL model fragment responsible for current part. func (s *Pattern) CurrentPart() *PatternPart { return s.Parts[len(s.Parts)-1] } +// Query groups SQL model state that must remain consistent while translating query. type Query struct { + // Parts supplies the parts input to the Query contract. Parts []*QueryPart } +// HasParts builds the SQL model fragment responsible for has parts. func (s *Query) HasParts() bool { return len(s.Parts) > 0 } +// AddPart builds the SQL model fragment responsible for add part. func (s *Query) AddPart(part *QueryPart) { s.Parts = append(s.Parts, part) } +// CurrentPart builds the SQL model fragment responsible for current part. func (s *Query) CurrentPart() *QueryPart { return s.Parts[len(s.Parts)-1] } +// QueryPart groups SQL model state that must remain consistent while translating query part. type QueryPart struct { - Model *pgsql.Query - Frame *Frame - Updates []*Mutations + // Model supplies the model input to the QueryPart contract. + Model *pgsql.Query + // Frame supplies the frame input to the QueryPart contract. + Frame *Frame + // Updates supplies the updates input to the QueryPart contract. + Updates []*Mutations + // SortItems supplies the sort items input to the QueryPart contract. SortItems []*pgsql.OrderBy - Skip pgsql.Expression - Limit pgsql.Expression - - numReadingClauses int + // Skip supplies the skip input to the QueryPart contract. + Skip pgsql.Expression + // Limit supplies the limit input to the QueryPart contract. + Limit pgsql.Expression + + // numReadingClauses retains the num reading clauses while QueryPart is assembled or evaluated. + numReadingClauses int + // numUpdatingClauses retains the num updating clauses while QueryPart is assembled or evaluated. numUpdatingClauses int // The fields below are meant to be used to build each component as the source AST is walked. There's some // repetition of some of the exported fields above which is intentional and may be a good refactor target // in the future - patternPredicates []*pgsql.Future[*Pattern] - pathEdgeIDArrayFutures []*pgsql.Future[*BoundIdentifier] - properties TranslatedProperties - currentPattern *Pattern - stashedPattern *Pattern - projections *Projections - mutations *Mutations - fromClauses []pgsql.FromClause - limitPushdownFrames *pgsql.IdentifierSet - referencedIdentifiers *pgsql.IdentifierSet + patternPredicates []*pgsql.Future[*Pattern] + // pathEdgeIDArrayFutures retains the path edge id array futures while QueryPart is assembled or evaluated. + pathEdgeIDArrayFutures []*pgsql.Future[*BoundIdentifier] + // properties retains the properties while QueryPart is assembled or evaluated. + properties TranslatedProperties + // currentPattern retains the current pattern while QueryPart is assembled or evaluated. + currentPattern *Pattern + // stashedPattern retains the stashed pattern while QueryPart is assembled or evaluated. + stashedPattern *Pattern + // projections retains the projections while QueryPart is assembled or evaluated. + projections *Projections + // mutations retains the mutations while QueryPart is assembled or evaluated. + mutations *Mutations + // fromClauses retains the from clauses while QueryPart is assembled or evaluated. + fromClauses []pgsql.FromClause + // limitPushdownFrames retains the limit pushdown frames while QueryPart is assembled or evaluated. + limitPushdownFrames *pgsql.IdentifierSet + // referencedIdentifiers retains the referenced identifiers while QueryPart is assembled or evaluated. + referencedIdentifiers *pgsql.IdentifierSet + // stashedExpressionTreeTranslator retains the stashed expression tree translator while QueryPart is assembled or evaluated. stashedExpressionTreeTranslator *ExpressionTreeTranslator - stashedQuantifierArray []pgsql.Expression - stashedQuantifierUseExists bool - quantifierIndex int - quantifierIdentifiers *pgsql.IdentifierSet - unwindClauses []UnwindClause - isCreating bool -} - + // stashedQuantifierArray retains the stashed quantifier array while QueryPart is assembled or evaluated. + stashedQuantifierArray []pgsql.Expression + // stashedQuantifierUseExists indicates whether stashed quantifier use exists applies. + stashedQuantifierUseExists bool + // quantifierIndex retains the quantifier index while QueryPart is assembled or evaluated. + quantifierIndex int + // quantifierIdentifiers retains the quantifier identifiers while QueryPart is assembled or evaluated. + quantifierIdentifiers *pgsql.IdentifierSet + // unwindClauses retains the unwind clauses while QueryPart is assembled or evaluated. + unwindClauses []UnwindClause + // isCreating indicates whether is creating applies. + isCreating bool +} + +// UnwindClause groups SQL model state that must remain consistent while translating unwind clause. type UnwindClause struct { + // Expression supplies the expression input to the UnwindClause contract. Expression pgsql.Expression - Binding *BoundIdentifier + // Binding supplies the binding input to the UnwindClause contract. + Binding *BoundIdentifier } +// TranslatedProperties groups SQL model state that must remain consistent while translating translated properties. type TranslatedProperties struct { - Map map[string]pgsql.Expression + // Map supplies the map input to the TranslatedProperties contract. + Map map[string]pgsql.Expression + // Parameter supplies the parameter input to the TranslatedProperties contract. Parameter pgsql.Expression } +// NewTranslatedProperties builds the SQL model fragment responsible for new translated properties. func NewTranslatedProperties() TranslatedProperties { return TranslatedProperties{ Map: map[string]pgsql.Expression{}, } } +// IsEmpty builds the SQL model fragment responsible for is empty. func (s TranslatedProperties) IsEmpty() bool { return len(s.Map) == 0 && s.Parameter == nil } +// NewQueryPart builds the SQL model fragment responsible for new query part. func NewQueryPart(numReadingClauses, numUpdatingClauses int) *QueryPart { return &QueryPart{ Model: &pgsql.Query{ @@ -654,10 +936,12 @@ func NewQueryPart(numReadingClauses, numUpdatingClauses int) *QueryPart { } } +// AddFromClause builds the SQL model fragment responsible for add from clause. func (s *QueryPart) AddFromClause(clause pgsql.FromClause) { s.fromClauses = append(s.fromClauses, clause) } +// ConsumeFromClauses builds the SQL model fragment responsible for consume from clauses. func (s *QueryPart) ConsumeFromClauses() []pgsql.FromClause { fromClauses := s.fromClauses s.fromClauses = nil @@ -665,41 +949,50 @@ func (s *QueryPart) ConsumeFromClauses() []pgsql.FromClause { return fromClauses } +// AllowLimitPushdown builds the SQL model fragment responsible for allow limit pushdown. func (s *QueryPart) AllowLimitPushdown(frameIdentifier pgsql.Identifier) { s.limitPushdownFrames.Add(frameIdentifier) } +// CanPushDownLimitTo builds the SQL model fragment responsible for can push down limit to. func (s *QueryPart) CanPushDownLimitTo(frameIdentifier pgsql.Identifier) bool { return s.limitPushdownFrames.Contains(frameIdentifier) } +// AddUnwindClause builds the SQL model fragment responsible for add unwind clause. func (s *QueryPart) AddUnwindClause(clause UnwindClause) { s.unwindClauses = append(s.unwindClauses, clause) } +// ConsumeUnwindClauses builds the SQL model fragment responsible for consume unwind clauses. func (s *QueryPart) ConsumeUnwindClauses() []UnwindClause { clauses := s.unwindClauses s.unwindClauses = nil return clauses } +// RestoreStashedPattern builds the SQL model fragment responsible for restore stashed pattern. func (s *QueryPart) RestoreStashedPattern() { s.currentPattern = s.stashedPattern s.stashedPattern = nil } +// StashCurrentPattern builds the SQL model fragment responsible for stash current pattern. func (s *QueryPart) StashCurrentPattern() { s.stashedPattern = s.ConsumeCurrentPattern() } +// AddPatternPredicateFuture builds the SQL model fragment responsible for add pattern predicate future. func (s *QueryPart) AddPatternPredicateFuture(predicateFuture *pgsql.Future[*Pattern]) { s.patternPredicates = append(s.patternPredicates, predicateFuture) } +// AddPathEdgeIDArrayFuture builds the SQL model fragment responsible for add path edge id array future. func (s *QueryPart) AddPathEdgeIDArrayFuture(pathEdgeIDArrayFuture *pgsql.Future[*BoundIdentifier]) { s.pathEdgeIDArrayFutures = append(s.pathEdgeIDArrayFutures, pathEdgeIDArrayFuture) } +// ConsumeCurrentPattern builds the SQL model fragment responsible for consume current pattern. func (s *QueryPart) ConsumeCurrentPattern() *Pattern { currentPattern := s.currentPattern s.currentPattern = &Pattern{} @@ -707,42 +1000,51 @@ func (s *QueryPart) ConsumeCurrentPattern() *Pattern { return currentPattern } +// HasProjections builds the SQL model fragment responsible for has projections. func (s *QueryPart) HasProjections() bool { return s.projections != nil && len(s.projections.Items) > 0 } +// PrepareProjections builds the SQL model fragment responsible for prepare projections. func (s *QueryPart) PrepareProjections(distinct bool) { s.projections = &Projections{ Distinct: distinct, } } +// PrepareMutations builds the SQL model fragment responsible for prepare mutations. func (s *QueryPart) PrepareMutations() { if s.mutations == nil { s.mutations = NewMutations() } } +// HasMutations builds the SQL model fragment responsible for has mutations. func (s *QueryPart) HasMutations() bool { return s.mutations != nil && s.mutations.Updates.Len() > 0 } +// HasDeletions builds the SQL model fragment responsible for has deletions. func (s *QueryPart) HasDeletions() bool { return s.mutations != nil && s.mutations.Deletions.Len() > 0 } +// PrepareProjection constructs the SQL model used for prepare projection. func (s *QueryPart) PrepareProjection() { s.projections.Items = append(s.projections.Items, &Projection{}) } +// CurrentProjection constructs the SQL model used for current projection. func (s *QueryPart) CurrentProjection() *Projection { return s.projections.Current() } +// HasProperties builds the SQL model fragment responsible for has properties. func (s *QueryPart) HasProperties() bool { return !s.properties.IsEmpty() } +// AddProperty builds the SQL model fragment responsible for add property. func (s *QueryPart) AddProperty(key string, expression pgsql.Expression) { if s.properties.Map == nil { s.properties.Map = map[string]pgsql.Expression{} @@ -751,10 +1053,12 @@ func (s *QueryPart) AddProperty(key string, expression pgsql.Expression) { s.properties.Map[key] = expression } +// AddPropertyParameter builds the SQL model fragment responsible for add property parameter. func (s *QueryPart) AddPropertyParameter(expression pgsql.Expression) { s.properties.Parameter = expression } +// ConsumeProperties builds the SQL model fragment responsible for consume properties. func (s *QueryPart) ConsumeProperties() TranslatedProperties { properties := s.properties s.properties = NewTranslatedProperties() @@ -762,77 +1066,122 @@ func (s *QueryPart) ConsumeProperties() TranslatedProperties { return properties } +// CurrentOrderBy builds the SQL model fragment responsible for current order by. func (s *QueryPart) CurrentOrderBy() *pgsql.OrderBy { return s.SortItems[len(s.SortItems)-1] } +// Projection groups SQL model state that must remain consistent while translating projection. type Projection struct { + // SelectItem supplies the select item input to the Projection contract. SelectItem pgsql.SelectItem - Alias models.Optional[pgsql.Identifier] + // Alias supplies the alias input to the Projection contract. + Alias models.Optional[pgsql.Identifier] } +// SetIdentifier builds the SQL model fragment responsible for set identifier. func (s *Projection) SetIdentifier(identifier pgsql.Identifier) { s.SelectItem = identifier } +// SetAlias builds the SQL model fragment responsible for set alias. func (s *Projection) SetAlias(alias pgsql.Identifier) { s.Alias = models.OptionalValue(alias) } +// Removal groups SQL model state that must remain consistent while translating removal. type Removal struct { + // Field supplies the field input to the Removal contract. Field string } +// LabelAssignment groups SQL model state that must remain consistent while translating label assignment. type LabelAssignment struct { + // Kinds supplies the kinds input to the LabelAssignment contract. Kinds pgsql.Expression } +// PropertyAssignment groups SQL model state that must remain consistent while translating property assignment. type PropertyAssignment struct { - Field string - Operator pgsql.Operator + // Field supplies the field input to the PropertyAssignment contract. + Field string + // Operator supplies the operator input to the PropertyAssignment contract. + Operator pgsql.Operator + // ValueExpression supplies the value expression input to the PropertyAssignment contract. ValueExpression pgsql.Expression } +// Update groups SQL model state that must remain consistent while translating update. type Update struct { - Frame *Frame - JoinConstraint pgsql.Expression - Projection []pgsql.SelectItem - TargetBinding *BoundIdentifier - UpdateBinding *BoundIdentifier - Removals *graph.IndexedSlice[string, Removal] + // Frame supplies the frame input to the Update contract. + Frame *Frame + // JoinConstraint supplies the join constraint input to the Update contract. + JoinConstraint pgsql.Expression + // Projection supplies the projection input to the Update contract. + Projection []pgsql.SelectItem + // TargetBinding supplies the target binding input to the Update contract. + TargetBinding *BoundIdentifier + // UpdateBinding supplies the update binding input to the Update contract. + UpdateBinding *BoundIdentifier + // Removals supplies the removals input to the Update contract. + Removals *graph.IndexedSlice[string, Removal] + // PropertyAssignments supplies the property assignments input to the Update contract. PropertyAssignments *graph.IndexedSlice[string, PropertyAssignment] - KindRemovals graph.Kinds - KindAssignments graph.Kinds + // KindRemovals supplies the kind removals input to the Update contract. + KindRemovals graph.Kinds + // KindAssignments supplies the kind assignments input to the Update contract. + KindAssignments graph.Kinds } +// Delete groups SQL model state that must remain consistent while translating delete. type Delete struct { - Frame *Frame + // Frame supplies the frame input to the Delete contract. + Frame *Frame + // TargetBinding supplies the target binding input to the Delete contract. TargetBinding *BoundIdentifier + // UpdateBinding supplies the update binding input to the Delete contract. UpdateBinding *BoundIdentifier } +// NodeCreate groups SQL model state that must remain consistent while translating node create. type NodeCreate struct { - Binding *BoundIdentifier + // Binding supplies the binding input to the NodeCreate contract. + Binding *BoundIdentifier + // Properties supplies the properties input to the NodeCreate contract. Properties TranslatedProperties - Kinds graph.Kinds + // Kinds supplies the kinds input to the NodeCreate contract. + Kinds graph.Kinds } +// EdgeCreate groups SQL model state that must remain consistent while translating edge create. type EdgeCreate struct { - Binding *BoundIdentifier + // Binding supplies the binding input to the EdgeCreate contract. + Binding *BoundIdentifier + // Properties supplies the properties input to the EdgeCreate contract. Properties TranslatedProperties - Kinds graph.Kinds - LeftNode *BoundIdentifier - RightNode *BoundIdentifier - Direction graph.Direction + // Kinds supplies the kinds input to the EdgeCreate contract. + Kinds graph.Kinds + // LeftNode supplies the left node input to the EdgeCreate contract. + LeftNode *BoundIdentifier + // RightNode supplies the right node input to the EdgeCreate contract. + RightNode *BoundIdentifier + // Direction selects the traversal orientation covered by the contract. + Direction graph.Direction } +// Mutations groups SQL model state that must remain consistent while translating mutations. type Mutations struct { - Deletions *graph.IndexedSlice[pgsql.Identifier, *Delete] - Updates *graph.IndexedSlice[pgsql.Identifier, *Update] - Creations *graph.IndexedSlice[pgsql.Identifier, *NodeCreate] + // Deletions supplies the deletions input to the Mutations contract. + Deletions *graph.IndexedSlice[pgsql.Identifier, *Delete] + // Updates supplies the updates input to the Mutations contract. + Updates *graph.IndexedSlice[pgsql.Identifier, *Update] + // Creations supplies the creations input to the Mutations contract. + Creations *graph.IndexedSlice[pgsql.Identifier, *NodeCreate] + // EdgeCreations supplies the edge creations input to the Mutations contract. EdgeCreations *graph.IndexedSlice[pgsql.Identifier, *EdgeCreate] } +// NewMutations builds the SQL model fragment responsible for new mutations. func NewMutations() *Mutations { return &Mutations{ Deletions: graph.NewIndexedSlice[pgsql.Identifier, *Delete](), @@ -842,6 +1191,7 @@ func NewMutations() *Mutations { } } +// AddDeletion builds the SQL model fragment responsible for add deletion. func (s *Mutations) AddDeletion(scope *Scope, targetIdentifier pgsql.Identifier, frame *Frame) (*Delete, error) { if targetBinding, bound := scope.Lookup(targetIdentifier); !bound { return nil, fmt.Errorf("invalid identifier: %s", targetIdentifier) @@ -859,6 +1209,7 @@ func (s *Mutations) AddDeletion(scope *Scope, targetIdentifier pgsql.Identifier, } } +// newIdentifierAssignment allocates a distinct update binding and empty assignment collections for targetBinding. func (s *Mutations) newIdentifierAssignment(scope *Scope, targetBinding *BoundIdentifier) (*Update, error) { if updateBinding, err := scope.DefineNew(targetBinding.DataType); err != nil { return nil, err @@ -877,6 +1228,7 @@ func (s *Mutations) newIdentifierAssignment(scope *Scope, targetBinding *BoundId } } +// getIdentifierMutation returns the existing update for targetIdentifier or creates its first assignment state. func (s *Mutations) getIdentifierMutation(scope *Scope, targetIdentifier pgsql.Identifier) (*Update, error) { if targetBinding, bound := scope.Lookup(targetIdentifier); !bound { return nil, fmt.Errorf("invalid identifier: %s", targetIdentifier) @@ -887,6 +1239,7 @@ func (s *Mutations) getIdentifierMutation(scope *Scope, targetIdentifier pgsql.I } } +// AddPropertyRemoval builds the SQL model fragment responsible for add property removal. func (s *Mutations) AddPropertyRemoval(scope *Scope, propertyLookup PropertyLookup) error { if mutation, err := s.getIdentifierMutation(scope, propertyLookup.Reference.Root()); err != nil { return err @@ -899,6 +1252,7 @@ func (s *Mutations) AddPropertyRemoval(scope *Scope, propertyLookup PropertyLook return nil } +// AddPropertyAssignment builds the SQL model fragment responsible for add property assignment. func (s *Mutations) AddPropertyAssignment(scope *Scope, propertyLookup PropertyLookup, operator pgsql.Operator, assignmentValueExpression pgsql.Expression) error { if mutation, err := s.getIdentifierMutation(scope, propertyLookup.Reference.Root()); err != nil { return err @@ -915,6 +1269,7 @@ func (s *Mutations) AddPropertyAssignment(scope *Scope, propertyLookup PropertyL return nil } +// AddKindAssignment builds the SQL model fragment responsible for add kind assignment. func (s *Mutations) AddKindAssignment(scope *Scope, targetIdentifier pgsql.Identifier, kinds graph.Kinds) error { if mutation, err := s.getIdentifierMutation(scope, targetIdentifier); err != nil { return err @@ -925,6 +1280,7 @@ func (s *Mutations) AddKindAssignment(scope *Scope, targetIdentifier pgsql.Ident return nil } +// AddKindRemoval builds the SQL model fragment responsible for add kind removal. func (s *Mutations) AddKindRemoval(scope *Scope, targetIdentifier pgsql.Identifier, kinds graph.Kinds) error { if mutation, err := s.getIdentifierMutation(scope, targetIdentifier); err != nil { return err @@ -935,22 +1291,31 @@ func (s *Mutations) AddKindRemoval(scope *Scope, targetIdentifier pgsql.Identifi return nil } +// Projections groups SQL model state that must remain consistent while translating projections. type Projections struct { - Distinct bool - Frame *Frame + // Distinct indicates whether distinct applies. + Distinct bool + // Frame supplies the frame input to the Projections contract. + Frame *Frame + // Constraints supplies the constraints input to the Projections contract. Constraints pgsql.Expression - Items []*Projection - GroupBy []pgsql.Expression + // Items supplies the items input to the Projections contract. + Items []*Projection + // GroupBy supplies the group by input to the Projections contract. + GroupBy []pgsql.Expression } +// Add builds the SQL model fragment responsible for add. func (s *Projections) Add(projection *Projection) { s.Items = append(s.Items, projection) } +// Current builds the SQL model fragment responsible for current. func (s *Projections) Current() *Projection { return s.Items[len(s.Items)-1] } +// extractIdentifierFromCypherExpression returns the variable or alias directly declared by a supported Cypher expression. func extractIdentifierFromCypherExpression(expression cypher.Expression) (pgsql.Identifier, bool, error) { if expression == nil { return "", false, nil @@ -988,17 +1353,22 @@ func extractIdentifierFromCypherExpression(expression cypher.Expression) (pgsql. return pgsql.Identifier(variableExpression.Symbol), true, nil } +// FromClauseBuilder groups SQL model state that must remain consistent while translating from clause builder. type FromClauseBuilder struct { - seen map[pgsql.Identifier]struct{} + // seen retains the seen while FromClauseBuilder is assembled or evaluated. + seen map[pgsql.Identifier]struct{} + // fromClauses retains the from clauses while FromClauseBuilder is assembled or evaluated. fromClauses []pgsql.FromClause } +// NewFromClauseBuilder builds the SQL model fragment responsible for new from clause builder. func NewFromClauseBuilder() *FromClauseBuilder { return &FromClauseBuilder{ seen: make(map[pgsql.Identifier]struct{}), } } +// AddIdentifier builds the SQL model fragment responsible for add identifier. func (s *FromClauseBuilder) AddIdentifier(frameID pgsql.Identifier) { if frameID == "" { return @@ -1014,12 +1384,14 @@ func (s *FromClauseBuilder) AddIdentifier(frameID pgsql.Identifier) { } } +// AddBinding builds the SQL model fragment responsible for add binding. func (s *FromClauseBuilder) AddBinding(binding *BoundIdentifier) { if binding != nil && binding.LastProjection != nil { s.AddIdentifier(binding.LastProjection.Binding.Identifier) } } +// Clauses builds the SQL model fragment responsible for clauses. func (s *FromClauseBuilder) Clauses() []pgsql.FromClause { return s.fromClauses } diff --git a/cypher/models/pgsql/translate/optimizer_safety_test.go b/cypher/models/pgsql/translate/optimizer_safety_test.go index eedc5028..8f51ab8b 100644 --- a/cypher/models/pgsql/translate/optimizer_safety_test.go +++ b/cypher/models/pgsql/translate/optimizer_safety_test.go @@ -2,6 +2,7 @@ package translate import ( "context" + "fmt" "strings" "testing" @@ -12,18 +13,20 @@ import ( "github.com/stretchr/testify/require" ) -const optimizerADCSQuery = ` -MATCH (n:Group) -WHERE n.objectid = 'S-1-5-21-2643190041-1319121918-239771340-513' -MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) -MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) -WHERE ct.authenticationenabled = true -AND ct.requiresmanagerapproval = false -AND ct.enrolleesuppliessubject = true -AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) +// optimizerFixedSuffixQuery exercises a bounded variable expansion followed by a selective three-edge suffix. +const optimizerFixedSuffixQuery = ` +MATCH (root:ExpansionRoot) +WHERE root.root_key = 'root' +MATCH p1 = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) +MATCH p2 = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) +WHERE predicate.eligible = true +AND predicate.requires_review = false +AND predicate.allows_direct = true +AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN p1, p2 ` +// optimizerSafetyKindMapper returns deterministic numeric IDs for the kinds used by optimizer-safety fixtures. func optimizerSafetyKindMapper() *pgutil.InMemoryKindMapper { mapper := pgutil.NewInMemoryKindMapper() @@ -31,23 +34,41 @@ func optimizerSafetyKindMapper() *pgutil.InMemoryKindMapper { "AllExtendedRights", "CertTemplate", "Domain", - "Enroll", - "EnterpriseCA", - "EnterpriseCAFor", + "SuffixEdgeOne", + "SuffixNodeOne", + "SuffixNodeOneFor", "GenericAll", "Group", "IssuedSignedBy", "MemberOf", - "NTAuthStore", - "NTAuthStoreFor", + "SuffixNodeTwo", + "SuffixEdgeThree", "PublishedTo", "RootCA", "RootCAFor", - "TrustedForNTAuth", + "SuffixEdgeTwo", "AdminTo", "Computer", "Tag_Tier_Zero", "User", + "ExpansionRoot", + "ExpansionNode", + "Expand", + "SuffixHead", + "EnterSuffix", + "SuffixMiddle", + "ContinueSuffix", + "SuffixTerminal", + "CompleteSuffix", + "OptionA", + "OptionB", + "OptionC", + "PredicateNode", + "JoinSuffix", + "HeadToBridge", + "HeadToAlternateBridge", + "BridgeNode", + "ReachTerminal", }) { mapper.Put(kind) } @@ -55,6 +76,7 @@ func optimizerSafetyKindMapper() *pgutil.InMemoryKindMapper { return mapper } +// optimizerSafetySQL translates cypherQuery and returns its rendered PostgreSQL text. func optimizerSafetySQL(t *testing.T, cypherQuery string) string { t.Helper() @@ -66,12 +88,14 @@ func optimizerSafetySQL(t *testing.T, cypherQuery string) string { return strings.Join(strings.Fields(formattedQuery), " ") } +// optimizerSafetyTranslation parses and translates cypherQuery with the optimizer-safety kind mapper. func optimizerSafetyTranslation(t *testing.T, cypherQuery string) Result { t.Helper() return optimizerSafetyTranslationWithParameters(t, cypherQuery, nil) } +// optimizerSafetyTranslationWithParameters parses and translates cypherQuery with the supplied parameter values. func optimizerSafetyTranslationWithParameters(t *testing.T, cypherQuery string, parameters map[string]any) Result { t.Helper() @@ -84,6 +108,7 @@ func optimizerSafetyTranslationWithParameters(t *testing.T, cypherQuery string, return translation } +// requireOptimizationLowering requires name to appear among the lowerings applied during translation. func requireOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -96,6 +121,7 @@ func requireOptimizationLowering(t *testing.T, summary OptimizationSummary, name require.Failf(t, "missing optimization lowering", "expected lowering %q in %#v", name, summary.Lowerings) } +// requireNoOptimizationLowering requires name to be absent from applied lowering diagnostics. func requireNoOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -104,6 +130,7 @@ func requireNoOptimizationLowering(t *testing.T, summary OptimizationSummary, na } } +// requirePlannedOptimizationLowering requires name to appear in the optimizer's planned lowerings. func requirePlannedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -116,6 +143,7 @@ func requirePlannedOptimizationLowering(t *testing.T, summary OptimizationSummar require.Failf(t, "missing planned optimization lowering", "expected planned lowering %q in %#v", name, summary.PlannedLowerings) } +// requireNoPlannedOptimizationLowering requires name to be absent from the optimizer's planned lowerings. func requireNoPlannedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -124,6 +152,7 @@ func requireNoPlannedOptimizationLowering(t *testing.T, summary OptimizationSumm } } +// requirePlanParameterContains requires at least one translated parameter value to contain expected. func requirePlanParameterContains(t *testing.T, translation Result, expected string) { t.Helper() @@ -136,6 +165,7 @@ func requirePlanParameterContains(t *testing.T, translation Result, expected str require.Failf(t, "missing plan parameter content", "expected a plan parameter to contain %q in %#v", expected, translation.Parameters) } +// requireSkippedOptimizationLowering requires a skipped-lowering diagnostic with the expected name and reason. func requireSkippedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string, reason string) { t.Helper() @@ -149,6 +179,7 @@ func requireSkippedOptimizationLowering(t *testing.T, summary OptimizationSummar require.Failf(t, "missing skipped optimization lowering", "expected skipped lowering %q in %#v", name, summary.SkippedLowerings) } +// requireSkippedOptimizationLoweringCount requires a skipped-lowering diagnostic with the expected occurrence count. func requireSkippedOptimizationLoweringCount(t *testing.T, summary OptimizationSummary, name string, count int) { t.Helper() @@ -162,6 +193,7 @@ func requireSkippedOptimizationLoweringCount(t *testing.T, summary OptimizationS require.Failf(t, "missing skipped optimization lowering", "expected skipped lowering %q in %#v", name, summary.SkippedLowerings) } +// requireNoSkippedOptimizationLowering requires name to be absent from skipped-lowering diagnostics. func requireNoSkippedOptimizationLowering(t *testing.T, summary OptimizationSummary, name string) { t.Helper() @@ -170,6 +202,7 @@ func requireNoSkippedOptimizationLowering(t *testing.T, summary OptimizationSumm } } +// TestOptimizerSafetyReportsPartiallySkippedLowerings verifies optimizer safety reports partially skipped lowerings behavior. func TestOptimizerSafetyReportsPartiallySkippedLowerings(t *testing.T) { t.Parallel() @@ -189,6 +222,1617 @@ func TestOptimizerSafetyReportsPartiallySkippedLowerings(t *testing.T) { requireSkippedOptimizationLoweringCount(t, translator.translation.Optimization, optimize.LoweringPredicatePlacement, 1) } +// TestFixedSuffixSearchStrategyIsPlannedButConservativelySkipped verifies that an unforced candidate remains diagnostic-only. +func TestFixedSuffixSearchStrategyIsPlannedButConservativelySkipped(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + + requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireNoOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, optimize.ExpansionSearchFallbackTournamentUnqualified) + require.Len(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy, 1) + require.True(t, translation.Optimization.LoweringPlan.ExpansionSearchStrategy[0].StructurallyEligible) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "fixed_suffix_expansion", outcome.Family) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Empty(t, outcome.EmittedPolicy) + require.Equal(t, []string{"EXPANSION-STEPWISE-FORWARD", "EXPANSION-LATE-HYDRATED-FORWARD", "EXPANSION-FACTORED-SUFFIX-FORWARD", "EXPANSION-SUFFIX-SEEDED-REVERSE", "EXPANSION-BACKWARD-VIABILITY-FORWARD"}, outcome.PlannedCandidates) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward)}, outcome.EmittedCandidates) + require.Equal(t, &optimize.ExpansionSearchProbeCaps{ + RootRowLimit: optimize.ExpansionSearchOrientationRootRowLimit, + ReverseSeedRowLimit: optimize.ExpansionSearchOrientationReverseSeedRowLimit, + DirectionalDegreeRowLimit: optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + }, outcome.ProbeCaps) + require.Equal(t, &optimize.ExpansionSearchAdmission{ + StateLimit: optimize.ExpansionSearchOrientationStateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, outcome.Admission) + require.Contains(t, outcome.EligibilityFacts, TargetEligibilityFact{ + Name: "qualified_fixed_suffix_topology", + Eligible: true, + }) + require.Equal(t, string(optimize.ExpansionSearchObservationFullPath), outcome.ObservationMode) + require.NotNil(t, outcome.Eligible) + require.True(t, *outcome.Eligible) + require.Equal(t, "incumbent_default", outcome.SelectionMode) + require.Equal(t, "fixed-suffix-static-v1", outcome.SelectorVersion) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Fallback) + require.Equal(t, optimize.ExpansionSearchFallbackTournamentUnqualified, outcome.SkipReason) +} + +// TestForcedSuffixSeededReverseEmitsNativeReverseTrailState verifies the reverse-search CTE and ordered edge-ID state emitted by a forced strategy. +func TestForcedSuffixSeededReverseEmitsNativeReverseTrailState(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + `) + require.NoError(t, err) + + plan, err := optimize.Optimize(regularQuery) + require.NoError(t, err) + require.NoError(t, applyToolOptions(&plan, ToolOptions{ + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + })) + require.Len(t, plan.LoweringPlan.ExpansionSearchStrategy, 1) + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + require.Equal(t, optimize.ExpansionSearchSuffixSeededReverse, decision.SelectedStrategy) + require.Empty(t, decision.EmittedPolicy) + require.Equal(t, []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse}, decision.EmittedCandidates) + require.Equal(t, "forced_tool", decision.SelectionMode) + require.Equal(t, "suffix-seeded-reverse-tool-v1", decision.SelectorVersion) + require.Empty(t, decision.FallbackReason) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "forced-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "with recursive") + require.Contains(t, formatted, "_suffix_seeded_suffix as materialized") + require.Contains(t, formatted, "_suffix_seeded_reverse(boundary_id, next_id, depth, path, node_path)") + require.Contains(t, formatted, "array_prepend(e0.id") + require.Contains(t, formatted, "array_prepend(e0.start_id") + require.Contains(t, formatted, "e0.id != all (s5_suffix_seeded_reverse.path)") + require.Contains(t, formatted, "e0.end_id = s5_suffix_seeded_reverse.next_id") + require.Contains(t, formatted, "s5_suffix_seeded_reverse.path && array [s5_suffix_seeded_suffix.e1, s5_suffix_seeded_suffix.e2, s5_suffix_seeded_suffix.e3]::int8[]") + require.Contains(t, formatted, "e2.id != e1.id") + require.Contains(t, formatted, "e3.id != e1.id") + require.Contains(t, formatted, "e3.id != e2.id") + require.Contains(t, formatted, "generate_subscripts(s5_suffix_seeded_reverse.node_path") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.NotContains(t, formatted, "s2(root_id, next_id, depth, satisfied, is_cycle, path)") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 1, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchSuffixSeededReverse), outcome.Applied) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV1), outcome.PlannedPolicy) + require.Empty(t, outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchSuffixSeededReverse)}, outcome.EmittedCandidates) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Empty(t, outcome.SkipReason) + requireOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) + requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy) +} + +// TestEndpointSeededReverseIsAutomaticallyGuardedAndApplied verifies that qualified endpoint seeding emits bounded probes and reports application. +func TestEndpointSeededReverseIsAutomaticallyGuardedAndApplied(t *testing.T) { + translation := optimizerSafetyTranslationWithParameters(t, ` + MATCH p = (c:Computer)-[:AdminTo]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix + RETURN p + LIMIT 1000 + `, map[string]any{"suffix": "-512"}) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "_endpoint_seeded_endpoints as materialized") + require.Contains(t, formatted, "limit 33") + require.Contains(t, formatted, "_endpoint_seeded_states as materialized") + require.Contains(t, formatted, "_endpoint_seeded_incumbent as materialized") + require.Contains(t, formatted, "limit 4097") + require.Contains(t, formatted, "array_prepend") + require.Contains(t, formatted, "_endpoint_seeded_reverse.next_id") + require.Contains(t, formatted, "offset 32 limit 1") + require.Contains(t, formatted, "offset 4096 limit 1") + require.Contains(t, formatted, "_endpoint_seeded_incumbent") + require.Contains(t, formatted, "_endpoint_seeded_states.path && array [") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 1, + }) + require.Equal(t, string(optimize.ExpansionSearchEndpointSeededReverse), outcome.Selected) + require.Equal(t, string(optimize.ExpansionSearchEndpointSeededReverse), outcome.Applied) + require.Equal(t, string(optimize.ExpansionSearchPolicyEndpointGuardV1), outcome.PlannedPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyEndpointGuardV1), outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward), string(optimize.ExpansionSearchEndpointSeededReverse)}, outcome.EmittedCandidates) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Equal(t, &optimize.ExpansionSearchProbeCaps{ReverseSeedRowLimit: 32}, outcome.ProbeCaps) + require.Equal(t, &optimize.ExpansionSearchAdmission{ + StateLimit: 4096, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + }, outcome.Admission) + require.Equal(t, int64(32), outcome.EndpointLimit) + require.Equal(t, int64(4096), outcome.StateLimit) + require.Equal(t, "property_ends_with", outcome.SeedPredicateClass) + require.Equal(t, 1, outcome.PrefixLength) + require.True(t, outcome.HasFinalLimit) +} + +// TestProductionEndpointSeededKillSwitchRestoresStepwiseSQL verifies production endpoint seeded kill switch restores stepwise sql behavior. +func TestProductionEndpointSeededKillSwitchRestoresStepwiseSQL(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = (c:Computer)-[:AdminTo]->(:User)-[:MemberOf*1..]->(g:Group) + WHERE g.objectid ENDS WITH $suffix + RETURN p LIMIT 1000 + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{"suffix": "-512"}, DefaultGraphID, ProductionOptions{ + DisableEndpointSeededReverse: true, + SelectorVersion: "endpoint-seeded-kill-switch-v1", + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.NotContains(t, formatted, "_endpoint_seeded_endpoints") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringExpansionSearchStrategy, optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 1, + }) + require.Equal(t, string(optimize.ExpansionSearchStepwiseForward), outcome.Selected) + require.Empty(t, outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ExpansionSearchStepwiseForward)}, outcome.EmittedCandidates) + require.Equal(t, "production_kill_switch", outcome.SelectionMode) + require.Equal(t, "inline_statement", outcome.ExecutionBoundary) +} + +// TestOrdinaryExpansionMayContinueAfterSelfLoop verifies that encountering a self-loop does not stop unrelated recursive expansion. +func TestOrdinaryExpansionMayContinueAfterSelfLoop(t *testing.T) { + formatted := optimizerSafetySQL(t, `MATCH p = (s)-[:MemberOf*1..3]->(g) RETURN p`) + require.Contains(t, formatted, "1, false, false, array [e0.id]") + require.NotContains(t, formatted, "e0.start_id = e0.end_id, array [e0.id]") +} + +// TestForcedSuffixSeededReverseEndpointSQLIsParameterStable verifies deterministic parameter numbering in forced reverse-search SQL. +func TestForcedSuffixSeededReverseEndpointSQLIsParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(head), id(terminal) + `) + require.NoError(t, err) + + translateForced := func(rootKey string) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": rootKey, + }, DefaultGraphID, ToolOptions{ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + first := translateForced("root-a") + second := translateForced("root-b") + require.Equal(t, first, second) + require.Contains(t, first, "s5_suffix_seeded_reverse.path") + require.Contains(t, first, "select s5.n2 as \"id(head)\", s5.n4 as \"id(terminal)\"") + require.NotContains(t, first, "node_path") + require.NotContains(t, first, "generate_subscripts") + require.NotContains(t, first, "ordered_edge_ids_to_path") + require.NotContains(t, first, "s2(root_id, next_id, depth, satisfied, is_cycle, path)") +} + +// TestForcedSuffixSeededReversePreservesBoundaryConstraints verifies that predicates attached at the suffix boundary survive reversal. +func TestForcedSuffixSeededReversePreservesBoundaryConstraints(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH (root)-[:Expand*0..16]->(boundary:ExpansionNode {enabled: true})-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(head), id(terminal) + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "root_key": "forced-fixed-suffix-root", + }, DefaultGraphID, ToolOptions{ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "_suffix_seeded_suffix as materialized") + require.Contains(t, formatted, "n1.kind_ids operator (pg_catalog.@>)") + require.Contains(t, formatted, "n1.properties -> 'enabled'") + require.Contains(t, formatted, "to_jsonb((true)::bool)") +} + +// TestForcedFixedSuffixSearchRejectsUnsupportedStrategy verifies that tooling cannot force a strategy outside the candidate family. +func TestForcedFixedSuffixSearchRejectsUnsupportedStrategy(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN id(head), id(terminal) + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), nil, DefaultGraphID, ToolOptions{ + ForceExpansionSearchStrategy: optimize.ExpansionSearchFactoredSuffixForward, + }) + require.ErrorContains(t, err, "unsupported forced expansion-search strategy") +} + +// TestForcedFixedSuffixSearchRejectsStructurallyIneligibleTarget verifies that forcing does not bypass structural qualification. +func TestForcedFixedSuffixSearchRejectsStructurallyIneligibleTarget(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead) + RETURN id(head) + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), nil, DefaultGraphID, ToolOptions{ + ForceExpansionSearchStrategy: optimize.ExpansionSearchSuffixSeededReverse, + }) + require.ErrorContains(t, err, "has no structurally eligible target") +} + +// TestForcedExpansionSearchRequiresExactlyOneEligibleTarget verifies that +// tooling fails closed before mutating any decision when a force is ambiguous. +func TestForcedExpansionSearchRequiresExactlyOneEligibleTarget(t *testing.T) { + plan := optimize.Plan{LoweringPlan: optimize.LoweringPlan{ + ExpansionSearchStrategy: []optimize.ExpansionSearchStrategyDecision{ + { + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + }, + { + CandidateStrategy: optimize.ExpansionSearchSuffixSeededReverse, + SelectedStrategy: optimize.ExpansionSearchStepwiseForward, + StructurallyEligible: true, + }, + }, + }} + before := append([]optimize.ExpansionSearchStrategyDecision(nil), plan.LoweringPlan.ExpansionSearchStrategy...) + + err := applyForcedExpansionSearchStrategy(&plan, optimize.ExpansionSearchSuffixSeededReverse) + require.ErrorContains(t, err, "matched 2 structurally eligible targets; expected exactly one") + require.Equal(t, before, plan.LoweringPlan.ExpansionSearchStrategy) +} + +// TestShortestDistanceExecutorIsAutomaticallySelectedAndReportedApplied verifies automatic scalar-distance selection and matching diagnostics. +func TestShortestDistanceExecutorIsAutomaticallySelectedAndReportedApplied(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..64]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + + requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringShortestPathExecutor) + requireOptimizationLowering(t, translation.Optimization, optimize.LoweringShortestPathExecutor) + requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringShortestPathExecutor) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "SP", outcome.Family) + require.Equal(t, []string{"SP-S0", "SP-S0-DIRECT", "SP-S1", "SP-S2", "SP-S3-U-D", "SP-S3-U-E+MAT-M0", "SP-S4-C-D", "SP-S4-C-WE+MAT-M0", "SP-I1-C-D", "SP-I2-C-D", "SP-I1-U-E+MAT-M0", "SP-I1-C-WE+MAT-M0", "SP-B1-C-ALT-NODE-D", "SP-B1-C-ALT-NODE-WE+MAT-M0", "SP-B2-C-MIN-LEVEL-D", "SP-B2-C-MIN-LEVEL-WE+MAT-M0"}, outcome.PlannedCandidates) + require.Equal(t, string(optimize.ShortestPathSchedulerSingleEndedLevel), outcome.Scheduler) + require.Contains(t, outcome.EligibilityFacts, TargetEligibilityFact{ + Name: "one_static_id_equality_per_endpoint", + Eligible: true, + }) + require.Equal(t, string(optimize.ShortestPathObservationDistance), outcome.ObservationMode) + require.NotNil(t, outcome.Eligible) + require.True(t, *outcome.Eligible) + require.Equal(t, "static", outcome.SelectionMode) + require.Equal(t, "sp-static-v3", outcome.SelectorVersion) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Applied) + require.Equal(t, string(optimize.ShortestPathExecutorIncumbentWorkspace), outcome.Fallback) + require.Empty(t, outcome.SkipReason) +} + +// TestImplicitMaximumShortestPathReportsPolicyDepthProvenance verifies a +// syntax-open shortest path is specialized without pretending its bound was +// written explicitly. +func TestImplicitMaximumShortestPathReportsPolicyDepthProvenance(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Applied) + require.Equal(t, optimize.ShortestPathSelectorStaticV7Contained, outcome.SelectorVersion) + require.NotNil(t, outcome.MaximumDepth) + require.Equal(t, int64(15), *outcome.MaximumDepth) + require.Equal(t, string(optimize.ShortestPathMaximumDepthPolicyDefault), outcome.MaximumDepthSource) +} + +// TestGreedyProjectionMaterializesShortestPathAndEntities verifies that RETURN * hydrates the path and every visible endpoint. +func TestGreedyProjectionMaterializesShortestPathAndEntities(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s:Group)-[:MemberOf*1..4]->(e:Group)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN * + `) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathObservationOnePath), outcome.ObservationMode) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "::pathcomposite") + require.Contains(t, formatted, "::nodecomposite") +} + +// TestGreedyProjectionMaterializesRelationships verifies that RETURN * hydrates relationship bindings. +func TestGreedyProjectionMaterializesRelationships(t *testing.T) { + formatted := optimizerSafetySQL(t, ` + MATCH (s:Group)-[r:MemberOf]->(e:Group) + RETURN * + `) + + require.Contains(t, formatted, "::nodecomposite") + require.Contains(t, formatted, "::edgecomposite") +} + +// TestGreedyWithProjectionCarriesFullShortestPath verifies that WITH * preserves a complete shortest-path value across query parts. +func TestGreedyWithProjectionCarriesFullShortestPath(t *testing.T) { + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s:Group)-[:MemberOf*1..4]->(e:Group)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH * + RETURN p + `) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "::pathcomposite") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "m0_hydrated") +} + +// TestShortestExecutorV4SelectsDeepInboundCompactDistance verifies canonical distance selection and inbound physical topology diagnostics. +func TestShortestExecutorV4SelectsDeepInboundCompactDistance(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf*1..8]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "shortest_path_compact") + require.NotContains(t, formatted, "sp_harness") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "inbound", outcome.Direction) + require.Equal(t, "end_id", outcome.PhysicalExpansion) + require.Equal(t, 1, outcome.RelationshipKindCount) + require.False(t, outcome.UntypedRelationship) + require.Equal(t, "physical_inbound_deep", outcome.TopologyClassification) + require.NotNil(t, outcome.Eligible) + require.True(t, *outcome.Eligible) + require.NotNil(t, outcome.StaticallyEligible) + require.True(t, *outcome.StaticallyEligible) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalDistance), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalDistance), outcome.Applied) + require.Empty(t, outcome.SkipReason) +} + +// TestShortestExecutorV4SelectsCompactMultiKindPathAndKeepsS3Distance verifies observation-dependent selection for multi-kind paths. +func TestShortestExecutorV4SelectsCompactMultiKindPathAndKeepsS3Distance(t *testing.T) { + for _, test := range []struct { + // observation is the return expression that consumes the shortest path. + observation string + // selected is the executor expected for that observation. + selected optimize.ShortestPathExecutor + // reason is the expected translation skip reason, if any. + reason string + }{ + { + observation: "p", + selected: optimize.ShortestPathExecutorS4CanonicalWitness, + }, + { + observation: "length(p)", + selected: optimize.ShortestPathExecutorS3Unidirectional, + }, + } { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf|SuffixEdgeOne*1..8]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.observation)) + require.NoError(t, err) + translation, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, 2, outcome.RelationshipKindCount) + require.Equal(t, string(test.selected), outcome.Selected) + require.Equal(t, test.reason, outcome.SkipReason) + if test.selected == optimize.ShortestPathExecutorS4CanonicalWitness { + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + } + } +} + +// TestAllShortestDAGIsAutomaticallySelectedAndUsesTypedStaticExecutor verifies typed predecessor-DAG execution for bound all-shortest paths. +func TestAllShortestDAGIsAutomaticallySelectedAndUsesTypedStaticExecutor(t *testing.T) { + translation := optimizerSafetyTranslationWithParameters(t, ` + MATCH p = allShortestPaths((s)-[*1..]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `, map[string]any{"start_id": int64(1), "end_id": int64(2)}) + + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "all_shortest_paths_dag") + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.NotContains(t, formatted, "bidirectional_asp_harness") + require.NotContains(t, formatted, "traversal_pair_filter") + require.Contains(t, formatted, "array []::int2[]") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "ASP", outcome.Family) + require.Equal(t, []string{"SP-S0", "ASP-A1-DAG", "ASP-I1-U-DAG+MAT-M0", "ASP-B1-DAG-ALT-NODE", "ASP-B2-DAG-MIN-LEVEL"}, outcome.PlannedCandidates) + require.Equal(t, string(optimize.ShortestPathSchedulerSingleEndedLevel), outcome.Scheduler) + require.Equal(t, string(optimize.ShortestPathObservationAllPaths), outcome.ObservationMode) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Applied) + require.Equal(t, "asp-static-v1", outcome.SelectorVersion) + require.Empty(t, outcome.SkipReason) +} + +// TestForcedAllShortestNoPathProbeUsesA1Fallback verifies that the negative +// probe has a distinct helper identity while preserving inline M0 hydration. +func TestForcedAllShortestNoPathProbeUsesA1Fallback(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..8]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPN1NegativeExhaustion}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "all_shortest_paths_no_path_probe") + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{QueryPartIndex: 0, ClauseIndex: 0, PatternIndex: 0, StepIndex: 0}) + require.Equal(t, string(optimize.ShortestPathExecutorASPN1NegativeExhaustion), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorASPN1NegativeExhaustion), outcome.Applied) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Fallback) +} + +// TestForcedCompactBidirectionalExecutorsUseTypedKernels verifies every SP B1/B2 +// identity reaches its scheduler wrapper without changing automatic selection. +func TestForcedCompactBidirectionalExecutorsUseTypedKernels(t *testing.T) { + tests := []struct { + // executor retains the executor while anonymous record is assembled or evaluated. + executor optimize.ShortestPathExecutor + // result retains the result while anonymous record is assembled or evaluated. + result string + // functionName identifies the function name. + functionName string + }{ + {optimize.ShortestPathExecutorB1AlternatingNodeDistance, "length(p)", "shortest_path_b1_strict_alternating"}, + {optimize.ShortestPathExecutorB1AlternatingNodeWitness, "p", "shortest_path_b1_strict_alternating"}, + {optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, "length(p)", "shortest_path_b2_smaller_current_level"}, + {optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, "p", "shortest_path_b2_smaller_current_level"}, + } + for _, test := range tests { + t.Run(string(test.executor), func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), fmt.Sprintf(` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN %s + `, test.result)) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: test.executor}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, test.functionName) + require.Equal(t, 3, strings.Count(formatted, "100000"), formatted) + require.NotContains(t, formatted, "bidirectional_sp_harness") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(test.executor), outcome.Selected) + require.Equal(t, string(test.executor), outcome.Applied) + require.Equal(t, string(test.executor.Scheduler()), outcome.Scheduler) + require.Equal(t, "forced_tool", outcome.SelectionMode) + }) + } +} + +// TestProductionCanaryShortestExecutorUsesVersionedSelectionMetadata verifies +// the production policy path emits the same qualified kernel while remaining +// distinguishable from tool forcing. +func TestProductionCanaryShortestExecutorUsesVersionedSelectionMetadata(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 1000, + PredecessorLimit: 1000, + EnumerationLimit: 1000, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + }, + }) + require.NoError(t, err) + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "production_canary", outcome.SelectionMode) + require.Equal(t, optimize.ShortestPathSelectorStaticV6, outcome.SelectorVersion) + require.Equal(t, string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), outcome.Applied) +} + +func TestProductionGuardedDistanceEmitsReversePhysicalCandidateAndExactFallback(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistanceV2, + SelectorVersion: optimize.ShortestPathSelectorStaticV9HiddenFanInTail, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: optimize.ShortestPathI2QualifiedStateLimit, + FrontierLimit: optimize.ShortestPathI2QualifiedFrontierLimit, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, + }, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "sp_i2_distance") + require.Contains(t, formatted, "e0.start_id = sp_i2_distance.node_id") + require.Contains(t, formatted, "sp_i2_distance.node_id != (select singleton_endpoints.root_id from singleton_endpoints)") + require.Contains(t, formatted, "shortest_path_compact(") + require.Contains(t, formatted, "sp_i2_candidate_marker") + require.Contains(t, formatted, "sp_i2_fallback_marker") + require.NotContains(t, formatted, "group by sp_i2_distance_bounded.depth") + require.Contains(t, formatted, "true as frontier_guard_dominated") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, optimize.TraversalStepTarget{}) + require.Equal(t, string(optimize.ShortestPathExecutorI2GuardedDistanceV2), outcome.Applied) + require.Equal(t, optimize.ShortestPathPolicyI2DistanceGuardedV2, outcome.EmittedPolicy) + require.Equal(t, []string{string(optimize.ShortestPathExecutorI2GuardedDistanceV2), string(optimize.ShortestPathExecutorS4CanonicalDistance)}, outcome.EmittedCandidates) + require.Equal(t, optimize.ShortestPathI2QualifiedStateLimit, outcome.StateLimit) + require.Equal(t, optimize.ShortestPathI2QualifiedFrontierLimit, outcome.FrontierLimit) + require.Zero(t, outcome.PredecessorLimit) + require.Zero(t, outcome.EnumerationLimit) + require.Zero(t, outcome.OutputBytesLimit) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) +} + +func TestProductionGuardedDistanceRejectsUnqualifiedCaps(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + tests := []struct { + name string + caps ProductionShortestPathCaps + }{ + { + name: "formerly accepted positive override", + caps: ProductionShortestPathCaps{StateLimit: 1000, FrontierLimit: 100}, + }, + { + name: "unauthorized cap dimension", + caps: ProductionShortestPathCaps{ + StateLimit: optimize.ShortestPathI2QualifiedStateLimit, + FrontierLimit: optimize.ShortestPathI2QualifiedFrontierLimit, + PredecessorLimit: 1, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistanceV2, + SelectorVersion: optimize.ShortestPathSelectorStaticV9HiddenFanInTail, + ShortestPathCaps: &test.caps, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "inbound", ObservationMode: "distance", MinimumDepth: 1, MaximumDepth: 32, RelationshipKindCount: 1, + }, + }) + require.ErrorContains(t, err, "requires exactly state_limit=100000 and frontier_limit=100000") + }) + } +} + +func TestProductionGuardedDistanceV1IsTerminallyRejected(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), guardedDistanceToolQuery) + require.NoError(t, err) + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI2GuardedDistance, + SelectorVersion: optimize.ShortestPathSelectorStaticV8HiddenFanIn, + }) + require.ErrorContains(t, err, "not production-canary eligible") +} + +// TestProductionCanonicalSPRequiresExactStaticV6Envelope verifies production canonical sp requires exact static v6 envelope behavior. +func TestProductionCanonicalSPRequiresExactStaticV6Envelope(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + base := ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 1000, + PredecessorLimit: 1000, + EnumerationLimit: 1000, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + }, + } + + tests := map[string]func(*ProductionOptions){ + "selector": func(options *ProductionOptions) { options.SelectorVersion = "sp-static-v5-contained" }, + "outbound": func(options *ProductionOptions) { options.AuthorizedBucket.Direction = "outbound" }, + "maximum": func(options *ProductionOptions) { options.AuthorizedBucket.MaximumDepth = 63 }, + "kinds": func(options *ProductionOptions) { options.AuthorizedBucket.RelationshipKindCount = 2 }, + "untyped": func(options *ProductionOptions) { + options.AuthorizedBucket.RelationshipKindCount = 0 + options.AuthorizedBucket.UntypedRelationship = true + }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + options := base + bucket := *base.AuthorizedBucket + options.AuthorizedBucket = &bucket + mutate(&options) + _, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.Error(t, err) + }) + } +} + +// TestProductionRejectsToolOnlyBidirectionalShortestExecutor verifies production rejects tool only bidirectional shortest executor behavior. +func TestProductionRejectsToolOnlyBidirectionalShortestExecutor(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + SelectorVersion: "traversal-production-g7", + }) + require.ErrorContains(t, err, "not production-canary eligible") +} + +// TestForcedBidirectionalASPExecutorsUseTypedKernels verifies the tool-only +// candidates reach their two-sided predecessor-DAG wrappers while automatic +// production selection remains ASP-A1-DAG. +func TestForcedBidirectionalASPExecutorsUseTypedKernels(t *testing.T) { + tests := []struct { + // executor retains the executor while anonymous record is assembled or evaluated. + executor optimize.ShortestPathExecutor + // functionName identifies the function name. + functionName string + }{ + {optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, "all_shortest_paths_b1_strict_alternating"}, + {optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, "all_shortest_paths_b2_smaller_current_level"}, + } + for _, test := range tests { + t.Run(string(test.executor), func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: test.executor}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, test.functionName) + require.NotContains(t, formatted, "bidirectional_asp_harness") + require.Equal(t, 4, strings.Count(formatted, "100000"), formatted) + require.Contains(t, formatted, "67108864") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(test.executor), outcome.Selected) + require.Equal(t, string(test.executor), outcome.Applied) + require.Equal(t, string(test.executor.Scheduler()), outcome.Scheduler) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Equal(t, "asp-tool-v1", outcome.SelectorVersion) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Fallback) + require.Equal(t, int64(100_000), outcome.EnumerationLimit) + require.Equal(t, int64(64*1024*1024), outcome.OutputBytesLimit) + }) + } +} + +// TestForcedInlineASPExecutorUsesGuardedTypedStatement verifies the I1 +// production-shaped emitter is forceable for qualification without changing +// the automatic ASP-A1 selection. +func TestForcedInlineASPExecutorUsesGuardedTypedStatement(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "asp_i1_distance") + require.Contains(t, formatted, "asp_i1_direct") + require.Contains(t, formatted, "asp_i1_predecessor_bounded") + require.Contains(t, formatted, "asp_i1_paths_bounded") + require.Contains(t, formatted, "asp_i1_admission") + require.Contains(t, formatted, "asp_i1_candidate_marker") + require.Contains(t, formatted, "asp_i1_fallback_marker") + require.Contains(t, formatted, "all_shortest_paths_dag") + require.Contains(t, formatted, "m0_hydrated") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "record_requested_traversal_runtime_attestation_v1") + require.Contains(t, formatted, "record_requested_traversal_runtime_attestation_v1(case when asp_i1_admission.overflow") + require.Contains(t, formatted, "end, asp_i1_admission.overflow, case when asp_i1_admission.overflow") + require.Equal(t, 7, strings.Count(formatted, "offset 100000 limit 1"), formatted) + require.Equal(t, 1, strings.Count(formatted, "67108864"), formatted) + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorASPI1DAG), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorASPI1DAG), outcome.Applied) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Equal(t, string(optimize.ShortestPathExecutorASPA1DAG), outcome.Fallback) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Equal(t, "asp-tool-v1", outcome.SelectorVersion) +} + +// TestProductionInlineASPUsesAuthorizedBucketAndImmutableCaps verifies production inline asp uses authorized bucket and immutable caps behavior. +func TestProductionInlineASPUsesAuthorizedBucketAndImmutableCaps(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + options := ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 31, + PredecessorLimit: 37, + EnumerationLimit: 41, + OutputBytesLimit: 43000, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "outbound", + ObservationMode: "all_paths", + MinimumDepth: 1, + MaximumDepth: 4, + RelationshipKindCount: 1, + UntypedRelationship: false, + }, + SelectorVersion: "asp-i1-canary-v1", + } + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + for _, limit := range []string{"31", "37", "41", "43000"} { + require.Contains(t, formatted, limit) + } + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, "production_canary", outcome.SelectionMode) + require.Equal(t, "asp-i1-canary-v1", outcome.SelectorVersion) + require.Equal(t, "asp-i1-guarded-v1", outcome.EmittedPolicy) + require.Equal(t, []string{"ASP-I1-U-DAG+MAT-M0", "ASP-A1-DAG"}, outcome.EmittedCandidates) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Zero(t, outcome.FrontierLimit) + + options.AuthorizedBucket.MaximumDepth = 8 + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.ErrorContains(t, err, "does not match its authorized promotion bucket") + + options.AuthorizedBucket = nil + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, options) + require.ErrorContains(t, err, "requires an exact authorized bucket") +} + +// TestForcedBidirectionalASPExecutorsFailClosedOutsideEnvelope verifies tool +// forcing cannot broaden the singleton, directed, predicate-free, read-only, +// minimum-depth-one all-path observation contract. +func TestForcedBidirectionalASPExecutorsFailClosedOutsideEnvelope(t *testing.T) { + tests := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. + query string + }{ + { + name: "wrong observation", + query: `MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + }, + { + name: "zero minimum", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*0..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + }, + { + name: "minimum two", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*2..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + }, + { + name: "maximum sixty five", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..65]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + }, + { + name: "directionless", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]-(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + }, + { + name: "path relationship predicate", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id AND all(r IN relationships(p) WHERE type(r) = 'MemberOf') RETURN p`, + }, + { + name: "optional", + query: `OPTIONAL MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + }, + { + name: "mutation", + query: `MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id SET s.flag = true RETURN p`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), test.query) + require.NoError(t, err) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + } { + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible all-paths target") + } + }) + } +} + +// TestForcedCompactBidirectionalExecutorsRejectUnsupportedDepth verifies the +// bounded maximum-depth envelope cannot be broadened by tool forcing. +func TestForcedCompactBidirectionalExecutorsRejectUnsupportedDepth(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..65]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + } { + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible distance-only target") + } +} + +// TestForcedShortestDistanceExecutorEmitsNativeScalarState verifies the scalar recursive state emitted by a forced distance executor. +func TestForcedShortestDistanceExecutorEmitsNativeScalarState(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + incumbent, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + incumbentSQL, err := Translated(incumbent) + require.NoError(t, err) + productionOutcome := requireTraversalTargetOutcome(t, incumbent.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), productionOutcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), productionOutcome.Applied) + require.Equal(t, "static", productionOutcome.SelectionMode) + require.Equal(t, "sp-static-v3", productionOutcome.SelectorVersion) + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Equal(t, incumbentSQL, forcedSQL) + require.Contains(t, forcedSQL, "with recursive") + require.Contains(t, forcedSQL, "s1(next_id, depth)") + require.NotContains(t, forcedSQL, "s1(root_id, next_id, depth)") + require.Contains(t, forcedSQL, "select singleton_endpoints.root_id, 0 from singleton_endpoints") + require.Contains(t, forcedSQL, "(select singleton_endpoints.root_id from singleton_endpoints) as n0") + require.NotContains(t, forcedSQL, "sp_harness") + require.NotContains(t, forcedSQL, "path)") + require.NotContains(t, forcedSQL, "is_cycle") + require.NotContains(t, forcedSQL, "cardinality") + require.Contains(t, forcedSQL, "order by") + require.Contains(t, forcedSQL, "depth limit 1") + require.NotContains(t, forcedSQL, "join node") + + outcome := requireTraversalTargetOutcome(t, forced.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3Unidirectional), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Empty(t, outcome.SkipReason) + requireOptimizationLowering(t, forced.Optimization, optimize.LoweringShortestPathExecutor) + requireNoSkippedOptimizationLowering(t, forced.Optimization, optimize.LoweringShortestPathExecutor) +} + +// TestForcedShortestIncumbentEmitsExactWorkspaceHarness verifies that forcing the incumbent preserves its workspace-table harness. +func TestForcedShortestIncumbentEmitsExactWorkspaceHarness(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorIncumbentWorkspace, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "sp_harness") + require.NotContains(t, formatted, "s1(next_id, depth)") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorIncumbentWorkspace), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorIncumbentWorkspace), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) +} + +// TestForcedShortestDirectPreflightGatesWorkspaceFallback verifies that direct preflight gates the incumbent workspace branch. +func TestForcedShortestDirectPreflightGatesWorkspaceFallback(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf|SuffixEdgeOne*1..8]-(s)) + WHERE id(e) = $end_id AND id(s) = $start_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "direct_shortest(root_id, next_id, depth, satisfied, is_cycle, path) as materialized") + require.Contains(t, formatted, "fallback_endpoints as (select * from singleton_endpoints where not exists") + require.Contains(t, formatted, "workspace_shortest(root_id, next_id, depth, satisfied, is_cycle, path)") + require.Contains(t, formatted, "from fallback_endpoints, bidirectional_sp_harness") + require.Contains(t, formatted, "select * from direct_shortest union all select * from workspace_shortest") + + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS0Direct), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS0Direct), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) +} + +// TestForcedShortestDirectPreflightRejectsZeroMinimumDepth verifies that forcing cannot bypass the direct executor's positive-depth requirement. +func TestForcedShortestDirectPreflightRejectsZeroMinimumDepth(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*0..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.ErrorContains(t, err, "no structurally eligible depth-one target") +} + +// TestForcedShortestDirectPreflightRejectsMutation verifies that statement mutation prevents direct shortest-path execution. +func TestForcedShortestDirectPreflightRejectsMutation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + CREATE (:Group) + RETURN p + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.ErrorContains(t, err, "no structurally eligible depth-one target") +} + +// TestForcedShortestDirectPreflightPreservesPathThroughWithAlias verifies that a path witness survives aliasing across WITH. +func TestForcedShortestDirectPreflightPreservesPathThroughWithAlias(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p AS q + RETURN q + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ + ForceShortestPathExecutor: optimize.ShortestPathExecutorS0Direct, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "direct_shortest") + require.Contains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "as q") +} + +// TestForcedShortestDistanceExecutorRejectsIneligibleObservation verifies that a path consumer cannot force distance-only execution. +func TestForcedShortestDistanceExecutorRejectsIneligibleObservation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.ErrorContains(t, err, "no structurally eligible distance-only target") +} + +// TestForcedShortestPathEdgeM0ExecutorEmitsNativeEdgeTrailAndMaterializer verifies ordered edge-trail state and deferred path hydration. +func TestForcedShortestPathEdgeM0ExecutorEmitsNativeEdgeTrailAndMaterializer(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + incumbent, err := Translate(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID) + require.NoError(t, err) + incumbentSQL, err := Translated(incumbent) + require.NoError(t, err) + productionOutcome := requireTraversalTargetOutcome(t, incumbent.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), productionOutcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), productionOutcome.Applied) + require.Equal(t, "static", productionOutcome.SelectionMode) + require.Equal(t, "sp-static-v5-contained", productionOutcome.SelectorVersion) + require.NotContains(t, incumbentSQL, "shortest_path_compact") + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Equal(t, incumbentSQL, forcedSQL, "forcing the contained S3 winner must reproduce the default SQL") + require.Contains(t, forcedSQL, "with recursive") + require.Contains(t, forcedSQL, "s1(next_id, depth, path)") + require.Contains(t, forcedSQL, "generate_subscripts(s1.path, 1)") + require.Equal(t, 1, strings.Count(forcedSQL, "generate_subscripts(s1.path, 1)"), forcedSQL) + require.Contains(t, forcedSQL, "array_agg((m0_terminal.id, m0_terminal.kind_ids, m0_terminal.properties)::nodecomposite order by m0_path_index)") + require.Contains(t, forcedSQL, "m0_hydrated.hydrated_count = cardinality(s1.path)") + require.Contains(t, forcedSQL, "m0_terminal.id = m0_edge.end_id") + require.Contains(t, forcedSQL, "::pathcomposite") + require.NotContains(t, forcedSQL, "sp_harness") + require.NotContains(t, forcedSQL, "ordered_edge_ids_to_path") + + outcome := requireTraversalTargetOutcome(t, forced.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), outcome.Selected) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), outcome.Applied) + require.Equal(t, "forced_tool", outcome.SelectionMode) + require.Empty(t, outcome.SkipReason) +} + +// TestForcedShortestPathEdgeM0ExecutorIsDirectionAware verifies that edge-trail recursion joins the correct physical endpoint for each direction. +func TestForcedShortestPathEdgeM0ExecutorIsDirectionAware(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf*1..8]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Contains(t, forcedSQL, "join edge e0 on e0.end_id = s1.next_id") + require.Contains(t, forcedSQL, "m0_terminal.id = m0_edge.start_id") +} + +// TestForcedShortestPathExecutorsRejectMismatchedObservation verifies tool +// forcing cannot broaden distance and witness observation contracts. +func TestForcedShortestPathExecutorsRejectMismatchedObservation(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.ErrorContains(t, err, "no structurally eligible one-path target") + + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + } { + _, err = TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible one-path target") + } + + witnessQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + for _, executor := range []optimize.ShortestPathExecutor{ + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + } { + _, err = TranslateForTool(context.Background(), witnessQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: executor}) + require.ErrorContains(t, err, "no structurally eligible distance-only target") + } +} + +// TestForcedShortestPathEdgeM0ExecutorPreservesPathThroughWithAlias verifies that a materialized witness survives aliasing across WITH. +func TestForcedShortestPathEdgeM0ExecutorPreservesPathThroughWithAlias(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH p AS q + RETURN q + `) + require.NoError(t, err) + + forced, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3EdgeM0}) + require.NoError(t, err) + forcedSQL, err := Translated(forced) + require.NoError(t, err) + + require.Contains(t, forcedSQL, "::pathcomposite") + require.Contains(t, forcedSQL, "as q") + require.NotContains(t, forcedSQL, "ordered_edge_ids_to_path") + + outcome := requireTraversalTargetOutcome(t, forced.Optimization, optimize.LoweringShortestPathExecutor, + optimize.TraversalStepTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + StepIndex: 0, + }) + require.Equal(t, string(optimize.ShortestPathExecutorS3EdgeM0), outcome.Applied) +} + +// TestForcedShortestDistanceExecutorIsDirectionAwareAndParameterStable verifies physical direction and deterministic parameters for scalar search. +func TestForcedShortestDistanceExecutorIsDirectionAwareAndParameterStable(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((e)<-[:MemberOf*1..8]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + + translateForced := func(startID, endID int64) string { + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": startID, "end_id": endID, + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + return formatted + } + + firstSQL := translateForced(1, 2) + secondSQL := translateForced(100, 200) + require.Equal(t, firstSQL, secondSQL) + require.Contains(t, firstSQL, "select e0.start_id, s1.depth + 1") + require.NotContains(t, firstSQL, "select s1.root_id, e0.start_id, s1.depth + 1") + require.Contains(t, firstSQL, "join edge e0 on e0.end_id = s1.next_id") +} + +// TestForcedShortestDistanceExecutorSupportsZeroDepthWithoutSelfEndpointError verifies legal same-endpoint zero-length paths. +func TestForcedShortestDistanceExecutorSupportsZeroDepthWithoutSelfEndpointError(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*0..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) AS distance + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(1), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.Contains(t, formatted, "s1.depth >= 0") + require.NotContains(t, formatted, "shortest_path_self_endpoint_error") + require.Contains(t, formatted, "(s0.ep0)::int as distance") +} + +// TestProductionRejectsUnderGuardedInlineDistanceExecutor verifies production rejects under guarded inline distance executor behavior. +func TestProductionRejectsUnderGuardedInlineDistanceExecutor(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN length(p) + `) + require.NoError(t, err) + _, err = TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalDistance, + SelectorVersion: "sp-i1-canary-v1", + }) + require.ErrorContains(t, err, "not production-canary eligible") +} + +// TestProductionInlineWitnessExecutorKeepsEdgeIDsAtMaterializationBoundary verifies production inline witness executor keeps edge i ds at materialization boundary behavior. +func TestProductionInlineWitnessExecutorKeepsEdgeIDsAtMaterializationBoundary(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p + `) + require.NoError(t, err) + translation, err := TranslateWithProductionOptions(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + ShortestPathCaps: &ProductionShortestPathCaps{ + StateLimit: 1000, + PredecessorLimit: 1000, + EnumerationLimit: 1000, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &ProductionTraversalBucket{ + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + }, + }) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + require.Contains(t, formatted, "with recursive") + require.Contains(t, formatted, "generate_subscripts(s1.path, 1)") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "shortest_path_compact") + outcome := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringShortestPathExecutor, optimize.TraversalStepTarget{}) + require.Equal(t, string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), outcome.Applied) + require.Equal(t, string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), outcome.Candidate) + require.Equal(t, optimize.ShortestPathPolicyI1CanonicalGuardedV1, outcome.EmittedPolicy) + require.Equal(t, []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + }, outcome.EmittedCandidates) + require.Equal(t, string(optimize.ShortestPathExecutorS4CanonicalWitness), outcome.Fallback) + require.Equal(t, "guarded_dual_arm", outcome.ExecutionBoundary) + require.Equal(t, "production_canary", outcome.SelectionMode) +} + +// TestForcedShortestDistanceExecutorPreservesDistanceThroughWithAlias verifies that scalar distance survives aliasing across WITH. +func TestForcedShortestDistanceExecutorPreservesDistanceThroughWithAlias(t *testing.T) { + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + WITH length(p) AS distance + RETURN distance + `) + require.NoError(t, err) + + translation, err := TranslateForTool(context.Background(), regularQuery, optimizerSafetyKindMapper(), map[string]any{ + "start_id": int64(1), "end_id": int64(2), + }, DefaultGraphID, ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional}) + require.NoError(t, err) + formatted, err := Translated(translation) + require.NoError(t, err) + + require.NotContains(t, formatted, "cardinality") + require.NotContains(t, formatted, "ordered_edge_ids_to_path") + require.Contains(t, formatted, "::int as i0") + require.Contains(t, formatted, "s0.i0 as distance") +} + +// requireTraversalTargetOutcome returns the diagnostic outcome for one lowering and traversal target. +func requireTraversalTargetOutcome(t *testing.T, summary OptimizationSummary, lowering string, target optimize.TraversalStepTarget) TargetLoweringOutcome { + t.Helper() + + for _, outcome := range summary.TargetOutcomes { + if outcome.Lowering == lowering && outcome.TraversalTarget != nil && *outcome.TraversalTarget == target { + return outcome + } + } + + require.FailNowf(t, "missing target outcome", "lowering %s target %+v", lowering, target) + return TargetLoweringOutcome{} +} + +// TestTraversalEnvelopeAnalysisHasExplicitTargetOutcomes verifies traversal envelope analysis has explicit target outcomes behavior. +func TestTraversalEnvelopeAnalysisHasExplicitTargetOutcomes(t *testing.T) { + t.Parallel() + + translation := optimizerSafetyTranslation(t, ` + MATCH p = shortestPath((s)-[:MemberOf*1..4]->(e)) + WHERE id(s) IN [1, 2] AND id(e) = 3 + AND all(n IN nodes(p) WHERE n.enabled = true) + RETURN p + `) + target := optimize.PatternTarget{ + QueryPartIndex: 0, + ClauseIndex: 0, + PatternIndex: 0, + }.TraversalStep(0) + + endpoint := requireTraversalTargetOutcome(t, translation.Optimization, optimize.LoweringEndpointResolution, target) + require.Equal(t, "endpoint_resolution", endpoint.TargetKind) + require.Equal(t, "endpoint_resolution", endpoint.Family) + require.Equal(t, "SP", endpoint.TraversalFamily) + require.Equal(t, string(optimize.EndpointResolutionPlanBounded), endpoint.Candidate) + require.Equal(t, string(optimize.EndpointResolutionPlanIncumbent), endpoint.Selected) + require.Equal(t, endpoint.Selected, endpoint.Applied) + require.Equal(t, "analysis_only", endpoint.SelectionMode) + require.Equal(t, optimize.EndpointResolutionFallbackPlannedOnly, endpoint.SkipReason) + require.NotNil(t, endpoint.EndpointRoot) + require.Equal(t, optimize.EndpointResolutionClassExplicitSmallSet, endpoint.EndpointRoot.Class) + require.Equal(t, 2, endpoint.EndpointRoot.StaticValueCount) + require.NotNil(t, endpoint.EndpointTerminal) + require.Equal(t, optimize.EndpointResolutionClassIDEquality, endpoint.EndpointTerminal.Class) + require.Equal(t, &optimize.EndpointResolutionCaps{ + SingletonLimit: optimize.EndpointResolutionSingletonLimit, + SingletonSentinel: optimize.EndpointResolutionSingletonSentinel, + SmallSetLimit: optimize.EndpointResolutionSmallSetLimit, + SmallSetSentinel: optimize.EndpointResolutionSmallSetSentinel, + }, endpoint.EndpointResolutionCaps) + + var predicate *TargetLoweringOutcome + for index := range translation.Optimization.TargetOutcomes { + outcome := &translation.Optimization.TargetOutcomes[index] + if outcome.Lowering == optimize.LoweringTraversalPredicateClassification && outcome.PredicateClass == optimize.TraversalPredicateClassUniversalAllNodes { + predicate = outcome + break + } + } + require.NotNil(t, predicate) + require.Equal(t, "traversal_predicate", predicate.TargetKind) + require.Equal(t, string(optimize.TraversalPredicatePlanStep), predicate.Candidate) + require.Equal(t, string(optimize.TraversalPredicatePlanIncumbent), predicate.Selected) + require.Equal(t, predicate.Selected, predicate.Applied) + require.Equal(t, optimize.TraversalPredicateFallbackPlannedOnly, predicate.SkipReason) + require.Equal(t, "analysis_only", predicate.SelectionMode) + require.NotNil(t, predicate.PredicateIndex) +} + +// requireSQLContainsInOrder requires each SQL fragment to occur after the preceding fragment. func requireSQLContainsInOrder(t *testing.T, sql string, parts ...string) { t.Helper() @@ -200,6 +1844,7 @@ func requireSQLContainsInOrder(t *testing.T, sql string, parts ...string) { } } +// TestOptimizerSafetyCountStoreFastPathUsesBaseNodeCount verifies unconstrained node counts use the graph-wide node count source. func TestOptimizerSafetyCountStoreFastPathUsesBaseNodeCount(t *testing.T) { t.Parallel() @@ -209,10 +1854,11 @@ func TestOptimizerSafetyCountStoreFastPathUsesBaseNodeCount(t *testing.T) { requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringCountStoreFastPath) requireOptimizationLowering(t, translation.Optimization, optimize.LoweringCountStoreFastPath) - require.Empty(t, translation.Optimization.SkippedLowerings) + requireSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringFieldRequirements, "analysis_metadata_only") require.Equal(t, "select count(*)::int8 from node n0;", strings.Join(strings.Fields(formattedQuery), " ")) } +// TestOptimizerSafetyCountStoreFastPathKeepsKindConstraintAndAlias verifies optimizer safety count store fast path keeps kind constraint and alias behavior. func TestOptimizerSafetyCountStoreFastPathKeepsKindConstraintAndAlias(t *testing.T) { t.Parallel() @@ -225,6 +1871,7 @@ func TestOptimizerSafetyCountStoreFastPathKeepsKindConstraintAndAlias(t *testing require.Equal(t, "select count(*)::int8 as total from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[];", strings.Join(strings.Fields(formattedQuery), " ")) } +// TestOptimizerSafetyCountStoreFastPathSupportsNodeCountStar verifies optimizer safety count store fast path supports node count star behavior. func TestOptimizerSafetyCountStoreFastPathSupportsNodeCountStar(t *testing.T) { t.Parallel() @@ -237,6 +1884,7 @@ func TestOptimizerSafetyCountStoreFastPathSupportsNodeCountStar(t *testing.T) { require.Equal(t, "select count(*)::int8 as total from node n0 where n0.kind_ids operator (pg_catalog.@>) array [8]::int2[];", strings.Join(strings.Fields(formattedQuery), " ")) } +// TestOptimizerSafetyCountStoreFastPathUsesBaseEdgeCount verifies optimizer safety count store fast path uses base edge count behavior. func TestOptimizerSafetyCountStoreFastPathUsesBaseEdgeCount(t *testing.T) { t.Parallel() @@ -250,10 +1898,11 @@ func TestOptimizerSafetyCountStoreFastPathUsesBaseEdgeCount(t *testing.T) { require.Equal(t, "select count(*)::int8 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [10]::int2[]);", strings.Join(strings.Fields(formattedQuery), " ")) } +// TestOptimizerSafetyCountStoreFastPathUsesSparseEdgeKindCount verifies a typed edge count reads only the selected kind's sparse count. func TestOptimizerSafetyCountStoreFastPathUsesSparseEdgeKindCount(t *testing.T) { t.Parallel() - translation := optimizerSafetyTranslation(t, `MATCH ()-[r:Enroll]->() RETURN count(r)`) + translation := optimizerSafetyTranslation(t, `MATCH ()-[r:SuffixEdgeOne]->() RETURN count(r)`) formattedQuery, err := Translated(translation) require.NoError(t, err) normalizedQuery := strings.Join(strings.Fields(formattedQuery), " ") @@ -266,6 +1915,7 @@ func TestOptimizerSafetyCountStoreFastPathUsesSparseEdgeKindCount(t *testing.T) require.Equal(t, "select count(*)::int8 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [4]::int2[]);", normalizedQuery) } +// TestOptimizerSafetyCountStoreFastPathUsesUntypedEdgeCount verifies optimizer safety count store fast path uses untyped edge count behavior. func TestOptimizerSafetyCountStoreFastPathUsesUntypedEdgeCount(t *testing.T) { t.Parallel() @@ -282,6 +1932,7 @@ func TestOptimizerSafetyCountStoreFastPathUsesUntypedEdgeCount(t *testing.T) { require.Equal(t, "select count(*)::int8 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id;", normalizedQuery) } +// TestOptimizerSafetyCountStoreFastPathSupportsEdgeCountStar verifies optimizer safety count store fast path supports edge count star behavior. func TestOptimizerSafetyCountStoreFastPathSupportsEdgeCountStar(t *testing.T) { t.Parallel() @@ -295,10 +1946,11 @@ func TestOptimizerSafetyCountStoreFastPathSupportsEdgeCountStar(t *testing.T) { require.Equal(t, "select count(*)::int8 from edge e0 join node n0 on n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [10]::int2[]);", strings.Join(strings.Fields(formattedQuery), " ")) } -func TestOptimizerSafetyADCSQueryPrunesExpansionEdgeCarry(t *testing.T) { +// TestOptimizerSafetyFixedSuffixQueryPrunesExpansionEdgeCarry verifies that unobserved expansion edges are omitted from recursive state. +func TestOptimizerSafetyFixedSuffixQueryPrunesExpansionEdgeCarry(t *testing.T) { t.Parallel() - translation := optimizerSafetyTranslation(t, optimizerADCSQuery) + translation := optimizerSafetyTranslation(t, optimizerFixedSuffixQuery) formattedQuery, err := Translated(translation) require.NoError(t, err) normalizedQuery := strings.Join(strings.Fields(formattedQuery), " ") @@ -315,13 +1967,15 @@ func TestOptimizerSafetyADCSQueryPrunesExpansionEdgeCarry(t *testing.T) { require.Contains(t, normalizedQuery, "select distinct (s9.n2).id as root_id from s9") require.Contains(t, normalizedQuery, "s5.ep0 as ep0") require.NotContains(t, normalizedQuery, "s5.e0 as e0") - require.Contains(t, normalizedQuery, "from unnest(s12.ep0)") - require.Contains(t, normalizedQuery, "from unnest(array [s12.e1]::int8[])") + require.Contains(t, normalizedQuery, "ordered_edge_ids_to_path(0, s12.n0, s12.ep0 || array [s12.e1]::int8[] || array [s12.e2]::int8[] || array [s12.e3]::int8[]") + require.Equal(t, 2, strings.Count(normalizedQuery, "ordered_edge_ids_to_path("), normalizedQuery) + require.NotContains(t, normalizedQuery, "ordered_edges_to_path(") + require.NotContains(t, normalizedQuery, "from unnest(") require.NotContains(t, normalizedQuery, "array [s12.e1]::edgecomposite[]") require.Contains(t, normalizedQuery, "from s5, s7") requireSQLContainsInOrder(t, normalizedQuery, "where s7.satisfied and exists (select 1 from edge e5 join node n6", - "properties -> 'authenticationenabled'", + "properties -> 'eligible'", "join edge e6 on n6.id = e6.start_id", "e6.end_id = (s5.n2).id", "and (s5.n0).id = s7.root_id", @@ -333,6 +1987,7 @@ func TestOptimizerSafetyADCSQueryPrunesExpansionEdgeCarry(t *testing.T) { ) } +// assertOptimizerSafetyRelationshipStaysComposite requires a relationship consumer to retain composite rather than scalar-ID state. func assertOptimizerSafetyRelationshipStaysComposite(t *testing.T, cypherQuery string) { t.Helper() @@ -344,6 +1999,7 @@ func assertOptimizerSafetyRelationshipStaysComposite(t *testing.T, cypherQuery s require.NotContains(t, normalizedQuery, "::int8[]") } +// TestOptimizerSafetyReferencedRelationshipStaysComposite verifies optimizer safety referenced relationship stays composite behavior. func TestOptimizerSafetyReferencedRelationshipStaysComposite(t *testing.T) { t.Parallel() @@ -353,11 +2009,14 @@ RETURN p, r `) } +// TestOptimizerSafetyRelationshipExpressionReferencesStayComposite verifies optimizer safety relationship expression references stay composite behavior. func TestOptimizerSafetyRelationshipExpressionReferencesStayComposite(t *testing.T) { t.Parallel() testCases := []struct { - name string + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. query string }{ { @@ -393,6 +2052,7 @@ RETURN p, startNode(r) } } +// TestOptimizerSafetyOptionalMatchPathStaysComposite verifies optimizer safety optional match path stays composite behavior. func TestOptimizerSafetyOptionalMatchPathStaysComposite(t *testing.T) { t.Parallel() @@ -406,6 +2066,7 @@ RETURN n, p require.NotContains(t, normalizedQuery, "::int8[]") } +// TestOptimizerSafetyFixedHopExpandIntoUsesBoundEndpoints verifies optimizer safety fixed hop expand into uses bound endpoints behavior. func TestOptimizerSafetyFixedHopExpandIntoUsesBoundEndpoints(t *testing.T) { t.Parallel() @@ -433,13 +2094,61 @@ RETURN p requireOptimizationLowering(t, translation.Optimization, "ExpandIntoDetection") } +// TestOptimizerSafetyFixedHopExpandIntoPreservesCarriedOuterMultiplicity verifies optimizer safety fixed hop expand into preserves carried outer multiplicity behavior. +func TestOptimizerSafetyFixedHopExpandIntoPreservesCarriedOuterMultiplicity(t *testing.T) { + t.Parallel() + + normalizedQuery := optimizerSafetySQL(t, ` + MATCH (a:Group), (b:User) + WITH a, b, [1, 2] AS copies + UNWIND copies AS copy + MATCH (a)-[:MemberOf|AdminTo]->(b) + RETURN copy + `) + + require.Contains(t, normalizedQuery, "from s0 join edge e0 on (s0.n0).id = e0.start_id and (s0.n1).id = e0.end_id, unnest(i0) as i1") + require.NotContains(t, normalizedQuery, "join node") +} + +// TestOptimizerSafetyFixedHopExpandIntoScopesNodeUnwindBeforePairPredicate verifies optimizer safety fixed hop expand into scopes node unwind before pair predicate behavior. +func TestOptimizerSafetyFixedHopExpandIntoScopesNodeUnwindBeforePairPredicate(t *testing.T) { + t.Parallel() + + normalizedQuery := optimizerSafetySQL(t, ` + MATCH (a:Group), (b:User) + WITH collect(a) AS sources, b + UNWIND sources AS source + MATCH (source)-[:MemberOf]->(b) + RETURN source + `) + + require.Contains(t, normalizedQuery, "from s0, edge e0, unnest(i0) as i1 where") + require.Contains(t, normalizedQuery, "i1.id = e0.start_id and (s0.n1).id = e0.end_id") + require.NotContains(t, normalizedQuery, "join edge e0 on i1.id") +} + +// TestOptimizerSafetyDirectionlessExpandIntoUsesPairwiseEndpoints verifies optimizer safety directionless expand into uses pairwise endpoints behavior. +func TestOptimizerSafetyDirectionlessExpandIntoUsesPairwiseEndpoints(t *testing.T) { + t.Parallel() + + normalizedQuery := optimizerSafetySQL(t, ` + MATCH (a:Group), (b:User) + MATCH (a)-[:MemberOf]-(b) + RETURN a, b + `) + + require.Contains(t, normalizedQuery, "(((s1.n0).id = e0.start_id and (s1.n1).id = e0.end_id) or ((s1.n1).id = e0.start_id and (s1.n0).id = e0.end_id))") + require.NotContains(t, normalizedQuery, "(s1.n0).id <> (s1.n1).id") +} + +// TestOptimizerSafetyReordersIndependentNodeAnchor verifies an independent selective node can become the traversal anchor without changing semantics. func TestOptimizerSafetyReordersIndependentNodeAnchor(t *testing.T) { t.Parallel() var ( normalizedQuery = optimizerSafetySQL(t, ` MATCH (a) - MATCH (b:EnterpriseCA {name: 'target'}) + MATCH (b:SuffixNodeOne {name: 'target'}) MATCH p = (a)-[:MemberOf]->(b) RETURN p `) @@ -454,11 +2163,12 @@ func TestOptimizerSafetyReordersIndependentNodeAnchor(t *testing.T) { require.Contains(t, normalizedQuery, "(s1.n0).id = e0.end_id") } +// TestOptimizerSafetyExpansionTerminalPushdownForFixedSuffix verifies an eligible fixed suffix is pushed into terminal expansion filtering. func TestOptimizerSafetyExpansionTerminalPushdownForFixedSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) RETURN p `) @@ -468,11 +2178,12 @@ RETURN p require.Contains(t, normalizedQuery, "n2.kind_ids operator (pg_catalog.@>) array [5]::int2[]") } -func TestOptimizerSafetyReversalAnchorsTerminalPredicateAtDriveRoot(t *testing.T) { +// TestOptimizerSafetySuffixPredicatePlacementStaysInsideTerminalExists verifies suffix predicates remain scoped to the terminal existence check. +func TestOptimizerSafetySuffixPredicatePlacementStaysInsideTerminalExists(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) WHERE ca.name = 'target' RETURN p `) @@ -489,11 +2200,12 @@ RETURN p require.Contains(t, normalizedQuery, "e1.kind_id = any (array [10]::int2[])") } +// TestOptimizerSafetyPredicatePlacementRecordsExpansionRootConstraint verifies root predicates are recorded and emitted at the expansion root. func TestOptimizerSafetyPredicatePlacementRecordsExpansionRootConstraint(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (src:Group)-[:MemberOf*1..]->(mid)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (src:Group)-[:MemberOf*1..]->(mid)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) WHERE src.name = 'source' RETURN p `) @@ -511,6 +2223,7 @@ RETURN p ) } +// TestOptimizerSafetyPredicatePlacementRecordsFixedTraversalConstraint verifies optimizer safety predicate placement records fixed traversal constraint behavior. func TestOptimizerSafetyPredicatePlacementRecordsFixedTraversalConstraint(t *testing.T) { t.Parallel() @@ -534,6 +2247,7 @@ RETURN dst ) } +// TestOptimizerSafetyPatternPredicateExistencePlacementIsPlanned verifies optimizer safety pattern predicate existence placement is planned behavior. func TestOptimizerSafetyPatternPredicateExistencePlacementIsPlanned(t *testing.T) { t.Parallel() @@ -552,11 +2266,12 @@ RETURN s requireOptimizationLowering(t, translation.Optimization, "PredicatePlacement") } +// TestOptimizerSafetyContinuationRelationshipsExcludePriorPathRelationships verifies suffix traversal cannot reuse relationships from the expanded prefix. func TestOptimizerSafetyContinuationRelationshipsExcludePriorPathRelationships(t *testing.T) { t.Parallel() expandedPrefixQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]-(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]-(ca:SuffixNodeOne) RETURN p `) @@ -564,18 +2279,19 @@ RETURN p require.Contains(t, expandedPrefixQuery, "ep0") fixedPrefixQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) RETURN p `) require.Contains(t, fixedPrefixQuery, "e1.id != s0.e0") } +// TestOptimizerSafetyDirectionBalancedExpansionDoesNotPlanStaleSuffixPushdown verifies reoriented traversal targets do not retain obsolete suffix decisions. func TestOptimizerSafetyDirectionBalancedExpansionDoesNotPlanStaleSuffixPushdown(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (n)-[:MemberOf*1..]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(d:Domain) +MATCH p = (n)-[:MemberOf*1..]->(ca:SuffixNodeOne)-[:SuffixEdgeTwo]->(d:Domain) RETURN p `) @@ -585,6 +2301,7 @@ RETURN p requireNoOptimizationLowering(t, translation.Optimization, "ExpansionSuffixPushdown") } +// TestOptimizerSafetyTraversalDirectionUsesRightEndpointPredicate verifies optimizer safety traversal direction uses right endpoint predicate behavior. func TestOptimizerSafetyTraversalDirectionUsesRightEndpointPredicate(t *testing.T) { t.Parallel() @@ -604,6 +2321,7 @@ RETURN p require.Contains(t, normalizedQuery, "join edge e0 on e0.end_id = s1_seed.root_id") } +// TestOptimizerSafetyExactOneHopRangeUsesFixedTraversal verifies optimizer safety exact one hop range uses fixed traversal behavior. func TestOptimizerSafetyExactOneHopRangeUsesFixedTraversal(t *testing.T) { t.Parallel() @@ -626,6 +2344,7 @@ RETURN p require.Contains(t, normalizedQuery, "join node n1") } +// TestOptimizerSafetyExactTwoHopRangeUsesFixedTraversal verifies optimizer safety exact two hop range uses fixed traversal behavior. func TestOptimizerSafetyExactTwoHopRangeUsesFixedTraversal(t *testing.T) { t.Parallel() @@ -648,11 +2367,12 @@ RETURN p require.Contains(t, normalizedQuery, "array [") } +// TestOptimizerSafetyExactTwoHopRangePreservesLaterSourceStepTargets verifies exact-range expansion does not renumber later source-step decisions. func TestOptimizerSafetyExactTwoHopRangePreservesLaterSourceStepTargets(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH (a)-[:MemberOf*2..2]->(b)-[:Enroll]->(c) +MATCH (a)-[:MemberOf*2..2]->(b)-[:SuffixEdgeOne]->(c) RETURN a `) formattedQuery, err := Translated(translation) @@ -661,11 +2381,12 @@ RETURN a requirePlannedOptimizationLowering(t, translation.Optimization, optimize.LoweringExactRangeExpansion) requireOptimizationLowering(t, translation.Optimization, optimize.LoweringExactRangeExpansion) - require.Contains(t, normalizedQuery, "on (s1.n2).id = e2.start_id") + require.Contains(t, normalizedQuery, "on s1.n2 = e2.start_id") require.NotContains(t, normalizedQuery, "on n2.id = e2.start_id") } -func TestOptimizerSafetyExactTwoHopRangeKeepsSyntheticIntermediateNode(t *testing.T) { +// TestOptimizerSafetyExactTwoHopRangeCarriesSyntheticIntermediateNodeID verifies that exact-range lowering retains the intermediate join identity. +func TestOptimizerSafetyExactTwoHopRangeCarriesSyntheticIntermediateNodeID(t *testing.T) { t.Parallel() normalizedQuery := strings.ToLower(optimizerSafetySQL(t, ` @@ -673,15 +2394,16 @@ MATCH (a)-[:MemberOf*2..2]->(b) RETURN a `)) - require.Contains(t, normalizedQuery, "on (s0.n1).id = e1.start_id") + require.Contains(t, normalizedQuery, "on s0.n1 = e1.start_id") require.NotContains(t, normalizedQuery, "on n1.id = e1.start_id") } +// TestOptimizerSafetyConsecutiveExactRangesUseSourceStepTargets verifies consecutive expansions retain their original source-step coordinates. func TestOptimizerSafetyConsecutiveExactRangesUseSourceStepTargets(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (a)-[:MemberOf*2..2]->(b)-[:Enroll*1..1]->(c) +MATCH p = (a)-[:MemberOf*2..2]->(b)-[:SuffixEdgeOne*1..1]->(c) RETURN p `) formattedQuery, err := Translated(translation) @@ -696,11 +2418,12 @@ RETURN p require.Contains(t, normalizedQuery, "join edge e2") } +// TestOptimizerSafetyExactRangePrefixPreservesSuffixPushdownTargets verifies prefix expansion leaves fixed-suffix decisions keyed to source coordinates. func TestOptimizerSafetyExactRangePrefixPreservesSuffixPushdownTargets(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` -MATCH p = (a)-[:MemberOf*2..2]->(b)-[:AdminTo*1..]->(c)-[:Enroll]->(d) +MATCH p = (a)-[:MemberOf*2..2]->(b)-[:AdminTo*1..]->(c)-[:SuffixEdgeOne]->(d) RETURN p `) @@ -711,6 +2434,7 @@ RETURN p requireNoSkippedOptimizationLowering(t, translation.Optimization, optimize.LoweringExpansionSuffixPushdown) } +// TestOptimizerSafetyPathRelationshipPredicateUsesPathIDs verifies optimizer safety path relationship predicate uses path i ds behavior. func TestOptimizerSafetyPathRelationshipPredicateUsesPathIDs(t *testing.T) { t.Parallel() @@ -732,6 +2456,7 @@ RETURN p require.NotContains(t, normalizedQuery, "from unnest(((select coalesce(array_agg") } +// TestOptimizerSafetyNonePathRelationshipPredicateUsesNotExists verifies optimizer safety none path relationship predicate uses not exists behavior. func TestOptimizerSafetyNonePathRelationshipPredicateUsesNotExists(t *testing.T) { t.Parallel() @@ -752,6 +2477,7 @@ RETURN p require.NotContains(t, normalizedQuery, "select count(*)::int from unnest") } +// TestOptimizerSafetyAggregateTraversalCountUsesIDOnlySourceAnchoredShape verifies optimizer safety aggregate traversal count uses id only source anchored shape behavior. func TestOptimizerSafetyAggregateTraversalCountUsesIDOnlySourceAnchoredShape(t *testing.T) { t.Parallel() @@ -792,6 +2518,7 @@ LIMIT 100 require.NotContains(t, lowerQuery, "::nodecomposite as n0 from") } +// TestOptimizerSafetyTraversalDirectionReportsKindOnlyTerminalSkip verifies optimizer safety traversal direction reports kind only terminal skip behavior. func TestOptimizerSafetyTraversalDirectionReportsKindOnlyTerminalSkip(t *testing.T) { t.Parallel() @@ -807,6 +2534,7 @@ RETURN count(c) requireSkippedOptimizationLowering(t, translation.Optimization, "TraversalDirectionSelection", "terminal kind-only estimate too broad") } +// TestOptimizerSafetyTraversalDirectionReportsSelectiveSourceSkip verifies optimizer safety traversal direction reports selective source skip behavior. func TestOptimizerSafetyTraversalDirectionReportsSelectiveSourceSkip(t *testing.T) { t.Parallel() @@ -822,6 +2550,7 @@ RETURN c requireSkippedOptimizationLowering(t, translation.Optimization, "TraversalDirectionSelection", "bound source estimate selective") } +// TestOptimizerSafetyTraversalDirectionReportsPriorLimitSourceSkip verifies optimizer safety traversal direction reports prior limit source skip behavior. func TestOptimizerSafetyTraversalDirectionReportsPriorLimitSourceSkip(t *testing.T) { t.Parallel() @@ -839,6 +2568,7 @@ RETURN c requireSkippedOptimizationLowering(t, translation.Optimization, "TraversalDirectionSelection", "bound source estimate selective") } +// TestOptimizerSafetyAggregateTraversalCountAcceptsRowCount verifies optimizer safety aggregate traversal count accepts row count behavior. func TestOptimizerSafetyAggregateTraversalCountAcceptsRowCount(t *testing.T) { t.Parallel() @@ -861,6 +2591,7 @@ LIMIT 100 require.Contains(t, normalizedQuery, "group by terminal_hits.root_id") } +// TestOptimizerSafetyAggregateTraversalCountHonorsExplicitDepthBounds verifies optimizer safety aggregate traversal count honors explicit depth bounds behavior. func TestOptimizerSafetyAggregateTraversalCountHonorsExplicitDepthBounds(t *testing.T) { t.Parallel() @@ -882,6 +2613,7 @@ LIMIT 100 require.Contains(t, normalizedQuery, "where traversal.depth >= 2") } +// TestOptimizerSafetyAggregateTraversalCountSupportsInboundSourceAnchoring verifies optimizer safety aggregate traversal count supports inbound source anchoring behavior. func TestOptimizerSafetyAggregateTraversalCountSupportsInboundSourceAnchoring(t *testing.T) { t.Parallel() @@ -903,6 +2635,7 @@ LIMIT 100 require.Contains(t, normalizedQuery, "e.end_id = traversal.next_id") } +// TestOptimizerSafetyAggregateTraversalCountReturnsCountAlias verifies optimizer safety aggregate traversal count returns count alias behavior. func TestOptimizerSafetyAggregateTraversalCountReturnsCountAlias(t *testing.T) { t.Parallel() @@ -925,6 +2658,7 @@ LIMIT 100 require.Contains(t, normalizedQuery, "order by ranked.admincount desc") } +// TestOptimizerSafetyAggregateTraversalCountFoldsTerminalFilter verifies optimizer safety aggregate traversal count folds terminal filter behavior. func TestOptimizerSafetyAggregateTraversalCountFoldsTerminalFilter(t *testing.T) { t.Parallel() @@ -948,6 +2682,7 @@ LIMIT 100 require.Contains(t, normalizedQuery, "join terminal_nodes on terminal_nodes.id = traversal.next_id") } +// TestOptimizerSafetyAggregateTraversalCountUsesDistinctPredicateParameters verifies optimizer safety aggregate traversal count uses distinct predicate parameters behavior. func TestOptimizerSafetyAggregateTraversalCountUsesDistinctPredicateParameters(t *testing.T) { t.Parallel() @@ -980,6 +2715,7 @@ LIMIT 100 require.ElementsMatch(t, []any{true, false}, parameterValues) } +// TestOptimizerSafetyAggregateTraversalCountReusesPredicateParameter verifies optimizer safety aggregate traversal count reuses predicate parameter behavior. func TestOptimizerSafetyAggregateTraversalCountReusesPredicateParameter(t *testing.T) { t.Parallel() @@ -1005,11 +2741,14 @@ LIMIT 100 require.Len(t, translation.Parameters, 1) } +// TestOptimizerSafetyAggregateTraversalCountSkipsUnsafeWideningCandidates verifies optimizer safety aggregate traversal count skips unsafe widening candidates behavior. func TestOptimizerSafetyAggregateTraversalCountSkipsUnsafeWideningCandidates(t *testing.T) { t.Parallel() testCases := []struct { - name string + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. query string }{{ name: "distinct terminal count", @@ -1091,6 +2830,7 @@ LIMIT 100 } } +// TestOptimizerSafetyAggregateTraversalCountSkipsParameterizedCorrelatedTerminalFilter verifies optimizer safety aggregate traversal count skips parameterized correlated terminal filter behavior. func TestOptimizerSafetyAggregateTraversalCountSkipsParameterizedCorrelatedTerminalFilter(t *testing.T) { t.Parallel() @@ -1111,6 +2851,7 @@ LIMIT 100 requireNoOptimizationLowering(t, translation.Optimization, optimize.LoweringAggregateTraversalCount) } +// TestOptimizerSafetyAggregateTraversalCountSkipsObservedTerminal verifies optimizer safety aggregate traversal count skips observed terminal behavior. func TestOptimizerSafetyAggregateTraversalCountSkipsObservedTerminal(t *testing.T) { t.Parallel() @@ -1128,6 +2869,7 @@ LIMIT 100 requireNoOptimizationLowering(t, translation.Optimization, optimize.LoweringAggregateTraversalCount) } +// TestOptimizerSafetyShortestPathStrategyUsesPlannedBidirectionalSearch verifies optimizer safety shortest path strategy uses planned bidirectional search behavior. func TestOptimizerSafetyShortestPathStrategyUsesPlannedBidirectionalSearch(t *testing.T) { t.Parallel() @@ -1148,6 +2890,7 @@ RETURN p requireOptimizationLowering(t, translation.Optimization, "ShortestPathFilterMaterialization") } +// TestOptimizerSafetyShortestPathTerminalFilterUsesPlannedMaterialization verifies optimizer safety shortest path terminal filter uses planned materialization behavior. func TestOptimizerSafetyShortestPathTerminalFilterUsesPlannedMaterialization(t *testing.T) { t.Parallel() @@ -1168,6 +2911,7 @@ RETURN p requireOptimizationLowering(t, translation.Optimization, "ShortestPathFilterMaterialization") } +// TestOptimizerSafetyShortestPathKindOnlyTerminalFilterUsesPlannedMaterialization verifies optimizer safety shortest path kind only terminal filter uses planned materialization behavior. func TestOptimizerSafetyShortestPathKindOnlyTerminalFilterUsesPlannedMaterialization(t *testing.T) { t.Parallel() @@ -1188,6 +2932,7 @@ LIMIT 1000 requireOptimizationLowering(t, translation.Optimization, "ShortestPathFilterMaterialization") } +// TestOptimizerSafetyLimitPushdownUsesPlannedTraversalFrame verifies optimizer safety limit pushdown uses planned traversal frame behavior. func TestOptimizerSafetyLimitPushdownUsesPlannedTraversalFrame(t *testing.T) { t.Parallel() @@ -1201,6 +2946,7 @@ LIMIT 1 requireOptimizationLowering(t, translation.Optimization, "LimitPushdown") } +// TestOptimizerSafetyShortestPathLimitPushdownUsesPlannedHarness verifies optimizer safety shortest path limit pushdown uses planned harness behavior. func TestOptimizerSafetyShortestPathLimitPushdownUsesPlannedHarness(t *testing.T) { t.Parallel() @@ -1215,6 +2961,7 @@ LIMIT 1 requireOptimizationLowering(t, translation.Optimization, "LimitPushdown") } +// TestOptimizerSafetyShortestPathRootCarriesUnwindSources verifies optimizer safety shortest path root carries unwind sources behavior. func TestOptimizerSafetyShortestPathRootCarriesUnwindSources(t *testing.T) { t.Parallel() @@ -1235,6 +2982,7 @@ func TestOptimizerSafetyShortestPathRootCarriesUnwindSources(t *testing.T) { requirePlanParameterContains(t, translation, "(n0.properties ->> 'name') = i0") } +// TestOptimizerSafetyShortestPathTerminalCarriesUnwindSources verifies optimizer safety shortest path terminal carries unwind sources behavior. func TestOptimizerSafetyShortestPathTerminalCarriesUnwindSources(t *testing.T) { t.Parallel() @@ -1254,11 +3002,12 @@ func TestOptimizerSafetyShortestPathTerminalCarriesUnwindSources(t *testing.T) { requirePlanParameterContains(t, translation, "(n1.properties ->> 'name') = i0") } +// TestOptimizerSafetyTranslationReportsOptimizerMetadata verifies translation reports planned, applied, and targeted lowering diagnostics. func TestOptimizerSafetyTranslationReportsOptimizerMetadata(t *testing.T) { t.Parallel() regularQuery, err := frontend.ParseCypher(frontend.NewContext(), ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) WHERE ca.name = 'target' RETURN p `) @@ -1285,11 +3034,12 @@ RETURN p requireOptimizationLowering(t, translation.Optimization, "PredicatePlacement") } +// TestOptimizerSafetyExpansionTerminalPushdownForZeroDepthExpansion verifies terminal filtering preserves the zero-edge expansion alternative. func TestOptimizerSafetyExpansionTerminalPushdownForZeroDepthExpansion(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:SuffixEdgeOne]->(ca:SuffixNodeOne) RETURN p `) @@ -1299,12 +3049,13 @@ RETURN p require.Contains(t, normalizedQuery, "n2.kind_ids operator (pg_catalog.@>) array [5]::int2[]") } +// TestOptimizerSafetyExpansionTerminalPushdownForBoundEndpointSuffixChain verifies a bound suffix endpoint is honored inside supplemental search. func TestOptimizerSafetyExpansionTerminalPushdownForBoundEndpointSuffixChain(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH (ca:EnterpriseCA {name: 'target'}) -MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ct:CertTemplate)-[:PublishedTo]->(ca) +MATCH (ca:SuffixNodeOne {name: 'target'}) +MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:SuffixEdgeOne]->(ct:CertTemplate)-[:PublishedTo]->(ca) WHERE ct.authenticationenabled = true RETURN p `) @@ -1324,12 +3075,13 @@ RETURN p ) } +// TestOptimizerSafetyExpansionTerminalPushdownIncludesConstrainedBoundEndpoint verifies bound-endpoint predicates are included in terminal filtering. func TestOptimizerSafetyExpansionTerminalPushdownIncludesConstrainedBoundEndpoint(t *testing.T) { t.Parallel() translation := optimizerSafetyTranslation(t, ` MATCH (ca) -MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:Enroll]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*0..]->(m)-[:SuffixEdgeOne]->(ct:CertTemplate)-[:PublishedTo]->(ca:SuffixNodeOne) RETURN p `) formattedQuery, err := Translated(translation) @@ -1346,12 +3098,13 @@ RETURN p require.Contains(t, normalizedQuery, "(s0.n0).kind_ids operator (pg_catalog.@>)") } +// TestOptimizerSafetyExpansionTerminalPushdownForBoundDomainSuffix verifies domain-bound suffix nodes remain constrained during supplemental search. func TestOptimizerSafetyExpansionTerminalPushdownForBoundDomainSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` MATCH (d:Domain {name: 'target'}) -MATCH p = (ca:EnterpriseCA)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(root:RootCA)-[:RootCAFor]->(d) +MATCH p = (ca:SuffixNodeOne)-[:IssuedSignedBy|SuffixNodeOneFor*1..]->(root:RootCA)-[:RootCAFor]->(d) RETURN p `) @@ -1362,11 +3115,12 @@ RETURN p require.Contains(t, normalizedQuery, "e1.end_id = (s0.n0).id") } +// TestOptimizerSafetyExpansionTerminalPushdownForInboundFixedSuffix verifies inbound suffix direction is preserved in terminal filtering. func TestOptimizerSafetyExpansionTerminalPushdownForInboundFixedSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (ca:EnterpriseCA)<-[:PublishedTo*1..]-(ct)<-[:Enroll]-(m:Group) +MATCH p = (ca:SuffixNodeOne)<-[:PublishedTo*1..]-(ct)<-[:SuffixEdgeOne]-(m:Group) RETURN p `) @@ -1376,11 +3130,12 @@ RETURN p require.Contains(t, normalizedQuery, "n2.kind_ids operator (pg_catalog.@>)") } +// TestOptimizerSafetyExpansionTerminalPushdownSkipsDirectionlessSuffix verifies undirected suffixes are excluded from terminal-filter pushdown. func TestOptimizerSafetyExpansionTerminalPushdownSkipsDirectionlessSuffix(t *testing.T) { t.Parallel() normalizedQuery := optimizerSafetySQL(t, ` -MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:Enroll]-(ca:EnterpriseCA) +MATCH p = (n:Group)-[:MemberOf*1..]->(m)-[:SuffixEdgeOne]-(ca:SuffixNodeOne) RETURN p `) diff --git a/cypher/models/pgsql/translate/path_functions.go b/cypher/models/pgsql/translate/path_functions.go index f9deacab..662cfc3a 100644 --- a/cypher/models/pgsql/translate/path_functions.go +++ b/cypher/models/pgsql/translate/path_functions.go @@ -6,6 +6,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// pathCompositeEdgesExpression returns the edge-array expression represented by a path binding. func pathCompositeEdgesExpression(scope *Scope, pathBinding *BoundIdentifier) (pgsql.Expression, error) { var edgeArrayReferences []pgsql.Expression @@ -45,6 +46,7 @@ func pathCompositeEdgesExpression(scope *Scope, pathBinding *BoundIdentifier) (p return pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray}, nil } +// pathCompositeEdgeIDArrayExpression returns ordered edge IDs from any supported carried path representation. func pathCompositeEdgeIDArrayExpression(scope *Scope, pathBinding *BoundIdentifier) (pgsql.Expression, error) { var edgeIDArrayReferences []pgsql.Expression @@ -88,6 +90,7 @@ func pathCompositeEdgeIDArrayExpression(scope *Scope, pathBinding *BoundIdentifi }, nil } +// buildPathEdgeIDArrayFutures records deferred replacements for path edge-ID references in a query part. func (s *Translator) buildPathEdgeIDArrayFutures() error { for _, future := range s.query.CurrentPart().pathEdgeIDArrayFutures { if edgeIDArrayExpression, err := pathCompositeEdgeIDArrayExpression(s.scope, future.Data); err != nil { @@ -100,6 +103,7 @@ func (s *Translator) buildPathEdgeIDArrayFutures() error { return nil } +// resolvePathCompositeFieldReference replaces a deferred path field with the expression that materializes it. func resolvePathCompositeFieldReference(scope *Scope, reference pgsql.RowColumnReference) (pgsql.Expression, bool, error) { identifier, isIdentifier := unwrapParenthetical(reference.Identifier).(pgsql.Identifier) if !isIdentifier { @@ -139,6 +143,7 @@ func resolvePathCompositeFieldReference(scope *Scope, reference pgsql.RowColumnR } } +// resolvePathCompositeFieldReferencesInProjection resolves deferred path fields in every projection item. func resolvePathCompositeFieldReferencesInProjection(scope *Scope, projection pgsql.Projection) (pgsql.Projection, error) { rewritten := make(pgsql.Projection, len(projection)) @@ -165,6 +170,7 @@ func resolvePathCompositeFieldReferencesInProjection(scope *Scope, projection pg return rewritten, nil } +// resolvePathCompositeFieldReferencesInFromClause resolves deferred path fields in a source and its join constraints. func resolvePathCompositeFieldReferencesInFromClause(scope *Scope, fromClause pgsql.FromClause) (pgsql.FromClause, error) { if resolvedSource, err := resolvePathCompositeFieldReferences(scope, fromClause.Source); err != nil { return pgsql.FromClause{}, err @@ -193,6 +199,7 @@ func resolvePathCompositeFieldReferencesInFromClause(scope *Scope, fromClause pg return fromClause, nil } +// resolvePathCompositeFieldReferencesInFromClauses resolves deferred path fields across all query sources. func resolvePathCompositeFieldReferencesInFromClauses(scope *Scope, fromClauses []pgsql.FromClause) ([]pgsql.FromClause, error) { rewritten := make([]pgsql.FromClause, len(fromClauses)) @@ -208,6 +215,7 @@ func resolvePathCompositeFieldReferencesInFromClauses(scope *Scope, fromClauses return rewritten, nil } +// resolvePathCompositeFieldReferences walks a query part and substitutes every recorded path-field future. func resolvePathCompositeFieldReferences(scope *Scope, expression pgsql.Expression) (pgsql.Expression, error) { switch typedExpression := expression.(type) { case nil: @@ -260,6 +268,16 @@ func resolvePathCompositeFieldReferences(scope *Scope, expression pgsql.Expressi typedExpression.Parameters[idx] = resolved } } + for _, orderBy := range typedExpression.OrderBy { + if orderBy == nil { + continue + } + resolved, err := resolvePathCompositeFieldReferences(scope, orderBy.Expression) + if err != nil { + return nil, err + } + orderBy.Expression = resolved + } return typedExpression, nil diff --git a/cypher/models/pgsql/translate/pattern.go b/cypher/models/pgsql/translate/pattern.go index 45450523..4254d486 100644 --- a/cypher/models/pgsql/translate/pattern.go +++ b/cypher/models/pgsql/translate/pattern.go @@ -1,15 +1,22 @@ package translate import ( + "fmt" + "github.com/specterops/dawgs/cypher/models/cypher" "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// BindingResult groups SQL model state that must remain consistent while translating binding result. type BindingResult struct { - Binding *BoundIdentifier + // Binding supplies the binding input to the BindingResult contract. + Binding *BoundIdentifier + // AlreadyBound indicates whether already bound applies. AlreadyBound bool } +// bindPatternExpression binds a completed traversal result to its pattern variable when one was declared. func (s *Translator) bindPatternExpression(cypherExpression cypher.Expression, dataType pgsql.DataType) (BindingResult, error) { if cypherBinding, hasCypherBinding, err := extractIdentifierFromCypherExpression(cypherExpression); err != nil { return BindingResult{}, err @@ -32,6 +39,7 @@ func (s *Translator) bindPatternExpression(cypherExpression cypher.Expression, d } } +// translatePatternPart dispatches shortest-path, variable-expansion, and fixed traversal patterns to their builders. func (s *Translator) translatePatternPart(patternPart *cypher.PatternPart) error { // We expect this to be a node select if there aren't enough pattern elements for a traversal newPatternPart := s.query.CurrentPart().currentPattern.NewPart() @@ -65,6 +73,7 @@ func (s *Translator) translatePatternPart(patternPart *cypher.PatternPart) error return nil } +// buildPatternPart finalizes a translated pattern part and exports its visible bindings. func (s *Translator) buildPatternPart(part *PatternPart) error { if part.IsTraversal { return s.buildTraversalPatternPart(part) @@ -73,6 +82,7 @@ func (s *Translator) buildPatternPart(part *PatternPart) error { } } +// buildTraversalPattern emits fixed traversal steps and applies any exact-range unrolling decisions. func (s *Translator) buildTraversalPattern(traversalStep *TraversalStep, isRootStep bool) error { if isRootStep { if traversalStepQuery, err := s.buildTraversalPatternRoot(traversalStep.Frame, traversalStep); err != nil { @@ -106,6 +116,7 @@ func (s *Translator) buildTraversalPattern(traversalStep *TraversalStep, isRootS return nil } +// buildExpansionPattern emits an ordinary variable expansion and any qualified specialized-search rewrite. func (s *Translator) buildExpansionPattern(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder) error { traversalStep := traversalStepContext.CurrentStep @@ -136,6 +147,7 @@ func (s *Translator) buildExpansionPattern(traversalStepContext TraversalStepCon return nil } +// buildShortestPathsExpansionPattern emits the selected shortest-path executor and its projection frame. func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext TraversalStepContext, expansion *ExpansionBuilder, allPaths bool) error { traversalStep := traversalStepContext.CurrentStep @@ -143,7 +155,36 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra expansion.SetUnwindClauses(s.query.CurrentPart().ConsumeUnwindClauses()) if allPaths { - if traversalStep.Expansion.UseBidirectionalSearch { + if compactShortestExecutor(traversalStep.Expansion.ShortestPathExecutor) { + var ( + traversalStepQuery pgsql.Query + err error + ) + switch traversalStep.Expansion.ShortestPathExecutor { + case optimize.ShortestPathExecutorASPA1DAG: + traversalStepQuery, err = expansion.BuildAllShortestPathsDAGRoot() + case optimize.ShortestPathExecutorASPN1NegativeExhaustion: + traversalStepQuery, err = expansion.BuildAllShortestPathsNoPathProbeRoot() + case optimize.ShortestPathExecutorASPI1DAG: + traversalStepQuery, err = expansion.BuildInlineAllShortestPathsDAGRoot() + case optimize.ShortestPathExecutorASPB1AlternatingNodeDAG: + traversalStepQuery, err = expansion.BuildB1AllShortestPathsDAGRoot() + case optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + traversalStepQuery, err = expansion.BuildB2AllShortestPathsDAGRoot() + default: + err = fmt.Errorf("compact executor %q does not implement all-shortest-path enumeration", traversalStep.Expansion.ShortestPathExecutor) + } + if err != nil { + return err + } + s.recordShortestPathExecutor(traversalStep.Expansion.ShortestPathTarget, traversalStep.Expansion.ShortestPathExecutor) + s.query.CurrentPart().Model.AddCTE(pgsql.CommonTableExpression{ + Alias: pgsql.TableAlias{ + Name: traversalStep.Frame.Binding.Identifier, + }, + Query: traversalStepQuery, + }) + } else if traversalStep.Expansion.UseBidirectionalSearch { if traversalStepQuery, err := expansion.BuildBiDirectionalAllShortestPathsRoot(); err != nil { return err } else { @@ -170,7 +211,28 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra err error ) - if traversalStep.Expansion.UseBidirectionalSearch { + if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3Unidirectional || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalDistance { + traversalStepQuery, err = expansion.BuildShortestDistanceRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistance { + traversalStepQuery, err = expansion.BuildInlineGuardedShortestDistanceRoot() + } else if isV2GuardedDistanceExecutor(traversalStep.Expansion.ShortestPathExecutor) { + traversalStepQuery, err = expansion.buildInlineGuardedShortestDistanceRoot( + traversalStep.Expansion.ShortestPathExecutor, + spI2DevelopmentArchitecture(traversalStep.Expansion.ShortestPathExecutor), + ) + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness { + traversalStepQuery, err = expansion.BuildShortestPathEdgeM0Root() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + traversalStepQuery, err = expansion.BuildInlineCanonicalShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB1AlternatingNodeDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB1AlternatingNodeWitness { + traversalStepQuery, err = expansion.BuildB1CompactShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness { + traversalStepQuery, err = expansion.BuildB2CompactShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS4CanonicalDistance || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS4CanonicalWitness { + traversalStepQuery, err = expansion.BuildCompactShortestPathRoot() + } else if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS0Direct { + traversalStepQuery, err = expansion.BuildBiDirectionalShortestPathsRootWithDirectPreflight() + } else if traversalStep.Expansion.UseBidirectionalSearch { traversalStepQuery, err = expansion.BuildBiDirectionalShortestPathsRoot() } else { traversalStepQuery, err = expansion.BuildShortestPathsRoot() @@ -179,6 +241,10 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra if err != nil { return err } + if traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3Unidirectional || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS3EdgeM0 || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalDistance || isGuardedDistanceExecutor(traversalStep.Expansion.ShortestPathExecutor) || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalWitness || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || compactShortestExecutor(traversalStep.Expansion.ShortestPathExecutor) || traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorS0Direct || + (traversalStep.Expansion.ShortestPathExecutor == optimize.ShortestPathExecutorIncumbentWorkspace && decisionIsForcedShortest(s, traversalStep.Expansion.ShortestPathTarget)) { + s.recordShortestPathExecutor(traversalStep.Expansion.ShortestPathTarget, traversalStep.Expansion.ShortestPathExecutor) + } s.query.CurrentPart().Model.AddCTE(pgsql.CommonTableExpression{ Alias: pgsql.TableAlias{ @@ -203,13 +269,24 @@ func (s *Translator) buildShortestPathsExpansionPattern(traversalStepContext Tra return nil } +// TraversalStepContext groups SQL model state that must remain consistent while translating traversal step context. type TraversalStepContext struct { + // PreviousStep supplies the previous step input to the TraversalStepContext contract. PreviousStep *TraversalStep - CurrentStep *TraversalStep - IsRootStep bool + // CurrentStep supplies the current step input to the TraversalStepContext contract. + CurrentStep *TraversalStep + // IsRootStep indicates whether is root step applies. + IsRootStep bool } +// buildTraversalPatternPart translates all steps in a non-expanding pattern chain. func (s *Translator) buildTraversalPatternPart(part *PatternPart) error { + firstCTE := len(s.query.CurrentPart().Model.CommonTableExpressions.Expressions) + fixedSuffixDecision, useFixedSuffixStrategy := selectedFixedSuffixDecision(part, s.expansionSearchStrategyDecisions) + suffixReverseGuardDecision, useSuffixReverseGuard := selectedSuffixReverseGuardDecision(part, s.expansionSearchStrategyDecisions) + guardedSuffixDecision, useGuardedSuffixStrategy := selectedGuardedFixedSuffixDecision(part, s.expansionSearchStrategyDecisions) + endpointSeededDecision, useEndpointSeededStrategy := selectedEndpointSeededDecision(part, s.expansionSearchStrategyDecisions) + for idx, traversalStep := range part.TraversalSteps { var ( isRootStep = idx == 0 @@ -224,7 +301,7 @@ func (s *Translator) buildTraversalPatternPart(part *PatternPart) error { } if traversalStep.Expansion != nil { - if expansion, err := NewExpansionBuilder(s.translation.Parameters, traversalStep); err != nil { + if expansion, err := NewExpansionBuilder(s.translation.Parameters, traversalStep, s.graphID); err != nil { return err } else if part.ShortestPath || part.AllShortestPaths { if err := s.buildShortestPathsExpansionPattern(traversalStepContext, expansion, part.AllShortestPaths); err != nil { @@ -240,5 +317,18 @@ func (s *Translator) buildTraversalPatternPart(part *PatternPart) error { s.allowLimitPushdownForStep(part, idx, traversalStep) } + if useSuffixReverseGuard { + return s.rewriteTraversalPatternAsSuffixReverseGuard(part, suffixReverseGuardDecision, firstCTE) + } + if useFixedSuffixStrategy { + return s.rewriteTraversalPatternAsSuffixSeededReverse(part, fixedSuffixDecision, firstCTE) + } + if useGuardedSuffixStrategy { + return s.rewriteTraversalPatternAsGuardedSuffixOrientation(part, guardedSuffixDecision, firstCTE) + } + if useEndpointSeededStrategy { + return s.rewriteTraversalPatternAsEndpointSeededReverse(part, endpointSeededDecision, firstCTE) + } + return nil } diff --git a/cypher/models/pgsql/translate/projection.go b/cypher/models/pgsql/translate/projection.go index 74adafa3..390b5e5f 100644 --- a/cypher/models/pgsql/translate/projection.go +++ b/cypher/models/pgsql/translate/projection.go @@ -1,9 +1,11 @@ package translate import ( + "bytes" "fmt" "github.com/specterops/dawgs/cypher/models/cypher" + cypherFormat "github.com/specterops/dawgs/cypher/models/cypher/format" "github.com/specterops/dawgs/cypher/models/walk" "github.com/specterops/dawgs/cypher/models" @@ -11,11 +13,16 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// BoundProjections pairs rendered select items with the identifiers they carry into the next frame. type BoundProjections struct { - Items pgsql.Projection + // Items contains the SQL expressions emitted by the projection. + Items pgsql.Projection + + // Bindings contains the scope bindings represented by Items. Bindings []*BoundIdentifier } +// rewriteConstraintIdentifierReferences resolves constraint bindings through the preceding projection frame. func rewriteConstraintIdentifierReferences(scope *Scope, frame *Frame, constraints []*Constraint) error { if frame.Previous == nil { return nil @@ -30,6 +37,7 @@ func rewriteConstraintIdentifierReferences(scope *Scope, frame *Frame, constrain return nil } +// buildExternalProjection renders user-visible projection expressions and applies their requested aliases. func buildExternalProjection(scope *Scope, projections []*Projection) (pgsql.Projection, error) { var sqlProjection pgsql.Projection @@ -82,6 +90,7 @@ func buildExternalProjection(scope *Scope, projections []*Projection) (pgsql.Pro return sqlProjection, nil } +// buildInternalProjection renders each distinct bound identifier required by an internal frame. func buildInternalProjection(scope *Scope, projectedBindings []*BoundIdentifier) (BoundProjections, error) { var ( boundProjections = BoundProjections{ @@ -113,6 +122,7 @@ func buildInternalProjection(scope *Scope, projectedBindings []*BoundIdentifier) return boundProjections, nil } +// buildVisibleProjections renders the bindings known to the current scope frame. func buildVisibleProjections(scope *Scope) (BoundProjections, error) { currentFrame := scope.CurrentFrame() @@ -123,7 +133,21 @@ func buildVisibleProjections(scope *Scope) (BoundProjections, error) { } } +// buildProjectionForExpansionPath projects an expansion's distance or accumulated edge-identifier path. func buildProjectionForExpansionPath(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.DistanceOnly { + reference := scope.CurrentFrame().Binding.Identifier + column := expansionDepth + if projected.LastProjection != nil { + reference = referenceFrame.Binding.Identifier + column = projected.Identifier + } + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{reference, column}, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if projected.LastProjection != nil { return []pgsql.SelectItem{ &pgsql.AliasedExpression{ @@ -141,6 +165,7 @@ func buildProjectionForExpansionPath(alias pgsql.Identifier, projected *BoundIde }, nil } +// concatenatePathCompositeParts joins ordered path fragments into one array expression. func concatenatePathCompositeParts(parts []pgsql.Expression) pgsql.Expression { if len(parts) == 0 { return nil @@ -154,6 +179,7 @@ func concatenatePathCompositeParts(parts []pgsql.Expression) pgsql.Expression { return joined } +// bindingFrameReference returns the qualified reference to a binding in its latest projection frame. func bindingFrameReference(scope *Scope, binding *BoundIdentifier) pgsql.CompoundIdentifier { frameIdentifier := scope.CurrentFrameBinding().Identifier if binding.LastProjection != nil { @@ -163,6 +189,7 @@ func bindingFrameReference(scope *Scope, binding *BoundIdentifier) pgsql.Compoun return pgsql.CompoundIdentifier{frameIdentifier, binding.Identifier} } +// pathBindingReference resolves a path binding against its latest available frame. func pathBindingReference(scope *Scope, binding *BoundIdentifier) pgsql.Expression { if binding.LastProjection != nil { return pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier} @@ -175,6 +202,7 @@ func pathBindingReference(scope *Scope, binding *BoundIdentifier) pgsql.Expressi return binding.Identifier } +// pathCompositeReference returns a projected path value or constructs a composite from table columns. func pathCompositeReference(scope *Scope, binding *BoundIdentifier, columns []pgsql.Identifier) pgsql.Expression { if binding.LastProjection != nil || scope.CurrentFrameBinding() != nil { return pathBindingReference(scope, binding) @@ -191,6 +219,7 @@ func pathCompositeReference(scope *Scope, binding *BoundIdentifier, columns []pg } } +// edgeCompositeValue constructs an edge composite from a table alias or row-valued expression. func edgeCompositeValue(expression pgsql.Expression) pgsql.CompositeValue { value := pgsql.CompositeValue{ DataType: pgsql.EdgeComposite, @@ -211,6 +240,7 @@ func edgeCompositeValue(expression pgsql.Expression) pgsql.CompositeValue { return value } +// pathCompositeColumnReference addresses a column of either a projected path composite or its source table. func pathCompositeColumnReference(scope *Scope, binding *BoundIdentifier, column pgsql.Identifier) pgsql.Expression { if binding.LastProjection != nil || scope.CurrentFrameBinding() != nil { return pgsql.RowColumnReference{ @@ -222,6 +252,7 @@ func pathCompositeColumnReference(scope *Scope, binding *BoundIdentifier, column return pgsql.CompoundIdentifier{binding.Identifier, column} } +// pathEdgeIDReference resolves the identifier of an edge used as a path component. func pathEdgeIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expression { if binding.LastProjection != nil || scope.CurrentFrameBinding() != nil { return pathBindingReference(scope, binding) @@ -230,23 +261,30 @@ func pathEdgeIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expressio return pgsql.CompoundIdentifier{binding.Identifier, pgsql.ColumnID} } -func pathEdgeArrayExpression(scope *Scope, edge *BoundIdentifier) pgsql.Expression { +// edgeArrayFromPathIDs creates a graph-scoped edge-array materializer for ordered edge identifiers. +func edgeArrayFromPathIDs(scope *Scope, pathIDs pgsql.Expression) *pgsql.EdgeArrayFromPathIDs { return &pgsql.EdgeArrayFromPathIDs{ - PathIDs: pgsql.ArrayLiteral{ - Values: []pgsql.Expression{ - pathEdgeIDReference(scope, edge), - }, - CastType: pgsql.Int8Array, - }, + PathIDs: pathIDs, + GraphID: pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), } } +// pathEdgeArrayExpression materializes one path-edge binding as an edge-composite array. +func pathEdgeArrayExpression(scope *Scope, edge *BoundIdentifier) pgsql.Expression { + return edgeArrayFromPathIDs(scope, pgsql.ArrayLiteral{ + Values: []pgsql.Expression{ + pathEdgeIDReference(scope, edge), + }, + CastType: pgsql.Int8Array, + }) +} + +// expansionPathEdgeArrayExpression materializes an expansion's edge-identifier path as edge composites. func expansionPathEdgeArrayExpression(scope *Scope, expansionPath *BoundIdentifier) (pgsql.Expression, error) { - return &pgsql.EdgeArrayFromPathIDs{ - PathIDs: pathBindingReference(scope, expansionPath), - }, nil + return edgeArrayFromPathIDs(scope, pathBindingReference(scope, expansionPath)), nil } +// optionalOr combines two predicates while treating a nil operand as absent. func optionalOr(leftOperand, rightOperand pgsql.Expression) pgsql.Expression { if leftOperand == nil { return rightOperand @@ -257,17 +295,19 @@ func optionalOr(leftOperand, rightOperand pgsql.Expression) pgsql.Expression { return pgsql.NewBinaryExpression(leftOperand, pgsql.OperatorOr, rightOperand) } +// expressionIsNull builds an SQL null test for an expression. func expressionIsNull(expression pgsql.Expression) pgsql.Expression { return pgsql.NewBinaryExpression(expression, pgsql.OperatorIs, pgsql.NullLiteral()) } +// pathCompositeDependencyNullGuard returns the null test appropriate for a path component binding. func pathCompositeDependencyNullGuard(scope *Scope, dependency *BoundIdentifier) pgsql.Expression { if dependency == nil { return nil } switch dependency.DataType { - case pgsql.ExpansionPath: + case pgsql.ExpansionPath, pgsql.PathComposite: return expressionIsNull(pathBindingReference(scope, dependency)) case pgsql.EdgeComposite: @@ -284,6 +324,7 @@ func pathCompositeDependencyNullGuard(scope *Scope, dependency *BoundIdentifier) } } +// nullGuardPathCompositeExpression yields SQL null instead of constructing a path when a dependency is null. func nullGuardPathCompositeExpression(expression, nullGuard pgsql.Expression) pgsql.Expression { if nullGuard == nil { return expression @@ -305,6 +346,7 @@ func reversePathCompositeExpressions(expressions []pgsql.Expression) { } } +// expressionForPathComposite assembles a path value from complete paths, node composites, and ordered edge components. func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql.Expression, error) { if projected.LastProjection != nil { return pgsql.CompoundIdentifier{projected.LastProjection.Binding.Identifier, projected.Identifier}, nil @@ -315,11 +357,27 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql nodeReferences []pgsql.Expression directNodeReferences []pgsql.Expression directEdgeReferences []pgsql.Expression + allRawPathIDParts []pgsql.Expression seenExpansionPath = false seenPathEdge = false + seenDirectEdge = false + directPath pgsql.Expression nullGuard pgsql.Expression + pendingPathIDParts []pgsql.Expression ) + flushPathIDParts := func() { + if len(pendingPathIDParts) == 0 { + return + } + + edgeArrayReferences = append(edgeArrayReferences, edgeArrayFromPathIDs( + scope, + concatenatePathCompositeParts(pendingPathIDParts), + )) + pendingPathIDParts = nil + } + // Path composite components are encoded as dependencies on the bound identifier representing the // path. This is not ideal as it escapes normal translation flow as driven by the structure of the // originating cypher AST. @@ -327,15 +385,21 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql nullGuard = optionalOr(nullGuard, pathCompositeDependencyNullGuard(scope, dependency)) switch dependency.DataType { + case pgsql.PathComposite: + if directPath != nil { + return nil, fmt.Errorf("path rendering contains multiple complete path dependencies") + } + directPath = pathBindingReference(scope, dependency) + case pgsql.ExpansionPath: seenExpansionPath = true - if edgeArrayReference, err := expansionPathEdgeArrayExpression(scope, dependency); err != nil { - return nil, err - } else { - edgeArrayReferences = append(edgeArrayReferences, edgeArrayReference) - } + pathIDs := pathBindingReference(scope, dependency) + pendingPathIDParts = append(pendingPathIDParts, pathIDs) + allRawPathIDParts = append(allRawPathIDParts, pathIDs) case pgsql.EdgeComposite: + seenDirectEdge = true + flushPathIDParts() directEdgeReference := pathCompositeReference(scope, dependency, pgsql.EdgeTableColumns) directEdgeReferences = append(directEdgeReferences, directEdgeReference) @@ -346,7 +410,12 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql case pgsql.PathEdge: seenPathEdge = true - edgeArrayReferences = append(edgeArrayReferences, pathEdgeArrayExpression(scope, dependency)) + pathIDs := pgsql.ArrayLiteral{ + Values: []pgsql.Expression{pathEdgeIDReference(scope, dependency)}, + CastType: pgsql.Int8Array, + } + pendingPathIDParts = append(pendingPathIDParts, pathIDs) + allRawPathIDParts = append(allRawPathIDParts, pathIDs) case pgsql.NodeComposite, pgsql.ExpansionRootNode, pgsql.ExpansionTerminalNode: directNodeReferences = append(directNodeReferences, pathCompositeReference(scope, dependency, pgsql.NodeTableColumns)) @@ -356,6 +425,13 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nil, fmt.Errorf("unsupported type for path rendering: %s", dependency.DataType) } } + flushPathIDParts() + if directPath != nil { + if seenExpansionPath || seenPathEdge || seenDirectEdge { + return nil, fmt.Errorf("complete path dependency cannot be mixed with edge path components") + } + return nullGuardPathCompositeExpression(directPath, nullGuard), nil + } // The optimizer reversed the originating pattern so the traversal could be driven from the // more selective terminal endpoint. The path dependencies were therefore accumulated in @@ -364,6 +440,7 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql // expansion step's PathReversed flag. if projected.PathDirectionReversed { reversePathCompositeExpressions(edgeArrayReferences) + reversePathCompositeExpressions(allRawPathIDParts) reversePathCompositeExpressions(nodeReferences) reversePathCompositeExpressions(directNodeReferences) reversePathCompositeExpressions(directEdgeReferences) @@ -394,6 +471,34 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nil, fmt.Errorf("expansion path %s does not contain a root node reference", projected.Identifier) } + knownNodes := pgsql.ArrayLiteral{ + Values: directNodeReferences, + CastType: pgsql.NodeCompositeArray, + } + + // Read expansions carry edge IDs in path order. When every edge + // component is still an ID, let the graph-scoped linear materializer + // hydrate and walk the stream once. A direct edge composite indicates a + // mixed or mutation-returning path and retains the conservative generic + // materializer below. + if !seenDirectEdge { + pathIDs := concatenatePathCompositeParts(allRawPathIDParts) + if pathIDs == nil { + pathIDs = pgsql.ArrayLiteral{CastType: pgsql.Int8Array} + } + + return nullGuardPathCompositeExpression(pgsql.FunctionCall{ + Function: pgsql.FunctionOrderedEdgeIDsToPath, + Parameters: []pgsql.Expression{ + pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), + directNodeReferences[0], + pathIDs, + knownNodes, + }, + CastType: pgsql.PathComposite, + }, nullGuard), nil + } + edgeArrayExpression := concatenatePathCompositeParts(edgeArrayReferences) if edgeArrayExpression == nil { edgeArrayExpression = pgsql.ArrayLiteral{CastType: pgsql.EdgeCompositeArray} @@ -402,12 +507,10 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nullGuardPathCompositeExpression(pgsql.FunctionCall{ Function: pgsql.FunctionOrderedEdgesToPath, Parameters: []pgsql.Expression{ + pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), directNodeReferences[0], edgeArrayExpression, - pgsql.ArrayLiteral{ - Values: directNodeReferences, - CastType: pgsql.NodeCompositeArray, - }, + knownNodes, }, CastType: pgsql.PathComposite, }, nullGuard), nil @@ -415,6 +518,7 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nullGuardPathCompositeExpression(pgsql.FunctionCall{ Function: pgsql.FunctionNodesToPath, Parameters: []pgsql.Expression{ + pgsql.NewLiteral(scope.GraphID(), pgsql.Int4), pgsql.Variadic{ Expression: pgsql.ArrayLiteral{ Values: nodeReferences, @@ -429,7 +533,21 @@ func expressionForPathComposite(projected *BoundIdentifier, scope *Scope) (pgsql return nil, fmt.Errorf("path variable does not contain valid components") } +// buildProjectionForPathComposite projects either a path's distance or its assembled composite value. func buildProjectionForPathComposite(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope) ([]pgsql.SelectItem, error) { + if projected.DistanceOnly { + reference := scope.CurrentFrame().Binding.Identifier + column := expansionDepth + if projected.LastProjection != nil { + reference = projected.LastProjection.Binding.Identifier + column = projected.Identifier + } + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{reference, column}, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if expression, err := expressionForPathComposite(projected, scope); err != nil { return nil, err } else { @@ -442,7 +560,20 @@ func buildProjectionForPathComposite(alias pgsql.Identifier, projected *BoundIde } } +// buildProjectionForExpansionNode projects an expansion endpoint as an identifier or hydrated node composite. func buildProjectionForExpansionNode(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.IDOnly { + var expression pgsql.Expression = pgsql.CompoundIdentifier{projected.Identifier, pgsql.ColumnID} + if projected.LastProjection != nil { + expression = pgsql.CompoundIdentifier{referenceFrame.Binding.Identifier, projected.Identifier} + } + + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: expression, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if projected.LastProjection != nil { return []pgsql.SelectItem{ &pgsql.AliasedExpression{ @@ -472,7 +603,20 @@ func buildProjectionForExpansionNode(alias pgsql.Identifier, projected *BoundIde }, nil } +// buildProjectionForNodeComposite projects an existing node binding as an identifier or node composite. func buildProjectionForNodeComposite(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.IDOnly { + var expression pgsql.Expression = pgsql.CompoundIdentifier{projected.Identifier, pgsql.ColumnID} + if projected.LastProjection != nil { + expression = pgsql.CompoundIdentifier{referenceFrame.Binding.Identifier, projected.Identifier} + } + + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: expression, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + if projected.LastProjection != nil { return []pgsql.SelectItem{ &pgsql.AliasedExpression{ @@ -499,6 +643,7 @@ func buildProjectionForNodeComposite(alias pgsql.Identifier, projected *BoundIde }, nil } +// buildProjectionForExpansionEdge materializes an expansion path's edge identifiers as edge composites. func buildProjectionForExpansionEdge(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope) ([]pgsql.SelectItem, error) { // Change the type to the edge composite now that this is projected projected.DataType = pgsql.EdgeComposite @@ -506,17 +651,16 @@ func buildProjectionForExpansionEdge(alias pgsql.Identifier, projected *BoundIde // Create a new final projection that's aliased to the visible binding's identifier return []pgsql.SelectItem{ &pgsql.AliasedExpression{ - Expression: &pgsql.EdgeArrayFromPathIDs{ - PathIDs: pgsql.CompoundIdentifier{ - scope.CurrentFrame().Binding.Identifier, - pgsql.ColumnPath, - }, - }, + Expression: edgeArrayFromPathIDs(scope, pgsql.CompoundIdentifier{ + scope.CurrentFrame().Binding.Identifier, + pgsql.ColumnPath, + }), Alias: pgsql.AsOptionalIdentifier(alias), }, }, nil } +// buildProjectionForEdgeComposite projects an edge binding from its latest frame or source columns. func buildProjectionForEdgeComposite(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { if projected.LastProjection != nil { return []pgsql.SelectItem{ @@ -536,6 +680,7 @@ func buildProjectionForEdgeComposite(alias pgsql.Identifier, projected *BoundIde }, nil } +// buildProjectionForPathEdge projects the identifier carried by a single-edge path component. func buildProjectionForPathEdge(alias pgsql.Identifier, projected *BoundIdentifier, referenceFrame *Frame) ([]pgsql.SelectItem, error) { var expression pgsql.Expression @@ -557,7 +702,21 @@ func buildProjectionForPathEdge(alias pgsql.Identifier, projected *BoundIdentifi }, nil } +// buildProjection dispatches a bound identifier to the projection form required by its data type. func buildProjection(alias pgsql.Identifier, projected *BoundIdentifier, scope *Scope, referenceFrame *Frame) ([]pgsql.SelectItem, error) { + if projected.DistanceOnly { + reference := scope.CurrentFrame().Binding.Identifier + column := expansionDepth + if projected.LastProjection != nil { + reference = referenceFrame.Binding.Identifier + column = projected.Identifier + } + return []pgsql.SelectItem{&pgsql.AliasedExpression{ + Expression: pgsql.CompoundIdentifier{reference, column}, + Alias: pgsql.AsOptionalIdentifier(alias), + }}, nil + } + switch projected.DataType { case pgsql.ExpansionPath: return buildProjectionForExpansionPath(alias, projected, scope, referenceFrame) @@ -598,6 +757,7 @@ func buildProjection(alias pgsql.Identifier, projected *BoundIdentifier, scope * } } +// buildInlineProjection renders a query part's prepared expressions directly into a select statement. func (s *Translator) buildInlineProjection(part *QueryPart) (pgsql.Select, error) { sqlSelect := pgsql.Select{ Distinct: part.projections.Distinct, @@ -659,6 +819,7 @@ func (s *Translator) buildInlineProjection(part *QueryPart) (pgsql.Select, error return sqlSelect, nil } +// collectProjectionFromFrames collects the frame sources required by projected bindings and path dependencies. func (s *Translator) collectProjectionFromFrames(projections []*Projection) []pgsql.FromClause { fromClauseBuilder := NewFromClauseBuilder() @@ -691,6 +852,7 @@ func (s *Translator) collectProjectionFromFrames(projections []*Projection) []pg return fromClauseBuilder.Clauses() } +// countLimitPushdownShortestPathHarnessCalls counts eligible shortest-path harness calls throughout a query's CTE tree. func countLimitPushdownShortestPathHarnessCalls(query pgsql.Query) int { var count int @@ -712,10 +874,12 @@ func countLimitPushdownShortestPathHarnessCalls(query pgsql.Query) int { return count } +// isLimitPushdownShortestPathHarness reports whether a function accepts the shortest-path limit parameter. func isLimitPushdownShortestPathHarness(function pgsql.Identifier) bool { return function == pgsql.FunctionUnidirectionalSPHarness || function == pgsql.FunctionBidirectionalSPHarness } +// appendLimitToShortestPathHarness passes a limit to each eligible harness and bounds its containing function scan. func appendLimitToShortestPathHarness(query *pgsql.Query, limit pgsql.Expression) { if query.CommonTableExpressions != nil { for idx := range query.CommonTableExpressions.Expressions { @@ -724,6 +888,7 @@ func appendLimitToShortestPathHarness(query *pgsql.Query, limit pgsql.Expression } if selectBody, isSelect := query.Body.(pgsql.Select); isSelect { + containsHarness := false for idx := range selectBody.From { if functionCall, isFunctionCall := selectBody.From[idx].Source.(pgsql.FunctionCall); isFunctionCall && isLimitPushdownShortestPathHarness(functionCall.Function) { @@ -732,13 +897,22 @@ func appendLimitToShortestPathHarness(query *pgsql.Query, limit pgsql.Expression // outer query will discard. functionCall.Parameters = append(functionCall.Parameters, pgsql.NewTypeCast(limit, pgsql.Int8)) selectBody.From[idx].Source = functionCall + containsHarness = true } } query.Body = selectBody + if containsHarness { + // Keep the internal limit so the BFS can stop early, and also bound + // the containing FunctionScan so downstream planning sees the same + // cardinality ceiling. In particular, LIMIT 0 must prevent invoking + // the harness because the harness uses zero to mean "unlimited". + query.Limit = limit + } } } +// selectContainsAggregate reports whether a select body contains an aggregate function call. func selectContainsAggregate(selectBody pgsql.Select) bool { containsAggregate := false @@ -753,6 +927,7 @@ func selectContainsAggregate(selectBody pgsql.Select) bool { return containsAggregate } +// compoundIdentifierEqual reports whether two qualified identifiers contain the same components. func compoundIdentifierEqual(left, right pgsql.CompoundIdentifier) bool { if len(left) != len(right) { return false @@ -767,6 +942,7 @@ func compoundIdentifierEqual(left, right pgsql.CompoundIdentifier) bool { return true } +// directShortestPathHarnessFrame returns the sole CTE that directly invokes an eligible shortest-path harness. func directShortestPathHarnessFrame(query pgsql.Query) (pgsql.Identifier, bool) { if query.CommonTableExpressions == nil { return "", false @@ -795,11 +971,13 @@ func directShortestPathHarnessFrame(query pgsql.Query) (pgsql.Identifier, bool) return harnessFrame, harnessFrame != "" } +// isCompoundIdentifierOperand reports whether an expression is the requested qualified identifier. func isCompoundIdentifierOperand(expression pgsql.Expression, identifier pgsql.CompoundIdentifier) bool { compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(expression).(pgsql.CompoundIdentifier) return isCompoundIdentifier && compoundIdentifierEqual(compoundIdentifier, identifier) } +// isEqualityBetweenCompoundIdentifiers recognizes equality between two qualified identifiers in either order. func isEqualityBetweenCompoundIdentifiers(expression pgsql.Expression, left, right pgsql.CompoundIdentifier) bool { binaryExpression, isBinaryExpression := unwrapParenthetical(expression).(*pgsql.BinaryExpression) if !isBinaryExpression || binaryExpression.Operator != pgsql.OperatorEquals { @@ -810,6 +988,7 @@ func isEqualityBetweenCompoundIdentifiers(expression pgsql.Expression, left, rig (isCompoundIdentifierOperand(binaryExpression.LOperand, right) && isCompoundIdentifierOperand(binaryExpression.ROperand, left)) } +// expansionEndpointJoin identifies a node-table join to the root or terminal column of a harness frame. func expansionEndpointJoin(join pgsql.Join, harnessFrame pgsql.Identifier) (pgsql.Identifier, pgsql.Identifier, bool) { tableReference, isTableReference := join.Table.(pgsql.TableReference) if !isTableReference || @@ -836,6 +1015,7 @@ func expansionEndpointJoin(join pgsql.Join, harnessFrame pgsql.Identifier) (pgsq return "", "", false } +// shortestPathEndpointAliases finds distinct node aliases joined to a shortest-path harness's root and terminal columns. func shortestPathEndpointAliases(query pgsql.Query) (pgsql.Identifier, pgsql.Identifier, bool) { harnessFrame, hasHarnessFrame := directShortestPathHarnessFrame(query) if !hasHarnessFrame { @@ -865,6 +1045,7 @@ func shortestPathEndpointAliases(query pgsql.Query) (pgsql.Identifier, pgsql.Ide return rootAlias, terminalAlias, rootAlias != "" && terminalAlias != "" && rootAlias != terminalAlias } +// harnessEndpointColumn recognizes a root or terminal identifier column belonging to a harness frame. func harnessEndpointColumn(expression pgsql.Expression, harnessFrame pgsql.Identifier) (pgsql.Identifier, bool) { compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(expression).(pgsql.CompoundIdentifier) if !isCompoundIdentifier || @@ -877,6 +1058,7 @@ func harnessEndpointColumn(expression pgsql.Expression, harnessFrame pgsql.Ident return compoundIdentifier[1], true } +// rowIDReferenceAlias extracts the row alias named by a composite identifier-field reference. func rowIDReferenceAlias(expression pgsql.Expression) (pgsql.Identifier, bool) { rowColumnReference, isRowColumnReference := unwrapParenthetical(expression).(pgsql.RowColumnReference) if !isRowColumnReference || rowColumnReference.Column != pgsql.ColumnID { @@ -891,6 +1073,7 @@ func rowIDReferenceAlias(expression pgsql.Expression) (pgsql.Identifier, bool) { return compoundIdentifier[1], true } +// sourceAliasMatchesEndpointColumn reports whether a source alias corresponds to a harness endpoint column. func sourceAliasMatchesEndpointColumn(sourceAlias, endpointColumn, rootAlias, terminalAlias pgsql.Identifier) bool { switch endpointColumn { case expansionRootID: @@ -902,6 +1085,7 @@ func sourceAliasMatchesEndpointColumn(sourceAlias, endpointColumn, rootAlias, te } } +// isBoundEndpointProjectionConstraint recognizes a shape-preserving equality between a harness endpoint and its node alias. func isBoundEndpointProjectionConstraint(expression pgsql.Expression, harnessFrame, rootAlias, terminalAlias pgsql.Identifier) bool { binaryExpression, isBinaryExpression := unwrapParenthetical(expression).(*pgsql.BinaryExpression) if !isBinaryExpression || binaryExpression.Operator != pgsql.OperatorEquals { @@ -917,6 +1101,7 @@ func isBoundEndpointProjectionConstraint(expression pgsql.Expression, harnessFra (rightIsEndpoint && leftIsRowIDReference && sourceAliasMatchesEndpointColumn(leftSourceAlias, rightEndpointColumn, rootAlias, terminalAlias)) } +// shortestPathSourceWhereTransparent reports whether a source CTE filters only by endpoint projection equalities. func shortestPathSourceWhereTransparent(query pgsql.Query, rootAlias, terminalAlias pgsql.Identifier) bool { harnessFrame, hasHarnessFrame := directShortestPathHarnessFrame(query) if !hasHarnessFrame { @@ -944,6 +1129,7 @@ func shortestPathSourceWhereTransparent(query pgsql.Query, rootAlias, terminalAl return true } +// endpointIDReference extracts an endpoint alias from an identifier-field reference in the source frame. func endpointIDReference(expression pgsql.Expression, sourceFrame pgsql.Identifier) (pgsql.Identifier, bool) { rowColumnReference, isRowColumnReference := unwrapParenthetical(expression).(pgsql.RowColumnReference) compoundIdentifier, isCompoundIdentifier := unwrapParenthetical(rowColumnReference.Identifier).(pgsql.CompoundIdentifier) @@ -958,11 +1144,13 @@ func endpointIDReference(expression pgsql.Expression, sourceFrame pgsql.Identifi return compoundIdentifier[1], true } +// isEndpointAliasPair reports whether two aliases are the root and terminal aliases in either order. func isEndpointAliasPair(leftAlias, rightAlias, rootAlias, terminalAlias pgsql.Identifier) bool { return (leftAlias == rootAlias && rightAlias == terminalAlias) || (leftAlias == terminalAlias && rightAlias == rootAlias) } +// isEndpointInequality recognizes a non-equality predicate between the source frame's root and terminal identifiers. func isEndpointInequality(expression pgsql.Expression, sourceFrame, rootAlias, terminalAlias pgsql.Identifier) bool { binaryExpression, isBinaryExpression := unwrapParenthetical(expression).(*pgsql.BinaryExpression) if !isBinaryExpression || @@ -976,6 +1164,7 @@ func isEndpointInequality(expression pgsql.Expression, sourceFrame, rootAlias, t return hasLeftAlias && hasRightAlias && isEndpointAliasPair(leftAlias, rightAlias, rootAlias, terminalAlias) } +// shortestPathLimitPushdownTransparentWhere permits only the endpoint anti-reflexive predicate above a transparent source CTE. func shortestPathLimitPushdownTransparentWhere(currentPart *QueryPart, sourceFrame pgsql.Identifier, where pgsql.Expression) bool { if where == nil { return true @@ -1003,6 +1192,7 @@ func shortestPathLimitPushdownTransparentWhere(currentPart *QueryPart, sourceFra return true } +// limitPushdownTailSource returns the sole pass-through source CTE when the tail select preserves limit semantics. func limitPushdownTailSource(currentPart *QueryPart, tailSelect pgsql.Select) (pgsql.Identifier, bool) { // Keep this intentionally narrow: LIMIT can move into the harness only when // the tail SELECT is a simple pass-through over one shortest-path CTE. Sorts, @@ -1047,6 +1237,7 @@ func limitPushdownTailSource(currentPart *QueryPart, tailSelect pgsql.Select) (p return sourceFrame, true } +// pushDownShortestPathLimit moves an outer limit into a single eligible shortest-path harness call. func pushDownShortestPathLimit(currentPart *QueryPart, tailSelect pgsql.Select) bool { sourceFrame, canPushDown := limitPushdownTailSource(currentPart, tailSelect) if !canPushDown { @@ -1065,6 +1256,7 @@ func pushDownShortestPathLimit(currentPart *QueryPart, tailSelect pgsql.Select) return false } +// findCTE returns the named top-level common table expression, if present. func findCTE(query *pgsql.Query, cteName pgsql.Identifier) *pgsql.CommonTableExpression { if query.CommonTableExpressions == nil { return nil @@ -1081,6 +1273,7 @@ func findCTE(query *pgsql.Query, cteName pgsql.Identifier) *pgsql.CommonTableExp return nil } +// applyLimitToCTE assigns a limit to the named common table expression. func applyLimitToCTE(query *pgsql.Query, cteName pgsql.Identifier, limit pgsql.Expression) bool { if cte := findCTE(query, cteName); cte != nil { cte.Query.Limit = limit @@ -1090,6 +1283,7 @@ func applyLimitToCTE(query *pgsql.Query, cteName pgsql.Identifier, limit pgsql.E return false } +// pushDownTraversalLimit moves an outer limit to a semantically transparent traversal CTE. func pushDownTraversalLimit(currentPart *QueryPart, tailSelect pgsql.Select) bool { sourceFrame, canPushDown := limitPushdownTailSource(currentPart, tailSelect) if !canPushDown || !currentPart.CanPushDownLimitTo(sourceFrame) { @@ -1099,6 +1293,7 @@ func pushDownTraversalLimit(currentPart *QueryPart, tailSelect pgsql.Select) boo return applyLimitToCTE(currentPart.Model, sourceFrame, currentPart.Limit) } +// projectionAliasBindings maps internal binding identifiers to their visible projection aliases. func projectionAliasBindings(scope *Scope, projections []*Projection) map[pgsql.Identifier]pgsql.Identifier { aliases := map[pgsql.Identifier]pgsql.Identifier{} @@ -1115,6 +1310,7 @@ func projectionAliasBindings(scope *Scope, projections []*Projection) map[pgsql. return aliases } +// rewriteOrderByProjectionAlias replaces an internal ORDER BY identifier with its visible projection alias. func rewriteOrderByProjectionAlias(orderBy *pgsql.OrderBy, aliases map[pgsql.Identifier]pgsql.Identifier) { identifier, isIdentifier := orderBy.Expression.(pgsql.Identifier) if !isIdentifier { @@ -1126,21 +1322,32 @@ func rewriteOrderByProjectionAlias(orderBy *pgsql.OrderBy, aliases map[pgsql.Ide } } +// pathCompositeReferenceCount records how a path and each of its component arrays are reused by a projection stage. type pathCompositeReferenceCount struct { + // binding is the unmaterialized path binding being counted. binding *BoundIdentifier - full int - nodes int - edges int + + // full counts references to the complete path value. + full int + + // nodes counts references to the path's node array. + nodes int + + // edges counts references to the path's edge array. + edges int } +// componentReferences returns the combined number of node-array and edge-array references. func (s pathCompositeReferenceCount) componentReferences() int { return s.nodes + s.edges } +// totalReferences returns the number of complete-path and component-array references. func (s pathCompositeReferenceCount) totalReferences() int { return s.full + s.componentReferences() } +// pathCompositeBinding resolves an identifier to an unmaterialized path-composite binding. func pathCompositeBinding(scope *Scope, identifier pgsql.Identifier) (*BoundIdentifier, bool) { binding, bound := scope.Lookup(identifier) if !bound { @@ -1154,6 +1361,7 @@ func pathCompositeBinding(scope *Scope, identifier pgsql.Identifier) (*BoundIden return binding, true } +// ensurePathCompositeReferenceCount returns the stable counter for a binding and records first-seen order. func ensurePathCompositeReferenceCount( counts map[pgsql.Identifier]*pathCompositeReferenceCount, orderedCounts *[]*pathCompositeReferenceCount, @@ -1173,6 +1381,7 @@ func ensurePathCompositeReferenceCount( return count } +// countPathCompositeComponents counts references to node and edge arrays of unmaterialized path composites. func countPathCompositeComponents(scope *Scope, expressions ...pgsql.Expression) ([]*pathCompositeReferenceCount, error) { var ( counts = map[pgsql.Identifier]*pathCompositeReferenceCount{} @@ -1215,6 +1424,7 @@ func countPathCompositeComponents(scope *Scope, expressions ...pgsql.Expression) return orderedCounts, nil } +// countPathCompositeProjectionReferences counts complete and component references made by projection items. func countPathCompositeProjectionReferences(scope *Scope, projections []*Projection) ([]*pathCompositeReferenceCount, error) { var ( counts = map[pgsql.Identifier]*pathCompositeReferenceCount{} @@ -1252,6 +1462,7 @@ func countPathCompositeProjectionReferences(scope *Scope, projections []*Project return orderedCounts, nil } +// tailPathCompositeStageBindings selects paths whose node arrays must be staged for a tail constraint. func tailPathCompositeStageBindings(scope *Scope, expression pgsql.Expression) ([]*BoundIdentifier, error) { counts, err := countPathCompositeComponents(scope, expression) if err != nil { @@ -1268,6 +1479,7 @@ func tailPathCompositeStageBindings(scope *Scope, expression pgsql.Expression) ( return bindings, nil } +// projectionPathCompositeStageBindings selects paths reused enough to warrant one intermediate materialization. func projectionPathCompositeStageBindings(scope *Scope, projections []*Projection) ([]*BoundIdentifier, error) { counts, err := countPathCompositeProjectionReferences(scope, projections) if err != nil { @@ -1289,6 +1501,7 @@ func projectionPathCompositeStageBindings(scope *Scope, projections []*Projectio return bindings, nil } +// mergePathCompositeStageBindings combines binding lists in first-seen order without duplicates. func mergePathCompositeStageBindings(bindingSets ...[]*BoundIdentifier) []*BoundIdentifier { var ( merged = make([]*BoundIdentifier, 0) @@ -1309,6 +1522,7 @@ func mergePathCompositeStageBindings(bindingSets ...[]*BoundIdentifier) []*Bound return merged } +// stagePathCompositeBindings adds lateral sources that materialize selected paths once for downstream reuse. func (s *Translator) stagePathCompositeBindings(fromClauses []pgsql.FromClause, bindings []*BoundIdentifier) ([]pgsql.FromClause, error) { for _, binding := range bindings { stageBinding, err := s.scope.DefineNew(pgsql.Scope) @@ -1348,6 +1562,7 @@ func (s *Translator) stagePathCompositeBindings(fromClauses []pgsql.FromClause, return fromClauses, nil } +// buildTailProjection renders the final select, stages reused paths, and applies grouping, ordering, skip, and limit. func (s *Translator) buildTailProjection() error { var ( currentPart = s.query.CurrentPart() @@ -1447,6 +1662,7 @@ func (s *Translator) buildTailProjection() error { return nil } +// ensureProjectionAliasBinding defines an inferred scope binding for an expression alias not already known. func (s *Translator) ensureProjectionAliasBinding(alias pgsql.Identifier, selectItem pgsql.SelectItem) error { if _, isBound := s.scope.AliasedLookup(alias); isBound { return nil @@ -1466,6 +1682,7 @@ func (s *Translator) ensureProjectionAliasBinding(alias pgsql.Identifier, select return nil } +// ensureSortItemProjectionAliases registers visible aliases that ORDER BY items may reference. func (s *Translator) ensureSortItemProjectionAliases() error { currentPart := s.query.CurrentPart() if currentPart.projections == nil { @@ -1489,7 +1706,53 @@ func (s *Translator) ensureSortItemProjectionAliases() error { return nil } +// isGreedyProjectionItem reports whether a Cypher projection item is the wildcard expression. +func isGreedyProjectionItem(projectionItem *cypher.ProjectionItem) bool { + variable, isVariable := projectionItem.Expression.(*cypher.Variable) + return isVariable && variable.Symbol == cypher.TokenLiteralAsterisk +} + +// translateGreedyProjection replaces a wildcard placeholder with every named binding visible in the frame. +func (s *Translator) translateGreedyProjection(scope *Scope) error { + currentPart := s.query.CurrentPart() + if _, err := s.treeTranslator.PopOperand(); err != nil { + return err + } + if currentPart.projections == nil || len(currentPart.projections.Items) == 0 { + return fmt.Errorf("greedy projection has no prepared projection item") + } + + // Entering the projection item reserves one slot. Replace that placeholder + // with a projection for every named binding visible at this boundary. + currentPart.projections.Items = currentPart.projections.Items[:len(currentPart.projections.Items)-1] + projected := 0 + for _, identifier := range scope.CurrentFrame().Known().Slice() { + binding, found := scope.Lookup(identifier) + if !found { + return fmt.Errorf("unable to resolve greedy projection binding %s", identifier) + } + for _, symbol := range scope.Symbols(binding) { + currentPart.projections.Items = append(currentPart.projections.Items, &Projection{ + SelectItem: binding.Identifier, + Alias: models.OptionalValue(symbol), + }) + projected++ + } + } + + if projected == 0 { + return fmt.Errorf("greedy projection requires at least one named binding") + } + currentPart.projections.Frame = scope.CurrentFrame() + return nil +} + +// translateProjectionItem records one translated select expression and establishes its explicit or implicit alias. func (s *Translator) translateProjectionItem(scope *Scope, projectionItem *cypher.ProjectionItem) error { + if isGreedyProjectionItem(projectionItem) { + return s.translateGreedyProjection(scope) + } + if alias, hasAlias, err := extractIdentifierFromCypherExpression(projectionItem); err != nil { return err } else if nextExpression, err := s.treeTranslator.PopOperand(); err != nil { @@ -1504,6 +1767,15 @@ func (s *Translator) translateProjectionItem(scope *Scope, projectionItem *cyphe s.query.CurrentPart().projections.Frame = s.scope.CurrentFrame() } + if !hasAlias { + var buffer bytes.Buffer + if err := cypherFormat.NewCypherEmitter(false).WriteExpression(&buffer, projectionItem.Expression); err != nil { + return fmt.Errorf("format implicit projection name: %w", err) + } + alias = pgsql.Identifier(buffer.String()) + hasAlias = true + } + switch typedSelectItem := unwrapParenthetical(selectItem).(type) { case pgsql.Identifier: // If this is an identifier then assume the identifier as the projection alias since the translator @@ -1548,6 +1820,7 @@ func (s *Translator) translateProjectionItem(scope *Scope, projectionItem *cyphe return nil } +// prepareProjection initializes a query part's projection state and validates literal SKIP and LIMIT values. func (s *Translator) prepareProjection(projection *cypher.Projection) error { currentPart := s.query.CurrentPart() currentPart.PrepareProjections(projection.Distinct) diff --git a/cypher/models/pgsql/translate/relationship.go b/cypher/models/pgsql/translate/relationship.go index 51ff6381..13cf867b 100644 --- a/cypher/models/pgsql/translate/relationship.go +++ b/cypher/models/pgsql/translate/relationship.go @@ -8,6 +8,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql/optimize" ) +// translateRelationshipPattern validates a relationship pattern and records its binding, kinds, and range. func (s *Translator) translateRelationshipPattern(relationshipPattern *cypher.RelationshipPattern) error { var ( currentQueryPart = s.query.CurrentPart() @@ -41,6 +42,11 @@ func (s *Translator) translateRelationshipPattern(relationshipPattern *cypher.Re return fmt.Errorf("failed to translate kinds: %w", err) } else { for _, edgeBinding := range edgeBindings { + for _, step := range patternPart.TraversalSteps { + if step.Edge == edgeBinding && step.Expansion != nil { + step.Expansion.RelationshipKindIDs = append([]int16(nil), kindIDs...) + } + } if err := s.treeTranslator.AddTranslationConstraint(pgsql.NewIdentifierSet().Add(edgeBinding.Identifier), pgsql.NewBinaryExpression( pgsql.CompoundIdentifier{edgeBinding.Identifier, pgsql.ColumnKindID}, pgsql.OperatorEquals, @@ -56,6 +62,7 @@ func (s *Translator) translateRelationshipPattern(relationshipPattern *cypher.Re return nil } +// collectCreateEdgePattern records the endpoints, kind, properties, and binding needed to create one edge. func (s *Translator) collectCreateEdgePattern(relationshipPattern *cypher.RelationshipPattern, part *PatternPart, bindingResult BindingResult) error { var ( queryPart = s.query.CurrentPart() @@ -99,6 +106,7 @@ func (s *Translator) collectCreateEdgePattern(relationshipPattern *cypher.Relati return nil } +// exactRangeExpansionDecision returns the planned exact-range unrolling decision for target. func (s *Translator) exactRangeExpansionDecision(sourceTarget optimize.TraversalStepTarget, hasSourceTarget bool, relationshipPattern *cypher.RelationshipPattern) (optimize.ExactRangeExpansionDecision, bool) { if !hasSourceTarget || relationshipPattern == nil { return optimize.ExactRangeExpansionDecision{}, false @@ -112,6 +120,7 @@ func (s *Translator) exactRangeExpansionDecision(sourceTarget optimize.Traversal return decision, true } +// translateExactRangeRelationshipPatternToSteps expands a fixed-depth relationship range into synthetic single-hop traversal steps. func (s *Translator) translateExactRangeRelationshipPatternToSteps( firstEdge *BoundIdentifier, part *PatternPart, @@ -191,6 +200,7 @@ func (s *Translator) translateExactRangeRelationshipPatternToSteps( return edgeBindings, nil } +// translateRelationshipPatternToStep attaches one translated relationship pattern to the current traversal step. func (s *Translator) translateRelationshipPatternToStep(bindingResult BindingResult, part *PatternPart, relationshipPattern *cypher.RelationshipPattern) ([]*BoundIdentifier, error) { var ( expansion *Expansion diff --git a/cypher/models/pgsql/translate/renamer.go b/cypher/models/pgsql/translate/renamer.go index f516cbbc..f8e2c191 100644 --- a/cypher/models/pgsql/translate/renamer.go +++ b/cypher/models/pgsql/translate/renamer.go @@ -7,6 +7,7 @@ import ( "github.com/specterops/dawgs/cypher/models/walk" ) +// rewriteCompositeTypeFieldReference rewrites the binding portion of a composite-field reference through mappings. func rewriteCompositeTypeFieldReference(scopeIdentifier pgsql.Identifier, compositeReference pgsql.CompoundIdentifier) pgsql.RowColumnReference { return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{scopeIdentifier, compositeReference.Root()}, @@ -14,6 +15,7 @@ func rewriteCompositeTypeFieldReference(scopeIdentifier pgsql.Identifier, compos } } +// rewriteIdentifierScopeReference replaces an identifier when mappings contains a scoped rename. func rewriteIdentifierScopeReference(scope *Scope, identifier pgsql.Identifier) (pgsql.SelectItem, error) { if !pgsql.IsReservedIdentifier(identifier) { if binding, bound := scope.Lookup(identifier); bound { @@ -27,9 +29,14 @@ func rewriteIdentifierScopeReference(scope *Scope, identifier pgsql.Identifier) return identifier, nil } +// rewriteCompoundIdentifierScopeReference replaces the root binding of a compound identifier through mappings. func rewriteCompoundIdentifierScopeReference(scope *Scope, identifier pgsql.CompoundIdentifier) (pgsql.SelectItem, error) { if binding, bound := scope.Lookup(identifier[0]); bound { if binding.LastProjection != nil { + if binding.IDOnly && len(identifier) == 2 && identifier[1] == pgsql.ColumnID { + return pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier}, nil + } + return pgsql.RowColumnReference{ Identifier: pgsql.CompoundIdentifier{binding.LastProjection.Binding.Identifier, binding.Identifier}, Column: identifier[1], @@ -41,6 +48,7 @@ func rewriteCompoundIdentifierScopeReference(scope *Scope, identifier pgsql.Comp return identifier, nil } +// rewriteExpressionScopeReference rewrites identifier-bearing expression variants through mappings. func rewriteExpressionScopeReference(scope *Scope, expression pgsql.Expression) (pgsql.Expression, bool, error) { switch typedExpression := expression.(type) { case pgsql.Identifier: @@ -62,6 +70,7 @@ type FrameBindingRewriter struct { scope *Scope } +// rewriteArraySlice rewrites identifier references in an array expression and its slice bounds. func (s *FrameBindingRewriter) rewriteArraySlice(slice *pgsql.ArraySlice) error { if slice == nil { return nil @@ -92,6 +101,7 @@ func (s *FrameBindingRewriter) rewriteArraySlice(slice *pgsql.ArraySlice) error return nil } +// rewriteArrayLiteral rewrites identifier references in every array literal element. func (s *FrameBindingRewriter) rewriteArrayLiteral(literal *pgsql.ArrayLiteral) error { if literal == nil { return nil @@ -106,6 +116,7 @@ func (s *FrameBindingRewriter) rewriteArrayLiteral(literal *pgsql.ArrayLiteral) return nil } +// rewriteExpression recursively rewrites every supported identifier-bearing SQL expression. func (s *FrameBindingRewriter) rewriteExpression(expression *pgsql.Expression) error { if expression == nil || *expression == nil { return nil @@ -148,6 +159,7 @@ func (s *FrameBindingRewriter) rewriteExpression(expression *pgsql.Expression) e return nil } +// rewriteCase rewrites identifier references in a CASE operand, branches, and fallback. func (s *FrameBindingRewriter) rewriteCase(caseExpression *pgsql.Case) error { if caseExpression == nil { return nil @@ -172,6 +184,7 @@ func (s *FrameBindingRewriter) rewriteCase(caseExpression *pgsql.Case) error { return s.rewriteExpression(&caseExpression.Else) } +// enter rewrites a node's inbound references and pushes aliases that become visible to its children. func (s *FrameBindingRewriter) enter(node pgsql.SyntaxNode) error { switch typedExpression := node.(type) { case pgsql.Case: @@ -450,7 +463,10 @@ func (s *FrameBindingRewriter) enter(node pgsql.SyntaxNode) error { } case *pgsql.EdgeArrayFromPathIDs: - return s.rewriteExpression(&typedExpression.PathIDs) + if err := s.rewriteExpression(&typedExpression.PathIDs); err != nil { + return err + } + return s.rewriteExpression(&typedExpression.GraphID) case *pgsql.AliasedExpression: switch typedInnerExpression := typedExpression.Expression.(type) { @@ -656,6 +672,7 @@ func (s *FrameBindingRewriter) Enter(node pgsql.SyntaxNode) { } } +// exit removes aliases whose scope ends after the visited node. func (s *FrameBindingRewriter) exit(node pgsql.SyntaxNode) error { switch node.(type) { } diff --git a/cypher/models/pgsql/translate/semantic_drift_test.go b/cypher/models/pgsql/translate/semantic_drift_test.go index 1efd5a6b..b17a3693 100644 --- a/cypher/models/pgsql/translate/semantic_drift_test.go +++ b/cypher/models/pgsql/translate/semantic_drift_test.go @@ -43,6 +43,7 @@ func TestTranslatorRejectsUnsupportedPropertyLookupSourcesDirectly(t *testing.T) require.Contains(t, err.Error(), "unsupported property lookup prop on expression type int8[]") } +// TestTranslatorRejectsEmptyPropertyLookupKeys verifies that invalid empty keys cannot reach SQL translation. func TestTranslatorRejectsEmptyPropertyLookupKeys(t *testing.T) { kindMapper := pgutil.NewInMemoryKindMapper() diff --git a/cypher/models/pgsql/translate/shortest_workspace_test.go b/cypher/models/pgsql/translate/shortest_workspace_test.go new file mode 100644 index 00000000..9d2bbdd0 --- /dev/null +++ b/cypher/models/pgsql/translate/shortest_workspace_test.go @@ -0,0 +1,22 @@ +package translate + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestShortestPathWorkspaceFragmentUsesDedicatedTablesAndConstraints verifies isolation and key constraints for each workspace relation. +func TestShortestPathWorkspaceFragmentUsesDedicatedTablesAndConstraints(t *testing.T) { + fragment := "insert into next_front select * from forward_front " + + "where not exists (select 1 from forward_visited) " + + "on conflict on constraint forward_visited_pkey do nothing" + + rewritten := shortestPathWorkspaceFragment(fragment) + require.Equal(t, + "insert into pg_temp.bsp_next_front select * from pg_temp.bsp_forward_front "+ + "where not exists (select 1 from pg_temp.bsp_forward_visited) "+ + "on conflict on constraint bsp_forward_visited_pkey do nothing", + rewritten, + ) +} diff --git a/cypher/models/pgsql/translate/tracking.go b/cypher/models/pgsql/translate/tracking.go index 8707cd06..d9a97191 100644 --- a/cypher/models/pgsql/translate/tracking.go +++ b/cypher/models/pgsql/translate/tracking.go @@ -2,6 +2,7 @@ package translate import ( "fmt" + "sort" "strconv" "github.com/specterops/dawgs/cypher/models" @@ -109,13 +110,30 @@ func (s *Frame) Reveal(identifier pgsql.Identifier) { // all visible projections. This is required when disambiguating references that otherwise belong to // a frame. type Scope struct { + // nextFrameID is the sequence value assigned to the next scope frame. nextFrameID int - stack []*Frame - generator IdentifierGenerator - aliases map[pgsql.Identifier]pgsql.Identifier + // graphID identifies the graph whose concrete partitions translation targets. + graphID int32 + // stack contains active scope frames from outermost to innermost. + stack []*Frame + // generator allocates collision-free PostgreSQL identifiers by data type. + generator IdentifierGenerator + // aliases maps Cypher-visible symbols to their canonical translated identifiers. + aliases map[pgsql.Identifier]pgsql.Identifier + // definitions maps canonical translated identifiers to their binding metadata. definitions map[pgsql.Identifier]*BoundIdentifier } +// SetGraphID sets the graph used for graph-scoped table references created in this scope. +func (s *Scope) SetGraphID(graphID int32) { + s.graphID = graphID +} + +// GraphID returns the graph used for graph-scoped table references in this scope. +func (s *Scope) GraphID() int32 { + return s.graphID +} + func NewScope() *Scope { return &Scope{ nextFrameID: 0, @@ -378,23 +396,33 @@ func (s *Scope) Define(identifier pgsql.Identifier, dataType pgsql.DataType) *Bo // will eagerly bind anonymous identifiers for traversal steps and rebind existing identifiers and their // aliases to prevent naming collisions. type BoundIdentifier struct { - Identifier pgsql.Identifier - Alias models.Optional[pgsql.Identifier] - Parameter *pgsql.Parameter + // Identifier is the canonical PostgreSQL name allocated for the binding. + Identifier pgsql.Identifier + // Alias is the optional source-visible name projected for the binding. + Alias models.Optional[pgsql.Identifier] + // Parameter is the translated SQL parameter represented by this binding, when applicable. + Parameter *pgsql.Parameter + // LastProjection is the most recent frame that materialized the binding. LastProjection *Frame - Dependencies []*BoundIdentifier - DataType pgsql.DataType - + // Dependencies are the bindings required to reconstruct this value. + Dependencies []*BoundIdentifier + // DataType is the PostgreSQL representation carried by the binding. + DataType pgsql.DataType + // IDOnly reports that the binding is represented by a scalar entity ID instead of a composite. + IDOnly bool // PathDirectionReversed marks a path composite binding whose dependency order was produced // from an optimizer-reversed pattern. Path materialization reverses the assembled node and // edge references so the path renders in its original left-to-right logical order. PathDirectionReversed bool + // DistanceOnly reports that the binding carries only shortest-path distance state. + DistanceOnly bool } func (s *BoundIdentifier) MaterializedBy(frame *Frame) { s.LastProjection = frame } +// Copy returns an independent binding whose dependency slice can be modified without affecting the source. func (s *BoundIdentifier) Copy() *BoundIdentifier { dependenciesCopy := make([]*BoundIdentifier, len(s.Dependencies)) copy(dependenciesCopy, s.Dependencies) @@ -406,8 +434,35 @@ func (s *BoundIdentifier) Copy() *BoundIdentifier { LastProjection: s.LastProjection, Dependencies: dependenciesCopy, DataType: s.DataType, + IDOnly: s.IDOnly, PathDirectionReversed: s.PathDirectionReversed, + DistanceOnly: s.DistanceOnly, + } +} + +// Symbol returns the first deterministic symbol that aliases binding. +func (s *Scope) Symbol(binding *BoundIdentifier) (pgsql.Identifier, bool) { + if symbols := s.Symbols(binding); len(symbols) > 0 { + return symbols[0], true + } + + return "", false +} + +// Symbols returns every symbol that aliases binding in lexical order. +func (s *Scope) Symbols(binding *BoundIdentifier) []pgsql.Identifier { + if binding == nil { + return nil + } + + var symbols []pgsql.Identifier + for symbol, identifier := range s.aliases { + if identifier == binding.Identifier { + symbols = append(symbols, symbol) + } } + sort.Slice(symbols, func(left, right int) bool { return symbols[left] < symbols[right] }) + return symbols } func (s *BoundIdentifier) Dematerialize() { diff --git a/cypher/models/pgsql/translate/translator.go b/cypher/models/pgsql/translate/translator.go index e7f354f2..81974982 100644 --- a/cypher/models/pgsql/translate/translator.go +++ b/cypher/models/pgsql/translate/translator.go @@ -12,9 +12,7 @@ import ( "github.com/specterops/dawgs/graph" ) -// DefaultGraphID is the graph_id used by callers that do not have a specific -// graph target available (tests, tooling, and visualization passes that only -// exercise translation output). +// DefaultGraphID selects graph zero for tests and tooling that do not target a concrete graph. const DefaultGraphID int32 = 0 // OptimizerMode controls whether PostgreSQL-specific rewrites and lowering @@ -53,41 +51,84 @@ func (s Options) normalized() (Options, error) { return s, nil } +// Translator walks an optimized Cypher AST and constructs the corresponding PostgreSQL AST. type Translator struct { + // Visitor supplies traversal control and error propagation for the Cypher walk. walk.Visitor[cypher.SyntaxNode] - ctx context.Context - kindMapper *contextAwareKindMapper - graphID int32 - parameters map[string]any - translation Result + // ctx carries cancellation and deadlines through translation. + ctx context.Context + // kindMapper resolves graph kind names within the translation context. + kindMapper *contextAwareKindMapper + // graphID identifies the concrete graph partitions targeted by generated SQL. + graphID int32 + // parameters is an isolated copy of the caller's Cypher parameter values. + parameters map[string]any + // translation accumulates the statement, generated parameters, and diagnostics. + translation Result + // parameterSources maps generated parameter identifiers back to caller symbols. parameterSources map[string]string - treeTranslator *ExpressionTreeTranslator - query *Query - scope *Scope - unwindTargets map[*cypher.Variable]struct{} - + // treeTranslator lowers the current Cypher expression tree into PostgreSQL expressions. + treeTranslator *ExpressionTreeTranslator + // query holds the PostgreSQL query model under construction. + query *Query + // scope tracks translated bindings and their materialization frames. + scope *Scope + // unwindTargets contains UNWIND variables awaiting source translation. + unwindTargets map[*cypher.Variable]struct{} + + // collectIDMembershipAliases identifies collect projections eligible to carry scalar entity IDs. collectIDMembershipAliases map[pgsql.Identifier]struct{} - collectIDProjectionDepth int - - appliedLoweringCounts map[string]int - patternTargets map[*cypher.PatternPart]optimize.PatternTarget - patternPredicateTargets map[*cypher.PatternPredicate]optimize.PatternTarget - projectionPruningDecisions map[optimize.TraversalStepTarget]optimize.ProjectionPruningDecision - latePathDecisions map[optimize.TraversalStepTarget][]optimize.LatePathMaterializationDecision - suffixPushdownDecisions map[optimize.TraversalStepTarget][]optimize.ExpansionSuffixPushdownDecision - predicatePlacementDecisions map[optimize.TraversalStepTarget][]optimize.PredicatePlacementDecision - expandIntoDecisions map[optimize.TraversalStepTarget]optimize.ExpandIntoDecision - traversalDirectionDecisions map[optimize.TraversalStepTarget]optimize.TraversalDirectionDecision - shortestPathStrategyDecisions map[optimize.TraversalStepTarget]optimize.ShortestPathStrategyDecision - shortestPathFilterDecisions map[optimize.TraversalStepTarget][]optimize.ShortestPathFilterDecision - limitPushdownDecisions map[optimize.TraversalStepTarget][]optimize.LimitPushdownDecision - patternPredicateDecisions map[optimize.TraversalStepTarget]optimize.PatternPredicatePlacementDecision - exactRangeExpansionDecisions map[optimize.TraversalStepTarget]optimize.ExactRangeExpansionDecision + // collectIDProjectionDepth tracks nesting within an ID-only collect projection. + collectIDProjectionDepth int + + // appliedLoweringCounts counts emitted applications of each planned lowering. + appliedLoweringCounts map[string]int + // appliedShortestPathExecutors retains the applied shortest path executors while Translator is assembled or evaluated. + appliedShortestPathExecutors map[optimize.TraversalStepTarget]optimize.ShortestPathExecutor + // appliedExpansionSearchStrategies retains the applied expansion search strategies while Translator is assembled or evaluated. + appliedExpansionSearchStrategies map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategy + // emittedExpansionSearchPolicies records runtime selection policies emitted for optimized expansions. + emittedExpansionSearchPolicies map[optimize.TraversalStepTarget]optimize.ExpansionSearchPolicy + // patternTargets maps source pattern parts to their stable optimizer coordinates. + patternTargets map[*cypher.PatternPart]optimize.PatternTarget + // patternPredicateTargets maps source pattern predicates to their stable optimizer coordinates. + patternPredicateTargets map[*cypher.PatternPredicate]optimize.PatternTarget + // projectionPruningDecisions indexes planned projection omissions by traversal target. + projectionPruningDecisions map[optimize.TraversalStepTarget]optimize.ProjectionPruningDecision + // latePathDecisions indexes deferred path-materialization decisions by traversal target. + latePathDecisions map[optimize.TraversalStepTarget][]optimize.LatePathMaterializationDecision + // suffixPushdownDecisions indexes fixed-suffix pushdown decisions by traversal target. + suffixPushdownDecisions map[optimize.TraversalStepTarget][]optimize.ExpansionSuffixPushdownDecision + // predicatePlacementDecisions indexes predicate attachment decisions by traversal target. + predicatePlacementDecisions map[optimize.TraversalStepTarget][]optimize.PredicatePlacementDecision + // expandIntoDecisions indexes bound-endpoint expansion choices by traversal target. + expandIntoDecisions map[optimize.TraversalStepTarget]optimize.ExpandIntoDecision + // traversalDirectionDecisions indexes physical traversal direction choices by traversal target. + traversalDirectionDecisions map[optimize.TraversalStepTarget]optimize.TraversalDirectionDecision + // shortestPathStrategyDecisions indexes directional shortest-path search choices by traversal target. + shortestPathStrategyDecisions map[optimize.TraversalStepTarget]optimize.ShortestPathStrategyDecision + // shortestPathFilterDecisions indexes shortest-path filter decisions by traversal target. + shortestPathFilterDecisions map[optimize.TraversalStepTarget][]optimize.ShortestPathFilterDecision + // shortestPathExecutorDecisions indexes planned shortest-path executor choices by traversal target. + shortestPathExecutorDecisions map[optimize.TraversalStepTarget]optimize.ShortestPathExecutorDecision + // expansionSearchStrategyDecisions indexes planned variable-expansion strategies by traversal target. + expansionSearchStrategyDecisions map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision + // limitPushdownDecisions indexes planned traversal limits by source target. + limitPushdownDecisions map[optimize.TraversalStepTarget][]optimize.LimitPushdownDecision + // patternPredicateDecisions indexes planned existence lowering by traversal target. + patternPredicateDecisions map[optimize.TraversalStepTarget]optimize.PatternPredicatePlacementDecision + // exactRangeExpansionDecisions indexes fixed-depth unrolling choices by source target. + exactRangeExpansionDecisions map[optimize.TraversalStepTarget]optimize.ExactRangeExpansionDecision + // pathRelationshipPredicateDecisions indexes path quantifier lowering by stable quantifier target. pathRelationshipPredicateDecisions map[optimize.QuantifierTarget]optimize.PathRelationshipPredicateDecision - quantifierTargets []optimize.QuantifierTarget + // fieldRequirementDecisions indexes binding representation requirements by query part and symbol. + fieldRequirementDecisions map[int]map[string]optimize.FieldRequirementDecision + // quantifierTargets records stable coordinates for visited quantified traversals. + quantifierTargets []optimize.QuantifierTarget } +// NewTranslator initializes translation state for the supplied graph and copies the caller's parameter map. func NewTranslator(ctx context.Context, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32) *Translator { if parameters == nil { parameters = map[string]any{} @@ -104,10 +145,12 @@ func NewTranslator(ctx context.Context, kindMapper pgsql.KindMapper, parameters ctxAwareKindMapper = newContextAwareKindMapper(ctx, kindMapper, translatedParameters) ) - return &Translator{ + translator := &Translator{ Visitor: walk.NewVisitor[cypher.SyntaxNode](), translation: Result{ - Parameters: translatedParameters, + Parameters: translatedParameters, + ParameterSources: parameterSources, + GraphID: graphID, }, ctx: ctx, kindMapper: ctxAwareKindMapper, @@ -119,8 +162,12 @@ func NewTranslator(ctx context.Context, kindMapper pgsql.KindMapper, parameters scope: NewScope(), unwindTargets: map[*cypher.Variable]struct{}{}, } + + translator.scope.SetGraphID(graphID) + return translator } +// SetOptimizationPlan indexes lowering decisions by their stable targets for use during AST traversal. func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { s.patternTargets = optimize.IndexPatternTargets(plan.Query) s.patternPredicateTargets = optimize.IndexPatternPredicateTargets(plan.Query) @@ -132,10 +179,13 @@ func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { s.traversalDirectionDecisions = map[optimize.TraversalStepTarget]optimize.TraversalDirectionDecision{} s.shortestPathStrategyDecisions = map[optimize.TraversalStepTarget]optimize.ShortestPathStrategyDecision{} s.shortestPathFilterDecisions = map[optimize.TraversalStepTarget][]optimize.ShortestPathFilterDecision{} + s.shortestPathExecutorDecisions = map[optimize.TraversalStepTarget]optimize.ShortestPathExecutorDecision{} + s.expansionSearchStrategyDecisions = map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategyDecision{} s.limitPushdownDecisions = map[optimize.TraversalStepTarget][]optimize.LimitPushdownDecision{} s.patternPredicateDecisions = map[optimize.TraversalStepTarget]optimize.PatternPredicatePlacementDecision{} s.exactRangeExpansionDecisions = map[optimize.TraversalStepTarget]optimize.ExactRangeExpansionDecision{} s.pathRelationshipPredicateDecisions = map[optimize.QuantifierTarget]optimize.PathRelationshipPredicateDecision{} + s.fieldRequirementDecisions = map[int]map[string]optimize.FieldRequirementDecision{} for _, decision := range plan.LoweringPlan.ProjectionPruning { s.projectionPruningDecisions[decision.Target] = decision @@ -169,6 +219,14 @@ func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { s.shortestPathFilterDecisions[decision.Target] = append(s.shortestPathFilterDecisions[decision.Target], decision) } + for _, decision := range plan.LoweringPlan.ShortestPathExecutor { + s.shortestPathExecutorDecisions[decision.Target] = decision + } + + for _, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + s.expansionSearchStrategyDecisions[decision.Target] = decision + } + for _, decision := range plan.LoweringPlan.LimitPushdown { s.limitPushdownDecisions[decision.Target] = append(s.limitPushdownDecisions[decision.Target], decision) } @@ -184,8 +242,18 @@ func (s *Translator) SetOptimizationPlan(plan optimize.Plan) { for _, decision := range plan.LoweringPlan.PathRelationshipPredicate { s.pathRelationshipPredicateDecisions[decision.Target] = decision } + + for _, decision := range plan.LoweringPlan.FieldRequirements { + bySymbol := s.fieldRequirementDecisions[decision.QueryPartIndex] + if bySymbol == nil { + bySymbol = map[string]optimize.FieldRequirementDecision{} + s.fieldRequirementDecisions[decision.QueryPartIndex] = bySymbol + } + bySymbol[decision.Symbol] = decision + } } +// Enter translates a Cypher syntax node when the walker reaches it. func (s *Translator) Enter(expression cypher.SyntaxNode) { switch typedExpression := expression.(type) { case *cypher.RegularQuery, *cypher.SingleQuery, *cypher.PatternElement, @@ -266,7 +334,9 @@ func (s *Translator) Enter(expression cypher.SyntaxNode) { } else { // Lift the parameter value into the parameters map s.translation.Parameters[parameterBinding.Identifier.String()] = negotiatedValue - s.parameterSources[parameterBinding.Identifier.String()] = typedExpression.Symbol + if typedExpression.Symbol != "" { + s.parameterSources[parameterBinding.Identifier.String()] = typedExpression.Symbol + } parameterBinding.Parameter = newParameter } @@ -278,7 +348,11 @@ func (s *Translator) Enter(expression cypher.SyntaxNode) { s.treeTranslator.PushOperand(binding.Parameter) case *cypher.Variable: - if binding, isUnwindTarget, err := s.prepareUnwindTarget(typedExpression); err != nil { + if typedExpression.Symbol == cypher.TokenLiteralAsterisk { + // Greedy projections are expanded to their named scope bindings when + // the enclosing projection item is completed. + s.treeTranslator.PushOperand(pgsql.Identifier(cypher.TokenLiteralAsterisk)) + } else if binding, isUnwindTarget, err := s.prepareUnwindTarget(typedExpression); err != nil { s.SetError(err) } else if isUnwindTarget { s.treeTranslator.PushOperand(binding.Identifier) @@ -374,6 +448,7 @@ func (s *Translator) Enter(expression cypher.SyntaxNode) { } } +// resolveParameterValue returns the caller-supplied value for a Cypher parameter or reports an unknown parameter. func (s *Translator) resolveParameterValue(parameter *cypher.Parameter) any { if value, hasValue := s.parameters[parameter.Symbol]; hasValue { return value @@ -382,6 +457,7 @@ func (s *Translator) resolveParameterValue(parameter *cypher.Parameter) any { return parameter.Value } +// coalescePropertyLookupExpression builds a coalesce call from a property lookup and translated fallback operands. func coalescePropertyLookupExpression(expression pgsql.Expression) pgsql.Expression { if propertyLookup, isPropertyLookup := expressionToPropertyLookupBinaryExpression(expression); isPropertyLookup { return pgsql.FunctionCall{ @@ -397,6 +473,7 @@ func coalescePropertyLookupExpression(expression pgsql.Expression) pgsql.Express return expression } +// rewriteNegatedStringPredicateExpression preserves Cypher null behavior when negating a string predicate. func rewriteNegatedStringPredicateExpression(expression pgsql.Expression) pgsql.Expression { switch typedExpression := expression.(type) { case *pgsql.Parenthetical: @@ -427,6 +504,7 @@ func rewriteNegatedStringPredicateExpression(expression pgsql.Expression) pgsql. return expression } +// Exit builds the SQL model fragment responsible for exit. func (s *Translator) Exit(expression cypher.SyntaxNode) { switch typedExpression := expression.(type) { @@ -679,27 +757,170 @@ func (s *Translator) Exit(expression cypher.SyntaxNode) { } } +// Result contains the translated PostgreSQL statement, parameters, graph target, and optimization diagnostics. type Result struct { - Statement pgsql.Statement - Parameters map[string]any + // Statement is the translated PostgreSQL AST. + Statement pgsql.Statement + // Parameters contains SQL parameters generated during translation. + Parameters map[string]any + // ParameterSources maps generated SQL parameter names back to Cypher parameter names. + ParameterSources map[string]string + // Optimization summarizes planned, applied, and skipped lowering decisions. Optimization OptimizationSummary + // GraphID identifies the graph partitions targeted by the statement. + GraphID int32 } +// OptimizationSummary records which optimizer decisions were planned, applied, or skipped during translation. type OptimizationSummary struct { - Rules []optimize.RuleResult `json:"rules,omitempty"` + // Rules contains the semantic optimizer rule results in execution order. + Rules []optimize.RuleResult `json:"rules,omitempty"` + // PredicateAttachments records optimizer-selected predicate scopes. PredicateAttachments []optimize.PredicateAttachment `json:"predicate_attachments,omitempty"` - PlannedLowerings []optimize.LoweringDecision `json:"planned_lowerings,omitempty"` - Lowerings []optimize.LoweringDecision `json:"lowerings,omitempty"` - SkippedLowerings []SkippedLowering `json:"skipped_lowerings,omitempty"` - LoweringPlan *optimize.LoweringPlan `json:"lowering_plan,omitempty"` + // PlannedLowerings summarizes lowering categories selected by the optimizer. + PlannedLowerings []optimize.LoweringDecision `json:"planned_lowerings,omitempty"` + // Lowerings summarizes lowering categories actually emitted by translation. + Lowerings []optimize.LoweringDecision `json:"lowerings,omitempty"` + // SkippedLowerings explains planned lowering applications that translation did not emit. + SkippedLowerings []SkippedLowering `json:"skipped_lowerings,omitempty"` + // TargetOutcomes reports selection and application results for each lowering target. + TargetOutcomes []TargetLoweringOutcome `json:"target_outcomes,omitempty"` + // LoweringPlan exposes the optimizer decisions used to translate the statement. + LoweringPlan *optimize.LoweringPlan `json:"lowering_plan,omitempty"` } +// TargetLoweringOutcome reports how one planned lowering target was qualified, selected, and applied. +type TargetLoweringOutcome struct { + // Lowering names the lowering pass that produced this outcome. + Lowering string `json:"lowering"` + // TargetKind identifies the kind of syntax or binding targeted by the lowering. + TargetKind string `json:"target_kind"` + // TraversalTarget locates a traversal-step target when the lowering applies to one. + TraversalTarget *optimize.TraversalStepTarget `json:"traversal_target,omitempty"` + // QueryPartIndex locates a query-part target when the lowering applies to one. + QueryPartIndex *int `json:"query_part_index,omitempty"` + // Symbol identifies a binding target when the lowering applies to one. + Symbol string `json:"symbol,omitempty"` + // Family names the candidate-selection family that produced this outcome. + Family string `json:"family,omitempty"` + // TraversalFamily preserves the SP/ASP family for analysis-only decisions + // whose outcome family must remain distinct from an executable traversal. + TraversalFamily string `json:"traversal_family,omitempty"` + // PlannedPolicy identifies the runtime policy intended for this candidate + // family, whether or not it was emitted. + PlannedPolicy string `json:"planned_policy,omitempty"` + // EmittedPolicy identifies a runtime policy present in translated SQL. A + // single incumbent or tool-forced arm has no emitted policy identity. + EmittedPolicy string `json:"emitted_policy,omitempty"` + // PlannedCandidates lists the candidates considered in preference order. + PlannedCandidates []string `json:"planned_candidates,omitempty"` + // EmittedCandidates lists the arms present in translated SQL. Runtime + // telemetry separately records which arm executed. + EmittedCandidates []string `json:"emitted_candidates,omitempty"` + // ProbeCaps records bounded evidence inputs for an expansion policy. + ProbeCaps *optimize.ExpansionSearchProbeCaps `json:"probe_caps,omitempty"` + // Admission supplies the admission input to the TargetLoweringOutcome contract. + Admission *optimize.ExpansionSearchAdmission `json:"admission,omitempty"` + // EndpointRoot and EndpointTerminal describe the bounded endpoint inputs + // considered by analysis without implying that translation emitted them. + EndpointRoot *optimize.EndpointResolutionInput `json:"endpoint_root,omitempty"` + // EndpointTerminal supplies the endpoint terminal input to the TargetLoweringOutcome contract. + EndpointTerminal *optimize.EndpointResolutionInput `json:"endpoint_terminal,omitempty"` + // EndpointPairClass records a correlation class when endpoint resolution + // must preserve a paired input rather than independent endpoint sets. + EndpointPairClass optimize.EndpointResolutionClass `json:"endpoint_pair_class,omitempty"` + // EndpointResolutionCaps records immutable 1/2/32/33 admission sentinels. + EndpointResolutionCaps *optimize.EndpointResolutionCaps `json:"endpoint_resolution_caps,omitempty"` + // PredicateClass and its source/index expose conservative traversal + // predicate placement analysis as a first-class target outcome. + PredicateClass optimize.TraversalPredicateClass `json:"predicate_class,omitempty"` + // PredicateSource supplies the predicate source input to the TargetLoweringOutcome contract. + PredicateSource string `json:"predicate_source,omitempty"` + // PredicateIndex supplies the predicate index input to the TargetLoweringOutcome contract. + PredicateIndex *int `json:"predicate_index,omitempty"` + // Scheduler identifies the selected shortest-path frontier scheduling policy. + Scheduler string `json:"scheduler,omitempty"` + // ExecutionBoundary identifies whether the selected executor is inline SQL, + // a stored helper, or a guarded multi-arm statement. + ExecutionBoundary string `json:"execution_boundary,omitempty"` + // Candidate is the specialized candidate proposed by analysis. + Candidate string `json:"candidate,omitempty"` + // EligibilityFacts records named qualification checks for the candidate. + EligibilityFacts []TargetEligibilityFact `json:"eligibility_facts,omitempty"` + // ObservationMode describes how downstream clauses consume the target. + ObservationMode string `json:"observation_mode,omitempty"` + // Direction selects the traversal orientation covered by the contract. + Direction string `json:"direction,omitempty"` + // PhysicalExpansion supplies the physical expansion input to the TargetLoweringOutcome contract. + PhysicalExpansion string `json:"physical_expansion,omitempty"` + // RelationshipKindCount is the number of statically resolved relationship kinds. + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + // UntypedRelationship reports whether the pattern omitted relationship kinds. + UntypedRelationship bool `json:"untyped_relationship,omitempty"` + // TopologyClassification summarizes logical direction, physical direction, and depth. + TopologyClassification string `json:"topology_classification,omitempty"` + // Eligible reports the structural qualification result when one is available. + Eligible *bool `json:"eligible,omitempty"` + // StaticallyEligible reports the literal- and kind-based qualification result when available. + StaticallyEligible *bool `json:"statically_eligible,omitempty"` + // SelectionMode records whether selection was automatic or forced by tooling. + SelectionMode string `json:"selection_mode,omitempty"` + // SelectorVersion identifies the policy version that ranked candidates. + SelectorVersion string `json:"selector_version,omitempty"` + // Fallback names the candidate used if the preferred lowering was not applied. + Fallback string `json:"fallback,omitempty"` + // MinimumDepth is the target's inclusive lower traversal-depth bound. + MinimumDepth *int64 `json:"minimum_depth,omitempty"` + // MaximumDepth is the target's inclusive upper traversal-depth bound when finite. + MaximumDepth *int64 `json:"maximum_depth,omitempty"` + // MaximumDepthSource distinguishes an explicit upper bound from the + // repository's existing effective cap for syntax-open shortest paths. + MaximumDepthSource string `json:"maximum_depth_source,omitempty"` + // StateLimit is the maximum intermediate-state count admitted by the candidate. + StateLimit int64 `json:"state_limit,omitempty"` + // FrontierLimit is the maximum current or queued frontier size admitted by a shortest-path candidate. + FrontierLimit int64 `json:"frontier_limit,omitempty"` + // PredecessorLimit is the maximum retained witness predecessor state admitted by a shortest-path candidate. + PredecessorLimit int64 `json:"predecessor_limit,omitempty"` + // EnumerationLimit is the maximum distinct ordered path count staged by an all-shortest-path candidate. + EnumerationLimit int64 `json:"enumeration_limit,omitempty"` + // OutputBytesLimit is the maximum staged ordered edge-array bytes admitted by an all-shortest-path candidate. + OutputBytesLimit int64 `json:"output_bytes_limit,omitempty"` + // EndpointLimit is the maximum endpoint-seed count admitted by the candidate. + EndpointLimit int64 `json:"endpoint_limit,omitempty"` + // SeedPredicateClass describes the predicate used to bound search seeds. + SeedPredicateClass string `json:"seed_predicate_class,omitempty"` + // PrefixLength is the number of fixed steps before the variable expansion. + PrefixLength int `json:"prefix_length,omitempty"` + // HasFinalLimit reports whether a final row limit influenced candidate selection. + HasFinalLimit bool `json:"has_final_limit,omitempty"` + // Selected names the candidate selected by the optimizer. + Selected string `json:"selected,omitempty"` + // Applied names the candidate actually emitted by translation. + Applied string `json:"applied,omitempty"` + // SkipReason explains why a planned candidate was not emitted. + SkipReason string `json:"skip_reason,omitempty"` +} + +// TargetEligibilityFact reports one named qualification result in a translated target outcome. +type TargetEligibilityFact struct { + // Name identifies the qualification check. + Name string `json:"name"` + // Eligible reports whether the target passed the named check. + Eligible bool `json:"eligible"` +} + +// SkippedLowering groups SQL model state that must remain consistent while translating skipped lowering. type SkippedLowering struct { - Name string `json:"name"` + // Name identifies the name. + Name string `json:"name"` + // Reason supplies the reason input to the SkippedLowering contract. Reason string `json:"reason"` - Count int `json:"count,omitempty"` + // Count records the number of count. + Count int `json:"count,omitempty"` } +// recordLowering increments the applied count for one lowering name. func (s *Translator) recordLowering(name string) { if s.appliedLoweringCounts == nil { s.appliedLoweringCounts = map[string]int{} @@ -717,6 +938,34 @@ func (s *Translator) recordLowering(name string) { }) } +// recordShortestPathExecutor builds the SQL model fragment responsible for record shortest path executor. +func (s *Translator) recordShortestPathExecutor(target optimize.TraversalStepTarget, executor optimize.ShortestPathExecutor) { + if s.appliedShortestPathExecutors == nil { + s.appliedShortestPathExecutors = map[optimize.TraversalStepTarget]optimize.ShortestPathExecutor{} + } + s.appliedShortestPathExecutors[target] = executor + s.recordLowering(optimize.LoweringShortestPathExecutor) +} + +// recordExpansionSearchStrategy builds the SQL model fragment responsible for record expansion search strategy. +func (s *Translator) recordExpansionSearchStrategy(target optimize.TraversalStepTarget, strategy optimize.ExpansionSearchStrategy) { + if s.appliedExpansionSearchStrategies == nil { + s.appliedExpansionSearchStrategies = map[optimize.TraversalStepTarget]optimize.ExpansionSearchStrategy{} + } + s.appliedExpansionSearchStrategies[target] = strategy + s.recordLowering(optimize.LoweringExpansionSearchStrategy) +} + +// recordExpansionSearchPolicy records a runtime expansion policy actually emitted for a traversal target. +func (s *Translator) recordExpansionSearchPolicy(target optimize.TraversalStepTarget, policy optimize.ExpansionSearchPolicy) { + if s.emittedExpansionSearchPolicies == nil { + s.emittedExpansionSearchPolicies = map[optimize.TraversalStepTarget]optimize.ExpansionSearchPolicy{} + } + s.emittedExpansionSearchPolicies[target] = policy + s.recordLowering(optimize.LoweringExpansionSearchStrategy) +} + +// appliedLoweringCountSnapshot merges optimizer-declared and translator-observed lowering counts into the snapshot used to diagnose unapplied plans. func (s *Translator) appliedLoweringCountSnapshot() map[string]int { applied := map[string]int{} @@ -731,12 +980,14 @@ func (s *Translator) appliedLoweringCountSnapshot() map[string]int { return applied } +// recordSkippedLowerings compares the plan with applied counts and emits aggregated skip diagnostics. func (s *Translator) recordSkippedLowerings() { if s.translation.Optimization.LoweringPlan == nil { return } applied := s.appliedLoweringCountSnapshot() + s.recordTargetOutcomes(*s.translation.Optimization.LoweringPlan) for _, planned := range plannedLoweringCounts(*s.translation.Optimization.LoweringPlan) { if planned.Count == 0 { @@ -756,6 +1007,271 @@ func (s *Translator) recordSkippedLowerings() { } } +// recordTargetOutcomes converts per-target plan decisions and applied choices into diagnostic outcomes. +func (s *Translator) recordTargetOutcomes(plan optimize.LoweringPlan) { + if len(s.translation.Optimization.TargetOutcomes) != 0 { + return + } + for _, decision := range plan.ShortestPathExecutor { + target := decision.Target + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + minimumDepth, maximumDepth := decision.MinimumDepth, decision.MaximumDepth + applied := string(s.appliedShortestPathExecutors[target]) + outcome := TargetLoweringOutcome{ + Lowering: optimize.LoweringShortestPathExecutor, + TargetKind: "traversal", + TraversalTarget: &target, + Family: decision.Family, + PlannedCandidates: shortestPathCandidateNames(decision.PlannedCandidates), + Scheduler: string(decision.Scheduler), + ExecutionBoundary: decision.ExecutionBoundary, + EligibilityFacts: shortestPathEligibilityFacts(decision.Eligibility), + ObservationMode: string(decision.ObservationMode), + Direction: decision.Direction.String(), + PhysicalExpansion: string(decision.PhysicalExpansion), + RelationshipKindCount: decision.RelationshipKindCount, + UntypedRelationship: decision.UntypedRelationship, + TopologyClassification: string(decision.TopologyClassification), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.SelectorVersion, + Selected: string(decision.SelectedExecutor), + Applied: applied, + Fallback: string(decision.FallbackExecutor), + SkipReason: decision.FallbackReason, + MinimumDepth: &minimumDepth, + MaximumDepth: &maximumDepth, + MaximumDepthSource: string(decision.MaximumDepthSource), + StateLimit: decision.StateLimit, + FrontierLimit: decision.FrontierLimit, + PredecessorLimit: decision.PredecessorLimit, + EnumerationLimit: decision.EnumerationLimit, + OutputBytesLimit: decision.OutputBytesLimit, + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorASPI1DAG && applied == string(optimize.ShortestPathExecutorASPI1DAG) { + outcome.Candidate = string(optimize.ShortestPathExecutorASPI1DAG) + outcome.EmittedPolicy = optimize.ShortestPathPolicyASPI1GuardedV1 + outcome.EmittedCandidates = []string{ + string(optimize.ShortestPathExecutorASPI1DAG), + string(optimize.ShortestPathExecutorASPA1DAG), + } + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness && applied == string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) { + outcome.Candidate = string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + outcome.EmittedPolicy = optimize.ShortestPathPolicyI1CanonicalGuardedV1 + outcome.EmittedCandidates = []string{ + string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), + string(optimize.ShortestPathExecutorS4CanonicalWitness), + } + } + if decision.SelectedExecutor == optimize.ShortestPathExecutorI2GuardedDistance && applied == string(optimize.ShortestPathExecutorI2GuardedDistance) { + outcome.Candidate = string(optimize.ShortestPathExecutorI2GuardedDistance) + outcome.EmittedPolicy = optimize.ShortestPathPolicyI2DistanceGuardedV1 + outcome.EmittedCandidates = []string{ + string(optimize.ShortestPathExecutorI2GuardedDistance), + string(optimize.ShortestPathExecutorS4CanonicalDistance), + } + } + if isV2GuardedDistanceExecutor(decision.SelectedExecutor) && applied == string(decision.SelectedExecutor) { + outcome.Candidate = string(decision.SelectedExecutor) + outcome.EmittedPolicy = optimize.ShortestPathPolicyI2DistanceGuardedV2 + outcome.EmittedCandidates = []string{ + string(decision.SelectedExecutor), + string(optimize.ShortestPathExecutorS4CanonicalDistance), + } + } + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, outcome) + } + for _, decision := range plan.ExpansionSearchStrategy { + target := decision.Target + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + minimumDepth, maximumDepth := decision.MinimumDepth, decision.MaximumDepth + applied := string(s.appliedExpansionSearchStrategies[target]) + probeCaps, admission := decision.ProbeCaps, decision.Admission + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringExpansionSearchStrategy, + TargetKind: "traversal", + TraversalTarget: &target, + Family: decision.Family, + PlannedPolicy: string(decision.PlannedPolicy), + EmittedPolicy: string(decision.EmittedPolicy), + PlannedCandidates: expansionSearchCandidateNames(decision.PlannedCandidates), + EmittedCandidates: expansionSearchCandidateNames(decision.EmittedCandidates), + ExecutionBoundary: decision.ExecutionBoundary, + ProbeCaps: &probeCaps, + Admission: &admission, + Candidate: string(decision.CandidateStrategy), + EligibilityFacts: expansionSearchEligibilityFacts(decision.EligibilityFacts), + ObservationMode: string(decision.ObservationMode), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.SelectorVersion, + Selected: string(decision.SelectedStrategy), + Applied: applied, + Fallback: string(decision.FallbackStrategy), + SkipReason: decision.FallbackReason, + MinimumDepth: &minimumDepth, + MaximumDepth: &maximumDepth, + StateLimit: decision.StateLimit, + EndpointLimit: decision.EndpointLimit, + SeedPredicateClass: decision.SeedPredicateClass, + PrefixLength: decision.PrefixLength, + HasFinalLimit: decision.HasFinalLimit, + }) + } + for _, decision := range plan.EndpointResolution { + target := decision.Target + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + root, terminal, caps := decision.Root, decision.Terminal, decision.Caps + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringEndpointResolution, + TargetKind: "endpoint_resolution", + TraversalTarget: &target, + Family: "endpoint_resolution", + TraversalFamily: decision.Family, + PlannedCandidates: endpointResolutionCandidateNames(decision.PlannedCandidates), + EndpointRoot: &root, + EndpointTerminal: &terminal, + EndpointPairClass: decision.PairClass, + EndpointResolutionCaps: &caps, + Candidate: string(decision.CandidatePlan), + EligibilityFacts: endpointResolutionEligibilityFacts(decision.EligibilityFacts), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.SelectorVersion, + Selected: string(decision.SelectedPlan), + Applied: string(decision.SelectedPlan), + Fallback: string(decision.FallbackPlan), + SkipReason: decision.FallbackReason, + }) + } + for _, decision := range plan.TraversalPredicate { + target, predicateIndex := decision.Target, decision.PredicateIndex + eligible, staticallyEligible := decision.StructurallyEligible, decision.StaticallyEligible + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringTraversalPredicateClassification, + TargetKind: "traversal_predicate", + TraversalTarget: &target, + Family: "traversal_predicate", + PlannedCandidates: traversalPredicateCandidateNames(decision.PlannedCandidates), + PredicateClass: decision.Class, + PredicateSource: decision.Source, + PredicateIndex: &predicateIndex, + Candidate: string(decision.CandidatePlan), + EligibilityFacts: traversalPredicateEligibilityFacts(decision.EligibilityFacts), + Eligible: &eligible, + StaticallyEligible: &staticallyEligible, + SelectionMode: decision.SelectionMode, + SelectorVersion: decision.ClassifierVersion, + Selected: string(decision.SelectedPlan), + Applied: string(decision.SelectedPlan), + Fallback: string(decision.FallbackPlan), + SkipReason: decision.FallbackReason, + }) + } + for _, decision := range plan.FieldRequirements { + queryPartIndex := decision.QueryPartIndex + s.translation.Optimization.TargetOutcomes = append(s.translation.Optimization.TargetOutcomes, TargetLoweringOutcome{ + Lowering: optimize.LoweringFieldRequirements, + TargetKind: "field_requirement", + QueryPartIndex: &queryPartIndex, + Symbol: decision.Symbol, + Selected: "analysis_only", + SkipReason: "analysis_metadata_only", + }) + } +} + +// shortestPathCandidateNames converts executor candidates to their stable diagnostic names. +func shortestPathCandidateNames(candidates []optimize.ShortestPathExecutor) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// expansionSearchCandidateNames converts expansion candidates to their stable diagnostic names. +func expansionSearchCandidateNames(candidates []optimize.ExpansionSearchStrategy) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// endpointResolutionCandidateNames converts analysis-only endpoint plans to +// their stable diagnostic identities. +func endpointResolutionCandidateNames(candidates []optimize.EndpointResolutionPlan) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// traversalPredicateCandidateNames converts predicate-placement plans to +// their stable diagnostic identities. +func traversalPredicateCandidateNames(candidates []optimize.TraversalPredicatePlan) []string { + names := make([]string, len(candidates)) + for idx, candidate := range candidates { + names[idx] = string(candidate) + } + return names +} + +// shortestPathEligibilityFacts converts executor qualification facts to public diagnostic records. +func shortestPathEligibilityFacts(facts []optimize.ShortestPathEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{ + Name: fact.Name, + Eligible: fact.Eligible, + } + } + return outcomes +} + +// expansionSearchEligibilityFacts converts search-strategy qualification facts to public diagnostic records. +func expansionSearchEligibilityFacts(facts []optimize.ExpansionSearchEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{ + Name: fact.Name, + Eligible: fact.Eligible, + } + } + return outcomes +} + +// endpointResolutionEligibilityFacts builds the SQL model fragment responsible for endpoint resolution eligibility facts. +func endpointResolutionEligibilityFacts(facts []optimize.EndpointResolutionEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{ + Name: fact.Name, + Eligible: fact.Eligible, + } + } + return outcomes +} + +// traversalPredicateEligibilityFacts builds the SQL model fragment responsible for traversal predicate eligibility facts. +func traversalPredicateEligibilityFacts(facts []optimize.TraversalPredicateEligibilityFact) []TargetEligibilityFact { + outcomes := make([]TargetEligibilityFact, len(facts)) + for idx, fact := range facts { + outcomes[idx] = TargetEligibilityFact{ + Name: fact.Name, + Eligible: fact.Eligible, + } + } + return outcomes +} + +// plannedLoweringCounts converts each lowering target collection into a named count so planned work can be reconciled with applied work. func plannedLoweringCounts(plan optimize.LoweringPlan) []SkippedLowering { return []SkippedLowering{ { @@ -790,6 +1306,10 @@ func plannedLoweringCounts(plan optimize.LoweringPlan) []SkippedLowering { Name: optimize.LoweringExpansionSuffixPushdown, Count: len(plan.ExpansionSuffixPushdown), }, + { + Name: optimize.LoweringExpansionSearchStrategy, + Count: len(plan.ExpansionSearchStrategy), + }, { Name: optimize.LoweringPredicatePlacement, Count: len(plan.PredicatePlacement) + len(plan.PatternPredicate), @@ -810,10 +1330,22 @@ func plannedLoweringCounts(plan optimize.LoweringPlan) []SkippedLowering { Name: optimize.LoweringAggregateTraversalCount, Count: len(plan.AggregateTraversalCount), }, + { + Name: optimize.LoweringFieldRequirements, + Count: len(plan.FieldRequirements), + }, + { + Name: optimize.LoweringShortestPathExecutor, + Count: len(plan.ShortestPathExecutor), + }, } } +// skippedLoweringReason explains why planned lowering work was not observed, including metadata-only analyses and lowerings superseded by a stronger fast path. func skippedLoweringReason(name string, applied map[string]int, plan optimize.LoweringPlan) string { + if name == optimize.LoweringFieldRequirements { + return "analysis_metadata_only" + } if applied[optimize.LoweringCountStoreFastPath] > 0 && name != optimize.LoweringCountStoreFastPath { return "superseded by CountStoreFastPath" } @@ -828,6 +1360,18 @@ func skippedLoweringReason(name string, applied map[string]int, plan optimize.Lo if reason := skippedTraversalDirectionReason(plan); reason != "" { return reason } + case optimize.LoweringExpansionSearchStrategy: + for _, decision := range plan.ExpansionSearchStrategy { + if decision.FallbackReason != "" { + return decision.FallbackReason + } + } + case optimize.LoweringShortestPathExecutor: + for _, decision := range plan.ShortestPathExecutor { + if decision.FallbackReason != "" { + return decision.FallbackReason + } + } default: return "planned lowering did not change the emitted SQL" } @@ -835,6 +1379,7 @@ func skippedLoweringReason(name string, applied map[string]int, plan optimize.Lo return "planned lowering did not change the emitted SQL" } +// skippedTraversalDirectionReason returns the first recorded reason a planned traversal direction was retained. func skippedTraversalDirectionReason(plan optimize.LoweringPlan) string { for _, decision := range plan.TraversalDirection { if !decision.Flip && decision.Reason != "" { @@ -845,13 +1390,143 @@ func skippedTraversalDirectionReason(plan optimize.LoweringPlan) string { return "" } +// ToolOptions controls experimental lowering selection exposed only to repository tooling. +type ToolOptions struct { + // ForceShortestPathExecutor requests a qualified shortest-path executor instead of automatic selection. + ForceShortestPathExecutor optimize.ShortestPathExecutor + // GuardedDistanceStateLimit overrides SP-I2's cap+1 state admission only for + // diagnostic tool forcing. It must be paired with a positive frontier limit. + GuardedDistanceStateLimit int64 + // GuardedDistanceFrontierLimit overrides SP-I2's per-level frontier admission + // only for diagnostic tool forcing. Production caps remain immutable. + GuardedDistanceFrontierLimit int64 + // ForceExpansionSearchStrategy requests a qualified variable-expansion strategy instead of automatic selection. + ForceExpansionSearchStrategy optimize.ExpansionSearchStrategy + // ExpansionOrientationPolicy selects the immutable orientation selector + // identity used by an enabled tournament or shadow mode. The zero value + // preserves orientation-probe-v1. + ExpansionOrientationPolicy optimize.ExpansionSearchPolicy + // EnableExpansionOrientationTournament emits a guarded orientation policy + // for one qualified fixed-suffix expansion. It defaults to + // orientation-probe-v1 and is intentionally tool-only while selectors are + // being shadow-qualified. + EnableExpansionOrientationTournament bool + // EnableExpansionOrientationShadow emits the same bounded orientation + // probes and SQL-visible would_select metadata while executing only the + // exact incumbent traversal arm. + EnableExpansionOrientationShadow bool + // EnableExpansionSuffixReverseGuard emits bounded fixed-suffix reverse + // execution with exact stepwise-forward fallback for one statically eligible + // full-path observation. It is deliberately tool-only during qualification. + EnableExpansionSuffixReverseGuard bool + // EnableExpansionSuffixReverseRetry emits only the bounded fixed-suffix + // reverse candidate. The PostgreSQL tool transaction owns exact forward + // retry; this option never installs a production selector. + EnableExpansionSuffixReverseRetry bool + // EnableExpansionSuffixRouteComponent emits one exact suffix-seeded reverse + // statement with a runtime receipt. It is a default-off GraphBench component + // arm and has no retry, probe, cache, or production policy. + EnableExpansionSuffixRouteComponent bool + // SuffixReverseGuardSuffixRowLimit overrides the tool-only fixed-suffix + // payload cap. Zero selects ExpansionSearchSuffixReverseGuardSuffixRowLimit. + SuffixReverseGuardSuffixRowLimit int64 + // SuffixReverseGuardStateLimit overrides the tool-only reverse-state cap. + // Zero selects ExpansionSearchSuffixReverseGuardStateLimit. + SuffixReverseGuardStateLimit int64 + // SuffixReverseRetryOutputRowLimit caps buffered candidate rows. Zero selects + // ExpansionSearchSuffixReverseRetryOutputRowLimit. + SuffixReverseRetryOutputRowLimit int64 + // SuffixReverseRetryOutputBytesLimit caps buffered candidate bytes. Zero + // selects ExpansionSearchSuffixReverseRetryOutputBytesLimit. + SuffixReverseRetryOutputBytesLimit int64 + // DisableEndpointSeededReverse is an emergency production rollback switch. + DisableEndpointSeededReverse bool +} + +// ProductionOptions contains the deliberately narrow subset of experimental +// lowerings that may be enabled by the PostgreSQL driver's versioned, +// query-allowlisted canary policy. The zero value preserves all incumbent +// production choices. +type ProductionOptions struct { + // ShortestPathExecutor supplies the shortest path executor input to the ProductionOptions contract. + ShortestPathExecutor optimize.ShortestPathExecutor + // ShortestPathCaps supplies the shortest path caps input to the ProductionOptions contract. + ShortestPathCaps *ProductionShortestPathCaps + // AuthorizedBucket supplies the authorized bucket input to the ProductionOptions contract. + AuthorizedBucket *ProductionTraversalBucket + // EnableExpansionOrientation indicates whether enable expansion orientation applies. + EnableExpansionOrientation bool + // EnableTopologyFixedSuffix enables the separately versioned production + // reverse-only fixed-suffix candidate. Callers must supply immutable caps; + // route selection and exact fallback remain driver responsibilities. + EnableTopologyFixedSuffix bool + // TopologyFixedSuffixCaps are the immutable candidate and output limits. + TopologyFixedSuffixCaps *ProductionFixedSuffixCaps + // ExpansionOrientationPolicy identifies the manifest-authorized immutable + // runtime formula. Production callers must set it explicitly when enabling + // orientation so v2 evidence cannot silently execute the v1 selector. + ExpansionOrientationPolicy optimize.ExpansionSearchPolicy + // DisableEndpointSeededReverse indicates whether disable endpoint seeded reverse applies. + DisableEndpointSeededReverse bool + // DisableInlineASPDAG indicates whether disable inline aspdag applies. + DisableInlineASPDAG bool + // DisableInlineSPWitness indicates whether disable inline sp witness applies. + DisableInlineSPWitness bool + // DisableInlineSPDistance is the emergency rollback switch for SP-I2-C-D. + DisableInlineSPDistance bool + // SelectorVersion identifies the schema version for selector version. + SelectorVersion string +} + +// ProductionFixedSuffixCaps bind the fixed-suffix candidate's SQL and driver +// buffering limits to one verified policy generation. +type ProductionFixedSuffixCaps struct { + SuffixRowLimit int64 `json:"suffix_row_limit"` + StateLimit int64 `json:"state_limit"` + OutputRowLimit int64 `json:"output_row_limit"` + OutputBytesLimit int64 `json:"output_bytes_limit"` +} + +// ProductionShortestPathCaps are immutable manifest-authorized limits. They +// are copied into the lowering decision and therefore into emitted SQL. +type ProductionShortestPathCaps struct { + // StateLimit supplies the state limit input to the ProductionShortestPathCaps contract. + StateLimit int64 `json:"state_limit"` + // FrontierLimit caps rows admitted at any one breadth-first level. + FrontierLimit int64 `json:"frontier_limit"` + // PredecessorLimit supplies the predecessor limit input to the ProductionShortestPathCaps contract. + PredecessorLimit int64 `json:"predecessor_limit"` + // EnumerationLimit supplies the enumeration limit input to the ProductionShortestPathCaps contract. + EnumerationLimit int64 `json:"enumeration_limit"` + // OutputBytesLimit supplies the output bytes limit input to the ProductionShortestPathCaps contract. + OutputBytesLimit int64 `json:"output_bytes_limit"` +} + +// ProductionTraversalBucket binds an exact-query authorization to the +// structural target characteristics independently qualified by evidence. +type ProductionTraversalBucket struct { + // Direction selects the traversal orientation covered by the contract. + Direction string `json:"direction"` + // ObservationMode identifies the observation mode. + ObservationMode string `json:"observation_mode"` + // MinimumDepth sets the inclusive lower traversal-depth bound. + MinimumDepth int64 `json:"minimum_depth"` + // MaximumDepth sets the inclusive upper traversal-depth bound. + MaximumDepth int64 `json:"maximum_depth"` + // RelationshipKindCount records the number of relationship kind count. + RelationshipKindCount int `json:"relationship_kind_count"` + // UntypedRelationship indicates whether untyped relationship applies. + UntypedRelationship bool `json:"untyped_relationship"` +} + +// Translate optimizes and translates a Cypher query for the selected graph using production lowering choices. func Translate(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32) (Result, error) { return TranslateWithOptions(ctx, cypherQuery, kindMapper, parameters, graphID, DefaultOptions()) } // TranslateWithOptions translates Cypher with an explicit optimizer policy. func TranslateWithOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options Options) (Result, error) { - translation, _, err := translateWithOptions(ctx, cypherQuery, kindMapper, parameters, graphID, options, false) + translation, _, err := translateWithOptions(ctx, cypherQuery, kindMapper, parameters, graphID, options, ToolOptions{}, false) return translation, err } @@ -860,10 +1535,89 @@ func TranslateWithOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, // The input tree is borrowed: it is copied only if an optimizer rule mutates // it. This is intended for compilation code that owns the input tree. func TranslateWithOptionsAndParameterSources(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options Options) (Result, map[string]string, error) { - return translateWithOptions(ctx, cypherQuery, kindMapper, parameters, graphID, options, true) + return translateWithOptions(ctx, cypherQuery, kindMapper, parameters, graphID, options, ToolOptions{}, true) } -func translateWithOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options Options, borrowQuery bool) (Result, map[string]string, error) { +// TranslateWithProductionOptions applies a validated canary policy. B +// executors remain unavailable unless the driver has independently established +// the required transaction snapshot; this function only controls lowering. +func TranslateWithProductionOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ProductionOptions) (Result, error) { + if options.SelectorVersion == "" { + return Result{}, fmt.Errorf("production traversal policy requires a selector version") + } + if options.EnableExpansionOrientation { + if !supportedExpansionOrientationPolicy(options.ExpansionOrientationPolicy) { + return Result{}, fmt.Errorf("production expansion orientation requires a supported explicit policy") + } + if options.SelectorVersion != string(options.ExpansionOrientationPolicy) { + return Result{}, fmt.Errorf("production expansion orientation selector %q does not match policy %q", options.SelectorVersion, options.ExpansionOrientationPolicy) + } + } + if options.ShortestPathExecutor != "" && !productionShortestPathExecutor(options.ShortestPathExecutor) { + return Result{}, fmt.Errorf("shortest-path executor %q is not production-canary eligible", options.ShortestPathExecutor) + } + if options.EnableTopologyFixedSuffix { + caps := options.TopologyFixedSuffixCaps + if caps == nil || caps.SuffixRowLimit <= 0 || caps.StateLimit <= 0 || caps.OutputRowLimit <= 0 || caps.OutputBytesLimit <= 0 { + return Result{}, fmt.Errorf("production topology fixed-suffix policy requires positive immutable caps") + } + if options.SelectorVersion != string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) && options.SelectorVersion != string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) { + return Result{}, fmt.Errorf("production topology fixed-suffix selector requires a supported versioned topology selector") + } + } + toolOptions := ToolOptions{ + ForceShortestPathExecutor: options.ShortestPathExecutor, + EnableExpansionOrientationTournament: options.EnableExpansionOrientation, + ExpansionOrientationPolicy: options.ExpansionOrientationPolicy, + DisableEndpointSeededReverse: options.DisableEndpointSeededReverse, + } + if options.EnableTopologyFixedSuffix { + toolOptions.EnableExpansionSuffixReverseRetry = true + toolOptions.SuffixReverseGuardSuffixRowLimit = options.TopologyFixedSuffixCaps.SuffixRowLimit + toolOptions.SuffixReverseGuardStateLimit = options.TopologyFixedSuffixCaps.StateLimit + toolOptions.SuffixReverseRetryOutputRowLimit = options.TopologyFixedSuffixCaps.OutputRowLimit + toolOptions.SuffixReverseRetryOutputBytesLimit = options.TopologyFixedSuffixCaps.OutputBytesLimit + } + optimizedPlan, err := optimize.Optimize(cypherQuery) + if err != nil { + return Result{}, err + } + if err := applyToolOptions(&optimizedPlan, toolOptions); err != nil { + return Result{}, err + } + if err := applyProductionTopologyFixedSuffixAuthorization(&optimizedPlan, options); err != nil { + return Result{}, err + } + applyProductionShortestPathRollback(&optimizedPlan, options) + if err := applyProductionShortestPathAuthorization(&optimizedPlan, options); err != nil { + return Result{}, err + } + for idx := range optimizedPlan.LoweringPlan.ShortestPathExecutor { + decision := &optimizedPlan.LoweringPlan.ShortestPathExecutor[idx] + if decision.SelectionMode == "forced_tool" { + decision.SelectionMode = "production_canary" + decision.SelectorVersion = options.SelectorVersion + } + } + for idx := range optimizedPlan.LoweringPlan.ExpansionSearchStrategy { + decision := &optimizedPlan.LoweringPlan.ExpansionSearchStrategy[idx] + if decision.SelectionMode == "guarded_tool" { + decision.SelectionMode = "production_canary" + decision.SelectorVersion = options.SelectorVersion + } + } + translation, _, err := translateOptimized(ctx, optimizedPlan, kindMapper, parameters, graphID, toolOptions) + return translation, err +} + +// TranslateForTool exposes qualified experimental lowerings to repository +// tooling without making them selectable through the production query API. +func TranslateForTool(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ToolOptions) (Result, error) { + translation, _, err := translateWithOptions(ctx, cypherQuery, kindMapper, parameters, graphID, DefaultOptions(), options, false) + return translation, err +} + +func translateWithOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options Options, toolOptions ToolOptions, borrowQuery bool) (Result, map[string]string, error) { options, err := options.normalized() if err != nil { return Result{}, nil, err @@ -883,6 +1637,15 @@ func translateWithOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, optimizedPlan.Query = cypherQuery } + if err := applyToolOptions(&optimizedPlan, toolOptions); err != nil { + return Result{}, nil, err + } + return translateOptimized(ctx, optimizedPlan, kindMapper, parameters, graphID, toolOptions) +} + +// translateOptimized builds the SQL model fragment responsible for translate optimized. +func translateOptimized(ctx context.Context, optimizedPlan optimize.Plan, kindMapper pgsql.KindMapper, parameters map[string]any, graphID int32, options ToolOptions) (Result, map[string]string, error) { + translator := NewTranslator(ctx, kindMapper, parameters, graphID) if membershipAliases, err := collectIDMembershipAliases(optimizedPlan.Query); err != nil { return Result{}, nil, err @@ -915,11 +1678,768 @@ func translateWithOptions(ctx context.Context, cypherQuery *cypher.RegularQuery, if err := walk.Cypher(optimizedPlan.Query, translator); err != nil { return Result{}, nil, err } + if options.ForceExpansionSearchStrategy != "" && len(translator.appliedExpansionSearchStrategies) == 0 { + return Result{}, nil, fmt.Errorf("forced expansion-search strategy %q was selected but not emitted", options.ForceExpansionSearchStrategy) + } + if options.EnableExpansionOrientationTournament && len(translator.emittedExpansionSearchPolicies) == 0 { + return Result{}, nil, fmt.Errorf("expansion orientation tournament was selected but not emitted") + } + if options.EnableExpansionOrientationShadow && len(translator.emittedExpansionSearchPolicies) == 0 { + return Result{}, nil, fmt.Errorf("expansion orientation shadow was selected but not emitted") + } + if options.EnableExpansionSuffixReverseGuard && len(translator.emittedExpansionSearchPolicies) == 0 { + return Result{}, nil, fmt.Errorf("expansion suffix reverse guard was selected but not emitted") + } + if options.EnableExpansionSuffixRouteComponent && len(translator.appliedExpansionSearchStrategies) == 0 { + return Result{}, nil, fmt.Errorf("suffix route component was selected but not emitted") + } + if options.ForceShortestPathExecutor != "" && len(translator.appliedShortestPathExecutors) == 0 { + return Result{}, nil, fmt.Errorf("forced shortest-path executor %q was selected but not emitted", options.ForceShortestPathExecutor) + } translator.recordSkippedLowerings() return translator.translation, translator.parameterSources, nil } +// productionShortestPathExecutor builds the SQL model fragment responsible for production shortest path executor. +func productionShortestPathExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorI2GuardedDistanceV2: + return true + default: + return false + } +} + +// applyProductionTopologyFixedSuffixAuthorization promotes the already +// structurally checked reverse-only tool lowering to its distinct production +// identity. It intentionally does not add a second arm to emitted SQL. +func applyProductionTopologyFixedSuffixAuthorization(plan *optimize.Plan, options ProductionOptions) error { + if !options.EnableTopologyFixedSuffix { + return nil + } + matching := 0 + for index := range plan.LoweringPlan.ExpansionSearchStrategy { + decision := &plan.LoweringPlan.ExpansionSearchStrategy[index] + if decision.Family != "fixed_suffix_expansion" || decision.SelectedStrategy != optimize.ExpansionSearchSuffixSeededReverse || decision.EmittedPolicy != optimize.ExpansionSearchPolicySuffixReverseRetryV1 { + continue + } + matching++ + decision.PlannedPolicy = optimize.ExpansionSearchPolicy(options.SelectorVersion) + decision.EmittedPolicy = optimize.ExpansionSearchPolicy(options.SelectorVersion) + decision.SelectionMode = "production_canary" + decision.SelectorVersion = options.SelectorVersion + } + if matching == 0 { + return fmt.Errorf("production topology fixed-suffix policy matched no candidates") + } + return nil +} + +// applyProductionShortestPathAuthorization applies production shortest path authorization. +func applyProductionShortestPathAuthorization(plan *optimize.Plan, options ProductionOptions) error { + if options.ShortestPathExecutor == "" { + return nil + } + if options.DisableInlineASPDAG && options.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG { + return fmt.Errorf("inline ASP DAG is disabled by production policy") + } + if options.DisableInlineSPDistance && options.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistanceV2 { + return fmt.Errorf("inline SP distance is disabled by production policy") + } + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if decision.SelectedExecutor != options.ShortestPathExecutor || decision.SelectionMode != "forced_tool" { + continue + } + if options.AuthorizedBucket != nil { + bucket := options.AuthorizedBucket + if decision.Direction.String() != bucket.Direction || + string(decision.ObservationMode) != bucket.ObservationMode || + decision.MinimumDepth != bucket.MinimumDepth || + decision.MaximumDepth != bucket.MaximumDepth || + decision.RelationshipKindCount != bucket.RelationshipKindCount || + decision.UntypedRelationship != bucket.UntypedRelationship { + return fmt.Errorf("production traversal target does not match its authorized promotion bucket") + } + } + if options.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG || options.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || options.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistanceV2 { + if options.AuthorizedBucket == nil { + return fmt.Errorf("guarded inline shortest-path production policy requires an exact authorized bucket") + } + if options.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + bucket := options.AuthorizedBucket + if options.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + return fmt.Errorf("canonical SP-I1 production policy requires selector %q", optimize.ShortestPathSelectorStaticV6) + } + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth != 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("canonical SP-I1 production policy requires the qualified inbound typed single-kind one-path depth 1..64 bucket") + } + } + if options.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistanceV2 { + bucket := options.AuthorizedBucket + if options.SelectorVersion != optimize.ShortestPathSelectorStaticV9HiddenFanInTail { + return fmt.Errorf("guarded SP-I2 V2 distance policy requires selector %q", optimize.ShortestPathSelectorStaticV9HiddenFanInTail) + } + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationDistance) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("guarded SP-I2 distance policy requires an exactly authorized inbound typed single-kind distance bucket") + } + } + if options.ShortestPathCaps == nil { + return fmt.Errorf("guarded inline shortest-path production policy requires immutable caps") + } + caps := options.ShortestPathCaps + if caps.StateLimit <= 0 || (options.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistanceV2 && caps.FrontierLimit <= 0) || + (options.ShortestPathExecutor != optimize.ShortestPathExecutorI2GuardedDistanceV2 && (caps.PredecessorLimit <= 0 || caps.EnumerationLimit <= 0 || caps.OutputBytesLimit <= 0)) { + return fmt.Errorf("guarded inline shortest-path production policy requires positive immutable caps") + } + if options.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistanceV2 && + (caps.StateLimit != optimize.ShortestPathI2QualifiedStateLimit || + caps.FrontierLimit != optimize.ShortestPathI2QualifiedFrontierLimit || + caps.PredecessorLimit != 0 || caps.EnumerationLimit != 0 || caps.OutputBytesLimit != 0) { + return fmt.Errorf( + "guarded SP-I2 distance production policy requires exactly state_limit=%d and frontier_limit=%d", + optimize.ShortestPathI2QualifiedStateLimit, + optimize.ShortestPathI2QualifiedFrontierLimit, + ) + } + decision.StateLimit = caps.StateLimit + decision.FrontierLimit = caps.FrontierLimit + decision.PredecessorLimit = caps.PredecessorLimit + decision.EnumerationLimit = caps.EnumerationLimit + decision.OutputBytesLimit = caps.OutputBytesLimit + decision.ExecutionBoundary = "guarded_dual_arm" + } + return nil + } + return fmt.Errorf("production shortest-path executor %q was not selected", options.ShortestPathExecutor) +} + +// applyProductionShortestPathRollback is deliberately post-optimization: an +// emergency switch must rewrite both a policy-forced candidate and any future +// statically preferred candidate. Returning to the exact incumbent also resets +// candidate-only limits and boundary metadata so cached SQL cannot retain a +// disabled guarded arm. +func applyProductionShortestPathRollback(plan *optimize.Plan, options ProductionOptions) { + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + switch { + case options.DisableInlineASPDAG && decision.SelectedExecutor == optimize.ShortestPathExecutorASPI1DAG: + decision.SelectedExecutor = optimize.ShortestPathExecutorASPA1DAG + decision.FallbackExecutor = optimize.ShortestPathExecutorIncumbentWorkspace + case options.DisableInlineSPWitness && decision.SelectedExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness: + decision.SelectedExecutor = optimize.ShortestPathExecutorS4CanonicalWitness + decision.FallbackExecutor = optimize.ShortestPathExecutorIncumbentWorkspace + case options.DisableInlineSPDistance && (decision.SelectedExecutor == optimize.ShortestPathExecutorI2GuardedDistance || decision.SelectedExecutor == optimize.ShortestPathExecutorI2GuardedDistanceV2): + decision.SelectedExecutor = optimize.ShortestPathExecutorS4CanonicalDistance + decision.FallbackExecutor = optimize.ShortestPathExecutorIncumbentWorkspace + default: + continue + } + decision.Scheduler = decision.SelectedExecutor.Scheduler() + decision.ExecutionBoundary = decision.SelectedExecutor.ExecutionBoundary() + decision.SelectionMode = "production_kill_switch" + decision.SelectorVersion = options.SelectorVersion + decision.FallbackReason = "disabled_by_production_policy" + decision.FrontierLimit = 0 + decision.PredecessorLimit = 0 + decision.EnumerationLimit = 0 + decision.OutputBytesLimit = 0 + } +} + +// applyToolOptions applies supported forced executor and expansion-strategy requests to an optimized plan. +func applyToolOptions(plan *optimize.Plan, options ToolOptions) error { + if options.EnableExpansionOrientationTournament && options.EnableExpansionOrientationShadow { + return fmt.Errorf("expansion orientation tournament and shadow modes are mutually exclusive") + } + suffixMode := options.EnableExpansionSuffixReverseGuard || options.EnableExpansionSuffixReverseRetry || options.EnableExpansionSuffixRouteComponent + if suffixMode && (options.EnableExpansionOrientationTournament || options.EnableExpansionOrientationShadow) { + return fmt.Errorf("expansion suffix reverse modes and orientation modes are mutually exclusive") + } + if (options.EnableExpansionOrientationTournament || options.EnableExpansionOrientationShadow) && options.ForceExpansionSearchStrategy != "" { + return fmt.Errorf("expansion orientation policy and forced expansion-search strategy are mutually exclusive") + } + if suffixMode && options.ForceExpansionSearchStrategy != "" { + return fmt.Errorf("expansion suffix reverse modes and forced expansion-search strategy are mutually exclusive") + } + if !suffixMode && (options.SuffixReverseGuardSuffixRowLimit != 0 || options.SuffixReverseGuardStateLimit != 0) { + return fmt.Errorf("expansion suffix reverse caps require a suffix mode to be enabled") + } + if !options.EnableExpansionSuffixReverseRetry && (options.SuffixReverseRetryOutputRowLimit != 0 || options.SuffixReverseRetryOutputBytesLimit != 0) { + return fmt.Errorf("expansion suffix reverse retry output caps require retry to be enabled") + } + if options.GuardedDistanceStateLimit != 0 || options.GuardedDistanceFrontierLimit != 0 { + if !isGuardedDistanceExecutor(options.ForceShortestPathExecutor) { + return fmt.Errorf("guarded distance cap overrides require forcing an SP-I2 distance executor") + } + if options.GuardedDistanceStateLimit <= 0 || options.GuardedDistanceFrontierLimit <= 0 { + return fmt.Errorf("guarded distance cap overrides require positive state and frontier limits") + } + } + orientationPolicy, err := requestedExpansionOrientationPolicy(options) + if err != nil { + return err + } + if err := applyForcedShortestPathExecutor(plan, options.ForceShortestPathExecutor); err != nil { + return err + } + if options.GuardedDistanceStateLimit > 0 { + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if isGuardedDistanceExecutor(decision.SelectedExecutor) && decision.SelectionMode == "forced_tool" { + decision.StateLimit = options.GuardedDistanceStateLimit + decision.FrontierLimit = options.GuardedDistanceFrontierLimit + } + } + } + if options.DisableEndpointSeededReverse { + for idx := range plan.LoweringPlan.ExpansionSearchStrategy { + decision := &plan.LoweringPlan.ExpansionSearchStrategy[idx] + if decision.SelectedStrategy == optimize.ExpansionSearchEndpointSeededReverse { + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.EmittedPolicy = "" + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + decision.SelectionMode = "production_kill_switch" + decision.SelectorVersion = "endpoint-seeded-disabled-v1" + decision.FallbackReason = "disabled_by_production_policy" + } + } + } + if options.EnableExpansionOrientationTournament { + return applyExpansionOrientationTournamentPolicy(plan, orientationPolicy) + } + if options.EnableExpansionOrientationShadow { + return applyExpansionOrientationShadowPolicy(plan, orientationPolicy) + } + if (options.EnableExpansionSuffixReverseGuard && options.EnableExpansionSuffixReverseRetry) || + (options.EnableExpansionSuffixReverseGuard && options.EnableExpansionSuffixRouteComponent) || + (options.EnableExpansionSuffixReverseRetry && options.EnableExpansionSuffixRouteComponent) { + return fmt.Errorf("expansion suffix reverse guard, transaction retry, and direct component modes are mutually exclusive") + } + if options.EnableExpansionSuffixReverseGuard { + return applyExpansionSuffixReverseGuardPolicy(plan, options.SuffixReverseGuardSuffixRowLimit, options.SuffixReverseGuardStateLimit) + } + if options.EnableExpansionSuffixReverseRetry { + return applyExpansionSuffixReverseRetryPolicy( + plan, + options.SuffixReverseGuardSuffixRowLimit, + options.SuffixReverseGuardStateLimit, + options.SuffixReverseRetryOutputRowLimit, + options.SuffixReverseRetryOutputBytesLimit, + ) + } + if options.EnableExpansionSuffixRouteComponent { + return applyExpansionSuffixRouteComponentPolicy(plan) + } + return applyForcedExpansionSearchStrategy(plan, options.ForceExpansionSearchStrategy) +} + +// applyExpansionSuffixRouteComponentPolicy selects one exact, reverse-only +// fixed-suffix arm for GraphBench component measurement. It deliberately has +// no admission caps or fallback because those belong to a later, separately +// frozen automatic-routing experiment. +func applyExpansionSuffixRouteComponentPolicy(plan *optimize.Plan) error { + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family == "fixed_suffix_expansion" && + decision.CandidateStrategy == optimize.ExpansionSearchSuffixSeededReverse && + decision.StructurallyEligible && decision.StaticallyEligible { + matching = append(matching, idx) + } + } + if len(matching) != 1 { + return fmt.Errorf("suffix route component matched %d statically eligible fixed-suffix targets; expected exactly one", len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.PlannedPolicy = "" + decision.EmittedPolicy = "" + decision.SelectedStrategy = optimize.ExpansionSearchSuffixSeededReverse + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + decision.ProbeCaps = optimize.ExpansionSearchProbeCaps{} + decision.Admission = optimize.ExpansionSearchAdmission{} + decision.StateLimit = 0 + decision.FallbackStrategy = "" + decision.SelectionMode = "component_tool" + decision.SelectorVersion = optimize.ExpansionSearchSelectorSuffixRouteComponentV1 + decision.FallbackReason = "" + return nil +} + +// applyExpansionSuffixReverseRetryPolicy selects one reverse-only full-path +// candidate. Exact forward fallback is deliberately absent from emitted SQL; +// the PostgreSQL tool transaction executes it only after a complete overflow. +func applyExpansionSuffixReverseRetryPolicy(plan *optimize.Plan, suffixRowLimit, stateLimit, outputRowLimit, outputBytesLimit int64) error { + if suffixRowLimit == 0 { + suffixRowLimit = optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit + } + if stateLimit == 0 { + stateLimit = optimize.ExpansionSearchSuffixReverseGuardStateLimit + } + if outputRowLimit == 0 { + outputRowLimit = optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit + } + if outputBytesLimit == 0 { + outputBytesLimit = optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit + } + if suffixRowLimit <= 0 || stateLimit <= 0 || outputRowLimit <= 0 || outputBytesLimit <= 0 { + return fmt.Errorf("expansion suffix reverse retry requires positive suffix, state, output-row, and output-byte limits") + } + + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family == "fixed_suffix_expansion" && + decision.CandidateStrategy == optimize.ExpansionSearchSuffixSeededReverse && + decision.StructurallyEligible && decision.StaticallyEligible && + decision.ObservationMode == optimize.ExpansionSearchObservationFullPath { + matching = append(matching, idx) + } + } + if len(matching) == 0 { + return fmt.Errorf("expansion suffix reverse retry matched no statically eligible full-path fixed-suffix targets") + } + for _, index := range matching { + decision := &plan.LoweringPlan.ExpansionSearchStrategy[index] + decision.PlannedPolicy = optimize.ExpansionSearchPolicySuffixReverseRetryV1 + decision.EmittedPolicy = optimize.ExpansionSearchPolicySuffixReverseRetryV1 + decision.SelectedStrategy = optimize.ExpansionSearchSuffixSeededReverse + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchSuffixSeededReverse} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryTransactionRetry + decision.ProbeCaps = optimize.ExpansionSearchProbeCaps{ReverseSeedRowLimit: suffixRowLimit} + decision.Admission = optimize.ExpansionSearchAdmission{ + StateLimit: stateLimit, + OutputRowLimit: outputRowLimit, + OutputBytesLimit: outputBytesLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + } + decision.StateLimit = stateLimit + decision.FallbackStrategy = optimize.ExpansionSearchStepwiseForward + decision.SelectionMode = "transaction_retry_tool" + decision.SelectorVersion = string(optimize.ExpansionSearchPolicySuffixReverseRetryV1) + decision.FallbackReason = "" + } + return nil +} + +// applyExpansionSuffixReverseGuardPolicy selects one full-path fixed-suffix +// target for bounded reverse execution. No endpoint-only observation or +// topology-scored orientation decision is admitted by this tool-only policy. +func applyExpansionSuffixReverseGuardPolicy(plan *optimize.Plan, suffixRowLimit, stateLimit int64) error { + if suffixRowLimit == 0 { + suffixRowLimit = optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit + } + if stateLimit == 0 { + stateLimit = optimize.ExpansionSearchSuffixReverseGuardStateLimit + } + if suffixRowLimit <= 0 || stateLimit <= 0 { + return fmt.Errorf("expansion suffix reverse guard requires positive suffix-row and state limits") + } + + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family != "fixed_suffix_expansion" || + decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse || + !decision.StructurallyEligible || !decision.StaticallyEligible || + decision.ObservationMode != optimize.ExpansionSearchObservationFullPath { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("expansion suffix reverse guard has no statically eligible full-path fixed-suffix target") + } + if len(matching) != 1 { + return fmt.Errorf("expansion suffix reverse guard matched %d statically eligible full-path fixed-suffix targets; expected exactly one", len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.PlannedPolicy = optimize.ExpansionSearchPolicySuffixReverseGuardV1 + decision.EmittedPolicy = optimize.ExpansionSearchPolicySuffixReverseGuardV1 + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchSuffixSeededReverse, + optimize.ExpansionSearchStepwiseForward, + } + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryGuardedDualArm + decision.ProbeCaps = optimize.ExpansionSearchProbeCaps{ReverseSeedRowLimit: suffixRowLimit} + decision.Admission = optimize.ExpansionSearchAdmission{ + StateLimit: stateLimit, + RequiresCompleteProbes: true, + FallbackStrategy: optimize.ExpansionSearchStepwiseForward, + } + decision.StateLimit = stateLimit + decision.FallbackStrategy = optimize.ExpansionSearchStepwiseForward + decision.SelectionMode = "guarded_tool" + decision.SelectorVersion = optimize.ExpansionSearchSelectorFixedSuffixPathV1 + decision.FallbackReason = "" + decision.EligibilityFacts = append(decision.EligibilityFacts, optimize.ExpansionSearchEligibilityFact{ + Name: "full_path_observation", + Eligible: true, + }) + return nil +} + +// requestedExpansionOrientationPolicy builds the SQL model fragment responsible for requested expansion orientation policy. +func requestedExpansionOrientationPolicy(options ToolOptions) (optimize.ExpansionSearchPolicy, error) { + policy := options.ExpansionOrientationPolicy + if policy == "" { + return optimize.ExpansionSearchPolicyOrientationProbeV1, nil + } + if !options.EnableExpansionOrientationTournament && !options.EnableExpansionOrientationShadow { + return "", fmt.Errorf("expansion orientation policy %q requires tournament or shadow mode", policy) + } + if !supportedExpansionOrientationPolicy(policy) { + return "", fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + return policy, nil +} + +// supportedExpansionOrientationPolicy reports whether production translation recognizes an orientation policy. +func supportedExpansionOrientationPolicy(policy optimize.ExpansionSearchPolicy) bool { + switch policy { + case optimize.ExpansionSearchPolicyOrientationProbeV1, + optimize.ExpansionSearchPolicyOrientationProbeV2: + return true + default: + return false + } +} + +// applyForcedShortestPathExecutor selects the requested executor only when exactly one qualified shortest-path target supports it. +func applyForcedShortestPathExecutor(plan *optimize.Plan, executor optimize.ShortestPathExecutor) error { + if executor == "" { + return nil + } + if !supportedForcedShortestPathExecutor(executor) { + return fmt.Errorf("unsupported forced shortest-path executor %q", executor) + } + if executor == optimize.ShortestPathExecutorIncumbentWorkspace || executor == optimize.ShortestPathExecutorS0Direct { + forced := 0 + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if !decision.StructurallyEligible { + continue + } + if executor == optimize.ShortestPathExecutorS0Direct && (decision.MinimumDepth != 1 || decision.MaximumDepth < 1) { + continue + } + decision.SelectedExecutor = executor + decision.Scheduler = executor.Scheduler() + decision.ExecutionBoundary = executor.ExecutionBoundary() + decision.SelectionMode = "forced_tool" + decision.SelectorVersion = "sp-tool-v1" + decision.FallbackReason = "" + forced++ + } + if forced == 0 { + if executor == optimize.ShortestPathExecutorS0Direct { + return fmt.Errorf("forced shortest-path executor %q has no structurally eligible depth-one target", executor) + } + return fmt.Errorf("forced shortest-path executor %q has no structurally eligible target", executor) + } + return nil + } + expectedObservation := optimize.ShortestPathObservationDistance + expectedDescription := "distance-only" + if executor == optimize.ShortestPathExecutorS3EdgeM0 || executor == optimize.ShortestPathExecutorS4CanonicalWitness || executor == optimize.ShortestPathExecutorI1CanonicalWitness || executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || executor == optimize.ShortestPathExecutorB1AlternatingNodeWitness || executor == optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness { + expectedObservation = optimize.ShortestPathObservationOnePath + expectedDescription = "one-path" + } else if executor == optimize.ShortestPathExecutorASPA1DAG || executor == optimize.ShortestPathExecutorASPN1NegativeExhaustion || executor == optimize.ShortestPathExecutorASPI1DAG || executor == optimize.ShortestPathExecutorASPB1AlternatingNodeDAG || executor == optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG { + expectedObservation = optimize.ShortestPathObservationAllPaths + expectedDescription = "all-paths" + } + + allShortestExecutor := executor == optimize.ShortestPathExecutorASPA1DAG || + executor == optimize.ShortestPathExecutorASPN1NegativeExhaustion || + executor == optimize.ShortestPathExecutorASPI1DAG || + executor == optimize.ShortestPathExecutorASPB1AlternatingNodeDAG || + executor == optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG + forced := 0 + for idx := range plan.LoweringPlan.ShortestPathExecutor { + decision := &plan.LoweringPlan.ShortestPathExecutor[idx] + if !decision.StructurallyEligible { + continue + } + if decision.ObservationMode != expectedObservation { + continue + } + if spI2ScalarProjectionExecutor(executor) && !spI2ScalarProjectionRequirementsEligible(plan.LoweringPlan.FieldRequirements, decision.Target.QueryPartIndex) { + continue + } + // Two-sided predecessor-DAG discovery is proven only for one distinct, + // directed singleton endpoint pair with minimum depth exactly one. The + // shared structural facts enforce every condition except this narrower + // minimum-depth check. Tool forcing must not broaden that envelope. + if allShortestExecutor && (decision.Family != "ASP" || decision.MinimumDepth != 1 || decision.MaximumDepth < 1 || decision.MaximumDepth > 64) { + continue + } + + decision.SelectedExecutor = executor + decision.ExecutionBoundary = executor.ExecutionBoundary() + if executor == optimize.ShortestPathExecutorASPI1DAG || executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || isGuardedDistanceExecutor(executor) { + decision.ExecutionBoundary = "guarded_dual_arm" + if !isGuardedDistanceExecutor(executor) { + decision.FrontierLimit = 0 + } + } + decision.Scheduler = executor.Scheduler() + decision.SelectionMode = "forced_tool" + decision.SelectorVersion = "sp-tool-v1" + decision.FallbackReason = "" + if allShortestExecutor { + decision.SelectorVersion = "asp-tool-v1" + if executor != optimize.ShortestPathExecutorASPA1DAG { + decision.FallbackExecutor = optimize.ShortestPathExecutorASPA1DAG + } + } + if executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + decision.SelectorVersion = "sp-i1-canonical-tool-v1" + decision.FallbackExecutor = optimize.ShortestPathExecutorS4CanonicalWitness + } + if executor == optimize.ShortestPathExecutorI2GuardedDistance { + decision.SelectorVersion = optimize.ShortestPathSelectorStaticV8HiddenFanIn + decision.FallbackExecutor = optimize.ShortestPathExecutorS4CanonicalDistance + decision.PredecessorLimit = 0 + decision.EnumerationLimit = 0 + decision.OutputBytesLimit = 0 + } + if isV2GuardedDistanceExecutor(executor) { + decision.SelectorVersion = optimize.ShortestPathSelectorStaticV9HiddenFanInTail + decision.FallbackExecutor = optimize.ShortestPathExecutorS4CanonicalDistance + decision.PredecessorLimit = 0 + decision.EnumerationLimit = 0 + decision.OutputBytesLimit = 0 + } + forced++ + } + if forced == 0 { + return fmt.Errorf("forced shortest-path executor %q has no structurally eligible %s target", executor, expectedDescription) + } + return nil +} + +// supportedForcedShortestPathExecutor reports whether production translation recognizes a shortest-path executor. +func supportedForcedShortestPathExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorIncumbentWorkspace, + optimize.ShortestPathExecutorS0Direct, + optimize.ShortestPathExecutorS3Unidirectional, + optimize.ShortestPathExecutorS3EdgeM0, + optimize.ShortestPathExecutorS4CanonicalDistance, + optimize.ShortestPathExecutorS4CanonicalWitness, + optimize.ShortestPathExecutorASPA1DAG, + optimize.ShortestPathExecutorASPN1NegativeExhaustion, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorI1CanonicalDistance, + optimize.ShortestPathExecutorI2GuardedDistance, + optimize.ShortestPathExecutorI2GuardedDistanceV2, + optimize.ShortestPathExecutorI2GuardedDistanceV2E0, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP, + optimize.ShortestPathExecutorI1CanonicalWitness, + optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG: + return true + default: + return false + } +} + +func isV2GuardedDistanceExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorI2GuardedDistanceV2, + optimize.ShortestPathExecutorI2GuardedDistanceV2E0, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1D, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1P, + optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP: + return true + default: + return false + } +} + +func isGuardedDistanceExecutor(executor optimize.ShortestPathExecutor) bool { + return executor == optimize.ShortestPathExecutorI2GuardedDistance || isV2GuardedDistanceExecutor(executor) +} + +func spI2ScalarProjectionExecutor(executor optimize.ShortestPathExecutor) bool { + return executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1P || + executor == optimize.ShortestPathExecutorI2GuardedDistanceV2E1DP +} + +func spI2ScalarProjectionRequirementsEligible(decisions []optimize.FieldRequirementDecision, queryPartIndex int) bool { + for _, decision := range decisions { + if decision.QueryPartIndex != queryPartIndex { + continue + } + for _, field := range decision.Fields { + switch field { + case optimize.FieldRequirementEntityID, optimize.FieldRequirementOrderedPathEdgeIDs: + continue + default: + return false + } + } + } + return true +} + +// applyForcedExpansionSearchStrategy selects the requested strategy only when exactly one qualified expansion target supports it. +func applyForcedExpansionSearchStrategy(plan *optimize.Plan, strategy optimize.ExpansionSearchStrategy) error { + if strategy == "" { + return nil + } + if strategy != optimize.ExpansionSearchSuffixSeededReverse && strategy != optimize.ExpansionSearchEndpointSeededReverse { + return fmt.Errorf("unsupported forced expansion-search strategy %q", strategy) + } + + var matching []int + for idx := range plan.LoweringPlan.ExpansionSearchStrategy { + decision := plan.LoweringPlan.ExpansionSearchStrategy[idx] + if !decision.StructurallyEligible { + continue + } + if strategy == optimize.ExpansionSearchSuffixSeededReverse && decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse { + continue + } + if strategy == optimize.ExpansionSearchEndpointSeededReverse && decision.CandidateStrategy != optimize.ExpansionSearchEndpointSeededReverse { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("forced expansion-search strategy %q has no structurally eligible target", strategy) + } + if len(matching) != 1 { + return fmt.Errorf("forced expansion-search strategy %q matched %d structurally eligible targets; expected exactly one", strategy, len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.SelectedStrategy = strategy + decision.SelectionMode = "forced_tool" + decision.EmittedPolicy = "" + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{strategy} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + if strategy == optimize.ExpansionSearchSuffixSeededReverse { + decision.SelectorVersion = "suffix-seeded-reverse-tool-v1" + } else { + decision.SelectorVersion = "endpoint-seeded-reverse-tool-v1" + decision.EmittedPolicy = optimize.ExpansionSearchPolicyEndpointGuardV1 + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchStepwiseForward, + optimize.ExpansionSearchEndpointSeededReverse, + } + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryGuardedDualArm + } + decision.FallbackReason = "" + + return nil +} + +// applyExpansionOrientationTournament emits orientation-probe-v1 only when a +// single already-qualified fixed-suffix target exists. It preserves the +// compile-time incumbent identity because the runtime arm is not known during +// translation. +func applyExpansionOrientationTournament(plan *optimize.Plan) error { + return applyExpansionOrientationTournamentPolicy(plan, optimize.ExpansionSearchPolicyOrientationProbeV1) +} + +// applyExpansionOrientationTournamentPolicy applies expansion orientation tournament policy. +func applyExpansionOrientationTournamentPolicy(plan *optimize.Plan, policy optimize.ExpansionSearchPolicy) error { + if !supportedExpansionOrientationPolicy(policy) { + return fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family != "fixed_suffix_expansion" || + decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse || + !decision.StructurallyEligible || !decision.StaticallyEligible { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("expansion orientation tournament has no structurally eligible fixed-suffix target") + } + if len(matching) != 1 { + return fmt.Errorf("expansion orientation tournament matched %d structurally eligible fixed-suffix targets; expected exactly one", len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.PlannedPolicy = policy + decision.SelectionMode = "guarded_tool" + decision.SelectorVersion = string(policy) + decision.EmittedPolicy = policy + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{ + optimize.ExpansionSearchStepwiseForward, + optimize.ExpansionSearchSuffixSeededReverse, + } + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryGuardedDualArm + decision.FallbackReason = "" + + return nil +} + +// applyExpansionOrientationShadow emits orientation-probe-v1 for one +// qualified fixed-suffix target while retaining the exact incumbent as the +// only emitted traversal arm. The generated policy CTE records which arm the +// selector would have chosen without dispatching it. +func applyExpansionOrientationShadow(plan *optimize.Plan) error { + return applyExpansionOrientationShadowPolicy(plan, optimize.ExpansionSearchPolicyOrientationProbeV1) +} + +// applyExpansionOrientationShadowPolicy applies expansion orientation shadow policy. +func applyExpansionOrientationShadowPolicy(plan *optimize.Plan, policy optimize.ExpansionSearchPolicy) error { + if !supportedExpansionOrientationPolicy(policy) { + return fmt.Errorf("unsupported expansion orientation policy %q", policy) + } + var matching []int + for idx, decision := range plan.LoweringPlan.ExpansionSearchStrategy { + if decision.Family != "fixed_suffix_expansion" || + decision.CandidateStrategy != optimize.ExpansionSearchSuffixSeededReverse || + !decision.StructurallyEligible || !decision.StaticallyEligible { + continue + } + matching = append(matching, idx) + } + if len(matching) == 0 { + return fmt.Errorf("expansion orientation shadow has no structurally eligible fixed-suffix target") + } + if len(matching) != 1 { + return fmt.Errorf("expansion orientation shadow matched %d structurally eligible fixed-suffix targets; expected exactly one", len(matching)) + } + + decision := &plan.LoweringPlan.ExpansionSearchStrategy[matching[0]] + decision.SelectedStrategy = optimize.ExpansionSearchStepwiseForward + decision.PlannedPolicy = policy + decision.SelectionMode = "shadow_tool" + decision.SelectorVersion = string(policy) + decision.EmittedPolicy = policy + decision.EmittedCandidates = []optimize.ExpansionSearchStrategy{optimize.ExpansionSearchStepwiseForward} + decision.ExecutionBoundary = optimize.ExpansionSearchExecutionBoundaryInlineStatement + decision.FallbackReason = "" + + return nil +} + +// decodeCypherStringLiteral decodes Cypher escape sequences by interpreting the token as a quoted Go string. func decodeCypherStringLiteral(raw string) (string, error) { if len(raw) < 2 { return "", fmt.Errorf("invalid cypher string literal: %q", raw) diff --git a/cypher/models/pgsql/translate/traversal.go b/cypher/models/pgsql/translate/traversal.go index 8b4e0a0f..3cb3aa07 100644 --- a/cypher/models/pgsql/translate/traversal.go +++ b/cypher/models/pgsql/translate/traversal.go @@ -10,23 +10,24 @@ import ( "github.com/specterops/dawgs/graph" ) -func boundEndpointIDReference(frame *Frame, binding *BoundIdentifier) pgsql.RowColumnReference { +// projectedNodeIDReference returns the scalar ID expression exposed for node by frame. +func projectedNodeIDReference(frameIdentifier pgsql.Identifier, binding *BoundIdentifier) pgsql.Expression { + if binding != nil && binding.IDOnly { + return pgsql.CompoundIdentifier{frameIdentifier, binding.Identifier} + } + return pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{frame.Binding.Identifier, binding.Identifier}, + Identifier: pgsql.CompoundIdentifier{frameIdentifier, binding.Identifier}, Column: pgsql.ColumnID, } } -func boundEndpointInequality(frame *Frame, traversalStep *TraversalStep) pgsql.Expression { - return pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - boundEndpointIDReference(frame, traversalStep.LeftNode), - pgsql.OperatorCypherNotEquals, - boundEndpointIDReference(frame, traversalStep.RightNode), - ), - ) +// boundEndpointIDReference returns the previous-frame scalar ID for a bound traversal endpoint. +func boundEndpointIDReference(frame *Frame, binding *BoundIdentifier) pgsql.Expression { + return projectedNodeIDReference(frame.Binding.Identifier, binding) } +// sourceTargetForTraversalStep returns optimizer coordinates for a step that originated in the source query. func sourceTargetForTraversalStep(part *PatternPart, stepIndex int) (optimize.TraversalStepTarget, bool) { if part == nil || stepIndex < 0 || stepIndex >= len(part.TraversalSteps) { return optimize.TraversalStepTarget{}, false @@ -43,6 +44,26 @@ func sourceTargetForTraversalStep(part *PatternPart, stepIndex int) (optimize.Tr return part.Target.TraversalStep(stepIndex), true } +// shortestPathExecutorDecision returns the planned physical executor for a source traversal step. +func (s *Translator) shortestPathExecutorDecision(part *PatternPart, stepIndex int) (optimize.ShortestPathExecutorDecision, bool) { + target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) + if !hasTarget { + return optimize.ShortestPathExecutorDecision{}, false + } + decision, hasDecision := s.shortestPathExecutorDecisions[target] + return decision, hasDecision +} + +// decisionIsForcedShortest reports whether tooling forced a non-incumbent shortest-path executor. +func decisionIsForcedShortest(translator *Translator, target optimize.TraversalStepTarget) bool { + if translator == nil { + return false + } + decision, found := translator.shortestPathExecutorDecisions[target] + return found && decision.SelectionMode == "forced_tool" +} + +// traversalStepIsFirstForSourceTarget reports whether step is the first translated step for its source target. func traversalStepIsFirstForSourceTarget(part *PatternPart, stepIndex int) bool { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget || stepIndex == 0 { @@ -53,6 +74,7 @@ func traversalStepIsFirstForSourceTarget(part *PatternPart, stepIndex int) bool return !previousHasTarget || previousTarget != target } +// traversalStepIsLastForSourceTarget reports whether step is the final translated step for its source target. func traversalStepIsLastForSourceTarget(part *PatternPart, stepIndex int) bool { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget || stepIndex+1 >= len(part.TraversalSteps) { @@ -63,6 +85,7 @@ func traversalStepIsLastForSourceTarget(part *PatternPart, stepIndex int) bool { return !nextHasTarget || nextTarget != target } +// shouldUseExpandInto reports whether a planned bound-endpoint traversal applies to this source step. func (s *Translator) shouldUseExpandInto(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { if traversalStep == nil || traversalStep.Expansion != nil || !traversalStep.LeftNodeBound || !traversalStep.RightNodeBound { return false @@ -79,6 +102,7 @@ func (s *Translator) shouldUseExpandInto(part *PatternPart, stepIndex int, trave return true } +// traversalDirectionDecision returns the planned direction choice for a source traversal step. func (s *Translator) traversalDirectionDecision(part *PatternPart, stepIndex int) (optimize.TraversalDirectionDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -89,6 +113,7 @@ func (s *Translator) traversalDirectionDecision(part *PatternPart, stepIndex int return decision, hasDecision } +// applyPatternConstraintBalance swaps endpoint constraints and reverses path state when the plan flips traversal direction. func (s *Translator) applyPatternConstraintBalance(part *PatternPart, stepIndex int, constraints *PatternConstraints, traversalStep *TraversalStep) error { if decision, hasDecision := s.traversalDirectionDecision(part, stepIndex); hasDecision { if decision.Flip { @@ -117,6 +142,7 @@ func (s *Translator) applyPatternConstraintBalance(part *PatternPart, stepIndex return nil } +// shortestPathStrategyDecision returns the planned unidirectional or bidirectional strategy for a source step. func (s *Translator) shortestPathStrategyDecision(part *PatternPart, stepIndex int) (optimize.ShortestPathStrategyDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -127,6 +153,7 @@ func (s *Translator) shortestPathStrategyDecision(part *PatternPart, stepIndex i return decision, hasDecision } +// useBidirectionalShortestPathStrategy reports whether a qualified plan selects bidirectional search for step. func (s *Translator) useBidirectionalShortestPathStrategy(part *PatternPart, stepIndex int, traversalStep *TraversalStep) (bool, error) { if decision, hasDecision := s.shortestPathStrategyDecision(part, stepIndex); hasDecision { if decision.Strategy != optimize.ShortestPathStrategyBidirectional { @@ -153,6 +180,7 @@ func (s *Translator) useBidirectionalShortestPathStrategy(part *PatternPart, ste return false, nil } +// shortestPathFilterDecisionsForStep returns every planned filter materialization for a source traversal step. func (s *Translator) shortestPathFilterDecisionsForStep(part *PatternPart, stepIndex int) []optimize.ShortestPathFilterDecision { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -162,6 +190,7 @@ func (s *Translator) shortestPathFilterDecisionsForStep(part *PatternPart, stepI return s.shortestPathFilterDecisions[target] } +// applyShortestPathFilterMaterialization enables terminal or endpoint-pair filters selected for the source step. func (s *Translator) applyShortestPathFilterMaterialization(part *PatternPart, stepIndex int, traversalStep *TraversalStep, expansionModel *Expansion) { for _, decision := range s.shortestPathFilterDecisionsForStep(part, stepIndex) { switch decision.Mode { @@ -180,6 +209,7 @@ func (s *Translator) applyShortestPathFilterMaterialization(part *PatternPart, s } } +// hasLimitPushdownDecision reports whether target has the requested limit-pushdown mode. func (s *Translator) hasLimitPushdownDecision(part *PatternPart, stepIndex int, mode optimize.LimitPushdownMode) bool { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -195,6 +225,7 @@ func (s *Translator) hasLimitPushdownDecision(part *PatternPart, stepIndex int, return false } +// allowLimitPushdownForStep authorizes the step's frame to consume a matching planned limit internally. func (s *Translator) allowLimitPushdownForStep(part *PatternPart, stepIndex int, traversalStep *TraversalStep) { if traversalStep == nil || traversalStep.Frame == nil { return @@ -215,14 +246,19 @@ func (s *Translator) allowLimitPushdownForStep(part *PatternPart, stepIndex int, } } +// buildBoundEndpointTraversalPattern emits a one-hop join between two endpoints already visible in the previous frame. func (s *Translator) buildBoundEndpointTraversalPattern(partFrame *Frame, traversalStep *TraversalStep) (pgsql.Query, error) { if partFrame == nil || partFrame.Previous == nil { return pgsql.Query{}, errors.New("expected previous frame for bound endpoint traversal") } var ( - previousFrame = partFrame.Previous - nextSelect = pgsql.Select{ + previousFrame = partFrame.Previous + edgeConstraint = pgsql.OptionalAnd( + traversalStep.EdgeJoinCondition, + traversalStep.RightNodeJoinCondition, + ) + nextSelect = pgsql.Select{ Projection: traversalStep.Projection, From: []pgsql.FromClause{{ Source: pgsql.TableReference{ @@ -234,25 +270,38 @@ func (s *Translator) buildBoundEndpointTraversalPattern(partFrame *Frame, traver Binding: models.OptionalValue(traversalStep.Edge.Identifier), }, JoinOperator: pgsql.JoinOperator{ - JoinType: pgsql.JoinTypeInner, - Constraint: pgsql.OptionalAnd( - traversalStep.EdgeJoinCondition, - traversalStep.RightNodeJoinCondition, - ), + JoinType: pgsql.JoinTypeInner, + Constraint: edgeConstraint, }, }}, }}, } ) + if traversalStep.Direction == graph.DirectionBoth { + edgeConstraint = buildDirectionlessPairwiseEdgeConstraintForRefs( + boundEndpointIDReference(previousFrame, traversalStep.LeftNode), + boundEndpointIDReference(previousFrame, traversalStep.RightNode), + traversalStep.Edge.Identifier, + ) + nextSelect.From[0].Joins[0].JoinOperator.Constraint = edgeConstraint + } + if referencesUnwind, err := expressionReferencesUnwindBinding(edgeConstraint, s.query.CurrentPart().unwindClauses); err != nil { + return pgsql.Query{}, err + } else if referencesUnwind { + // An UNWIND alias is appended as a comma source after this builder + // returns. PostgreSQL JOIN ... ON cannot see a later comma source, while + // WHERE can see the complete FROM list. Keep the exact pair predicate + // and edge scan together in that shared scope. + edgeJoin := nextSelect.From[0].Joins[0] + nextSelect.From[0].Joins = nil + nextSelect.From = append(nextSelect.From, pgsql.FromClause{Source: edgeJoin.Table}) + nextSelect.Where = pgsql.OptionalAnd(edgeConstraint, nextSelect.Where) + } nextSelect.Where = pgsql.OptionalAnd(traversalStep.LeftNodeConstraints, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(traversalStep.EdgeConstraints.Expression, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(traversalStep.RightNodeConstraints, nextSelect.Where) - if traversalStep.Direction == graph.DirectionBoth && traversalStep.LeftNode.Identifier != traversalStep.RightNode.Identifier { - nextSelect.Where = pgsql.OptionalAnd(boundEndpointInequality(previousFrame, traversalStep), nextSelect.Where) - } - return pgsql.Query{ Body: nextSelect, }, nil @@ -368,12 +417,16 @@ func (s *Translator) buildTraversalPatternRootWithOuterCorrelation(partFrame *Fr } } +// buildTraversalPatternRoot emits the first node source, constraints, and projection for a traversal pattern. func (s *Translator) buildTraversalPatternRoot(partFrame *Frame, traversalStep *TraversalStep) (pgsql.Query, error) { if traversalStep.Direction == graph.DirectionBoth { return s.buildDirectionlessTraversalPatternRoot(traversalStep) } - if traversalStep.UseExpandInto { + // Dual-bound fixed hops must always use the exact pair join. The optimizer + // decision records and measures this shape, but correctness must not depend + // on that analysis recognizing every supported binding source. + if traversalStep.UseExpandInto || (traversalStep.LeftNodeBound && traversalStep.RightNodeBound) { return s.buildBoundEndpointTraversalPattern(partFrame, traversalStep) } @@ -558,8 +611,12 @@ func (s *Translator) buildTraversalPatternRoot(partFrame *Frame, traversalStep * }, nil } +// buildTraversalPatternStep emits one relationship join, terminal node join, constraints, and projection frame. func (s *Translator) buildTraversalPatternStep(partFrame *Frame, traversalStep *TraversalStep) (pgsql.Query, error) { - if traversalStep.UseExpandInto { + // Keep the dual-bound semantic fallback independent of optimizer coverage; + // otherwise a missed decision can introduce an uncorrelated terminal-node + // join and multiply the outer bag. + if traversalStep.UseExpandInto || (traversalStep.LeftNodeBound && traversalStep.RightNodeBound) { return s.buildBoundEndpointTraversalPattern(partFrame, traversalStep) } @@ -626,6 +683,7 @@ func (s *Translator) buildTraversalPatternStep(partFrame *Frame, traversalStep * }, nil } +// translateTraversalPatternPart prepares source targets, constraints, and state for translating one pattern part. func (s *Translator) translateTraversalPatternPart(part *PatternPart, isolatedProjection bool, allowProjectionPruning bool) error { var scopeSnapshot *Scope @@ -679,6 +737,7 @@ func (s *Translator) translateTraversalPatternPart(part *PatternPart, isolatedPr return nil } +// applyExpansionSuffixPushdown attaches planned fixed-suffix predicates and records any applied predicate placement. func (s *Translator) applyExpansionSuffixPushdown(part *PatternPart) (int, error) { if part == nil || !part.HasTarget { return applyExpansionSuffixPushdown(part) @@ -699,6 +758,7 @@ func (s *Translator) applyExpansionSuffixPushdown(part *PatternPart) (int, error for _, decision := range decisions { if decision.SuffixLength <= 0 || + !decision.ApplySupplemental || decision.SuffixStartStep <= target.StepIndex || decision.SuffixEndStep < decision.SuffixStartStep || decision.SuffixEndStep-decision.SuffixStartStep+1 != decision.SuffixLength { @@ -750,10 +810,120 @@ func (s *Translator) applyExpansionSuffixPushdown(part *PatternPart) (int, error return applied, nil } +// traversalStepHasContinuation reports whether another translated step follows in the pattern part. func traversalStepHasContinuation(part *PatternPart, stepIndex int) bool { return part != nil && stepIndex+1 < len(part.TraversalSteps) } +// fieldRequirementAllowsIDOnly reports whether all external uses of symbol can consume a scalar entity ID. +func fieldRequirementAllowsIDOnly(decision optimize.FieldRequirementDecision) bool { + observesID := false + for _, use := range decision.Uses { + for _, field := range use.Fields { + if !use.Internal && field == optimize.FieldRequirementEntityID { + observesID = true + } + + if !use.Internal && field != optimize.FieldRequirementEntityID { + return false + } + + if field == optimize.FieldRequirementFullEntity || field == optimize.FieldRequirementFullPath { + return false + } + } + } + + return observesID +} + +// fieldRequirementAllowsIDOnlyContinuation reports whether later pattern use can continue from scalar ID state. +func fieldRequirementAllowsIDOnlyContinuation(decision optimize.FieldRequirementDecision) bool { + for _, use := range decision.Uses { + for _, field := range use.Fields { + if field == optimize.FieldRequirementFullEntity || field == optimize.FieldRequirementFullPath { + return false + } + + if !use.Internal && field != optimize.FieldRequirementEntityID { + return false + } + } + } + + return true +} + +// traversalStepContinuesFromBinding reports whether the next step starts from binding. +func traversalStepContinuesFromBinding(part *PatternPart, stepIndex int, binding *BoundIdentifier) bool { + if part == nil || binding == nil || stepIndex < 0 || stepIndex+1 >= len(part.TraversalSteps) { + return false + } + + currentStep := part.TraversalSteps[stepIndex] + nextStep := part.TraversalSteps[stepIndex+1] + + return currentStep != nil && nextStep != nil && + currentStep.RightNode == binding && nextStep.LeftNode == binding +} + +// applyIDOnlyNodeProjection replaces an eligible node composite projection with its scalar ID. +func (s *Translator) applyIDOnlyNodeProjection(part *PatternPart, stepIndex int, binding *BoundIdentifier) bool { + if part == nil || binding == nil || !part.HasTarget { + return false + } + + var ( + isContinuation = traversalStepContinuesFromBinding(part, stepIndex, binding) + isTerminal = !traversalStepHasContinuation(part, stepIndex) + ) + if !isContinuation && !isTerminal { + return false + } + + if part.PatternBinding != nil { + for _, pathSymbol := range s.scope.Symbols(part.PatternBinding) { + if decision, found := s.fieldRequirementDecisions[part.Target.QueryPartIndex][pathSymbol.String()]; found { + for _, field := range decision.Fields { + if field == optimize.FieldRequirementFullPath { + return false + } + } + } + } + } + + foundDecision := false + for _, symbol := range s.scope.Symbols(binding) { + if decision, found := s.fieldRequirementDecisions[part.Target.QueryPartIndex][symbol.String()]; found { + foundDecision = true + allowsIDOnly := fieldRequirementAllowsIDOnly(decision) + if isContinuation { + allowsIDOnly = fieldRequirementAllowsIDOnlyContinuation(decision) + } + + if !allowsIDOnly { + return false + } + } + } + if foundDecision { + binding.IDOnly = true + return true + } + + // Anonymous or otherwise unobserved intermediate nodes have no source-level + // field-requirement decision. Their identity is still required to join the + // next relationship, so carry that identity as a scalar between steps. + if isContinuation && !foundDecision { + binding.IDOnly = true + return true + } + + return false +} + +// relationshipIDReference returns the scalar relationship ID exposed by a composite or ID-only binding. func relationshipIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expression { if binding != nil && binding.DataType == pgsql.EdgeComposite { return pathCompositeColumnReference(scope, binding, pgsql.ColumnID) @@ -762,6 +932,7 @@ func relationshipIDReference(scope *Scope, binding *BoundIdentifier) pgsql.Expre return pathEdgeIDReference(scope, binding) } +// relationshipIDNotInPath builds the edge-uniqueness predicate for a relationship and accumulated path. func relationshipIDNotInPath(edgeID, pathIDs pgsql.Expression) pgsql.Expression { return pgsql.NewBinaryExpression( edgeID, @@ -770,6 +941,7 @@ func relationshipIDNotInPath(edgeID, pathIDs pgsql.Expression) pgsql.Expression ) } +// previousRelationshipUniquenessConstraint excludes a relationship ID already used by a prior fixed step. func previousRelationshipUniquenessConstraint(scope *Scope, part *PatternPart, stepIndex int, traversalStep *TraversalStep) pgsql.Expression { if scope == nil || part == nil || stepIndex <= 0 || traversalStep == nil || traversalStep.Edge == nil { return nil @@ -840,6 +1012,7 @@ func expansionPreviousRelationshipUniquenessConstraint(scope *Scope, part *Patte return constraint } +// projectionPruningDecision returns the planned omitted fields for a source traversal step. func (s *Translator) projectionPruningDecision(part *PatternPart, stepIndex int) (optimize.ProjectionPruningDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -850,6 +1023,7 @@ func (s *Translator) projectionPruningDecision(part *PatternPart, stepIndex int) return decision, hasDecision } +// prepareProjectionPruning builds the SQL model fragment responsible for prepare projection pruning. func (s *Translator) prepareProjectionPruning(part *PatternPart, stepIndex int, traversalStep *TraversalStep) { decision, hasDecision := s.projectionPruningDecision(part, stepIndex) if !hasDecision || traversalStep == nil { @@ -873,6 +1047,7 @@ func (s *Translator) prepareProjectionPruning(part *PatternPart, stepIndex int, } } +// latePathMaterializationDecision returns the requested deferred materialization mode for target. func (s *Translator) latePathMaterializationDecision(part *PatternPart, stepIndex int, mode optimize.LatePathMaterializationMode) (optimize.LatePathMaterializationDecision, bool) { target, hasTarget := sourceTargetForTraversalStep(part, stepIndex) if !hasTarget { @@ -888,6 +1063,7 @@ func (s *Translator) latePathMaterializationDecision(part *PatternPart, stepInde return optimize.LatePathMaterializationDecision{}, false } +// applyPathEdgeIDMaterialization replaces a path binding with ordered edge-ID state for later hydration. func (s *Translator) applyPathEdgeIDMaterialization(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { if traversalStep == nil || traversalStep.Edge == nil || @@ -903,6 +1079,7 @@ func (s *Translator) applyPathEdgeIDMaterialization(part *PatternPart, stepIndex return true } +// unexportFrameBinding removes binding and its alias from a frame's exported identifiers. func unexportFrameBinding(frame *Frame, identifier pgsql.Identifier) bool { if frame == nil { return false @@ -913,6 +1090,7 @@ func unexportFrameBinding(frame *Frame, identifier pgsql.Identifier) bool { return exported } +// traversalStepBindingBound reports whether binding is an endpoint or relationship already bound for step. func traversalStepBindingBound(traversalStep *TraversalStep, binding *BoundIdentifier) bool { if traversalStep == nil || binding == nil { return false @@ -929,6 +1107,7 @@ func traversalStepBindingBound(traversalStep *TraversalStep, binding *BoundIdent return false } +// unexportPrunedNodeBinding removes a pruned node and its aliases unless another step still requires the binding. func unexportPrunedNodeBinding(traversalStep *TraversalStep, binding *BoundIdentifier) bool { if binding == nil || traversalStepBindingBound(traversalStep, binding) { return false @@ -937,6 +1116,7 @@ func unexportPrunedNodeBinding(traversalStep *TraversalStep, binding *BoundIdent return unexportFrameBinding(traversalStep.Frame, binding.Identifier) } +// pruneTraversalStepProjectionExports removes planned node, relationship, and path exports from a fixed step. func pruneTraversalStepProjectionExports(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { var applied bool @@ -949,6 +1129,7 @@ func pruneTraversalStepProjectionExports(part *PatternPart, stepIndex int, trave return applied } +// pruneExpansionStepProjectionExports removes planned node, relationship, and path exports from an expansion step. func pruneExpansionStepProjectionExports(part *PatternPart, stepIndex int, traversalStep *TraversalStep) bool { if traversalStep == nil || traversalStep.Expansion == nil { return false @@ -966,6 +1147,7 @@ func pruneExpansionStepProjectionExports(part *PatternPart, stepIndex int, trave return applied } +// translateTraversalPatternPartWithoutExpansion emits each fixed step, applying pruning and scalar-ID continuation where qualified. func (s *Translator) translateTraversalPatternPartWithoutExpansion(part *PatternPart, stepIndex int, traversalStep *TraversalStep, allowProjectionPruning bool) error { isFirstTraversalStep := stepIndex == 0 @@ -1062,6 +1244,12 @@ func (s *Translator) translateTraversalPatternPartWithoutExpansion(part *Pattern } } + leftNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.LeftNode) + rightNodeIDOnly := s.applyIDOnlyNodeProjection(part, stepIndex, traversalStep.RightNode) + if leftNodeIDOnly || rightNodeIDOnly { + s.recordLowering(optimize.LoweringFieldRequirements) + } + if boundProjections, err := buildVisibleProjections(s.scope); err != nil { return err } else { diff --git a/cypher/models/pgsql/translate/traversal_directionless.go b/cypher/models/pgsql/translate/traversal_directionless.go index 7f51c0ba..2402f122 100644 --- a/cypher/models/pgsql/translate/traversal_directionless.go +++ b/cypher/models/pgsql/translate/traversal_directionless.go @@ -10,15 +10,23 @@ import ( // directionlessSingleBoundPlan is a "generic" plan to hold the details necessary for constructing // a single-bound traversal step type directionlessSingleBoundPlan struct { - boundNodeConstraints pgsql.Expression + // boundNodeConstraints retains the bound node constraints while directionlessSingleBoundPlan is assembled or evaluated. + boundNodeConstraints pgsql.Expression + // boundNodeJoinCondition retains the bound node join condition while directionlessSingleBoundPlan is assembled or evaluated. boundNodeJoinCondition pgsql.Expression - nodeJoinBinding pgsql.Identifier - nodeJoinConstraint pgsql.Expression - whereConstraint pgsql.Expression - boundNode *BoundIdentifier - unboundNodeIdentifier pgsql.Identifier + // nodeJoinBinding retains the node join binding while directionlessSingleBoundPlan is assembled or evaluated. + nodeJoinBinding pgsql.Identifier + // nodeJoinConstraint retains the node join constraint while directionlessSingleBoundPlan is assembled or evaluated. + nodeJoinConstraint pgsql.Expression + // whereConstraint retains the where constraint while directionlessSingleBoundPlan is assembled or evaluated. + whereConstraint pgsql.Expression + // boundNode retains the bound node while directionlessSingleBoundPlan is assembled or evaluated. + boundNode *BoundIdentifier + // unboundNodeIdentifier retains the unbound node identifier while directionlessSingleBoundPlan is assembled or evaluated. + unboundNodeIdentifier pgsql.Identifier } +// buildDirectionlessSingleBoundPlan builds directionless single bound plan. func buildDirectionlessSingleBoundPlan(traversalStep *TraversalStep) directionlessSingleBoundPlan { // Partition node constraints rightJoinLocal, rightJoinExternal := partitionConstraintByLocality( @@ -78,6 +86,7 @@ func (s *Translator) buildDirectionlessTraversalPatternRoot(traversalStep *Trave return s.buildUnboundDirectionlessTraversalPatternRoot(traversalStep) } +// buildDirectionlessPairwiseEdgeConstraintForRefs builds directionless pairwise edge constraint for refs. func buildDirectionlessPairwiseEdgeConstraintForRefs(left pgsql.Expression, right pgsql.Expression, edge pgsql.Identifier) pgsql.Expression { // ((left).id = (eN).start_id AND (right).id = (eN).end_id) leftToRight := pgsql.NewParenthetical( @@ -174,11 +183,6 @@ func (s *Translator) buildPairwiseDirectionlessTraversalPatternRoot(traversalSte nextSelect.Where = pgsql.OptionalAnd(leftJoinExternal, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(rightJoinExternal, nextSelect.Where) - // Only apply endpoint inequality when the bound nodes are different, to allow for self-referential relationships - if traversalStep.LeftNode.Identifier != traversalStep.RightNode.Identifier { - nextSelect.Where = pgsql.OptionalAnd(boundEndpointInequality(traversalStep.Frame.Previous, traversalStep), nextSelect.Where) - } - return pgsql.Query{Body: nextSelect}, nil } @@ -244,15 +248,11 @@ func (s *Translator) buildUnboundDirectionlessTraversalPatternRoot(traversalStep nextSelect.Where = pgsql.OptionalAnd(leftJoinExternal, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(traversalStep.EdgeConstraints.Expression, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(rightJoinExternal, nextSelect.Where) - - // AND (n0.id <> n1.id) - ensures edges are properly constrained to the specified nodes nextSelect.Where = pgsql.OptionalAnd( - pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - pgsql.CompoundIdentifier{traversalStep.LeftNode.Identifier, pgsql.ColumnID}, - pgsql.OperatorCypherNotEquals, - pgsql.CompoundIdentifier{traversalStep.RightNode.Identifier, pgsql.ColumnID}, - ), + buildDirectionlessPairwiseEdgeConstraintForRefs( + pgsql.CompoundIdentifier{traversalStep.LeftNode.Identifier, pgsql.ColumnID}, + pgsql.CompoundIdentifier{traversalStep.RightNode.Identifier, pgsql.ColumnID}, + traversalStep.Edge.Identifier, ), nextSelect.Where, ) @@ -316,18 +316,11 @@ func (s *Translator) buildSingleBoundDirectionlessTraversalRoot(traversalStep *T }) nextSelect.Where = plan.whereConstraint - - // selected node is not joined here, so the guard must reference the bound node through the previous frame nextSelect.Where = pgsql.OptionalAnd( - pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - pgsql.RowColumnReference{ - Identifier: pgsql.CompoundIdentifier{previousFrame.Binding.Identifier, plan.boundNode.Identifier}, - Column: pgsql.ColumnID, - }, - pgsql.OperatorCypherNotEquals, - pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, - ), + buildDirectionlessPairwiseEdgeConstraintForRefs( + boundEndpointIDReference(previousFrame, plan.boundNode), + pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, + traversalStep.Edge.Identifier, ), nextSelect.Where, ) @@ -337,6 +330,7 @@ func (s *Translator) buildSingleBoundDirectionlessTraversalRoot(traversalStep *T }, nil } +// buildSelfReferentialDirectionlessTraversalRoot builds self referential directionless traversal root. func (s *Translator) buildSelfReferentialDirectionlessTraversalRoot(traversalStep *TraversalStep) (pgsql.Query, error) { var ( // Partition node constraints @@ -392,6 +386,7 @@ func (s *Translator) buildSelfReferentialDirectionlessTraversalRoot(traversalSte // UNDIRECTED TRAVERSALS **WITH** OUTER CORRELATION // +// buildDirectionlessTraversalPatternRootWithOuterCorrelation builds directionless traversal pattern root with outer correlation. func (s *Translator) buildDirectionlessTraversalPatternRootWithOuterCorrelation(traversalStep *TraversalStep) (pgsql.Query, error) { if traversalStep.UseExpandInto { return s.buildBoundEndpointTraversalPattern(traversalStep.Frame, traversalStep) @@ -410,6 +405,7 @@ func (s *Translator) buildDirectionlessTraversalPatternRootWithOuterCorrelation( return s.buildUnboundDirectionlessTraversalPatternRoot(traversalStep) } +// buildSingleBoundDirectionlessTraversalRootWithOuterCorrelation builds single bound directionless traversal root with outer correlation. func (s *Translator) buildSingleBoundDirectionlessTraversalRootWithOuterCorrelation(traversalStep *TraversalStep) (pgsql.Query, error) { previousFrame, hasPreviousFrame := s.previousValidFrame(traversalStep.Frame) @@ -449,15 +445,11 @@ func (s *Translator) buildSingleBoundDirectionlessTraversalRootWithOuterCorrelat nextSelect.Where = pgsql.OptionalAnd(plan.boundNodeConstraints, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(plan.boundNodeJoinCondition, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(plan.whereConstraint, nextSelect.Where) - - // selected node is not joined here, so the guard must reference the bound node through the previous frame nextSelect.Where = pgsql.OptionalAnd( - pgsql.NewParenthetical( - pgsql.NewBinaryExpression( - boundEndpointIDReference(previousFrame, plan.boundNode), - pgsql.OperatorCypherNotEquals, - pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, - ), + buildDirectionlessPairwiseEdgeConstraintForRefs( + boundEndpointIDReference(previousFrame, plan.boundNode), + pgsql.CompoundIdentifier{plan.unboundNodeIdentifier, pgsql.ColumnID}, + traversalStep.Edge.Identifier, ), nextSelect.Where, ) @@ -504,10 +496,5 @@ func (s *Translator) buildPairwiseDirectionlessTraversalPatternRootWithOuterCorr nextSelect.Where = pgsql.OptionalAnd(leftJoinExternal, nextSelect.Where) nextSelect.Where = pgsql.OptionalAnd(rightJoinExternal, nextSelect.Where) - // Only apply endpoint inequality when the bound nodes are different, to allow for self-referential relationships - if traversalStep.LeftNode.Identifier != traversalStep.RightNode.Identifier { - nextSelect.Where = pgsql.OptionalAnd(boundEndpointInequality(traversalStep.Frame.Previous, traversalStep), nextSelect.Where) - } - return pgsql.Query{Body: nextSelect}, nil } diff --git a/cypher/models/pgsql/translate/traversal_test.go b/cypher/models/pgsql/translate/traversal_test.go new file mode 100644 index 00000000..5cff5407 --- /dev/null +++ b/cypher/models/pgsql/translate/traversal_test.go @@ -0,0 +1,54 @@ +package translate + +import ( + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// TestDualBoundTraversalUsesExactPairJoinWithoutOptimizerMarker verifies fixed +// traversal correctness does not depend on ExpandInto analysis being exhaustive. +func TestDualBoundTraversalUsesExactPairJoinWithoutOptimizerMarker(t *testing.T) { + previousFrame := &Frame{Binding: &BoundIdentifier{Identifier: "s0"}} + currentFrame := &Frame{ + Previous: previousFrame, + Binding: &BoundIdentifier{Identifier: "s1"}, + } + left := &BoundIdentifier{Identifier: "n0"} + right := &BoundIdentifier{Identifier: "n1"} + edge := &BoundIdentifier{Identifier: "e0"} + step := &TraversalStep{ + Frame: currentFrame, + Direction: graph.DirectionOutbound, + LeftNode: left, + LeftNodeBound: true, + Edge: edge, + EdgeConstraints: &Constraint{}, + EdgeJoinCondition: pgsql.NewBinaryExpression( + boundEndpointIDReference(previousFrame, left), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{edge.Identifier, pgsql.ColumnStartID}, + ), + RightNode: right, + RightNodeBound: true, + RightNodeJoinCondition: pgsql.NewBinaryExpression( + boundEndpointIDReference(previousFrame, right), + pgsql.OperatorEquals, + pgsql.CompoundIdentifier{edge.Identifier, pgsql.ColumnEndID}, + ), + } + + translator := &Translator{query: &Query{Parts: []*QueryPart{{}}}} + query, err := translator.buildTraversalPatternRoot(currentFrame, step) + require.NoError(t, err) + + selectBody, ok := query.Body.(pgsql.Select) + require.True(t, ok) + require.Len(t, selectBody.From, 1) + require.Len(t, selectBody.From[0].Joins, 1, "dual-bound fallback must not add an uncorrelated terminal-node join") + edgeTable, ok := selectBody.From[0].Joins[0].Table.(pgsql.TableReference) + require.True(t, ok) + require.Equal(t, pgsql.CompoundIdentifier{pgsql.TableEdge}, edgeTable.Name) +} diff --git a/cypher/models/pgsql/translate/with.go b/cypher/models/pgsql/translate/with.go index 38860366..624afddb 100644 --- a/cypher/models/pgsql/translate/with.go +++ b/cypher/models/pgsql/translate/with.go @@ -6,6 +6,7 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// translateWith closes the current query part, projects WITH items, and opens the scope consumed by the next part. func (s *Translator) translateWith() error { currentPart := s.query.CurrentPart() @@ -14,6 +15,7 @@ func (s *Translator) translateWith() error { } else { var ( projectedItems = pgsql.NewIdentifierSet() + materialized []*BoundIdentifier // aggregatedItems contains a set of symbols of projected aggregate functions. aggregatedItems = pgsql.NewSymbolTable() @@ -124,8 +126,11 @@ func (s *Translator) translateWith() error { currentPart.projections.Items[idx].Alias = pgsql.AsOptionalIdentifier(projectedBinding.Identifier) } - // Assign the frame to the binding's last projection backref - projectedBinding.MaterializedBy(currentPart.Frame) + // Delay the back-reference update until every select item has + // been built. Path projections may depend on node bindings that + // appear earlier in a greedy WITH projection, and those + // dependencies must still reference the input frame here. + materialized = append(materialized, projectedBinding) // Reveal and export the identifier in the current multipart query part's frame currentPart.Frame.Reveal(projectedBinding.Identifier) @@ -143,8 +148,7 @@ func (s *Translator) translateWith() error { // Track this projected item for scope pruning projectedItems.Add(binding.Identifier) - // Assign the frame to the binding's last projection backref - binding.LastProjection = currentPart.Frame + materialized = append(materialized, binding) // Reveal and export the identifier in the current multipart query part's frame currentPart.Frame.Reveal(binding.Identifier) @@ -156,6 +160,9 @@ func (s *Translator) translateWith() error { } } } + for _, binding := range materialized { + binding.MaterializedBy(currentPart.Frame) + } if !aggregatedItems.IsEmpty() { currentPart.projections.GroupBy = append(currentPart.projections.GroupBy, groupByItems...) diff --git a/cypher/models/walk/walk_pgsql.go b/cypher/models/walk/walk_pgsql.go index f3ac3945..6ab08c84 100644 --- a/cypher/models/walk/walk_pgsql.go +++ b/cypher/models/walk/walk_pgsql.go @@ -6,10 +6,12 @@ import ( "github.com/specterops/dawgs/cypher/models/pgsql" ) +// pgsqlSyntaxNodeSliceTypeConvert widens a concrete PostgreSQL syntax-node slice for the generic walker without changing element order. func pgsqlSyntaxNodeSliceTypeConvert[F any, FS []F](fs FS) ([]pgsql.SyntaxNode, error) { return ConvertSliceType[pgsql.SyntaxNode](fs) } +// newSQLCaseWalkCursor creates a cursor that visits a CASE operand, conditions, results, and fallback in SQL order. func newSQLCaseWalkCursor(node pgsql.SyntaxNode, caseExpr pgsql.Case) (*Cursor[pgsql.SyntaxNode], error) { if len(caseExpr.Conditions) != len(caseExpr.Then) { return nil, fmt.Errorf("case expression has %d conditions and %d then expressions", len(caseExpr.Conditions), len(caseExpr.Then)) @@ -34,6 +36,7 @@ func newSQLCaseWalkCursor(node pgsql.SyntaxNode, caseExpr pgsql.Case) (*Cursor[p return nextCursor, nil } +// newSQLWalkCursor creates a structural cursor for the concrete PostgreSQL AST node type. func newSQLWalkCursor(node pgsql.SyntaxNode) (*Cursor[pgsql.SyntaxNode], error) { if isNilNode(node) { return nil, fmt.Errorf("unable to negotiate sql type %T into a translation cursor", node) @@ -210,15 +213,22 @@ func newSQLWalkCursor(node pgsql.SyntaxNode) (*Cursor[pgsql.SyntaxNode], error) }, nil case *pgsql.EdgeArrayFromPathIDs: + branches := []pgsql.SyntaxNode{typedNode.PathIDs} + if typedNode.GraphID != nil { + branches = append(branches, typedNode.GraphID) + } return &Cursor[pgsql.SyntaxNode]{ Node: node, - Branches: []pgsql.SyntaxNode{typedNode.PathIDs}, + Branches: branches, }, nil case pgsql.FunctionCall: if branches, err := pgsqlSyntaxNodeSliceTypeConvert(typedNode.Parameters); err != nil { return nil, err } else { + for _, orderBy := range typedNode.OrderBy { + branches = append(branches, orderBy) + } return &Cursor[pgsql.SyntaxNode]{ Node: node, Branches: branches, @@ -229,6 +239,9 @@ func newSQLWalkCursor(node pgsql.SyntaxNode) (*Cursor[pgsql.SyntaxNode], error) if branches, err := pgsqlSyntaxNodeSliceTypeConvert(typedNode.Parameters); err != nil { return nil, err } else { + for _, orderBy := range typedNode.OrderBy { + branches = append(branches, orderBy) + } return &Cursor[pgsql.SyntaxNode]{ Node: node, Branches: branches, diff --git a/cypher/test/cases/mutation_tests.json b/cypher/test/cases/mutation_tests.json index dc73b031..3b8338c1 100644 --- a/cypher/test/cases/mutation_tests.json +++ b/cypher/test/cases/mutation_tests.json @@ -44,7 +44,7 @@ "name": "JD's Create User Example", "type": "string_match", "details": { - "query": "merge (x:Base {objectid: '\u003cobjId\u003e'}) set x:User, x.name = 'BOB@TEST.LAB' set x += {arr: ['abc', 'def', 'ghi']} return x", + "query": "merge (x:Base {objectid: ''}) set x:User, x.name = 'BOB@TEST.LAB' set x += {arr: ['abc', 'def', 'ghi']} return x", "fitness": 6 } }, @@ -52,7 +52,7 @@ "name": "JD's Create Edges Example", "type": "string_match", "details": { - "query": "match (x) match (y) merge (x)-[:Edge]-\u003e(y)", + "query": "match (x) match (y) merge (x)-[:Edge]->(y)", "fitness": 1 } }, @@ -100,7 +100,7 @@ "name": "Create relationship", "type": "string_match", "details": { - "query": "create p = (:Label {p: '1234'})-[:Link {r: 1234}]-\u003e(b {p: '4321'}) return p", + "query": "create p = (:Label {p: '1234'})-[:Link {r: 1234}]->(b {p: '4321'}) return p", "fitness": 12 } }, @@ -108,7 +108,7 @@ "name": "Create relationship with decimal properties parameter", "type": "string_match", "details": { - "query": "create p = (:Label {p: '1234'})-[:Link $1]-\u003e(b {p: '4321'}) return p", + "query": "create p = (:Label {p: '1234'})-[:Link $1]->(b {p: '4321'}) return p", "fitness": 9 } }, @@ -116,7 +116,7 @@ "name": "Create relationship with named properties parameter", "type": "string_match", "details": { - "query": "create p = (:Label {p: '1234'})-[:Link $named]-\u003e(b {p: '4321'}) return p", + "query": "create p = (:Label {p: '1234'})-[:Link $named]->(b {p: '4321'}) return p", "fitness": 9 } }, @@ -124,7 +124,7 @@ "name": "Create relationship with matching", "type": "string_match", "details": { - "query": "match (a), (b) where a.name = 'a' and b.linked = id(a) create p = (a)-[:Linked]-\u003e(b) return p", + "query": "match (a), (b) where a.name = 'a' and b.linked = id(a) create p = (a)-[:Linked]->(b) return p", "fitness": 12 } }, @@ -248,6 +248,86 @@ "query": "match (a:Thing1), (b:Thing2) detach delete a, b return b", "fitness": 4 } + }, + { + "name": "LOGIC-04 filtered relationship delete preserves mutation binding", + "type": "string_match", + "details": { + "query": "match (s:RegressionKind05)-[r:RegressionKind06]->(e:RegressionKind07) where e.objectid = $object_id and r.shoulddelete = $should_delete delete r", + "fitness": 16 + } + }, + { + "name": "LOGIC-04 filtered detach node delete preserves mutation binding", + "type": "string_match", + "details": { + "query": "match (n:RegressionKind08) where n.objectid = $object_id detach delete n", + "fitness": 9 + } + }, + { + "name": "REC-01 inbound reconciliation delete with thirty relationship kinds", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->(e:RegressionKind31) where e.objectid = $object_id delete r", + "fitness": 8 + } + }, + { + "name": "REC-02 outbound reconciliation delete with thirty relationship kinds", + "type": "string_match", + "details": { + "query": "match (s:RegressionKind31)-[r:RegressionKind01|RegressionKind02|RegressionKind03|RegressionKind04|RegressionKind05|RegressionKind06|RegressionKind07|RegressionKind08|RegressionKind09|RegressionKind10|RegressionKind11|RegressionKind12|RegressionKind13|RegressionKind14|RegressionKind15|RegressionKind16|RegressionKind17|RegressionKind18|RegressionKind19|RegressionKind20|RegressionKind21|RegressionKind22|RegressionKind23|RegressionKind24|RegressionKind25|RegressionKind26|RegressionKind27|RegressionKind28|RegressionKind29|RegressionKind30]->() where s.objectid = $object_id delete r", + "fitness": 8 + } + }, + { + "name": "REC-03 inbound primary group relationship delete", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid = $object_id and r.isprimarygroup = $flag delete r", + "fitness": 14 + } + }, + { + "name": "REC-03 outbound primary group relationship delete", + "type": "string_match", + "details": { + "query": "match (s:RegressionKind31)-[r:RegressionKind32]->() where s.objectid = $object_id and r.isprimarygroup = $flag delete r", + "fitness": 14 + } + }, + { + "name": "REC-04 endpoint object ID list relationship delete", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind32]->(e:RegressionKind31) where e.objectid in $object_ids delete r", + "fitness": 8 + } + }, + { + "name": "REC-06 delegated enrollment relationship delete by endpoint IDs", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind37]->(e:RegressionKind35) where id(e) in $template_ids delete r", + "fitness": 4 + } + }, + { + "name": "REC-07 HostsCAService relationship delete", + "type": "string_match", + "details": { + "query": "match ()-[r:RegressionKind39]->(e:RegressionKind38) where e.objectid = $object_id delete r", + "fitness": 10 + } + }, + { + "name": "REC-08 AD entity detach delete by object ID list", + "type": "string_match", + "details": { + "query": "match (n:RegressionKind31) where n.objectid in $object_ids detach delete n", + "fitness": 7 + } } ] } diff --git a/cypher/test/cases/positive_tests.json b/cypher/test/cases/positive_tests.json index cedfccdc..b27bcd6a 100644 --- a/cypher/test/cases/positive_tests.json +++ b/cypher/test/cases/positive_tests.json @@ -36,7 +36,7 @@ "name": "Support filter and quantifier expressions", "type": "string_match", "details": { - "query": "match (g:GPO) optional match (g)-[r1:GPLink {enforced: false}]-\u003e(container1) with g, container1 optional match (g)-[r2:GPLink {enforced: true}]-\u003e(container2) with g, container1, container2 optional match p1 = (g)-[r1:GPLink]-\u003e(container1)-[r2:Contains*1..]-\u003e(n1:Computer) where none(x in nodes(p1) where x.blocksinheritance = true and labels(x) = 'OU') with g, p1, container2, n1 optional match p2 = (g)-[r1:GPLink]-\u003e(container2)-[r2:Contains*1..]-\u003e(n2:Computer) return p1, p2", + "query": "match (g:GPO) optional match (g)-[r1:GPLink {enforced: false}]->(container1) with g, container1 optional match (g)-[r2:GPLink {enforced: true}]->(container2) with g, container1, container2 optional match p1 = (g)-[r1:GPLink]->(container1)-[r2:Contains*1..]->(n1:Computer) where none(x in nodes(p1) where x.blocksinheritance = true and labels(x) = 'OU') with g, p1, container2, n1 optional match p2 = (g)-[r1:GPLink]->(container2)-[r2:Contains*1..]->(n2:Computer) return p1, p2", "fitness": -6 } }, @@ -90,34 +90,34 @@ } }, { - "name": "Filter nodes using WHERE clause with \u003c operator", + "name": "Filter nodes using WHERE clause with < operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003c 50 return p", + "query": "match (p:Person) where p.age < 50 return p", "fitness": 3 } }, { - "name": "Filter nodes using WHERE clause with \u003e operator", + "name": "Filter nodes using WHERE clause with > operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003e 50 return p", + "query": "match (p:Person) where p.age > 50 return p", "fitness": 3 } }, { - "name": "Filter nodes using WHERE clause with \u003c= operator", + "name": "Filter nodes using WHERE clause with <= operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003c= 50 return p", + "query": "match (p:Person) where p.age <= 50 return p", "fitness": 3 } }, { - "name": "Filter nodes using WHERE clause with \u003e= operator", + "name": "Filter nodes using WHERE clause with >= operator", "type": "string_match", "details": { - "query": "match (p:Person) where p.age \u003e= 50 return p", + "query": "match (p:Person) where p.age >= 50 return p", "fitness": 3 } }, @@ -125,7 +125,7 @@ "name": "Filter nodes using WHERE clause with not equal to", "type": "string_match", "details": { - "query": "match (p:Person) where p.name \u003c\u003e 'Tom Hanks' return p", + "query": "match (p:Person) where p.name <> 'Tom Hanks' return p", "fitness": 5 } }, @@ -149,7 +149,7 @@ "name": "Traverse relationship by specifying edge type, filter query using where clause", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where p.name = 'Tom Hanks' return m", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where p.name = 'Tom Hanks' return m", "fitness": 12 } }, @@ -157,7 +157,7 @@ "name": "Traverse relationship by specifying edge type, filter query using property matcher", "type": "string_match", "details": { - "query": "match (p:Person {name: 'Tom Hanks'})-[:ACTED_IN]-\u003e(m:Movie) return m", + "query": "match (p:Person {name: 'Tom Hanks'})-[:ACTED_IN]->(m:Movie) return m", "fitness": 9 } }, @@ -165,7 +165,7 @@ "name": "Traverse relationship by specifying multiple edge types", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN|DIRECTED]-\u003e(m:Movie) return m", + "query": "match (p:Person)-[:ACTED_IN|DIRECTED]->(m:Movie) return m", "fitness": 4 } }, @@ -173,7 +173,7 @@ "name": "Specify left to right relationship", "type": "string_match", "details": { - "query": "match (p:Person)-[]-\u003e(m:Movie) return m", + "query": "match (p:Person)-[]->(m:Movie) return m", "fitness": 3 } }, @@ -181,7 +181,7 @@ "name": "Specify right to left relationship", "type": "string_match", "details": { - "query": "match (p:Person)\u003c-[]-(m:Movie) return m", + "query": "match (p:Person)<-[]-(m:Movie) return m", "fitness": 3 } }, @@ -197,7 +197,7 @@ "name": "Filter query by specifying node labels in the where clause", "type": "string_match", "details": { - "query": "match (p)-[:ACTED_IN]-\u003e(m) where p:Person and m:Movie and m.title = 'The Matrix' return p.name", + "query": "match (p)-[:ACTED_IN]->(m) where p:Person and m:Movie and m.title = 'The Matrix' return p.name", "fitness": 9 } }, @@ -205,7 +205,7 @@ "name": "Filter using ranges in where clause", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where 2000 \u003c m.released \u003c 2003 and 100 \u003e m.last \u003c 200 return p.name", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where 2000 < m.released < 2003 and 100 > m.last < 200 return p.name", "fitness": 10 } }, @@ -285,7 +285,7 @@ "name": "Filter by list inclusion: list comes from the edge property named `r.roles`", "type": "string_match", "details": { - "query": "match (p:Person)-[r:ACTED_IN]-\u003e(m:Movie) where 'Neo' in r.roles return p.name", + "query": "match (p:Person)-[r:ACTED_IN]->(m:Movie) where 'Neo' in r.roles return p.name", "fitness": 6 } }, @@ -301,7 +301,7 @@ "name": "Query for the properties of an edge using keys()", "type": "string_match", "details": { - "query": "match ()-[e:EDGE_OF_INTEREST]-\u003e() return keys(e)", + "query": "match ()-[e:EDGE_OF_INTEREST]->() return keys(e)", "fitness": 1 } }, @@ -373,7 +373,7 @@ "name": "Eliminate duplicate rows returned", "type": "string_match", "details": { - "query": "match (p:Person)-[]-\u003e(m:Movie) return distinct p.name, m.title", + "query": "match (p:Person)-[]->(m:Movie) return distinct p.name, m.title", "fitness": 4 } }, @@ -413,7 +413,7 @@ "name": "Aggregation using collect() to return a list", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) return p.name, collect(m.title)", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) return p.name, collect(m.title)", "fitness": 5 } }, @@ -421,7 +421,7 @@ "name": "Eliminate duplication in lists", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where m.year = 1920 return collect(distinct (m.title))", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where m.year = 1920 return collect(distinct (m.title))", "fitness": 8 } }, @@ -429,7 +429,7 @@ "name": "Collecting nodes", "type": "string_match", "details": { - "query": "match (p:Person)-[:ACTED_IN]-\u003e(m:Movie) where p.name = 'tom cruise' return collect(m) as tomCruiseMovies", + "query": "match (p:Person)-[:ACTED_IN]->(m:Movie) where p.name = 'tom cruise' return collect(m) as tomCruiseMovies", "fitness": 12 } }, @@ -485,7 +485,7 @@ "name": "Conjunction", "type": "string_match", "details": { - "query": "match (n) where n.indexed \u003e= 1 and n.other_1 = 2 return n", + "query": "match (n) where n.indexed >= 1 and n.other_1 = 2 return n", "fitness": 5 } }, @@ -493,7 +493,7 @@ "name": "Multiple conjunctions", "type": "string_match", "details": { - "query": "match (n) where n.indexed \u003e= 1 and n.other_1 = 2 and n.other_2 = 3 return n", + "query": "match (n) where n.indexed >= 1 and n.other_1 = 2 and n.other_2 = 3 return n", "fitness": 8 } }, @@ -501,7 +501,7 @@ "name": "Conjunction with disjunction", "type": "string_match", "details": { - "query": "match (n) where n.indexed \u003e= 1 and (n.other_1 = 2 or n.other_2 = 3) return n", + "query": "match (n) where n.indexed >= 1 and (n.other_1 = 2 or n.other_2 = 3) return n", "fitness": 7 } }, @@ -509,7 +509,7 @@ "name": "Disjunction", "type": "string_match", "details": { - "query": "match (n) where (n.indexed \u003e= 1 or n.other_1 = 2) return n", + "query": "match (n) where (n.indexed >= 1 or n.other_1 = 2) return n", "fitness": 3 } }, @@ -517,7 +517,7 @@ "name": "Multiple disjunctions", "type": "string_match", "details": { - "query": "match (n) where (n.indexed \u003e= 1 or n.other_1 = 2 or n.other_2 = 3) return n", + "query": "match (n) where (n.indexed >= 1 or n.other_1 = 2 or n.other_2 = 3) return n", "fitness": 6 } }, @@ -557,7 +557,7 @@ "name": "Match patterns with range literal", "type": "string_match", "details": { - "query": "match (n)-[:NestedEdge*]-\u003e() where id(n) = 1 return n", + "query": "match (n)-[:NestedEdge*]->() where id(n) = 1 return n", "fitness": 1 } }, @@ -565,7 +565,7 @@ "name": "Match patterns with range literal with at least one edge", "type": "string_match", "details": { - "query": "match (n)-[:NestedEdge*1..]-\u003e() where id(n) = 1 return n", + "query": "match (n)-[:NestedEdge*1..]->() where id(n) = 1 return n", "fitness": 5 } }, @@ -573,7 +573,7 @@ "name": "Match patterns with range literal with 1 to 2 edges", "type": "string_match", "details": { - "query": "match (n)-[:NestedEdge*1..2]-\u003e() where id(n) = 1 return n", + "query": "match (n)-[:NestedEdge*1..2]->() where id(n) = 1 return n", "fitness": 3 } }, @@ -581,7 +581,7 @@ "name": "Match patterns with where and return clauses", "type": "string_match", "details": { - "query": "match (n {property: true})\u003c-[r {property: n.name}]-(s)-[v]-\u003e() where n.indexed = false return n, r.other", + "query": "match (n {property: true})<-[r {property: n.name}]-(s)-[v]->() where n.indexed = false return n, r.other", "fitness": 2 } }, @@ -613,7 +613,7 @@ "name": "Find All Domain Admins", "type": "string_match", "details": { - "query": "match p = (n:Group)\u003c-[:MemberOf*1..]-(m) where n.objectid =~ '(?i)S-1-5-.*-512' return p", + "query": "match p = (n:Group)<-[:MemberOf*1..]-(m) where n.objectid =~ '(?i)S-1-5-.*-512' return p", "fitness": 10 } }, @@ -621,7 +621,7 @@ "name": "Map Domain Trusts", "type": "string_match", "details": { - "query": "match p = (n:Domain)-[]-\u003e(m:Domain) return p", + "query": "match p = (n:Domain)-[]->(m:Domain) return p", "fitness": 3 } }, @@ -629,7 +629,7 @@ "name": "Find principals with DCSync rights", "type": "string_match", "details": { - "query": "match p = ()-[:DCSync|AllExtendedRights|GenericAll]-\u003e(:Domain {name: 'DOMAIN.PAIN'}) return p", + "query": "match p = ()-[:DCSync|AllExtendedRights|GenericAll]->(:Domain {name: 'DOMAIN.PAIN'}) return p", "fitness": 6 } }, @@ -637,7 +637,7 @@ "name": "Principals with Foreign Domain Group Membership", "type": "string_match", "details": { - "query": "match p = (n:Base)-[:MemberOf]-\u003e(m:Group) where n.domain = 'DOMAIN.PAIN' and m.domain \u003c\u003e n.domain return p", + "query": "match p = (n:Base)-[:MemberOf]->(m:Group) where n.domain = 'DOMAIN.PAIN' and m.domain <> n.domain return p", "fitness": 8 } }, @@ -645,7 +645,7 @@ "name": "Find Computers where Domain Users are Local Admin", "type": "string_match", "details": { - "query": "match p = (m:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:AdminTo]-\u003e(n:Computer) return p", + "query": "match p = (m:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:AdminTo]->(n:Computer) return p", "fitness": 9 } }, @@ -653,7 +653,7 @@ "name": "Find Computers where Domain Users can read LAPS passwords", "type": "string_match", "details": { - "query": "match p = (Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:MemberOf*0..]-\u003e(g:Group)-[:AllExtendedRights|ReadLAPSPassword]-\u003e(n:Computer) return p", + "query": "match p = (Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:MemberOf*0..]->(g:Group)-[:AllExtendedRights|ReadLAPSPassword]->(n:Computer) return p", "fitness": 4 } }, @@ -661,7 +661,7 @@ "name": "Find All Paths from Domain Users to High Value Targets", "type": "string_match", "details": { - "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]-\u003e(n {highvalue: true})) where g \u003c\u003e n return p", + "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]->(n {highvalue: true})) where g <> n return p", "fitness": 13 } }, @@ -669,7 +669,7 @@ "name": "Find all shortest paths to workstations where Domain Users can RDP", "type": "string_match", "details": { - "query": "match p = allShortestPaths((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]-\u003e(c:Computer)) where not (c.operatingsystem contains 'Server') return p", + "query": "match p = allShortestPaths((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]->(c:Computer)) where not (c.operatingsystem contains 'Server') return p", "fitness": 14 } }, @@ -677,7 +677,7 @@ "name": "Find Workstations where Domain Users can RDP", "type": "string_match", "details": { - "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]-\u003e(c:Computer) where not (c.operatingsystem contains 'Server') return p", + "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]->(c:Computer) where not (c.operatingsystem contains 'Server') return p", "fitness": 10 } }, @@ -685,7 +685,7 @@ "name": "Find Servers where Domain Users can RDP", "type": "string_match", "details": { - "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]-\u003e(c:Computer) where c.operatingsystem contains 'Server' return p", + "query": "match p = (g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[:CanRDP]->(c:Computer) where c.operatingsystem contains 'Server' return p", "fitness": 11 } }, @@ -693,7 +693,7 @@ "name": "Find Dangerous Privileges for Domain Users Groups", "type": "string_match", "details": { - "query": "match p = (m:Group)-[:Owns|GenericAll|GenericWrite|WriteOwner|WriteDacl|MemberOf|ForceChangePassword|AllExtendedRights|AddMember|HasSession|CanApplyGPO|AllowedToDelegate|CoerceToTGT|SameForestTrust|AllowedToAct|AdminTo|CanPSRemote|CanRDP|ExecuteDCOM|HasSIDHistory|AddSelf|DCSync|ReadLAPSPassword|ReadGMSAPassword|DumpSMSAPassword|SQLAdmin|AddAllowedToAct|WriteSPN|AddKeyCredentialLink|SyncLAPSPassword|WriteAccountRestrictions|GoldenCert|ADCSESC1|ADCSESC3|ADCSESC4|ADCSESC5|ADCSESC6a|ADCSESC6b|ADCSESC7|ADCSESC9a|ADCSESC9b|ADCSESC10a|ADCSESC10b|ADCSESC13|DCFor|SyncedToEntraUser]-\u003e(n:Base) where m.objectid ends with '-513' return p", + "query": "match p = (m:Group)-[:Owns|GenericAll|GenericWrite|WriteOwner|WriteDacl|MemberOf|ForceChangePassword|AllExtendedRights|AddMember|HasSession|CanApplyGPO|AllowedToDelegate|CoerceToTGT|SameForestTrust|AllowedToAct|AdminTo|CanPSRemote|CanRDP|ExecuteDCOM|HasSIDHistory|AddSelf|DCSync|ReadLAPSPassword|ReadGMSAPassword|DumpSMSAPassword|SQLAdmin|AddAllowedToAct|WriteSPN|AddKeyCredentialLink|SyncLAPSPassword|WriteAccountRestrictions|GoldenCert|ADCSESC1|ADCSESC3|ADCSESC4|ADCSESC5|ADCSESC6a|ADCSESC6b|ADCSESC7|ADCSESC9a|ADCSESC9b|ADCSESC10a|ADCSESC10b|ADCSESC13|DCFor|SyncedToEntraUser]->(n:Base) where m.objectid ends with '-513' return p", "fitness": 9 } }, @@ -701,7 +701,7 @@ "name": "Find Domain Admins Logons to non-Domain Controllers", "type": "string_match", "details": { - "query": "match (dc)-[r:MemberOf*0..]-\u003e(g:Group) where g.objectid ends with '-516' with collect(dc) as exclude match p = (c:Computer)-[n:HasSession]-\u003e(u:User)-[r2:MemberOf*1..]-\u003e(g:Group) where g.objectid ends with '-512' and not (c in exclude) return p", + "query": "match (dc)-[r:MemberOf*0..]->(g:Group) where g.objectid ends with '-516' with collect(dc) as exclude match p = (c:Computer)-[n:HasSession]->(u:User)-[r2:MemberOf*1..]->(g:Group) where g.objectid ends with '-512' and not (c in exclude) return p", "fitness": 17 } }, @@ -789,7 +789,7 @@ "name": "Find Kerberoastable Users with most privileges", "type": "string_match", "details": { - "query": "match (u:User {hasspn: true}) optional match (u)-[:AdminTo]-\u003e(c1:Computer) optional match (u)-[:MemberOf*1..]-\u003e(:Group)-[:AdminTo]-\u003e(c2:Computer) with u, collect(c1) + collect(c2) as tempVar unwind tempVar as comps return u.name, count(distinct (comps)) order by count(distinct (comps)) desc", + "query": "match (u:User {hasspn: true}) optional match (u)-[:AdminTo]->(c1:Computer) optional match (u)-[:MemberOf*1..]->(:Group)-[:AdminTo]->(c2:Computer) with u, collect(c1) + collect(c2) as tempVar unwind tempVar as comps return u.name, count(distinct (comps)) order by count(distinct (comps)) desc", "fitness": 2 } }, @@ -797,7 +797,7 @@ "name": "Find Kerberoastable Members of High Value Groups", "type": "string_match", "details": { - "query": "match p = shortestPath((n:User)-[:MemberOf]-\u003e(g:Group)) where g.highvalue = true and n.hasspn = true return p", + "query": "match p = shortestPath((n:User)-[:MemberOf]->(g:Group)) where g.highvalue = true and n.hasspn = true return p", "fitness": 17 } }, @@ -805,7 +805,7 @@ "name": "Shortest Paths to Unconstrained Delegation Systems", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", "fitness": 13 } }, @@ -813,7 +813,7 @@ "name": "Shortest Paths from Kerberoastable Users", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Computer {unconstraineddelegation: true})) where not (n = m) return p", "fitness": 13 } }, @@ -821,7 +821,7 @@ "name": "Shortest Paths to Domain Admins from Kerberoastable Users", "type": "string_match", "details": { - "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", + "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", "fitness": 17 } }, @@ -829,7 +829,7 @@ "name": "Shortest Paths from Owned Principals", "type": "string_match", "details": { - "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", + "query": "match p = shortestPath((n:User {hasspn: true})-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) return p", "fitness": 17 } }, @@ -837,7 +837,7 @@ "name": "Shortest Paths to High Value Targets", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[*1..]-\u003e(m {highvalue: true})) where m.domain = 'DOMAIN.PAIN' and m \u003c\u003e n return p", + "query": "match p = shortestPath((n)-[*1..]->(m {highvalue: true})) where m.domain = 'DOMAIN.PAIN' and m <> n return p", "fitness": 11 } }, @@ -853,7 +853,7 @@ "name": "Shortest Paths from Domain Users to High Value Targets", "type": "string_match", "details": { - "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]-\u003e(n {highvalue: true})) where g.objectid ends with '-513' and g \u003c\u003e n return p", + "query": "match p = shortestPath((g:Group {name: 'DOMAIN USERS@DOMAIN.PAIN'})-[*1..]->(n {highvalue: true})) where g.objectid ends with '-513' and g <> n return p", "fitness": 20 } }, @@ -861,7 +861,7 @@ "name": "Find Shortest Paths to Domain Admins", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*1..]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", "fitness": 14 } }, @@ -869,7 +869,7 @@ "name": "Find Shortest Paths to Domain Admins with Traversal Limit", "type": "string_match", "details": { - "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*5..1]-\u003e(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", + "query": "match p = shortestPath((n)-[:HasSession|AdminTo|Contains|AZLogicAppContributor*5..1]->(m:Group {name: 'DOMAIN ADMINS@DOMAIN.PAIN'})) where not (n = m) return p", "fitness": 17 } }, diff --git a/cypher/test/test.go b/cypher/test/test.go index b4c340cf..b94ac08d 100644 --- a/cypher/test/test.go +++ b/cypher/test/test.go @@ -21,13 +21,18 @@ import ( "github.com/stretchr/testify/require" ) +// testCaseFiles embeds the parser and analyzer fixture cases consumed by Runner. +// //go:embed cases var testCaseFiles embed.FS type Type = string const ( - TypeStringMatch Type = "string_match" + // TypeStringMatch identifies a case that compares formatted query text. + TypeStringMatch Type = "string_match" + + // TypeNegativeCase identifies a case that expects parsing or analysis errors. TypeNegativeCase Type = "negative_case" ) @@ -208,6 +213,7 @@ func LoadFixture(t *testing.T, filename string) Cases { return fixture } +// testRunner loads one embedded fixture and dispatches it to the runner selected by its case type. func testRunner[T Runner](testCase Case) func(t *testing.T) { return func(t *testing.T) { // Run the test case if it isn't ignored @@ -221,6 +227,7 @@ func testRunner[T Runner](testCase Case) func(t *testing.T) { } } +// testCase parses one named JSON fixture from fs into the concrete case type requested by its metadata. func testCase(test Case) func(t *testing.T) { switch test.Type { case TypeStringMatch: @@ -236,6 +243,7 @@ func testCase(test Case) func(t *testing.T) { } } +// updatedCasesDir returns the caller-provided fixture update directory or an isolated temporary directory. func updatedCasesDir() (string, error) { if workingDir, err := os.Getwd(); err != nil { return "", err @@ -250,6 +258,7 @@ func updatedCasesDir() (string, error) { } } +// UpdatePositiveTestCasesFitness rewrites positive fixtures with their current PostgreSQL translations. func UpdatePositiveTestCasesFitness() error { if updatedCasesPath, err := updatedCasesDir(); err != nil { return err @@ -291,10 +300,13 @@ func UpdatePositiveTestCasesFitness() error { } else { details.ExpectedFitness = &complexity.RelativeFitness - if updatedDetails, err := json.Marshal(details); err != nil { + var updatedDetails bytes.Buffer + encoder := json.NewEncoder(&updatedDetails) + encoder.SetEscapeHTML(false) + if err := encoder.Encode(details); err != nil { return fmt.Errorf("error marshalling test case details: %v", err) } else { - nextCase.Details = updatedDetails + nextCase.Details = bytes.TrimSpace(updatedDetails.Bytes()) } } @@ -309,7 +321,10 @@ func UpdatePositiveTestCasesFitness() error { } else { defer output.Close() - if err := json.NewEncoder(output).Encode(updatedCases); err != nil { + encoder := json.NewEncoder(output) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + if err := encoder.Encode(updatedCases); err != nil { return err } } diff --git a/docs/cysql_traversal_priorities.md b/docs/cysql_traversal_priorities.md new file mode 100644 index 00000000..e0a69a29 --- /dev/null +++ b/docs/cysql_traversal_priorities.md @@ -0,0 +1,1004 @@ +# CySQL traversal performance priorities + +Date: 2026-08-12 + +Status: implementation complete; promotion evidence pending + +The code and current activation disposition are recorded in +[`experiments/traversal_priority_implementation_status_v1.md`](experiments/traversal_priority_implementation_status_v1.md). +New production promotion remains evidence-gated as specified below. + +This plan turns the fresh CySQL/PostgreSQL versus Cypher/Neo4j benchmark and +source review into an implementation and qualification program. It focuses on +ordinary variable-length traversal orientation, bound `shortestPath` (SP), +`allShortestPaths` (ASP), and fixed one-hop `ExpandInto` behavior. + +The principal decision is to build one exact, observable traversal-selection +framework rather than add another isolated lowering. The first production +targets are a topology-aware forward/reverse orientation tournament and compact +bidirectional SP candidates. Bidirectional ASP follows after the shared search +kernel and telemetry are qualified. Fixed one-hop `ExpandInto` is a narrow, +measure-first opportunity. A persistent topology synopsis is deferred until +runtime probes prove that its maintenance and cache complexity are warranted. + +## Executive priority order + +Engineering effort should proceed in this order: + +| Priority | Work | Reason | +| --- | --- | --- | +| P0 | Shared telemetry, matched plan deltas, and frozen qualification corpus | Current PostgreSQL function scans hide traversal work, while Neo4j 4.4 SP/ASP profiles do not count internal relationship traversal. Selector work is not explainable or safely promotable without independent counters. | +| P1 | General ordinary-expansion orientation tournament | The measured fixed-suffix crossover is the largest ordinary-traversal opportunity: reverse is dramatically better on sparse terminal topology and materially worse under high reverse fan-in. | +| P2 | Compact SP architecture and scheduler tournament | Current S4 witness and deep/inbound execution is the main SP loss, while exact inline references show that the gap is not inherent to PostgreSQL storage. | +| P3 | Compact bidirectional ASP predecessor DAG | Recursive ASP is materially slower than Neo4j and currently lacks independent predecessor/output gates. It should reuse the proven SP search and telemetry foundation. | +| P4 | Bounded endpoint resolution and step-local predicate support | The current singleton-ID envelope excludes unique property seeks, small endpoint sets, and safe universal predicates that Neo4j can prepare before traversal. | +| P5 | Fixed one-hop `ExpandInto` endpoint choice and pair reuse | Neo4j's lower-degree scan and pair cache are useful hypotheses, but PostgreSQL may already choose an efficient plan for the current bound-pair join, including a parameterized index lookup or `Memoize`; this must be measured before adding probes. | +| P6 | Optional versioned topology synopsis | Persistent estimates may reduce probe cost, but they are advisory, mutation-sensitive, and absent from the current translation-cache identity. Runtime evidence comes first. | + +This is the engineering-priority order, not necessarily the automatic-promotion +order. A semantically narrow fixed-hop candidate may graduate before a recursive +candidate if it independently passes every gate. Orientation and SP reference +work can proceed in parallel after P0. ASP depends on the common bidirectional +state model and its counters. + +## Outcomes and success measures + +The program should deliver: + +1. Exact runtime selection between forward and reverse ordinary expansions for + qualified shapes, with a same-statement forward incumbent on uncertainty or + overflow. +2. Exact SP comparison among the current single-ended compact executor, + Neo4j-4.4-style strict per-node alternation, and current-Neo4j-style + smaller-current-level expansion. +3. Exact ASP comparison among the current single-ended predecessor DAG and two + bidirectional predecessor-DAG schedulers, with independent discovery, + predecessor, and output-enumeration gates. +4. Executor-reported work metrics that explain a choice in terms of seeds, + directional degree, frontier growth, edge scans, reconvergence, + predecessor multiplicity, meeting width, fallback, and hydration. +5. Matched PostgreSQL/Neo4j plan-delta reports that identify starting side, + physical direction, predicate placement, estimate error, and traversal + setup without treating unlike backend operator counters as equivalent. +6. Versioned selectors, reference identities, negative-result records, and a + reversible rollout path. + +Promotion is not defined as "beat Neo4j everywhere." Neo4j is an exact-result +and descriptive latency oracle. A CySQL candidate is promoted only when it is +exact, beats or contains its PostgreSQL incumbent on predeclared topology +buckets, and stays within resource and operational limits. + +For tied singleton SP, "exact" means the same minimum distance and one valid +minimum relationship-unique witness, not the same arbitrary witness as Neo4j +or another CySQL executor. ASP and bag-valued ordinary traversals require their +complete logical result multisets. + +## Scope and explicit non-goals + +The initial scope is read-only, directed, bounded traversal with a single +variable region or one statically proven endpoint pair. It includes the current +endpoint-seeded and three-hop fixed-suffix envelopes, singleton bound SP/ASP, +and fixed one-hop `ExpandInto`. + +The first program does not: + +- implement a general IDP query-graph solver or reorder arbitrary Cypher + components; +- infer correctness from planner estimates or make mutable statistics a + translation-time dependency; +- change trail, bag, tie, optional-match, mutation, or predicate semantics; +- make legacy full-trail bidirectional harnesses production candidates; +- revive the retired suffix keyset-continuation design; +- force one SP/ASP scheduler across every topology or observation mode; +- use Neo4j latency or opaque 4.4 `ShortestPath` DB hits as a CySQL release + threshold. + +## Baseline evidence to freeze + +The 2026-08-12 discovery capture used PostgreSQL 17.10 and Neo4j 4.4.44. It +contained two backend-order-balanced rounds, ten warmups, and thirty measured +samples per round. These results motivate the work, but they are not a release +gate and must be recaptured as milestone M0. + +| Shape | Discovery result | Planning implication | +| --- | --- | --- | +| Bounded outbound SP distance | CySQL S3 was about 6-30x faster | Preserve S3 as a real tournament arm; do not replace it globally. | +| SP witness and deep physical-inbound search | Neo4j was about 5-16x faster | Tournament both execution boundary and bidirectional scheduler. | +| Recursive ASP at depths 3 and 16 | Neo4j was about 5.7-13.1x faster | Shallow two-hop fixtures are insufficient; exercise the predecessor workspace. | +| Sparse fixed suffix | Neo4j was about 51x faster than production CySQL | General orientation selection has high expected value. | +| Forced CySQL suffix reverse on that sparse case | About 460x faster than forward endpoint-ID output | The reverse implementation is viable when topology is favorable. | +| High reverse fan-in | CySQL forward was about 3.7-4.4x faster than Neo4j; forced reverse was about 3.4x slower than forward | Static "always reverse" is unsafe as a performance policy. | +| Exact inline PostgreSQL references | About 2.5-45.7x faster than corresponding compact production functions on selected cases | Function/workspace overhead and algorithm must be separated in the tournament. | + +The source capture, raw benchmark records, and local review currently live +under `.coverage/fresh-plan-delta-20260812`. M0 must create a checksummed capture +bundle and commit only compact, credential-free decision records; raw +environment-specific artifacts remain ignored. + +## Neo4j lessons to use deliberately + +The primary source target is the measured Neo4j 4.4.44 tag at commit +[`17d7609`](https://github.com/neo4j/neo4j/tree/17d7609361109bd9b08ea149a5ed5966f1115324). +Current upstream behavior is pinned separately to the reviewed 2026.06 commit +[`eccd584`](https://github.com/neo4j/neo4j/tree/eccd584a64d468af3daeab421478fe78567c518f). +Current behavior must not be projected backward onto the measured server. + +The source review establishes these design inputs: + +- Ordinary relationship planning creates candidates from both endpoints and + lets bounded IDP retain the cheapest orientation. The suffix-first benchmark + plan is a general enumeration result, not a special suffix rule. See + [`SingleComponentPlanner`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/cypher-planner/src/main/scala/org/neo4j/cypher/internal/compiler/planner/logical/idp/SingleComponentPlanner.scala#L215-L244). +- Neo4j 4.4 statistics contain global node, label, relationship-step, and index + selectivity values, but no endpoint-local degree, frontier survival, + reconvergence, meeting-cut width, or predecessor/output multiplicity. See + [`GraphStatistics`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/planner-spi/src/main/scala/org/neo4j/cypher/internal/planner/spi/GraphStatistics.scala#L27-L66). +- Generic `VarLengthExpand(All/Into)` is a single-ended stack-based DFS in its + planned orientation. `Into` checks the bound target when emitting; it does + not become target-directed or bidirectional. See + [`VarLengthExpandPipe`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/interpreted-runtime/src/main/scala/org/neo4j/cypher/internal/runtime/interpreted/pipes/VarLengthExpandPipe.scala#L50-L135). +- Fixed one-hop `ExpandInto` is different: Neo4j can scan the lower-degree + endpoint and cache a node-pair result. See + [`CachingExpandInto`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/cypher/runtime-util/src/main/java/org/neo4j/internal/kernel/api/helpers/CachingExpandInto.java#L139-L207). +- Bound SP/ASP is attached only after both endpoints are available. Neo4j + 4.4's specialized bidirectional BFS alternates one newly discovered node per + side and retains same-depth predecessor relationships. See + [`ShortestPath`](https://github.com/neo4j/neo4j/blob/17d7609361109bd9b08ea149a5ed5966f1115324/community/graph-algo/src/main/java/org/neo4j/graphalgo/impl/path/ShortestPath.java#L207-L343). +- Current Neo4j expands a complete level from the side with the smaller current + level, a materially different scheduler. See + [`BiDirectionalBFSImpl`](https://github.com/neo4j/neo4j/blob/eccd584a64d468af3daeab421478fe78567c518f/community/cypher/runtime-util/src/main/java/org/neo4j/internal/kernel/api/helpers/traversal/BiDirectionalBFSImpl.java#L167-L195). +- Neo4j 4.4's Cypher profiler does not expose internal SP relationship reads. + Raw `ShortestPath` DB-hit counts must be marked opaque, not compared to + PostgreSQL recursive rows or edge probes. + +The plan adopts orientation enumeration, endpoint binding, bidirectional BFS, +frontier-aware scheduling, and two-sided predecessor reconstruction as +candidate ideas. It does not adopt Neo4j's global-average cost blindness, +opaque SP/ASP telemetry, or generic DFS behavior as CySQL requirements. + +## Architecture and decision boundaries + +The target decision flow is: + +```text +Cypher shape analysis + | + v +exact candidate envelope + observation classification + | + +---------------- compile-time diagnostics ----------------+ + | | + v v +same-statement capped probes or executor frontier state plan-delta record + | + v +versioned runtime policy + | + +---------+-----------+------------------+ + | | | | + v v v v +forward/reverse SP arm ASP arm fixed-hop arm + | | | | + +---------+-----------+------------------+ + | + v + exact gated output or incumbent fallback + | + v + late hydration + runtime telemetry +``` + +Compile-time facts and runtime facts must remain distinct: + +- The optimizer records the correctness envelope, candidates, observation + mode, selector version, caps, and fallback policy. +- The emitted SQL or executor records probes performed, scheduler decisions, + runtime arm, work, overflow, and fallback actually executed. +- GraphBench must not claim that a compile-time candidate ran merely because it + was planned or emitted. +- Tool forcing may choose among structurally eligible candidates; it may never + broaden their correctness envelope. + +The current translation cache is keyed by normalized query text, graph ID, and +parameter-name/type shape. Mutable parameter values or graph statistics must +therefore be consulted inside the generated statement. If a future selector +embeds a synopsis value at translation time, a statistics generation and +invalidation contract must first be added to the cache key. + +Mutable rollout policy is subject to the same rule. Feature-gate state, +selector version, and caps are not in the current cache key. A policy that can +change during a driver's lifetime must be supplied at execution time, add an +explicit cache generation, or invalidate affected translations. Otherwise a +rollback can leave cached tournament SQL active. Immutable caps may be SQL +literals; planner-created SQL parameters without `ParameterSources` currently +make a translation non-cacheable and need explicit rebinding/cache support if +that behavior is not desired. + +## Non-negotiable semantic contract + +Every candidate, probe, and fallback must preserve: + +- graph partition and resolved relationship-kind filtering; +- logical direction and the correct physical adjacency index; +- inclusive minimum and maximum depth, including qualified zero-length paths; +- relationship-trail uniqueness while permitting repeated nodes where Cypher + permits them; +- ordered relationship and node IDs in logical source-to-target order; +- prefix/suffix relationship non-reuse across stitched path regions; +- duplicate root rows, endpoint rows, suffix rows, and output bag + multiplicity; +- SP's one arbitrary valid minimum trail and ASP's complete set of + relationship-distinct minimum trails; +- predicate null behavior, locality, determinism, and evaluation count; +- optional-match and mutation visibility rules; +- one top-level SQL statement for probes, candidate, and fallback, plus an + explicit snapshot contract. SQL-only CTE arms share a statement snapshot; + `VOLATILE` PL/pgSQL internal statements under `READ COMMITTED` must not be + assumed to do so. Function-backed candidates require a deliberate mechanism + such as repeatable-read execution, or an independently proven equivalent, + before claiming snapshot-stable fallback; +- no candidate row exposure until every fallback-triggering gate has passed; +- prompt cancellation, rollback recovery, and clean reuse of a pooled session. + +The singleton SP tie policy remains the contract in +[`shortest_path_tie_policy.md`](shortest_path_tie_policy.md). Physical edge ID +or insertion order is not public. ASP may not use the singleton tie policy to +discard equal-depth predecessors. + +The PostgreSQL schema currently has a unique +`(start_id, end_id, kind_id, graph_id)` relationship constraint. Same-kind +parallel physical relationships cannot be represented in the current backend. +Cross-kind parallel relationships must be covered now; same-kind parallel-edge +parity remains an explicit storage boundary, not a silently skipped test. + +## Workstream 0: observability and matched plan deltas + +This is the prerequisite for every selector change. + +### 0.1 PostgreSQL executor telemetry + +Add a versioned `TraversalExecutionTelemetry` schema to GraphBench records and +PostgreSQL full-comparator records. Preserve `PostgresPlanMetrics` for measured +plan facts, but do not infer hidden PL/pgSQL work from a `Function Scan` loop. + +Use two telemetry levels: + +- A lightweight summary: requested/planned/emitted/runtime/applied identity, + selector and scheduler version, caps, runtime branch, overflow, and fallback. +- A tool-only diagnostic replay on the same connection: per-level and + per-stage executor counters. It runs outside the timed sample block so + detailed instrumentation does not contaminate latency evidence. + +Replay counters describe that untimed invocation, not a particular timed +sample. Store them in a separate diagnostic boundary and do not combine their +resource values with the production timing record. + +Missing required telemetry is a qualification failure, not a zero value. Every +derived field carries provenance naming the function, CTE, or executor metric +that produced it. + +Record at minimum: + +| Family | Required runtime counters | +| --- | --- | +| Ordinary DFS/recursive CTE | roots, edge candidates, admitted states, relationship-repeat rejects, recursive rows, peak state, emitted trails, hydration rows | +| Orientation policy | forward/reverse seeds, duplicate seeds, suffix rows, distinct boundaries, typed directional degree samples, shallow survival, probe rows/time/buffers, scores, selected side, sentinel overflow, branch loops | +| SP | scheduler actions, per-side depth/frontier, candidate edges, distinct new nodes, seen/frontier/queue peaks, meeting candidates, frozen distance, witness rows, fallback | +| ASP | SP counters plus same-depth predecessor additions, predecessor peak, meeting nodes, cut depth, saturating path-count estimate, enumerated candidates, duplicate rejects, output paths/edge cells/bytes | +| Hydration | path count, node/edge lookups, loops, rows, time, and bytes separately from discovery | + +Candidate workspace metrics must be invocation-keyed and session-local so +concurrent pooled sessions cannot collide. Cancellation and SQL errors +propagate; they are not converted into performance fallbacks. + +### 0.2 Neo4j read profiling + +Extend GraphBench to run a read-only `PROFILE` pass after the timed block while +retaining `EXPLAIN` for writes. Persist: + +- planner and runtime version; +- ordered operator tree and child order; +- estimated and actual rows, loops, DB hits, page-cache hits/misses, and + operator time where the server exposes them; +- leaf variables, access predicates, expansion direction, and starting side; +- an explicit `internal_traversal_work=opaque` marker for 4.4 SP/ASP. + +Normalize the current doubled `@neo4j` operator suffix and verify endpoint-child +fidelity. Neo4j profile data remains descriptive and must not become a CySQL +release gate. + +### 0.3 Paired PlanCorpus record + +Add a versioned PostgreSQL/Neo4j plan-delta record keyed by dataset, case, +workload hash, source revision, and backend plan fingerprints. It should +compare semantic stages rather than raw operator names: + +- starting and terminal access; +- logical and physical traversal direction; +- predicate placement and endpoint binding; +- ordinary expand versus SP/ASP operator family; +- estimated seeds, traversal multiplier/frontier, output, and Q-error; +- PostgreSQL planned/emitted/runtime/fallback identities; +- whether Neo4j reordered the pattern and whether the chosen side did less + observed work. + +Rank opposite-side choices, largest estimate disagreements, predicate moves, +fallback/cap cases, and hydration deltas. Incomplete pairs must be explicit; +they must not disappear through intersection-only reporting. PlanCorpus remains +the plan inventory and GraphBench remains the runtime authority. + +## Workstream 1: ordinary traversal orientation tournament + +The strategy should be general in framework and deliberately narrow at first +activation. + +### 1.1 Candidate model + +Introduce runtime policy identity `orientation-probe-v1`. Keep executed arm +identities separate: + +- `EXPANSION-STEPWISE-FORWARD` is the permanent exact incumbent. +- `EXPANSION-SUFFIX-SEEDED-REVERSE` is the exact fixed-suffix reverse arm. +- `EXPANSION-ENDPOINT-SEEDED-REVERSE` remains the exact terminal-seeded arm. +- factored-forward and backward-viability arms remain references until they + independently qualify. + +Do not overload compile-time `SelectedStrategy` to imply a runtime choice. Add +emitted-policy, probe-cap, admission, and candidate fields to the typed +`ExpansionSearchStrategyDecision` and translation outcome. Record the actual +arm, probe results, overflow, and fallback only in execution/GraphBench +telemetry; translation cannot know them, and a translation-cache hit does not +reconstruct a fresh runtime outcome. + +Initial eligibility remains conservative: + +- one read-only, non-optional ordinary pattern region; +- one directed, bounded variable expansion with maximum depth at most 64; +- a bound/safely materializable seed region on each considered side; +- no relationship variable or relationship/path-dependent predicate; +- no cross-region correlation or limit-pushdown conflict; +- endpoint-ID, ordered-ID, or full-path observation with proven projection + alignment. + +The first suffix activation must reproduce the current envelope exactly: a +bound root; one outbound, single-kind variable expansion; exactly three +outbound, single-kind fixed suffix hops; exactly one right-node kind on every +suffix hop; and the existing dependency, observation, and no-function-call +restrictions. Endpoint-seeded migration likewise preserves its current +identity-function exception and all other restrictions. "Deterministic" is not +enough to broaden expression eligibility because repeated probing can change +evaluation count and exception behavior. Other predicates or contiguous fixed +regions wait for the predicate-class workstream and their own decision record. + +### 1.2 Same-statement probe and branch design + +Emit one statement containing: + +1. A capped forward-root materialization. +2. A capped reverse seed materialization: terminal endpoints or exact suffix + rows plus distinct boundary nodes. +3. Capped typed directional-degree probes using the existing covering + `(start_id, kind_id)` and `(end_id, kind_id)` indexes. +4. An optional, statically enabled one-level survival probe with an explicit + row/edge cap; its cost envelope is qualified offline. +5. A versioned score and hysteresis decision CTE. +6. A reverse-state admission relation capped at `state_limit + 1`. +7. Strictly disjoint reverse and forward-incumbent branches. + +Every cap uses a `cap + 1` sentinel. Probe relations must actually contain an +explicit bound. The existing unused `buildFixedSuffixProbeCTE` helper is not +currently limited despite its comment; bounding or replacing it is a +prerequisite, not evidence that suffix probing is already safe. + +Capped relations are evidence, not automatically exact query inputs. Keep an +uncapped exact source for the incumbent. A candidate may consume a capped root, +endpoint, or suffix relation only after its sentinel proves that the relation +is complete; overflow must not feed truncated rows to either arm. If a complete +probe relation is reused to avoid duplicate work, tests must prove that it +retains the exact duplicate and suffix-bag multiplicity required by that arm. + +Record: + +- distinct and duplicate roots; +- reverse seed rows and distinct seed nodes; +- suffix row multiplicity and distinct boundary count; +- first-hop typed adjacency rows, maximum sampled degree, and a high percentile + when the seed set is small; +- one-level admitted-next-node ratio; +- reverse states consumed before admission; +- total probe latency and buffers. + +Latency and buffers are post-execution telemetry used to qualify the policy; +plain CTE SQL cannot observe them in time to choose a branch within that same +statement. + +The initial policy is dominance-based, not a fragile learned formula: + +- choose reverse only when required probes are complete below their caps and + its versioned score beats forward by a qualified hysteresis margin; +- choose forward on overflow, missing evidence, ties, or ambiguous + correlation; +- if reverse-state admission crosses its sentinel, discard all candidate state + and run the exact forward incumbent before returning a row. + +Thresholds are derived from predeclared GraphBench training buckets and frozen +before the holdout is opened. Parameter values and topology stay runtime inputs, +so cached SQL remains safe. + +### 1.3 Implementation sequence + +1. Refactor fixed-prefix and fixed-suffix analysis in + `cypher/models/pgsql/optimize/lowering_plan.go` into a common contiguous + orientation-candidate analyzer while retaining specific fallback reasons. +2. Extend typed decisions in `cypher/models/pgsql/optimize/lowering.go` and + outcomes in `cypher/models/pgsql/translate/translator.go`. +3. Add `cypher/models/pgsql/translate/expansion_orientation.go` and extract + reusable seed, reverse recursion, projection alignment, overflow, and + incumbent-gating helpers from `expansion_endpoint_seeded.go` and + `expansion_suffix_seeded.go`. +4. Emit the incumbent first, then wrap it with probes and disjoint gates in + `pattern.go`. Distinguish tournament emission from runtime arm execution. +5. Migrate endpoint-seeded reverse to the common framework without changing + its current 32-endpoint/4096-state behavior. +6. Add guarded suffix reverse; keep the existing force seams as independent + A/B controls. +7. Run shadow selection before changing production. The shadow can compute + `would_select` while executing the incumbent; regret comes from separate + matched GraphBench runs that execute the exact forced arms. + +The retired keyset-continuation experiment is not a candidate. Its confirmed +negative result remains authoritative unless a materially different design is +given a new identity and hypothesis. + +## Workstream 2: compact SP scheduler tournament + +SP must tournament algorithm, scheduler, and execution boundary. Current +production winners remain controls: + +- `SP-S3-U-D` for qualified outbound distance and shallow physical-inbound + distance; +- `SP-S4-C-D` for qualified deep physical-inbound distance; +- `SP-S4-C-WE+MAT-M0` for qualified one-path witnesses; +- `SP-S0` as the exact broad-envelope incumbent. + +The specialized SP envelope requires an explicit bounded maximum depth at most +64. The current ASP envelope differs: an omitted maximum is admitted as depth +15, while minimum depth must be one for `ASP-A1-DAG`. Preserve those distinctions +in candidate eligibility, comparator choice, and serialized decisions. + +Reserve stable candidate identities before capture: + +| Candidate | Scheduler ID | Observation | Reference arm | +| --- | --- | --- | --- | +| `SP-B1-C-ALT-NODE-D` | `strict_alternating_node` | distance | `sp_b1_strict_alternating_distance` | +| `SP-B1-C-ALT-NODE-WE+MAT-M0` | `strict_alternating_node` | one witness | `sp_b1_strict_alternating_witness_m0` | +| `SP-B2-C-MIN-LEVEL-D` | `smaller_current_level` | distance | `sp_b2_smaller_frontier_distance` | +| `SP-B2-C-MIN-LEVEL-WE+MAT-M0` | `smaller_current_level` | one witness | `sp_b2_smaller_frontier_witness_m0` | + +Add a typed scheduler field to `ShortestPathExecutorDecision`; scheduler +behavior must not be inferred from a display name. Freeze +`single_ended_level` for S3/S4/A1 as well as the two candidate scheduler values +before the first artifact. + +### 2.1 Shared compact kernel + +Prototype a typed, graph-scoped bound-pair kernel with distinct forward and +backward structures: + +- node/depth frontier and next-front state; +- minimum-depth seen state per side; +- one deterministic predecessor/successor per accepted node for SP witness; +- per-node FIFO queue state for strict alternation; +- invocation telemetry and independently versioned limits. + +Keep relationship and node IDs only until one late hydration boundary. Preserve +logical source-to-target relationship order even when physical search begins at +the target. Outbound logical search uses `start_id -> end_id` forward and +`end_id -> start_id` backward; inbound search reverses those accesses. + +The legacy `bidirectional_sp_harness` already contains smaller-frontier control +logic, but it retains full path arrays, executes generated SQL text, and uses +generic pathspace tables. Reuse its control-flow lessons only. Do not promote or +rename it as a compact candidate. + +Strict alternation must dequeue one accepted node from each side in turn; +alternating whole SQL levels is a different scheduler. Smaller-frontier must +expand a complete level and use a deterministic tie break. Both schedulers need +a documented lower-bound termination proof: do not stop merely at the first +intersection, and complete enough depth on both sides to prove that no shorter +path remains. + +Retain exact zero-, one-, and two-hop arms before workspace allocation. Their +latency is a setup control, not evidence that distinguishes recursive +schedulers. + +### 2.2 Architecture boundary tournament + +The discovery references show that inline recursive SQL can be much faster than +the current session-workspace functions. Therefore: + +- retain exact inline S3/S4/ASP full comparators; +- implement compact bidirectional references with explicit internal counters; +- compare a typed function/workspace boundary to the smallest viable inline or + SQL-visible boundary where the scheduler permits it; +- attribute search, workspace reset, predecessor reconstruction, and hydration + separately. + +Do not select a scheduler based on a comparison that also changes hydration or +public observation. Each pair must share the same output boundary. + +### 2.3 Gates and fallback + +SP admission gates are separate counters: + +- total distinct seen nodes across both sides; +- current/next frontier or queue rows; +- retained witness-predecessor rows; +- optionally bounded meeting candidates. + +No recursive result is emitted until all gates pass. Overflow invokes +the production incumbent for the candidate's bucket in the same top-level +statement: S3 for S3 distance buckets, and S4 for deep-inbound distance or +witness buckets. Alternatively, restrict the first B1/B2 production activation +to S4 buckets. Candidate workspace names must be distinct from the current +`spd_*` workspace so nested fallback cannot corrupt state. Record the complete +fallback chain when S4 invokes its relationship-trail fallback, and establish +the function snapshot contract described above before calling the chain +snapshot-stable. + +After confirmation, a new `sp-static-v5` may select candidates only for the +topology and observation buckets that pass. A global scheduler winner is not +required: S3 or S4 may remain best for shallow or selective shapes. + +Before shadow or production use, define a versioned mapping from facts available +to the real query—query shape, observation, physical direction, depth, bounded +endpoint/degree probes, or executor frontier state—to each selectable topology +bucket. Fixture metadata and post-run telemetry label evaluation strata; they +cannot drive production selection. If a bucket cannot be recognized from +runtime inputs, it remains a diagnostic classification. + +## Workstream 3: bidirectional ASP predecessor DAG + +ASP begins only after the shared bidirectional search kernel, termination proof, +and SP telemetry pass qualification. + +Reserve: + +| Candidate | Scheduler ID | Reference arm | +| --- | --- | --- | +| `ASP-B1-DAG-ALT-NODE` | `strict_alternating_node` | `asp_b1_bidirectional_dag_strict_m0` | +| `ASP-B2-DAG-MIN-LEVEL` | `smaller_current_level` | `asp_b2_bidirectional_dag_smaller_frontier_m0` | + +The current `ASP-A1-DAG` remains the single-ended exact production control. +The legacy `bidirectional_asp_harness` carries complete trails and is not the +new candidate. + +### 3.1 State and reconstruction + +Each side retains: + +- minimum reached depth per node; +- every relationship-distinct predecessor or successor that reaches that node + at the same minimum depth; +- frontier state and scheduler order independently from predecessor state. + +When minimum distance `L` is proven, select one deterministic completed meeting +cut `k`. Enumerate source predecessor paths to nodes at depth `k`, target +successor paths from the same nodes at depth `L-k`, and stitch ordered edge ID +arrays. Using one cut ensures that a complete path is not emitted once per +overlap level. For the initial singleton pair, uniquely stage ordered +`edge_ids` and assert relationship uniqueness before public output. Endpoint +broadening must key uniqueness by input-pair identity plus `edge_ids`, then +reapply duplicate input-pair multiplicity; otherwise repeated endpoint rows +would be collapsed. + +Within the initial distinct-endpoint, minimum-depth-one envelope, an unweighted +minimum path cannot repeat a node because removing the intervening cycle would +make it shorter. This justifies minimum-node-depth discovery for this envelope +only. It does not justify directionless traversal, positive-minimum self cycles, +whole-path predicates, or broader trail semantics. + +### 3.2 Independent resource gates + +ASP has three different explosion modes and therefore three limits: + +1. Discovery: distinct seen/frontier nodes. +2. Predecessors: same-minimum-depth relationship-distinct predecessor rows. +3. Enumeration: distinct ordered edge arrays and materialized bytes. + +Before enumeration, calculate a saturating path-count bound over the predecessor +DAG. Stage output under `limit + 1` sentinels. Any overflow clears candidate +state and invokes `all_shortest_paths_dag` before exposing a row. This fallback +uses the same top-level statement, but still requires the deliberate function +snapshot contract before it can be described as one-snapshot execution. + +These are candidate-admission guards, not public result limits. ASP may never +silently truncate a required path set. If the exact incumbent itself cannot +complete within an external statement/resource policy, propagate that error; +do not relabel truncation as fallback success. + +After independent confirmation, `asp-static-v2` may select a qualified +bidirectional arm. If enumeration dominates total latency or no candidate +contains predecessor/output risk, retain A1 and record the new arm as a frozen +negative result. + +## Workstream 4: endpoints and predicate classes + +The first SP/ASP candidates retain the current one-literal-ID-per-endpoint +envelope. Broaden only after their core algorithms are stable. + +### 4.1 Bounded endpoint resolution + +Materialize endpoint resolution once with explicit 1/2/32/33 sentinels and exact +fallback. Qualify independently: + +- ID equality; +- unique indexed property equality; +- nonunique property equality that returns a small bounded set; +- explicitly supplied small endpoint sets; +- endpoint pairs whose correlation must be preserved rather than treated as a + Cartesian product. + +Record input rows, distinct endpoint IDs, duplicate multiplicity, pair count, +resolution plan/index, and overflow. Endpoint cardinality is runtime evidence; +predicate syntax alone is not selectivity proof. + +Keep the compact bidirectional ASP kernel singleton-only until a wrapper assigns +stable input-pair identities, deduplicates paths within each pair, and reapplies +duplicate pair-row multiplicity. Endpoint broadening must not make global +`edge_ids` uniqueness collapse the Cypher result bag. + +### 4.2 Predicate classification + +Add an explicit classifier for: + +- step-local node predicates; +- step-local relationship predicates; +- universal `ALL`/`NONE` predicates over path nodes or relationships that can + be evaluated on each expansion step; +- whole-path predicates requiring a complete materialized candidate. + +Only step-local or proven universal predicates may enter the compact expander. +Whole-path predicates retain an exact fallback-capable exhaustive plan. Each +predicate class needs mutation and translation fixtures because placement can +change evaluation and output semantics. + +## Workstream 5: fixed one-hop `ExpandInto` + +This work applies only when both endpoints of a fixed, one-hop relationship are +bound. It must not be generalized to variable-length `Into`. + +Start with a three-way plan study: + +1. Current bound-endpoint edge join, recording the plan PostgreSQL actually + chooses (for example, parameterized index lookup, hash join, or another + shape). +2. Typed lower-degree endpoint probe followed by adjacency scan and opposite + endpoint check. +3. The bound-pair join plus PostgreSQL `Memoize` or an explicit + statement-local distinct-pair cache for repeated input pairs. + +Measure wildcard and multi-kind cases separately. An actual parameterized pair +index plan may make lower-degree probing redundant for singleton typed pairs, +while pair reuse may matter only with duplicate outer rows. Add policy metadata +to the currently marker-only `ExpandIntoDecision` only if a candidate +demonstrates a real crossover. + +Pair caching stores or reproduces all matching relationship rows, not only a +connectivity boolean. It must preserve relationship IDs/properties, one-per-kind +multiplicity, wildcard/multi-kind and directionless behavior, self-loops, and +duplicate outer-row multiplicity even when it deduplicates lookup work. Qualify +cache hit/miss, missing endpoints, cross-kind parallel relationships, +cancellation, and generic/custom plans. + +## Workstream 6: statistics and probe roadmap + +Runtime capped probes are the first authority because they use the current +parameters and graph contents in the executing statement. Function-backed +search and fallback remain subject to the explicit snapshot contract above. +The useful evidence is: + +| Evidence | Primary use | +| --- | --- | +| Root/terminal endpoint rows and distinct IDs | Bound pair count and seed cost | +| Typed directional degree at each endpoint | First-step orientation and frontier risk | +| Suffix rows, distinct boundaries, and path multiplicity | Reverse seed and reconstruction cost | +| One-level survival and distinct-next ratio | Predicate selectivity and reconvergence hint | +| Per-level frontier and candidate edges | Adaptive SP/ASP scheduler choice | +| Seen-to-frontier and candidate-to-new-node ratios | Cycle/reconvergence cost | +| Same-depth predecessor additions | ASP predecessor memory risk | +| Meeting-node count and cut width | Bidirectional reconstruction cost | +| Saturating returned-path count and edge cells | ASP output/hydration risk | + +An optional synopsis is a later optimization, never a correctness proof. A +versioned synopsis may contain: + +- node counts by graph and kind; +- relationship counts by graph, direction, kind, and endpoint kind; +- distinct start/end counts and most-common endpoints; +- directional degree quantiles and heavy hitters; +- observed frontier survival/reconvergence buckets by depth; +- predecessor and output multiplicity buckets for qualified generated shapes. + +Node multi-kind membership makes endpoint-kind estimates overlapping rather +than additive. Sampling, refresh cadence, mutation overhead, stale-data +behavior, and graph drop/reload handling require an explicit design record. The +runtime guard remains authoritative. Prefer reading a synopsis at execution +time; embedding it in translated SQL requires a synopsis epoch in +`cypherTranslationCacheKey` and mutation-safe invalidation. + +## Qualification corpus + +Preserve the scale corpus's `normal`, `envelope`, and `stress` tiers. Gate normal +and envelope; use stress for exact fallback and failure-mode diagnosis. Expand +the existing deterministic generators before adding a new generator family. + +| Area | Required axes | +| --- | --- | +| Orientation | root and terminal seeds `0/1/2/32/33/128/512/513`; independent forward/reverse typed degree `0/1/4/32/128/1000/16000`; productive fraction `0/sparse/half/all`; mirrored fan-out/fan-in; hidden spike at first/middle/final depth | +| Common traversal | depth `0/1/2/4/8/16/32/64`; outbound/inbound/directionless; one/multiple kinds; fixed prefix/suffix `0/1/3`; disconnected decoys; cycles; self-loops; convergence; payload | +| SP | direct and two-hop controls; highly asymmetric endpoints; alternating-frontier crossovers; shallow target plus huge continuation; disconnected exhaustion; intermediate skew; one/equal witnesses; distance and path observations | +| ASP | depths `3/8/16`; diamond width and path count `1/2/16/128+`; same node count with different predecessor density; multiple meeting nodes; merge-then-split DAG; modest state with explosive output; large predecessor state with modest output | +| `ExpandInto` | asymmetric degrees; typed/wildcard/multi-kind; missing endpoints; self-loop; repeated pair hit/miss; duplicate outer rows | +| Endpoints/predicates | ID, unique property, nonunique property, small sets; local node/edge universal and whole-path predicates | +| Limits | every probe/state/predecessor/output cap at `N-1/N/N+1`, including current `32/33` and `4096/4097` boundaries | +| Output | scalar/count, endpoint IDs, ordered witness, full path/hydration, `LIMIT` absent/one/small | + +Freeze a topology holdout before selector thresholds are tuned. Include textually +permuted multi-`MATCH` and multi-pattern forms to compare Neo4j reorder +invariance with CySQL clause ordering. Record unsupported same-kind parallel +edges as a storage boundary while covering cross-kind multiplicity. + +## Tests required for every behavior change + +### Unit, translation, and mutation coverage + +- Optimizer table tests for candidate lists, exact eligibility facts, physical + direction, policy/scheduler versions, caps, and stable fallback reasons. +- SQL-shape tests for materialized probes, explicit `LIMIT cap+1`, disjoint + branch dependencies, ID-only state, edge-index orientation, and late + hydration. +- Fail-closed forcing tests for wrong observation, predicates, mutation, + correlation, optional match, directionless traversal, multiple calls, and + unsupported depth. +- Reverse path-order, relationship-overlap, suffix bag multiplicity, duplicate + roots, parameter rebinding, and generic/custom-plan tests. +- Source translation-case updates plus generated artifacts and mutation tests + for parsing, lowering, rendering, and predicate placement changes. + +### Semantic integration + +- Shared backend-equivalent cases validate logical stable observations; no + driver-specific expected values or skips belong in the shared corpus. +- PostgreSQL-scoped tests validate candidate branch loops, exact fallback, + workspace state, edge indexes, caps, buffers, and function invocation. +- Cover missing/null/equal endpoints, zero depth, maximum-depth miss, both + directions, cycles, repeated nodes without repeated relationships, suffix + multiplicity, empty/disconnected sides, and every accepted/rejected predicate + class. +- For singleton SP ties, compare distance and validate that each returned trail + is minimum and relationship-unique; use unique-witness cases when an exact + ordered-ID reference comparator is required. +- For ASP, compare the full stable path multiset and predecessor/output cap + boundaries, not only row count. + +### Operational integration + +- Pool sizes `1/2/8` and concurrency `1/8/16`. +- Prompt cancellation followed by successful rollback and reuse of the same + PostgreSQL backend PID. +- A concurrent-writer semantic test proving the selected snapshot mechanism or + rejecting function-backed fallback under the default isolation behavior. +- Low `work_mem`, forced generic plan, forced custom plan, and normal `auto` + plan modes. +- No cross-invocation workspace or telemetry contamination. +- Schema-up/schema-down symmetry and upgrade coverage for every new helper or + temporary workspace. + +## Performance and resource gates + +Use the existing balanced GraphBench protocols: + +- Discovery: at least five independently reloaded rounds, five warmups, and ten + samples per arm. +- Confirmation: 10-20 independently reloaded rounds, at least 20 warmups and + 50 samples per arm, seeded 97.5% intervals, and balanced arm order. +- Before accepting three-arm SP or ASP evidence, add and freeze a balanced + three-arm Latin/Williams schedule. The current non-five-arm forward/reverse + ordering leaves the middle arm in the middle and is not carryover-balanced. +- A/A calibration: derive per-host p50/p95 absolute and ratio resolution before + applying materiality. +- Complete declarations only: filtered or adaptive artifacts are diagnostic and + cannot pass a release gate. + +Initial promotion thresholds are policy inputs and must be versioned: + +- target p50 candidate/incumbent ratio upper bound at most `0.95`, or absolute + saving lower bound at least `100us`; +- no p95 regression beyond the greater of host A/A noise and 5%, using the + greater of A/A absolute noise and `100us` for very fast cases; +- no confirmed normal/envelope regression outside that same noise band; +- selector regret versus the fastest exact arm: ratio upper bound at most + `1.10` or within the A/A absolute floor; +- probe overhead versus the forced selected arm: at most 10% or `100us`; +- production/reference closure: retain the existing `1.10` ratio/A/A floor; +- Neo4j latency and PROFILE remain descriptive. + +Extend the resource gate to enforce numeric envelopes, not only spill classes: + +- probe rows at or below `cap + 1`; +- frontier, queue, seen, predecessor, output, and bytes at declared ceilings; +- no executor temp-file read/write or WAL for non-mutating candidates; +- local workspace only for explicitly workspace-qualified architectures; +- measured per-session and pool memory ceilings; +- no unexpected fallback in admitted normal/envelope buckets; +- exactly attributed fallback in stress buckets. + +The identities must form a valid chain: the translation-applied policy matches +the planned candidate set, the runtime arm belongs to that emitted policy, and +any runtime fallback matches the declared incumbent chain. Probes execute at +most once, unselected arms show zero work, and fallback executes once before any +output. Any missing or contradictory attribution fails the gate. + +## Milestones and exit criteria + +| Milestone | Deliverables | Exit criterion | +| --- | --- | --- | +| M0: freeze baseline | Clean-source capture bundle; PostgreSQL/Neo4j environment fingerprints; current plans; A/A calibration; stable candidate IDs; topology holdout split | Checksummed artifacts reproduce exact observations and the discovery findings without credentials. | +| M1: observability | `TraversalExecutionTelemetry`; PostgreSQL diagnostic counters; Neo4j read `PROFILE`; paired PlanCorpus deltas; numeric resource schema | No result change; measured telemetry overhead is within A/A noise or disabled outside diagnostic replay; missing counters fail qualification. | +| M2: orientation framework | Common candidate analyzer; bounded seed/degree probes; endpoint-reverse migration; guarded suffix reverse; forced and shadow modes | Exact parity across semantic/cap cases; disjoint branches; selector-regret and probe-overhead reports exist. Production still uses the incumbent except the already qualified endpoint family. | +| M3: SP references | Strict-alternating and smaller-level compact reference arms; typed scheduler metadata; balanced three-arm schedule; formal termination invariant; inline/function boundary comparison | Exact distance/witness results, bounded state, cancellation/reuse, and discovery report across asymmetric topology buckets. | +| M4: SP production qualification | Incumbent-specific same-statement fallback; snapshot contract; complete confirmation/holdout/resource/reference-closure reports; `sp-static-v5` policy | Only runtime-recognizable, passing topology/observation buckets select a new arm; all other shapes preserve S3/S4/S0 with precise reasons. | +| M5: ASP references and qualification | Two-sided predecessor state; canonical meeting cut; three independent gates; full multiset comparator; ASP stress corpus | Exact ASP output, no truncation, bounded candidate state, confirmation and holdout pass; otherwise freeze a negative result and retain A1. | +| M6: envelope broadening | Bounded property/small-set endpoints; step-local/universal predicates; fixed one-hop `ExpandInto` study and any qualified policy | Each class has its own eligibility, exact fallback, corpus, and decision record. No broadening by tool forcing. | +| M7: optional synopsis | Synopsis ADR, schema/refresh/cache design, shadow comparison against runtime probes | Implement only if it materially reduces probe/selector regret and its mutation/cache cost passes independent gates. | + +M2 and M3 may proceed in parallel after M1. M5 begins after the shared SP +kernel and telemetry stabilize. M6's `ExpandInto` plan study may run earlier, +but automatic behavior still requires its own evidence. + +## Repository implementation map + +| Concern | Primary files | +| --- | --- | +| Typed decisions and selectors | `cypher/models/pgsql/optimize/lowering.go`, `lowering_plan.go`, `optimizer_test.go` | +| Ordinary orientation emission | `cypher/models/pgsql/translate/expansion_orientation.go` (new), `expansion_endpoint_seeded.go`, `expansion_suffix_seeded.go`, `pattern.go`, `traversal.go`, `translator.go` | +| SP/ASP builders and dispatch | `cypher/models/pgsql/translate/expansion.go`, `pattern.go`, `optimizer_safety_test.go`, `cypher/models/pgsql/functions.go` | +| Compact workspaces/functions | `drivers/pg/query/sql/schema_up.sql`, `schema_down.sql`, `drivers/pg/query/sql_workspace_test.go`, schema-upgrade integration tests | +| Translation cache contract | `drivers/pg/translation_cache.go` and tests; change if mutable rollout policy is translated rather than supplied at execution, or if a synopsis is embedded | +| GraphBench telemetry/references | `cmd/graphbench/results.go`, `postgres_plan.go`, `neo4j.go`, `references.go`, `datasets.go`, `main.go` and tests | +| Gates and reports | `cmd/graphbench/resource_gate.go`, `perf_gate.go`, reference-pair/closure reports, backend-delta report | +| Matched plan deltas | `cmd/plancorpus/types.go`, `report.go`, capture/report tests | +| Deterministic topology generators | `testutil/perf_shortest_v2.go`, `perf_endpoint_seeded.go`, `perf_fixtures.go` | +| Scale declarations | `benchmark/testdata/scale/cases/generated_shortest_paths_v2.json`, `generated_endpoint_seeded_expansion_v1.json`, `generated_fixed_suffix_expansion.json` | +| Semantic fixtures | `integration/testdata/cases`, `integration/testdata/templates`, PostgreSQL-scoped plan-invariant tests | +| Documentation and evidence | this plan, `recursive_descent_cost_controls.md`, `postgresql_translation.md`, GraphBench/scale READMEs, and versioned `docs/experiments` records | + +Changes should be sliced so telemetry, candidate implementation, selector +activation, and envelope broadening are separately reviewable. Do not combine a +new algorithm, new semantic support, and automatic selection in one change. + +## Rollout and rollback + +Every candidate follows the same stages: + +1. Telemetry only; no selection change. +2. Exact benchmark reference arm with a frozen implementation ID. +3. Tool-forced production emitter, failing closed outside its envelope. +4. Shadow selection that records `would_select` while executing the incumbent; + matched diagnostic arms calculate regret. +5. Explicit opt-in with same-statement exact fallback and an established + snapshot contract for function-backed arms. +6. Narrow automatic selection for named, passing topology buckets. +7. One-bucket-at-a-time expansion after new holdout confirmation. + +Keep the incumbent selector and previous function/schema identity available for +at least one release after automatic activation. A feature gate must be able to +return all traffic to the incumbent without a data migration, and changing it +must invalidate cached translated SQL or be an execution-time policy input. + +Immediately disable automatic selection on: + +- any correctness or ASP multiplicity mismatch; +- planned/emitted/runtime attribution disagreement; +- cap breach, partial candidate output, unexpected spill, or read-query WAL; +- cancellation poisoning or workspace/telemetry cross-talk; +- unstable SQL/plan fingerprint outside a declared change; +- abnormal fallback frequency in a qualified bucket; +- a confirmed p95 regression outside the A/A/materiality envelope. + +Do not retune a failed identity post hoc. Preserve the failed arm and compact +evidence in `docs/experiments`, assign a new ID to a materially changed design, +and reopen discovery with a new hypothesis. + +## Risk register + +| Risk | Mitigation | +| --- | --- | +| Probe overhead erases the orientation win | Cap every probe, materialize once, measure probe-only cost, use hysteresis, and keep forward on ambiguous small gains. | +| Reverse admission plus fallback doubles expensive work | Gate before output, measure fallback regret explicitly, lower admission caps, and qualify overflow buckets separately. | +| Mutable topology or rollout policy invalidates cached SQL | Keep topology values inside same-statement probes; make mutable policy an execution input or cache generation; require a synopsis epoch before embedding statistics. | +| Bidirectional search stops at a nonminimal first meeting | Require a documented lower-bound termination proof and adversarial asymmetric/reconvergent tests. | +| ASP predecessor or output explosion is hidden by node-state counts | Enforce separate discovery, predecessor, path-count, output-row, and byte gates. | +| Session workspaces consume excessive pool memory or collide | Use invocation/session isolation, explicit per-session/pool ceilings, concurrency tests, and prompt cleanup on error/cancel. | +| Detailed telemetry changes the measured algorithm | Keep detailed counters in untimed diagnostic replay; separately measure lightweight summary overhead. | +| Fixed `ExpandInto` copies a Neo4j optimization that PostgreSQL does not need | Compare direct pair index lookup, lower-degree scan, and `Memoize`/pair cache before implementation. | +| Predicate pushdown changes evaluation semantics | Classify locality/universality, retain exact fallback, and require mutation plus cross-backend semantic fixtures. | +| Aggregate benchmark wins hide topology regressions | Gate by predeclared buckets, worst-case containment, and a frozen holdout rather than aggregate median alone. | +| Neo4j version differences corrupt interpretation | Pin source commits and server version in every artifact; keep 4.4 strict alternation and current smaller-level scheduling as separate arms. | + +## Validation and evidence workflow + +After code changes, run formatting and unit validation: + +```bash +make format +make test +make lint +``` + +Run backend-specific full validation separately: + +```bash +CONNECTION_STRING="$PG_CONNECTION_STRING" make test_all + +CONNECTION_STRING="$NEO4J_CONNECTION_STRING" make test_all +``` + +Then run both-backend PlanCorpus and the staged GraphBench workflow: + +1. plan corpus and matched delta capture; +2. discovery plus exact reference comparisons; +3. A/A calibration; +4. 10-20-round confirmation; +5. numeric resource gate; +6. production/reference closure; +7. concurrency, cancellation, and session-reuse cases; +8. topology holdout; +9. descriptive backend delta; +10. complete performance gate and capture bundle checksum. + +Never place connection strings, endpoint IDs from sensitive graphs, query +parameters, or credentials in durable artifacts. Existing-graph confirmation +uses the current redacted anchor-manifest workflow and cannot substitute for the +deterministic correctness corpus. + +For every accepted or rejected candidate, add +`docs/experiments/_vN.md` containing: + +- immutable implementation and selector IDs; +- source and artifact SHA-256 values; +- backend versions and relevant settings; +- corpus declaration and holdout identity; +- rounds, warmups, samples, order balancing, and confidence policy; +- correctness, performance, resource, fallback, concurrency, and cancellation + results; +- the promotion/rejection decision and unchanged incumbent behavior. + +Raw captures remain under `.coverage`; compact canonical reports may be +committed when they contain no secrets or unstable physical identifiers. + +## Definition of done + +This priority plan is complete when: + +- traversal decisions and runtime execution are separately observable and + matched across plan records; +- Neo4j read plans include actual evidence with SP/ASP opacity represented + honestly; +- ordinary orientation, SP, and ASP each have exact incumbent and candidate + arms with stable identities; +- every candidate has bounded probes/state, disjoint output/fallback behavior, + and precise machine-readable fallback reasons; +- semantic, cap-boundary, operational, resource, performance, and holdout gates + run reproducibly; +- production selectors enable only independently passing topology/observation + buckets and remain quickly reversible; +- nonwinning candidates are retired with durable negative evidence rather than + left as ambiguous code paths; +- documentation describes current production behavior separately from future + candidates and their qualification status. + +Success may legitimately conclude that S3/S4/A1 or direct PostgreSQL pair +lookup remains best for some or all buckets. The required outcome is a measured, +exact, explainable selector program—not a predetermined Neo4j-shaped executor. diff --git a/docs/development.md b/docs/development.md index b39bcfe7..8815fcb8 100644 --- a/docs/development.md +++ b/docs/development.md @@ -34,6 +34,11 @@ export CONNECTION_STRING="postgresql://dawgs:weneedbetterpasswords@localhost:654 export CONNECTION_STRING="neo4j://neo4j:weneedbetterpasswords@localhost:7687" ``` +Integration and fixture-loading GraphBench runs mutate the selected database. +GraphBench `-existing-graph` mode rejects writes and validates before/after +cardinalities. Its PostgreSQL sessions remain read-write so temporary traversal +workspaces retain production behavior. + Use backend-specific targets when needed: ```bash @@ -59,7 +64,17 @@ Run: make format ``` -The target uses `goimports`; install it locally if it is missing from your environment. +The target uses `goimports`; install it locally if it is missing from your +environment. Sandboxed or nonstandard installations can supply its explicit +path without changing `PATH`: + +```bash +make format GOIMPORTS_CMD=/absolute/path/to/goimports +``` + +`make lint` runs the standard Go vet analyzers across the repository. The unreachable-code analyzer is rerun only for +handwritten packages because ANTLR emits intentional terminal branches in `cypher/parser`; generated parser code still +receives every other vet analyzer. ## Quality And Metrics @@ -108,7 +123,8 @@ The defaults can be adjusted with `CYCLO_TOP`, `CYCLO_OVER`, `CRAP_TOP`, `CRAP_O `make plan_corpus` captures plan diagnostics for the shared Cypher integration corpus. It accepts either `CONNECTION_STRING` for one backend or `PG_CONNECTION_STRING` and `NEO4J_CONNECTION_STRING` for both backends, then -writes JSONL captures and markdown/JSON summaries under `.coverage/`. +writes JSONL captures and markdown/JSON summaries under `.coverage/`. Fixture loading requires the same destructive +acknowledgement and exact credential-free allowlist entries as integration testing. Run it when changing PostgreSQL Cypher planning, lowering, or SQL emission. The summaries rank expensive PostgreSQL plans and report recursive CTEs, `SubPlan`, `Function Scan on unnest`, planned/applied optimizer lowerings, and @@ -120,13 +136,67 @@ See [Plan Corpus Capture](../cmd/plancorpus/README.md) for flags and review guid `go run ./cmd/graphbench` captures runtime diagnostics for the scale corpus under `benchmark/testdata/scale`. -Current modes are: +Implemented modes are: - `postgres_sql` -- `local_traversal` - `neo4j` +`local_traversal` emits non-gating `not_implemented` diagnostics only; it is not an implemented executor. + AGE is reference-design input only and is not a direct comparison mode. The command can emit JSONL records plus Markdown and JSON summaries, and can compare current timings against a previous JSONL baseline. +The tool-only `-postgres-expansion-suffix-reverse-retry` mode measures the +reverse-only fixed-suffix P1 candidate with exact forward retry inside one +Repeatable Read transaction. It requires diagnostic traversal telemetry and a +pool size of one; see the +[frozen development protocol](experiments/suffix_reverse_retry_v1.md) before +selecting cases or overriding caps. + +The tool-only `-postgres-expansion-suffix-route-component` mode measures one +exact suffix-seeded reverse statement for the default-off SQL-routing +preflight. It requires Repeatable Read, diagnostic traversal telemetry, and a +pool size of one. It forbids probes, retries, cap overrides, cache behavior, +and production manifests; see the +[preimplementation contract](experiments/sql_strategy_routing_preflight_v1.md). + +`-postgres-suffix-route-component-closure` is the separately frozen, +measurement-only closure for that preflight. It is valid for either the +ordinary incumbent or the explicitly forced direct component, but requires the +same Repeatable Read/diagnostic/size-one-pool contract plus positive declared +session and pool workspace ceilings. It records fresh prepared-miss, +same-session prepared-hit, and release/reacquisition strata with client and +raw-PGX waterfalls. Each raw sample records a normalized public-observation +SHA-256 that must agree with the primary CySQL result and every closure +stratum. It cannot be combined with reference, concurrency, +orientation, retry, guard, force, or production-manifest modes and does not +select an executor. The exact roster and acceptance conditions are in the +[closure protocol](../benchmark/testdata/scale/protocols/sql_strategy_routing_component_closure_v1.json). + +The PostgreSQL scale-plan correctness gate shares the scale runner. It checks the +required stable query-form IDs, declared read/write cardinalities, rollback-safe +mutation post-state, `EXPLAIN ANALYZE` capture, and stable plan invariants. It +runs under `make test_all` for PostgreSQL or can be selected directly: + +```bash +CONNECTION_STRING="$PG_CONNECTION_STRING" \ + go test -tags manual_integration ./cmd/graphbench \ + -run 'Test(PostgreSQLScalePlanInvariants|ScaleCorpusRequiredRepresentativesDeclareCardinality)' \ + -count=1 +``` + +Store graphbench and plan-corpus captures under `.coverage/`; they are +environment-specific review artifacts, not committed correctness goldens. + See [Graph Benchmark Capture](../cmd/graphbench/README.md) for command examples. + +## BloodHound Source-Parity Audits + +When the reviewed BHE or BHCE snapshots change, repeat the call-site inventory, +active-entry-point trace, normalized query-form mapping, and commit recording in +[BloodHound Regression Source Parity](regression_source_parity.md). + +Dormant `FUTURE-*` forms stay manifest-only until a reviewed caller is enabled. +The unit suites reject dormant IDs from both shared plan inputs and scale cases; +activating a form requires updating those gates together with its required +semantic, plan, and scale coverage. diff --git a/docs/experiments/asp_a1_diagnostic_prerequisite_v1.md b/docs/experiments/asp_a1_diagnostic_prerequisite_v1.md new file mode 100644 index 00000000..36902ff4 --- /dev/null +++ b/docs/experiments/asp_a1_diagnostic_prerequisite_v1.md @@ -0,0 +1,71 @@ +# A1 all-shortest diagnostic prerequisite v1 + +Status: implemented and PostgreSQL-integration-validated. A clean source +commit and fresh P4 capture are still required before this can produce any +performance evidence. It does not change A1 selection, resource caps, or +public results. + +## Observed gap + +The P4 V1 first-round replay reached `ASP-A1-DAG` exactly, but PostgreSQL's +outer `Function Scan` hid all invocation-local work. The P3 B1/B2 diagnostic +workspace cannot be relabeled as A1 evidence: A1 is a separate single-ended +predecessor-DAG executor with its own `spd_*` workspace and scheduler. + +## Implemented boundary + +The implementation adds a session-local A1 diagnostic reader used only by +GraphBench's untimed Repeatable Read replay. Its begin step resets `spd_seen`, +`spd_candidate`, and `spd_predecessor`, records an invocation ID in a dedicated +temporary telemetry table, then executes the translated A1 statement once. Its +reader verifies that the same session produced exactly one A1 call and reports +only values observed in that replay: + +- per-depth candidate, admitted-node, and predecessor counts from the three + `spd_*` relations, with cumulative seen and predecessor peaks; +- single-ended scheduler, target depth/no-path branch, and no-fallback A1 + runtime identity; +- path count and edge cells derived from the exact replayed public path set; +- serialized output bytes from the exact GraphBench path observation; +- outer hydration loops/rows/time from the untimed timing-on plan; and +- session and pool workspace high water from `pg_total_relation_size` over + `spd_*` only, excluding telemetry tables. + +Depth-one and depth-two returns leave no `spd_*` rows by design. The reader +must label those exact branches explicitly and derive their path count/depth +from the replayed path set; it must never reuse rows from an earlier call. +No-path results likewise require a cleared workspace and an explicit no-path +receipt. + +The A1 receipt has its own `a1_single_ended` schema and validation. It reuses +GraphBench's invocation-local replay transaction, all-shortest counter types, +hydration accounting, and workspace counter types, but does not call or +reinterpret the bidirectional B1/B2 diagnostic API. + +The stored A1 function reads the local +`dawgs.asd_diagnostic_invocation_id` setting once per call. With that setting +absent, it performs no telemetry-table writes or workspace-count queries. With +the setting present, it records the invocation-local receipt. This is a small +production SQL instrumentation change, so the recapture must start from a +clean commit and must not be compared with the stopped V1 artifact. + +## Validation and clean-recapture gate + +The implementation includes SQL-shape tests for opt-in, session-local A1 +telemetry and symmetric teardown; GraphBench unit tests for complete and +stale/contradictory receipts; and PostgreSQL integration coverage for one-hop, +two-hop, recursive inbound, reconvergent, and no-path records. The integration +contract checks the forced `ASP-A1-DAG` identity, exact public observation, +complete A1/hydration/workspace counters, Repeatable Read replay, two-session +isolation for a shared invocation key, cancellation rollback, and reuse of the +same backend PID. + +The targeted A1 integration tests and the full PostgreSQL `make +test_integration` suite pass. Missing invocation identity, multiple calls, +hidden/stale counters, a mismatched public path count, or a workspace +measurement that includes telemetry relations fails closed. + +Only after this implementation is committed from a clean source may the P4 +open baseline V1 be recaptured from round one. The existing stop artifact, I1 +archives, B1/B2 functions, all holdouts, diagnostic/stress cases, manifests, +and selectors remain outside that authorization. diff --git a/docs/experiments/asp_a1_inline_hydration_v1.md b/docs/experiments/asp_a1_inline_hydration_v1.md new file mode 100644 index 00000000..c9d70eb8 --- /dev/null +++ b/docs/experiments/asp_a1_inline_hydration_v1.md @@ -0,0 +1,32 @@ +# ASP A1 inline hydration disposition v1 + +The `allShortestPaths` diamond regression was attributed with parameterized +`EXPLAIN (ANALYZE, BUFFERS, SETTINGS, FORMAT JSON)`. The previous benchmark +explainer translated without scenario parameters and therefore captured an +empty endpoint plan; this disposition uses the corrected exact-parameter +capture. + +The A1 predecessor-DAG function was not the dominant cost. The generated SQL +materialized each returned edge-ID path through `ordered_edge_ids_to_path`, a +separately planned generic helper. `ASP-A1-DAG` now uses the existing inline +M0 hydration relation used by compact shortest-path executors: it hydrates the +ordered edges and terminal nodes once at the outer statement boundary. + +On the local `traversal_shapes` corpus with one V2 connection, two warmups, +and ten timed samples: + +| Arm | Diamond median | Disconnected median | +| --- | ---: | ---: | +| Previous A1 generic hydration | 24.22 ms | 1.46 ms | +| A1 inline M0 hydration | 0.65 ms | 1.60 ms | +| B1 bidirectional component | 1.50 ms | 9.30 ms | +| B2 bidirectional component | 1.40 ms | 6.60 ms | +| Neo4j reference | 1.23 ms | 1.35 ms | + +The A1 diamond improvement is approximately 37x and places PostgreSQL V2 +ahead of the observed Neo4j median. Neither B1 nor B2 clears the required 20% +win over A1 and both are frozen as tool-only negative results for this workload. + +No new ASP production selector is installed. The existing automatic A1 route +remains the conservative production control, while the corrected benchmark +stage capture remains available for future deeper or wider topology studies. diff --git a/docs/experiments/asp_i1_inline_v1.md b/docs/experiments/asp_i1_inline_v1.md new file mode 100644 index 00000000..057e3aed --- /dev/null +++ b/docs/experiments/asp_i1_inline_v1.md @@ -0,0 +1,75 @@ +# Inline all-shortest-path predecessor DAG v1 + +Date: 2026-08-12 + +Status: implemented as a default-off production canary; automatic selection +withheld pending clean qualification evidence + +`ASP-I1-U-DAG+MAT-M0` is the typed, inline PostgreSQL comparator for qualified +`allShortestPaths` queries. It is intentionally distinct from the stored +helper implementation `ASP-A1-DAG` so benchmark arms and production receipts +identify the executable code path rather than only the algorithm family. + +## Correctness and resource boundary + +The emitter accepts one read-only, non-optional, directed endpoint pair with +static singleton endpoint IDs, minimum depth one, and a bounded maximum depth +from 1 through 64. It discovers minimum node distances, retains every +relationship-distinct predecessor at that minimum layer, and enumerates the +predecessor DAG into ordered relationship-ID arrays. Existing outer +translation performs path hydration. + +The production emitter resolves exact one- and two-hop targets first. These +bounded preflight rows participate in the enumeration cap+1 gate, and recursive +distance discovery runs only when no early target exists. + +Every recursive producer is consumed through a materialized cap+1 relation. +Separate immutable limits cover discovered states, predecessor rows, all +intermediate enumeration states, and serialized output bytes. The guarded +decision is complete before either public-output arm opens. A cap overflow +selects exact `ASP-A1-DAG` in the same statement and stable snapshot; candidate +rows cannot mix with fallback rows. + +Materialized candidate and fallback markers provide singular plan evidence. +The runtime attestation receipt schema v2 records an ordered event chain. A +non-nested I1 execution records one of: + +- `inline_predecessor_dag` with runtime identity `ASP-I1-U-DAG+MAT-M0`; +- `inline_no_path` with runtime identity `ASP-I1-U-DAG+MAT-M0`; +- `exact_a1_fallback` with runtime identity `ASP-A1-DAG`. + +GraphBench replays distance, predecessor, enumeration, output, marker, and +branch-row counters. Qualification fails when attribution is absent, +contradictory, over cap, or shows rows from the inactive output arm. + +## Production policy + +The driver can select I1 only under Repeatable Read or Serializable isolation. +The verified schema-v2 promotion manifest must name the candidate and exact A1 fallback, +use `guarded_dual_arm`, declare all four positive caps, and authorize the exact +normalized-query SHA plus direction, all-path observation, depth, +relationship-kind count, and typed/untyped bucket. Every evidence report must +repeat that complete authorization identity. Query allowlisting and the +policy generation partition the translation cache. Read Committed, unmatched +queries, and the zero policy retain the incumbent. `DisableInlineASPDAG` +provides an evidence-free immediate rollback switch. + +Tool forcing remains available for controlled comparison but does not broaden +the structural envelope. B1/B2 shortest and ASP experiments remain tool-only; +the production allowlist is centralized on the implemented inline families. + +## Qualification sequence + +1. Capture balanced A/A and A1-versus-I1 runs from a clean source tree. +2. Require exact full path-multiset parity on training, frozen holdout, and + diagnostic cases, including inbound, disconnected, parallel-kind, + early-target, diamond, cycle, and self-loop topologies. +3. Pass confirmation materiality/p95, selector-regret, resource, + reference-closure, cancellation, concurrency, and session-isolation gates. +4. Generate a checksummed manifest for only the independently passing query + and topology buckets, then canary at stable isolation. +5. Expand allowlisted buckets only with new clean evidence. Keep A1 automatic + and retain the kill switch until post-canary production telemetry closes. + +No result from a dirty diagnostic tree is promotion evidence, and this +implementation does not change the automatic `asp-static-v1` selector. diff --git a/docs/experiments/asp_n1_negative_exhaustion_v1.md b/docs/experiments/asp_n1_negative_exhaustion_v1.md new file mode 100644 index 00000000..a81d0f34 --- /dev/null +++ b/docs/experiments/asp_n1_negative_exhaustion_v1.md @@ -0,0 +1,27 @@ +# ASP N1 negative-exhaustion disposition v1 + +`ASP-N1-NEGATIVE-EXHAUSTION` is a default-off, bounded target-side reachability +preflight for the existing A1 all-shortest-path executor. It can return an +empty result only after one of two exact proofs: + +- no eligible relationship enters the target in the logical traversal + direction; or +- reverse breadth-first discovery exhausts before the configured maximum depth. + +When the source is reached, or the reverse state sentinel is exceeded, N1 +discards its temporary state and invokes `ASP-A1-DAG`. It never produces a +positive path itself, so complete predecessor-DAG enumeration and path +hydration remain A1 responsibilities. + +The local `traversal_shapes` smoke run used one PostgreSQL V2 connection, two +warmups, and ten samples. The disconnected query measured 1.2 ms median and +1.7 ms p95, compared with the most recent Neo4j reference of 1.34 ms and +1.90 ms. A globally forced N1 executor regressed the reachable diamond because +the target-side probe is inconclusive there and must call A1 afterwards. + +N1 therefore remains tool-only. A future production selector must establish +the degree-zero condition before choosing N1; it must not use query shape or a +global topology synopsis as a proxy for a negative proof. The candidate's +fallback states are recorded as `asp_n1_target_degree_zero`, +`asp_n1_reverse_exhausted`, `asp_n1_source_reached_a1`, and +`asp_n1_state_cap_a1` for that qualification work. diff --git a/docs/experiments/asp_p4_i1_disconnected_preflight_v1.md b/docs/experiments/asp_p4_i1_disconnected_preflight_v1.md new file mode 100644 index 00000000..f81c7d6b --- /dev/null +++ b/docs/experiments/asp_p4_i1_disconnected_preflight_v1.md @@ -0,0 +1,101 @@ +# All-shortest P4 I1 disconnected preflight v1 + +Status: current I1 arm terminally rejected. The capture was a small, +training-only telemetry preflight, not a power study, performance +qualification, holdout opening, selector change, or promotion decision. + +## Scope and hypothesis + +The clean P4 A1 baseline selected +`GSPV2-TRAINING-disconnected-all-shortest-max64`: its four matched +PostgreSQL/Neo4j median ratios were 7.041×, 5.825×, 10.560×, and 5.228×, and +the invocation-local A1 receipt consistently used `search_no_path`. The case +has no public paths, but A1 still records 32 candidate edges, a seen peak of +33, and a predecessor peak of 32. + +`ASP-I1-U-DAG+MAT-M0` is the only candidate in this roster. It is a distinct, +default-off inline predecessor-DAG executor with a typed `inline_no_path` +receipt, complete guarded-branch evidence, and exact A1 fallback. A second +bidirectional search is not a justified first response to the selected +single-ended no-path target. This preflight tests whether I1 can execute its +own inline no-path path exactly and observably before any broader candidate +work is considered. + +## Frozen roster and schedule + +The target is `GSPV2-TRAINING-disconnected-all-shortest-max64`. The adverse +controls are `GSPV2-TRAINING-early-depth1-all-shortest-max16`, +`GSPV2-TRAINING-early-depth2-all-shortest-max64`, and +`GSPV2-TRAINING-reconvergent-all-shortest-max16`. They keep shallow early +returns and relationship-distinct multipath behavior in the preflight rather +than allowing a no-path result to obscure a shallow regression. + +Capture only PostgreSQL, with pool size one, Repeatable Read, GraphBench +diagnostic traversal telemetry, one warm-up, and five timed samples. Every +case receives forced `ASP-A1-DAG` and forced `ASP-I1-U-DAG+MAT-M0` in four +carryover-balanced orders: A1/I1, I1/A1, A1/I1, I1/A1. The capture therefore +contains 32 exact case/arm/round records, 160 timed samples, and 32 excluded +warm-up samples. No cap override, reference arm, concurrency exercise, or +other candidate is permitted. + +The complete machine-readable contract is +[`benchmark/testdata/scale/protocols/asp_p4_i1_disconnected_preflight_v1.json`](../../benchmark/testdata/scale/protocols/asp_p4_i1_disconnected_preflight_v1.json). +The GraphBench binary must be built from this clean roster commit; results are +not pooled with the P4 A1/Neo4j baseline or any historical I1 archive. + +## Acceptance and stop gate + +Every record must preserve the exact public all-shortest path multiset and +complete hydration/workspace evidence. A1 records must identify `ASP-A1-DAG` +and supply complete invocation-local A1 telemetry. I1 records must identify +`ASP-I1-U-DAG+MAT-M0`, expose complete `asp-i1-guarded-v1` typed telemetry, +and execute no fallback. The disconnected target must report `inline_no_path`; +each control must report `inline_predecessor_dag`. + +Any inexact result, missing/hidden/contradictory counter, wrong identity or +branch, fallback, undeclared cap behavior, or adverse-control failure stops +the preflight. A complete capture still authorizes neither formal power, +additional candidate timing, protected cases, nor a selector/manifest change: +a separately frozen next-stage decision is required. + +## Capture result and stop + +The capture ran from clean source +`307e62f4d0e102384752e031f9c2850d6a73dbfe` with GraphBench SHA-256 +`ce101efdc55d173176aaa221b3ca3a18b4d40e3fb3a970fa346fdc98c125557c`. +The ignored artifact directory `.coverage/p4-i1-disconnected-307e62f` +contains the separate arm streams `a1.jsonl` (SHA-256 +`03114fd4b2cfdc50959697eebbfe08e4932ab307c38845b88415d8a287a3778f`) +and `i1.jsonl` (SHA-256 +`f147df180b8b351bf14ed83977105beaf48b96a822a143a33229afbb979083b6`). +Its capture ledger hashes to +`9baa5f3d25f47d5efd1a496ac266fee013edd419d9a696eb3f953e65c40a29f5`. + +It contains the frozen 32 records, 160 timed samples, and 32 excluded warm-up +samples. All source-diff hashes are empty. Every record is exact, and A1/I1 +public path multisets match for every case/round pair. A1 has complete +invocation-local telemetry. Every I1 record has complete `asp-i1-guarded-v1` +telemetry, the expected I1 runtime identity, and zero fallback; the target +uses `inline_no_path` and all controls use `inline_predecessor_dag`. + +The first I1 command tried to append its rows to the existing A1 JSONL. +GraphBench correctly rejected the different arm at serialization, so those +rows were not accepted as an artifact. The identical frozen I1 round was +immediately rerun into its own stream before validation; the rejected output +is excluded from all counts and comparisons. A later typo in the disposable +target allowlist was rejected before database work. The host reported a +`powersave` governor, so this remains diagnostic-only evidence. + +| Case | I1/A1 pooled median | I1/A1 pooled p95 | Result | +| --- | ---: | ---: | --- | +| `GSPV2-TRAINING-disconnected-all-shortest-max64` | 0.192× | 0.246× | target improves | +| `GSPV2-TRAINING-early-depth1-all-shortest-max16` | 1.365× | 1.516× | fails control | +| `GSPV2-TRAINING-early-depth2-all-shortest-max64` | 1.410× | 1.070× | fails control | +| `GSPV2-TRAINING-reconvergent-all-shortest-max16` | 1.200× | 0.896× | fails control | + +The no-path target win cannot average away three adverse-control median +regressions. Under P4's frozen stop gate, `ASP-I1-U-DAG+MAT-M0` is terminally +rejected for this generation before any power study, broader I1 timing, +holdout, selector, or manifest work. Reopening P4 requires a distinct executor +and a separately frozen clean-source roster; this artifact cannot be retuned, +pooled, or promoted. diff --git a/docs/experiments/asp_p4_open_baseline_v1.md b/docs/experiments/asp_p4_open_baseline_v1.md new file mode 100644 index 00000000..c5e83a12 --- /dev/null +++ b/docs/experiments/asp_p4_open_baseline_v1.md @@ -0,0 +1,118 @@ +# All-shortest P4 open baseline v1 + +Status: completed training-only baseline; one open target selected for a +separately frozen candidate preflight. This is not an ASP candidate comparison, +power study, qualification, or selector change. + +## Why this is a fresh baseline + +The retained clean P0 corpus has one all-shortest PostgreSQL/Neo4j ratio, for +the protected shallow diamond. Its 1.70x descriptive ratio neither establishes +the 4.8-5.4x P4 opportunity nor may it select a target. Earlier ASP-I1 archive +captures also mixed protected holdouts, so they are historical diagnostics and +cannot serve as P4 evidence. + +This generation therefore measures only the nine declared +`generated_shortest_paths_v2` all-shortest training cases. They cover shallow +outbound, early target, inbound hidden-fan-in, cycle/dead-tail, reconvergent +multi-path, and disconnected behavior. All generated holdout, diagnostic, and +stress declarations remain unopened. + +## Frozen schedule + +After P3 commit `7f5d0f9dcc7bd1a86dd2846e7180e06b7795c13c`, capture from the +clean commit that contains this roster. Capture provenance records that exact +source commit and binary digest. Run four +independent rounds with one warm-up and five timed samples per case/backend. +Each round runs the PostgreSQL `ASP-A1-DAG` incumbent under Repeatable Read, +pool size one, and diagnostic traversal telemetry, alongside the ordinary +Neo4j reference. GraphBench alternates requested backend order on even rounds. +The capture consequently contains 72 exact case/backend/round records and +360 timed samples. PostgreSQL exact path-multiset, all-shortest, hydration, +workspace, plan-resource, runtime identity, and fallback evidence must all be +present; Neo4j records must be exact and present. + +The machine-readable roster is +[`benchmark/testdata/scale/protocols/asp_p4_open_baseline_v1.json`](../../benchmark/testdata/scale/protocols/asp_p4_open_baseline_v1.json). +The forced A1 identity is an attribution control: `asp-static-v1` already +selects A1 for these shapes, and this run does not exercise an automatic +candidate policy. + +## Stop and next gate + +Any non-exact record, missing telemetry, fallback, incomplete counters, or +unexpected PostgreSQL runtime identity stops the baseline and permits no +candidate timing. A completed baseline only authorizes a separately frozen +A1-versus-one-candidate P4 preflight if an open training case is materially +behind Neo4j. It does not authorize ASP-I1/B1/B2 timing, a power study, +protected cases, a manifest, or automatic selection. + +## V1 stop result + +The first clean combined-backend round ran from +`3a74d14be83f2c99b1694109d54840501ebbc3f5` with GraphBench binary SHA-256 +`4e47611c0e06d508e81011cc35d348a167c4c9a1d863095e3b72add821780c91`. +Its artifact `.coverage/p4-open-baseline-3a74d14/round-1.jsonl` hashes to +`b6c09fbd9dd1e46bd262af224469c12a9a69367f5ad7e504e853da5247fb53f6`. +All 18 records (nine PostgreSQL and nine Neo4j) had exact public observations; +PostgreSQL reported `ASP-A1-DAG`, the selected branch, and no fallback. + +The diagnostic replay failed the frozen telemetry condition for every A1 +record. The outer `Function Scan` exposes neither invocation-local search nor +predecessor, enumeration, hydration, and workspace counters, and therefore +reported `hidden_counters_unavailable`. The remaining three rounds were not +run. The first round is retained only as a stop artifact and cannot select a +P4 target. [`docs/experiments/asp_a1_diagnostic_prerequisite_v1.md`](asp_a1_diagnostic_prerequisite_v1.md) +defines the distinct prerequisite required before a clean recapture. + +## Clean-recapture boundary + +The separate A1 invocation-local diagnostic is now implemented and validated +for shallow, recursive, reconvergent, inbound, no-path, session-isolation, and +cancellation/rollback paths. The clean capture below used a commit that +contains that diagnostic and restarted at round one. It does not merge, pool, +or compare the stopped V1 round with the new capture. + +## Clean recapture result + +The fresh capture ran from `bf055b3aaeda1f887e652a399b065290db560236` with +GraphBench SHA-256 +`8a8cdd0998ef9391776ee4a6de3689f31f0a2133675bc96d9cffd95d6caceb31`. +The appended four-round artifact +`.coverage/p4-open-baseline-bf055b3/round-1.jsonl` hashes to +`69c29c0a79e2ca566bbd54e533c896138a7770b245d030dc347ebc9413e6b6fe`. +It contains 72 exact records (36 PostgreSQL and 36 Neo4j), 360 timed samples, +and 72 excluded warm-up samples. The source diff hash is empty in every record. + +Every PostgreSQL record has runtime/applied identity `ASP-A1-DAG`, no fallback, +and complete all-shortest, hydration, and `spd_*` workspace receipts. Every +Neo4j record is exact and present. The capture host reported the `powersave` +CPU governor, so the deltas below are target-selection diagnostics only—not +performance qualification evidence. + +| Open case | Matched median PostgreSQL/Neo4j | Matched p95 PostgreSQL/Neo4j | Branch | +| --- | ---: | ---: | --- | +| `GSPV2-TRAINING-disconnected-all-shortest-max64` | 5.825× | 5.263× | `search_no_path` | +| `GSPV2-NORMAL-outbound-all-shortest-depth3` | 3.801× | 2.942× | `single_ended_search` | +| `GSPV2-TRAINING-early-depth3-all-shortest-max16` | 3.076× | 3.766× | `single_ended_search` | +| `GSPV2-TRAINING-inbound-early-depth3-all-shortest-max64` | 2.852× | 2.703× | `single_ended_search` | +| shallow/reconvergent controls | 1.125–1.206× | 1.018–1.242× | preflight | + +The cycle/dead-tail case is exact on each backend but its serialized public +observations do not match across backends, so it is visible in the artifact but +excluded from the matched backend-delta ranking. The descriptive delta report +is `.coverage/p4-open-baseline-bf055b3/backend-delta.json` (SHA-256 +`e127d5d09bf68bd809c41f1c849044f2d61f5dc6c0d363c6a8d33eab28765fa0`). + +## Selected target and next gate + +`GSPV2-TRAINING-disconnected-all-shortest-max64` is the sole selected open +target: all four matched median ratios exceed 5.22×, its four-round median is +5.825×, and its p95 ratio is 5.263×. Its A1 receipt consistently reports the +`search_no_path` branch with 32 candidate edges, seen peak 33, predecessor peak +32, zero output paths, and 229,376 bytes of `spd_*` workspace. + +This selection authorizes only a new, immutable A1-versus-one-candidate P4 +telemetry preflight roster. It does not authorize candidate timing beyond that +separately frozen preflight, a power study, holdouts, a selector change, or +reuse of the stopped V1 round. diff --git a/docs/experiments/fixed_suffix_cardinality_metadata_audit.md b/docs/experiments/fixed_suffix_cardinality_metadata_audit.md new file mode 100644 index 00000000..d19adb62 --- /dev/null +++ b/docs/experiments/fixed_suffix_cardinality_metadata_audit.md @@ -0,0 +1,40 @@ +# Fixed-suffix cardinality metadata audit + +Status: **no hard pre-translation bound is currently available**. + +This audit asks whether production translation can directly select +`EXPANSION-SUFFIX-SEEDED-REVERSE` only when it can prove both physical suffix +rows and reverse states are at most 512, without executing the retired runtime +probe/fallback design. + +## Existing inputs + +- The public translator receives the Cypher AST, kind mapper, parameters, and + graph ID. It has no database connection or graph-cardinality provider. +- Graph schema metadata describes names, kinds, indexes, and constraints. It + does not contain degree, suffix-row, path, or reverse-state bounds. +- The PostgreSQL `graph` catalog contains only graph ID and name. Partition + models contain table names, indexes, and constraints. +- `OptimizeStorage` reads approximate live/dead tuple counts for vacuum + decisions. These counts are database-storage statistics, not per-root or + per-kind hard bounds. +- PostgreSQL planner statistics and `pg_class.reltuples` are estimates. They + are neither correctness-grade upper bounds nor available to the optimizer + before SQL emission. +- Translation caching is keyed by query text, graph ID, and parameter types. + A selector dependent on parameter values or mutable graph cardinality would + require new invalidation and cache-identity rules. + +## Finding + +Suffix rows and reverse states depend on the selected root, relationship kinds, +query depth, physical trail multiplicity, and current graph contents. Global +node/edge counts or planner estimates cannot prove either 512 ceiling. No +existing schema constraint establishes these limits, and no maintained +per-graph or per-root synopsis supplies conservative upper bounds. + +Therefore the S511/S512 wins do not currently support automatic production +dispatch. Production must continue selecting `EXPANSION-STEPWISE-FORWARD` for +this family. A future attempt would require a new proof-bearing metadata/API +contract plus mutation-safe maintenance, cache invalidation, and independent +qualification; that work is outside this completed audit. diff --git a/docs/experiments/fixed_suffix_composite_lowering_v1.md b/docs/experiments/fixed_suffix_composite_lowering_v1.md new file mode 100644 index 00000000..fa574c8b --- /dev/null +++ b/docs/experiments/fixed_suffix_composite_lowering_v1.md @@ -0,0 +1,17 @@ +# Composite fixed-suffix lowering v1 + +The suffix-reverse retry lowering now applies to every independently eligible, +full-path fixed-suffix region in one optimized query. Previously it rejected a +plan containing more than one such region solely because the tool boundary +expected one target. + +All selected regions receive the same immutable reverse-state, suffix-row, +output-row, and output-byte limits. Candidate results are buffered as one query +result; any guard overflow or candidate failure retries the original query as a +whole in the same stable transaction. This preserves query-level semantics and +avoids mixing partial candidate and incumbent results. + +Automatic production routing remains conservative: a future composite selector +must bind the joint shape and its exact fallback behavior in a new manifest +version. This lowering is the execution substrate for that selector, not an +authorization to reinterpret single-target v4 or v5 manifests. diff --git a/docs/experiments/guarded_suffix_keyset_continuation_v1.md b/docs/experiments/guarded_suffix_keyset_continuation_v1.md new file mode 100644 index 00000000..f32a9cef --- /dev/null +++ b/docs/experiments/guarded_suffix_keyset_continuation_v1.md @@ -0,0 +1,41 @@ +# Guarded suffix keyset continuation v1 + +Status: **rejected and retired negative result**. The historical implementation +identity `t16_s512_r512_e1_e2_e3_boundary_keyset_v1` is frozen in these +artifacts. Its GraphBench reference arm and experiment-specific telemetry have +been removed, and it was never part of production translation. +This confirmation is the canonical upstream record; later local reruns are not +part of the submitted evidence set. + +The confirmation run used an isolated PostgreSQL 18.4 database with +`plan_cache_mode=auto`, 10 matched reload rounds, 20 warmups per round, and 50 +measurements per arm per round (500 samples per arm). Intervals are paired +97.5% confidence intervals. The source artifact SHA-256 is +`e6aa00733de4861b9684d8f1276e922ff1e8059671e57400703a8266ca88ee25`. + +| Case | Baseline p50 | Candidate p50 | Median ratio (97.5% CI) | Candidate shared hits | Interpretation | +| --- | ---: | ---: | ---: | ---: | --- | +| S511 | 11.254 ms | 4.758 ms | 0.416 [0.407, 0.439] | 6,866 | Existing bounded reverse branch wins | +| S512 | 11.165 ms | 4.748 ms | 0.428 [0.408, 0.453] | 6,879 | Existing bounded reverse branch wins | +| S513 | 11.247 ms | 20.065 ms | 1.791 [1.752, 1.875] | 54,020 | Continuation is 79% slower | +| S600 | 11.566 ms | 68.950 ms | 5.898 [5.649, 6.462] | 55,523 | Non-empty continuation is 490% slower | + +S511 and S512 do not validate keyset continuation: they select the previously +known bounded reverse branch. S513 and S600 are the cases that exercise the new +continuation path, and both regress decisively. The reconstruction after the +prefix/remainder probes accounts for roughly 45,093 shared-buffer hits in both +overflow cases, so tuning the keyset predicate alone is not a credible next +step. + +The experiment's unpublished resource gate v5 passed all 40 candidate records: +there was no temporary or +local workspace, WAL, sentinel-budget violation, or inactive-branch execution. +Correctness and structured-plan checks also passed under `auto`, +`force_custom_plan`, and `force_generic_plan`. This makes the rejection a +performance decision rather than a correctness or spill failure. + +The compact machine-readable evidence is preserved in +`guarded_suffix_keyset_continuation_v1_pair.json` and +`guarded_suffix_keyset_continuation_v1_resources.json`. The JSON retains the +historical case names for artifact comparability; the active corpus uses +generic `GFSE-BOUNDARY-*` names for these fixed-suffix expansion holdouts. diff --git a/docs/experiments/guarded_suffix_keyset_continuation_v1_pair.json b/docs/experiments/guarded_suffix_keyset_continuation_v1_pair.json new file mode 100644 index 00000000..9da571fa --- /dev/null +++ b/docs/experiments/guarded_suffix_keyset_continuation_v1_pair.json @@ -0,0 +1,58 @@ +{ + "version": 1, + "decision": "rejected", + "implementation_id": "t16_s512_r512_e1_e2_e3_boundary_keyset_v1", + "source_artifact_sha256": "e6aa00733de4861b9684d8f1276e922ff1e8059671e57400703a8266ca88ee25", + "environment": "isolated local PostgreSQL 18.4, plan_cache_mode=auto", + "protocol": { + "reload_rounds": 10, + "warmups_per_round": 20, + "samples_per_arm_per_round": 50, + "samples_per_arm": 500, + "confidence_level": 0.975 + }, + "baseline": "complete_reference", + "candidate": "guarded_suffix_keyset_continuation", + "cases": [ + { + "name": "GFSE-GUARDED-S511-admitted-suffix-limit-minus-one", + "baseline_p50_ns": 11254109, + "candidate_p50_ns": 4758027, + "baseline_p95_ns": 12290083, + "candidate_p95_ns": 5301188, + "median_ratio": {"estimate": 0.41555464313812607, "lower": 0.40705827827777386, "upper": 0.4385313114263852}, + "p95_ratio": {"estimate": 0.43133866549151867, "lower": 0.42443171773646776, "upper": 0.44452360755289955}, + "median_change_ns": {"estimate": -6587660, "lower": -6747944, "upper": -6252829} + }, + { + "name": "GFSE-GUARDED-S512-admitted-suffix-limit-exact", + "baseline_p50_ns": 11164527, + "candidate_p50_ns": 4747871, + "baseline_p95_ns": 12142325, + "candidate_p95_ns": 5565890, + "median_ratio": {"estimate": 0.427560127676012, "lower": 0.40752095743455247, "upper": 0.4528344352178298}, + "p95_ratio": {"estimate": 0.4583874999227907, "lower": 0.4396809723294706, "upper": 0.4702413504336159}, + "median_change_ns": {"estimate": -6335235, "lower": -6732953, "upper": -6004886} + }, + { + "name": "GFSE-GUARDED-S513-admitted-suffix-limit-plus-one", + "baseline_p50_ns": 11247438, + "candidate_p50_ns": 20064748, + "baseline_p95_ns": 12188292, + "candidate_p95_ns": 21819920, + "median_ratio": {"estimate": 1.7906082254074516, "lower": 1.7518498795013928, "upper": 1.8748309114021493}, + "p95_ratio": {"estimate": 1.7902360724537942, "lower": 1.7578466359917437, "upper": 1.827209494526714}, + "median_change_ns": {"estimate": 8847280, "lower": 8396170, "upper": 9581455} + }, + { + "name": "GFSE-KEYSET-S600-productive-nonempty-remainder", + "baseline_p50_ns": 11565607, + "candidate_p50_ns": 68949562, + "baseline_p95_ns": 12595535, + "candidate_p95_ns": 75773743, + "median_ratio": {"estimate": 5.898025791399814, "lower": 5.649302617882263, "upper": 6.462465544247631}, + "p95_ratio": {"estimate": 6.015920959292321, "lower": 5.936216979766291, "upper": 6.159786543310964}, + "median_change_ns": {"estimate": 56902893, "lower": 56055344, "upper": 60770683} + } + ] +} diff --git a/docs/experiments/guarded_suffix_keyset_continuation_v1_resources.json b/docs/experiments/guarded_suffix_keyset_continuation_v1_resources.json new file mode 100644 index 00000000..38e80112 --- /dev/null +++ b/docs/experiments/guarded_suffix_keyset_continuation_v1_resources.json @@ -0,0 +1,24 @@ +{ + "version": 5, + "decision": "rejected", + "implementation_id": "t16_s512_r512_e1_e2_e3_boundary_keyset_v1", + "source_artifact_sha256": "e6aa00733de4861b9684d8f1276e922ff1e8059671e57400703a8266ca88ee25", + "passed": true, + "evaluated_records": 120, + "candidate_records": 40, + "candidate_failures": 0, + "candidate_cases": 4, + "candidate_shared_hit_blocks": { + "GFSE-GUARDED-S511-admitted-suffix-limit-minus-one": 6866, + "GFSE-GUARDED-S512-admitted-suffix-limit-exact": 6879, + "GFSE-GUARDED-S513-admitted-suffix-limit-plus-one": 54020, + "GFSE-KEYSET-S600-productive-nonempty-remainder": 55523 + }, + "observed": { + "temporary_blocks": 0, + "local_blocks": 0, + "wal_records": 0, + "sentinel_budget_violations": 0, + "inactive_branch_executions": 0 + } +} diff --git a/docs/experiments/p5_adjacency_materialization_feasibility_v1.md b/docs/experiments/p5_adjacency_materialization_feasibility_v1.md new file mode 100644 index 00000000..39a22b35 --- /dev/null +++ b/docs/experiments/p5_adjacency_materialization_feasibility_v1.md @@ -0,0 +1,129 @@ +# P5 adjacency materialization feasibility v1 + +Status: rejected for incomplete trigger-WAL attribution. The architecture and +all state oracles remain valid, but V1's completed clean-source artifact cannot +support a feasibility disposition. Its successor is +[`P5 adjacency materialization feasibility v2`](p5_adjacency_materialization_feasibility_v2.md). +Neither version authorizes a Cypher candidate, query routing, a production +schema migration, protected-corpus timing, or a promotion claim. + +## Chosen architecture + +The feasibility audit deferred alternative architectures because the topology +synopsis lacks a graph mutation epoch and cache contract. The remaining locally +testable option is a shadow, graph-scoped directed adjacency materialization. + +`public.p5_adjacency_v1` will contain exactly two rows for every base edge: +one outbound `(start_id, end_id)` row and one inbound `(end_id, start_id)` row, +both retaining graph, kind, and edge identity. Its only proposed lookup index +is `(graph_id, direction, anchor_id, kind_id, edge_id) INCLUDE (neighbor_id)`. +It is deliberately compared with the existing base-edge covering indexes, not +with an artificial sequential-scan floor. + +The table is shadow-only. No Cypher translation, shortest-path executor, +runtime policy, translation-cache key, or production selector may read it. +This boundary lets the work establish maintenance and storage feasibility +before it can change a public read path. + +## Frozen measurement roster + +The roster uses the existing direct-write fixture at sizes 1, 1,000, and 2,000 +for relationship create, conflict/upsert, property-only update, relationship +delete, node-delete cascade, graph reload, and graph drop. Four +counterbalanced blocks use one warm-up and five timed samples. Each mutation +validates its post-state then rolls back; committed calibration runs separately +measure WAL LSN deltas and relation/index bytes. + +Every base edge must have exactly one outbound and one inbound shadow row, and +every shadow row must map back to one matching base edge. Property-only updates +must retain row identity. Cancellation, rollback, pool reuse, reload, and +graph drop must leave no stale committed row. Raw parameterized adjacency +lookups may collect plan/buffer/cardinality measurements only after these +oracles pass; their rows never become a Cypher result. + +The complete contract is +[`benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1.json`](../../benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1.json). +It freezes what to measure, but intentionally freezes no pass/fail latency or +write budget: the first report must expose the physical trade-off before a +separate budget decision can authorize any candidate experiment. + +## Stop boundary + +Any shadow/base mismatch, stale committed row, non-transactional maintenance, +unattributed write/WAL/storage metric, or read-path access by a Cypher +candidate stops the study. Even a complete report authorizes only a separately +frozen budget decision. It cannot authorize a production schema, automatic +selector, cache-key change, protected corpus, or performance claim. + +## Implemented shadow boundary + +The shadow is an explicit `query.On(tx).InstallP5AdjacencyShadow` action; +ordinary `schema_up.sql` and normal driver startup neither install nor read it. +The paired remove action leaves core graph storage intact. Installation creates +graph partitions for existing graphs, backfills each edge into one outbound and +one inbound row, and then enables same-transaction triggers for edge insert, +endpoint/kind update, and delete. A property-only edge update does not touch +the shadow rows. The base edge foreign key and node-delete edge cascade remove +shadow rows on relationship/node/graph deletion. + +The unit SQL boundary test and targeted PostgreSQL lifecycle test pass. The +lifecycle test covers backfill, insert, endpoint update, property-only +non-rewrite, node-delete cascade, rollback, canceled statement recovery through +a replacement pooled connection, graph deletion, and shadow removal. It does +not provide any P5 latency, WAL, storage, or query-performance result; those +still require a clean source capture under this roster. + +## Capture runner + +`graphbench -p5-adjacency-feasibility-output ` is the sole capture +entry point for this roster. It requires the disposable PostgreSQL guard, a +clean source tree, and a one-connection pool. It fixes the four counterbalanced +blocks, one warm-up, five timed rollback-only samples, fixture sizes, and +mutation roster internally; it rejects Cypher corpus selectors and normal +GraphBench result outputs. + +The runner first captures base-only blocks with the shadow relation removed, +then alternates shadow and base conditions by dropping or reinstalling the +explicit shadow schema at each block boundary. It creates a fresh graph-scoped +fixture per condition, verifies exact base/shadow mapping around every write, +and records raw base and shadow adjacency probes separately. Its committed +calibration graph runs report setup and per-mutation WAL deltas; they are never +timed Cypher observations. The graph-clear/reload rollback sample clears the +fixture within its transaction and uses the required rollback to restore the +same fixture before the next sample. + +Before it starts, the runner removes any abandoned `p5_adjacency_*` graphs +from an interrupted earlier P5 capture, after first dropping any residual +shadow relation. That namespace is reserved for this disposable experiment; +the runner never selects or removes an arbitrary application graph. + +For example, build the artifact from a clean commit and write the report to an +ignored workspace location: + +```bash +go build -trimpath -o .coverage/graphbench-p5 ./cmd/graphbench +CONNECTION_STRING='postgresql://dawgs:weneedbetterpasswords@localhost:65432/dawgs' \ + ./.coverage/graphbench-p5 \ + -p5-adjacency-feasibility-output .coverage/p5-adjacency-feasibility.json +``` + +The JSON report contains the protocol checksum, clean source and binary +identity, per-operation p50/p95 values, committed WAL deltas, relation bytes, +raw parameterized lookup plans with buffers, and cancellation/pool-reuse +evidence. A successful report remains a feasibility record only: it cannot set +a write budget or authorize a Cypher read-path experiment. + +The first clean-source execution completed every state oracle but was rejected +as a feasibility artifact: background autovacuum inflated its global LSN +deltas, making several WAL values unattributable. A second clean-source run +disabled and restored autovacuum successfully, but exposed a different +attribution defect: `EXPLAIN (ANALYZE, WAL)` reported base-plan WAL only and +omitted the shadow's row-trigger writes. For a 1,000-edge shadow delete it +reported 54,000 bytes, while an isolated `pg_stat_statements` probe recorded +162,000 bytes for the same top-level mutation. The exact artifact and its +rejection are frozen in +[`p5_adjacency_materialization_feasibility_v1_rejection.json`](../../benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1_rejection.json). + +V2 replaces the invalid plan-WAL measurement with tagged +`pg_stat_statements` WAL deltas and retains quiescent LSN only as a diagnostic +cross-check. V1 makes no feasibility claim and creates no budget decision. diff --git a/docs/experiments/p5_adjacency_materialization_feasibility_v2.md b/docs/experiments/p5_adjacency_materialization_feasibility_v2.md new file mode 100644 index 00000000..b061a90e --- /dev/null +++ b/docs/experiments/p5_adjacency_materialization_feasibility_v2.md @@ -0,0 +1,81 @@ +# P5 adjacency materialization feasibility v2 + +Status: captured and terminally not advanced. The valid physical evidence does +not authorize a Cypher candidate, query routing, a production schema migration, +protected-corpus timing, or a promotion claim. + +V2 preserves the shadow-only architecture and the frozen base/shadow roster +from V1. V1's completed artifact is rejected by +[`p5_adjacency_materialization_feasibility_v1_rejection.json`](../../benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v1_rejection.json): PostgreSQL's +`EXPLAIN (ANALYZE, WAL)` plan-node counter omitted the WAL emitted by the +shadow's row triggers. It therefore could not attribute the complete physical +write cost. + +## Measurement correction + +For each committed calibration V2 executes the ordinary mutation once with an +operation-specific no-op CTE marker, then reads that statement's before/after +one-call delta from `pg_stat_statements`. PostgreSQL attributes the mutation's +trigger maintenance to that statement, so the artifact separately records WAL +records, full-page images, and bytes. The report also preserves quiescent LSN +deltas as diagnostics, but does not use them as the attributed mutation result. + +The capture-only runner may install `pg_stat_statements` in its disposable +database when the extension is absent, and removes it again only if it created +it. PostgreSQL must already preload the module; an unavailable preload setting +is a clear capture precondition failure. This measurement dependency is not a +driver or production-schema dependency. + +## Frozen boundary and roster + +The shadow remains `public.p5_adjacency_v1`, containing two graph-scoped rows +per base edge, with no Cypher translator, executor, policy, cache key, or +production selector allowed to read it. The runner fixes four counterbalanced +base/shadow blocks at sizes 1, 1,000, and 2,000; one warm-up and five timed +rollback-only samples cover relationship create, conflict/upsert, +property-only update, relationship delete, node-delete cascade, graph reload, +and graph drop. Raw parameterized adjacency reads collect physical probes only. + +Exact mapping, property-update identity, rollback, cancellation/pool reuse, +graph cleanup, autovacuum quiescence, and clean-source checks remain mandatory. +A passed artifact still authorizes only a separate, frozen resource-budget +decision; it cannot authorize a candidate, selector, production schema, or +performance claim. + +The complete V2 contract is +[`p5_adjacency_materialization_feasibility_v2.json`](../../benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2.json). + +## Captured result and disposition + +The clean-source capture completed on PostgreSQL 17.10 at commit `d257cd9` +with all 24 timed conditions, 42 committed calibrations, exact-state oracles, +cancellation/pool-reuse proof, quiescent LSN checks, and final cleanup passing. +The immutable ignored-workspace artifact has SHA-256 +`c8c4d7f88d0904bd00d1b85355f2f5806dcb74cf84a41d80237bf081a7222d04`. +Its statement WAL fields are one-call `pg_stat_statements` deltas, including +trigger maintenance; this is distinct from V1's invalid plan-node counter. + +The following values are upper medians across the four per-block medians. The +storage rows compare base edge storage with base plus the additional shadow +relation; the latency rows compare shadow with base. + +| Measurement | 1,000 targets | 2,000 targets | +| --- | ---: | ---: | +| Combined storage / base | 1.92x | 1.95x | +| Attributed WAL: batch create | 2.22x | 2.22x | +| Attributed WAL: relationship delete | 3.00x | 3.00x | +| Relationship-delete latency | 605x | 1,324x | +| Node-delete-cascade latency | 182x | 524x | +| Graph-clear/reload latency | 245x | 640x | +| Graph-drop latency | 224x | 559x | + +The raw shadow lookup probe was faster than the base physical probe (0.66x at +1,000 and 0.53x at 2,000), but it is intentionally not a Cypher result and +does not offset the storage, WAL, or mutation cost. No separate write-budget +decision is created and no candidate experiment is authorized. The complete +non-promotional record is +[`p5_adjacency_materialization_feasibility_v2_disposition.json`](../../benchmark/testdata/scale/protocols/p5_adjacency_materialization_feasibility_v2_disposition.json). + +No P5 successor is currently selected. The native-extension feasibility path +was withdrawn; it does not revive, retune, or reuse this materialization as a +candidate. diff --git a/docs/experiments/p5_architecture_feasibility_v1.md b/docs/experiments/p5_architecture_feasibility_v1.md new file mode 100644 index 00000000..7fc792ee --- /dev/null +++ b/docs/experiments/p5_architecture_feasibility_v1.md @@ -0,0 +1,69 @@ +# P5 architecture feasibility v1 + +Status: deferred before candidate implementation. This is a read-only +repository and platform inventory, not a performance study, schema migration, +or production-design decision. + +## Why P5 was assessed + +P1's static suffix-retry generation and P3's B1/B2 shortest-path generation +are terminally rejected. The sole current P4 I1 preflight is also terminally +rejected because its no-path target win regressed every shallow/reconvergent +control. That meets the performance plan's condition to assess an +architectural lane, but it does not itself justify a persistent copy of graph +data or native server code. + +## Inventory + +The PostgreSQL layout is already graph-partitioned. Every edge partition has +covering B-tree indexes for both physical adjacency directions: +`(start_id, kind_id) INCLUDE (id, end_id)` and `(end_id, kind_id) INCLUDE +(id, start_id)`, plus a kind-first covering index. A denormalized adjacency +relation would therefore duplicate data already available to the recursive +emitters and would have to prove a benefit after its read, write, WAL, and +storage costs. + +The mutation path performs direct batched node/edge inserts, upserts, updates, +and deletes in PostgreSQL transactions. Neither the schema nor the driver +maintains a graph mutation epoch. The translation-cache key contains normalized +query text, graph ID, parameter-type shape, and policy identity—but no graph +data generation. Thus a mutable adjacency copy or synopsis cannot safely be +embedded in generated SQL, and a maintained lookup needs an explicit epoch, +publication, staleness, and cache contract first. + +There is also no native PostgreSQL extension package in this repository: no C +sources, control file, PGXS build, versioned SQL install scripts, or deployment +matrix. The local build environment supplies PostgreSQL 18.4 PGXS, while the +live PostgreSQL target used for this work is 17.10. Building a server extension +locally would not produce a testable binary for that target. The live server +has only `intarray`, `pg_trgm`, and `plpgsql` installed; `pg_stat_statements` +is preloaded but is not an installed execution extension. + +The credential-free inventory is +`.coverage/p5-feasibility-4db030c/inventory.json`. It records the source +commit, server/build versions, existing indexes, installed extensions, and the +cache/mutation findings; its SHA-256 is +`6913d46209601b5bd8bd955b28738f1040dc6c14a514d95aeae081d2977ffb92`. +It is a platform observation rather than portable performance evidence. + +## Disposition + +Do not add a topology synopsis, duplicate adjacency table, or application-side +traversal service under the current P5 scope: + +- The versioned topology synopsis remains deferred under + [`traversal_topology_synopsis_adr_v1.md`](traversal_topology_synopsis_adr_v1.md). + Its required graph mutation epoch, atomic refresh publication, stale-read + behavior, mutation/WAL budget, and cache-key proof do not yet exist. +- A duplicate adjacency relation has no predeclared exact target or measured + read advantage over the existing direction-specific covering indexes. Adding + it first would expose write and storage costs without an admission case. +- An application-side service cannot be considered until it can preserve the + driver's Repeatable Read snapshot and exact fallback contract. + +There is no selected P5 successor. Any future, non-native architecture must +first freeze a feasibility protocol that specifies its mutation fixture and +write/WAL/storage budget, graph-generation and cache behavior, +rollback/removal path, and a read-only exact baseline. Only after that +prerequisite passes may it create a candidate roster. P4 artifacts and the +current platform inventory cannot tune or promote such a successor. diff --git a/docs/experiments/production_wide_sql_selection_v1.md b/docs/experiments/production_wide_sql_selection_v1.md new file mode 100644 index 00000000..ae7db93e --- /dev/null +++ b/docs/experiments/production_wide_sql_selection_v1.md @@ -0,0 +1,113 @@ +# Production-wide SQL selection v1 + +Status: implementation contract. This work expands verified SQL selection from +one exact-query canary to structurally eligible PostgreSQL traversal queries. +It does not make an unqualified candidate automatic, weaken an existing +promotion gate, revive terminally rejected selector identities, or permit a +stale topology decision to change query results. + +## Outcome + +For a verified policy generation, every structurally eligible query is matched +to one of these outcomes: + +1. a graph-independent qualified candidate; +2. a snapshot-bound topology-selected qualified candidate; or +3. the incumbent. + +The incumbent is required for every unknown, stale, unsupported, disabled, +unqualified, malformed, cancelled, or resource-limited selection. Candidates +retain their existing admission sentinels, exact fallback, and dedicated +rollback switch. + +## Frozen safety contract + +Selection has two distinct stages: + +```text +Cypher -> structural shape -> verified policy bucket -> StrategySelection + | | + | +-> selected arm + v + transaction synopsis state +``` + +`StrategySelection` is immutable metadata. It includes the policy generation, +selector version, bucket, candidate, fallback, immutable caps, and selection +reason. Topology-sensitive selections additionally carry the observed graph +mutation epoch and synopsis generation. It must never retain rows, graph +values, a snapshot identifier, a transaction, a connection, a result, or a +caller-owned parameter map. + +Translation cache entries are partitioned by the effective policy identity and +selected arm. A routing decision is not a translation-cache entry. Until a +separate evidence-backed admission proves otherwise, topology decisions are +owned by one read-only Repeatable Read or Serializable transaction and are +discarded before that transaction closes. + +## Candidate scope + +The implementation order is deliberately narrow: + +| Family | Initial selection type | Current boundary | +| --- | --- | --- | +| ASP-I1 predecessor DAG | static structural | exact-query canary becomes a verified structural bucket after clean evidence | +| Canonical SP-I1 | static structural | qualified shape bucket only | +| Endpoint-seeded reverse | existing static envelope | preserves current rollback semantics | +| Fixed-suffix reverse | topology-sensitive | new identity only; rejected orientation and suffix-guard identities remain terminal | + +SP-I2, B1, B2, adjacency materialization, result caching, cross-transaction +routing caches, and any persistent route cache are outside this version. + +## Admission SLOs + +Every production candidate requires exact semantic observations, complete +resource and execution receipts, and the normal GraphBench evidence closure. +The selection layer additionally requires all of the following on its declared +training and frozen holdout cohorts: + +- p50 materially improves by at least 5% or 100 microseconds; +- p95 is at most 1.05 times the incumbent after selector cost; +- selector overhead is at most 1.10 times or 100 microseconds of the selected + exact arm; +- no selected candidate exposes output before its declared fallback gates pass; +- stale, absent, malformed, partial, or incompatible selector state selects + the incumbent and emits a precise reason; +- cancellation, rollback, pool reuse, concurrent mutation, and schema reset + preserve exact observations and leave no reusable decision state. + +Stress cases prove correctness, caps, and fallback only. They cannot tune a +selector or promote a bucket. + +## Phased delivery + +1. Add a versioned structural traversal identity and typed shadow selection. +2. Extend policy manifests and GraphBench evidence to bind structural buckets. +3. Promote graph-independent qualified buckets through the existing policy + generation and rollback path. +4. Add graph mutation epochs, then a versioned topology synopsis in + shadow-only mode. +5. Admit one snapshot-bound selected SQL arm only after synopsis evidence + passes. +6. Qualify a new fixed-suffix selector identity and roll it out by policy + generation and structural bucket. + +Each phase must be independently reversible. No later phase may silently make +an earlier diagnostic, shadow, or tool-only candidate automatic. + +## Rollout + +Policy generations are the rollout unit. Deployment begins with shadow-only +selection, then moves through explicit graph/bucket cohorts. A matching +candidate rollback switch or a zero policy immediately produces a distinct +incumbent cache identity. Automatic rollback decisions are deliberately out of +scope; the first release uses operator-controlled generation changes and +query-text-free telemetry. + +## Definition of done + +Production-wide SQL selection is complete only when all qualified structural +buckets route without exact-query enumeration, topology-dependent selection is +snapshot-bound and stale-safe, each selected statement has one primary arm and +an exact fallback, and the PostgreSQL and Neo4j validation matrix plus clean +GraphBench evidence pass for every activated bucket. diff --git a/docs/experiments/remaining_outlier_delivery_v1.md b/docs/experiments/remaining_outlier_delivery_v1.md new file mode 100644 index 00000000..38c5abc5 --- /dev/null +++ b/docs/experiments/remaining_outlier_delivery_v1.md @@ -0,0 +1,349 @@ +# Remaining SP and traversal outlier delivery + +This delivery turns the remaining PostgreSQL-versus-Neo4j outlier plan into +independently reversible slices. A code-complete candidate is not a promoted +candidate: an eligible, non-terminal candidate still requires clean-source +training and holdout evidence plus every schema-v2 manifest role. A terminally +rejected generation cannot be revived by recapture or rebinding. + +## Delivered sequence + +1. Backend-delta reports now aggregate matched successful rounds, retain SQL + and runtime identities, and rank only repeated PostgreSQL regressions. +2. The driver/runtime seam can stage `orientation-probe-v2` only when the + manifest candidate and selector are both v2, preserving its diagnostic and + guarded statement tooling. Final authorization terminally rejects that + immutable generation because its training overhead gate failed; further work + requires a new policy generation. The same verifier rejects v1 because its + legacy evidence schema cannot bind source, corpus, and frozen-cohort + identity. The v1 and v2 formulas remain immutable and independently + reversible for diagnostics. +3. Syntax-open singleton shortest paths use the repository's effective depth + 15 in contained S3/S4 selection. Diagnostics distinguish `policy_default` + from `explicit` depth through `maximum_depth_source` and selector + `sp-static-v7-contained`. +4. `SP-I2-C-D` addresses hidden fan-in with reverse-physical ID-only distance + discovery. Total state and each breadth-first level are capped before any + candidate row is visible; overflow invokes exact S4 in the same statement. + Runtime markers, plan-replay counters, production manifests, stable-snapshot + enforcement, query-SHA buckets, and `DisableInlineSPDistance` are wired. +5. Existing ASP-I1 qualification and promotion tooling remains a separate + later evidence lane; it is not a substitute for the next clean SP-I2 + discovery decision. A1 stays automatic because the current focused evidence + has known shallow regressions and does not authorize broad I1 selection. +6. B2 remains conditional. Its kernels and tournament arms stay available, + but no production policy accepts B2 until a frozen cohort proves a stable + win over both incumbent and B1 with resource and closure gates passing. + +## SP-I2 diagnostic checkpoint + +The 2026-08-13 two-round diagnostic run for +`GSPV2-STRESS-hidden-fanin-distance` exposed and corrected a fail-closed +admission defect: the SQL formatter emitted `GROUP BY` but omitted the +frontier sentinel's `HAVING count(*) > cap`, so every non-empty candidate +incorrectly selected S4 fallback. Formatter and translation regressions now +require the `HAVING` predicate. + +After correction, both rounds matched Neo4j observations, executed +`SP-I2-C-D` with 17 state rows, and recorded zero fallback loops. PostgreSQL +median latency was 0.330-0.357 ms versus Neo4j's 2.191-3.528 ms, or a +6.14-10.68x descriptive advantage. This validates the candidate's intended +hidden-fan-in search shape, but it is not promotion evidence by itself: the +full training/holdout, cap-overflow, resource, closure, and operational matrix +below remains mandatory. + +A subsequent two-round expansion covered shallow hidden fan-in, the depth-16 +stress case, and disconnected exhaustion. All six matched backend comparisons +were semantically equal. PostgreSQL was 2.62-4.77x faster on the shallow case, +6.32-10.53x faster on stress, and 1.89-2.60x faster on the disconnected +control. Candidate receipts reported four or 17 states for reachable searches +and the expected no-path branch for exhaustion, with no fallback loops. + +Formal qualification is now implemented as the independent +`sp-i2-distance-v1` staged protocol. It seals six training declarations and +four unopened holdouts behind exact tags/case identities, declaration and +resolved-selection digests, clean source/archive/binary identity, and +recomputed training evidence before any holdout database setup. Discovery +requires 5-20 alternating `SP-S4-C-D`/`SP-I2-C-D` rounds with at least five +warmups and ten samples per arm; confirmation requires 10-20 rounds with at +least 20 warmups and 50 samples. Every normal case must preserve exact scalar +observations and timed runtime receipts, execute the guarded distance branch +without fallback, pass the state/frontier resource contract, meet median ratio +upper `<= 0.95` or saving lower `>= 100us`, and keep p95 ratio upper `<= 1.05`. +The preregistered cycle control instead uses the bounded-overhead gate: median +ratio upper `<= 1.10` or absolute overhead upper `<= 100us`, with the same p95 +limit. +The preregistered production-form state and frontier limits are both 100,000. +They are immutable protocol inputs, not qualified caps; qualification requires +the passing clean-source discovery freeze that does not yet exist. +The complete commands and artifact sequence are documented in GraphBench's +`Frozen SP-I2 distance qualification` section. No protected timing is +authorized until a clean committed tree produces a passing discovery freeze. + +The implementation workflow was then exercised with the minimum five +alternating training rounds, six cases, and 50 timed samples per arm/case. +That live rehearsal caught two capture-recipe defects before they could enter +promotion evidence: `block` must equal `round`, and even rounds must physically +execute I2 before S4 rather than merely swapping the arm-order labels. The +documented capture loop and a chronology regression test now enforce both +rules. A fresh 60-record capture passed corpus, schedule, runtime-receipt, and +resource validation and then stopped at the intended clean-source/archive +barrier before either a report or freeze was written. No holdout was selected +or timed. + +The first dirty-tree rehearsal exposed a cycle-control regression: after +reaching the requested root at depth one, the recursive candidate expanded +back out of that completed target until the maximum depth. Target-terminal +pruning now prevents those strictly longer descendants while retaining all +unrelated branches, cap admission, and same-statement exact fallback. The +cycle plan consequently fell from 65 recursive states to two in every +diagnostic replay. + +A fresh five-round, order-balanced training-only capture on 2026-08-14 retained +zero fallback across all six cases. Pooled candidate median ratios were +`0.033-0.759` for the five target cases. The cycle control measured `1.015` +with about `4.7us` overhead, and its pooled p95 ratio was `0.894`; these point +estimates are inside the preregistered control bounds. The clean-source check +still stopped report and freeze creation, as required. These dirty-tree +results solve the observed control mechanism but do not authorize holdout +access or production activation; an authoritative confidence-bound decision +requires a clean committed recapture. + +The production-manifest seam was also exercised independently. An inbound, +typed, distance-only disconnected case executed the no-path candidate branch +with nine states and no fallback; PostgreSQL's 0.260 ms median compared with +Neo4j's 1.184 ms. A deliberately reduced state/frontier cap of ten admitted +exactly the cap+1 sentinel, exposed zero candidate rows, and recorded the full +`SP-I2-C-D -> SP-S4-C-D -> SP-S3-U-E+MAT-M0` exact fallback receipt chain. + +## Required evidence order + +For SP-I2, ASP-I1, or a newly preregistered candidate generation, capture in +this order: order-balanced A/A, training discovery and freeze, unopened holdout +confirmation, matched performance, resource gate, reference closure, and +operational cancellation/concurrency/session-reuse evidence. This also records +the historical orientation-v2 protocol order, but that frozen generation is +terminally rejected and must not be recaptured, retuned, or promoted. Bind each +report to one promotion identity, then verify the final manifest before +installing a driver policy. Any missing runtime receipt, inactive-arm proof, +cap counter, or exact query bucket fails closed. + +Promotion binding embeds the exact native producer bytes for A/A, resource, and +reference-closure reports. SP confirmation names the exact native resource +digest; confirmation, performance, and resource bind the same candidate +artifact. For every promotion case, resource evidence must contain exactly the +performance round count, and its flattened candidate receipt-chain set must +equal performance's complete set. Reference closure deliberately uses its own +raw-pgx/comparator capture, while matching candidate/source/binary/corpus +identity, exact query/dataset/name/split cohort, thresholds, and independently +valid production receipt chains. Each reference workload must also match exactly +one native PostgreSQL A/A case by dataset, name, and workload digest; its +independent invocation IDs are not equated with the performance/resource set. +Confirmation and performance do not yet embed raw benchmark samples, so a +future producer schema is required for independent bootstrap replay. + +Formal operational capture is two-pass. A non-promotional preflight may omit +`operational_candidate_sql_sha256` only long enough to derive the exact SQL +digest emitted by the provisional production policy. Freeze that digest in the +manifest, discard the preflight records, then recapture formal evidence. The +operational requirements and every non-overflow record must equal the manifest +anchor; the runner, final verifier, and driver reject a populated anchor that +is not canonical, and the runner additionally rejects a canonical anchor that +does not match generated SQL. An anchored schema-v2 manifest therefore admits +exactly one unique query digest; cohort variation remains in bound parameters +and fixtures. + +The operational command validates an already assembled 32-record source +document; no standalone producer currently generates it. Release engineering +must preserve the complete native GraphBench worker/iteration, cancellation, +snapshot, isolation, overflow, optimization, plan-replay, and fixture evidence +when assembling `OperationalGateInput`. A summarized or hand-authored pass claim +cannot satisfy the validator. + +## Rollback boundaries + +- orientation: `DisableExpansionOrientation` or a zero policy; +- endpoint-seeded reverse: `DisableEndpointSeededReverse`; +- canonical witness: `DisableInlineSPWitness`; +- guarded distance: `DisableInlineSPDistance`; +- ASP-I1: `DisableInlineASPDAG`. + +If a manifest-backed candidate carries a rollback switch, it may carry exactly +one and it must be dedicated to that candidate: orientation with +`DisableExpansionOrientation`, ASP-I1 with +`DisableInlineASPDAG`, canonical SP-I1 witness with `DisableInlineSPWitness`, +or SP-I2 distance with `DisableInlineSPDistance`. An unrelated or second switch +is rejected. `DisableEndpointSeededReverse` is standalone-only. Every standalone +rollback policy must have no manifest candidate and must leave its manifest +digest, manifest JSON, and query allowlist empty +(`promotion_manifest_sha256`, `promotion_manifest_json`, and +`query_sha256_allowlist`). + +Changing a policy generation changes the translation-cache identity +immediately. For a manifest-backed emergency rollback, install the disable +switch under a new nonzero generation. The effective rollback copy clears the +candidate SQL-anchor comparison so incumbent SQL can execute, while the stored +manifest and its candidate anchor remain immutable. A zero policy returns every +query directly to its incumbent identity. B1/B2 have no production activation +boundary and therefore need no production rollback switch yet. + +## Fixed-suffix checkpoint + +Earlier orientation-v2 training diagnostics show that its topology choice is +mostly correct, but the guarded statement adds a roughly fixed 187-376 +microseconds of probe and dispatch work. That overhead dominates the shallow +training cases, so v2 is terminally rejected; its immutable formula must not be +retuned or recaptured in place. + +A fresh exact-reverse comparison isolates the next boundary. On +`GFSE-V2-D16-F1000-R1-X1-M1-sparse_endpoint_ids`, PostgreSQL returned the same +stable observations as Neo4j in 0.274 ms versus 1.401 ms, a 5.11x advantage. +The complete-path form was also semantically equal, but took 3.290 ms versus +Neo4j's 1.616 ms. PostgreSQL plan execution rose from 0.236 ms for endpoint IDs +to 2.975 ms for paths while search state stayed at 19 rows. The remaining +sparse-path deficit is therefore path hydration, not reverse discovery. + +The V2 sparse-path fixture now declares both deterministic path rows instead +of row count alone, making future backend reports fail closed on real path +semantic differences. The next implementation slice should preserve the +proven reverse ID-only arm and batch or specialize directed path hydration; +only after that should a new selector generation attempt to amortize or avoid +v2's fixed guarded-probe cost. + +That hydration slice is now implemented. A component run measured generic +`ordered_edge_ids_to_path` hydration at 1.979 ms median and 350 shared-buffer +hits for one long path, while direct hydration from precomputed ordered node +and edge IDs measured 0.143 ms and 21 hits. Reverse traversal carries ordered +node IDs only when a complete path is observed and hydrates both arrays in the +translated statement. On the sparse path case, exact reverse fell from 3.290 +ms to 0.278 ms median versus Neo4j at 1.117 ms, with the exact two-path oracle +matching. Endpoint-only output retains the narrower reverse state. + +The guarded orientation-v2 candidate uses the same ordered-ID hydration while +the exact forward fallback retains the established generic materializer. Its +initial end-to-end medians were 0.930 ms for endpoint IDs and 1.021 ms for full +paths, versus Neo4j at 1.177 ms and 0.988 ms respectively. Plan decomposition +then showed that the degree probes materialized one boolean tuple per adjacency +and the metrics CTE rescanned those tuples. The probes now aggregate the same +cap+1-limited streams to scalar counts; the selector formula, overflow boundary, +duplicate-root contribution, and exact fallback behavior are unchanged. + +In the subsequent 30-sample diagnostic, guarded PostgreSQL medians fell to +0.468 ms for endpoint IDs and 0.671 ms for full paths, versus Neo4j at 1.00 ms +and 1.0 ms. PostgreSQL plan execution for the path case fell from 0.926 ms to +0.635 ms with the same 275 inclusive shared-buffer hits. Telemetry still +reported 1,000 forward and one reverse degree samples, selected exact reverse, +and recorded no overflow or fallback. Targeted live integration also retained +the probe-overflow and reverse-state-overflow fail-closed receipts. These are +diagnostic results from a dirty development tree, not qualification evidence; +they cannot revive orientation-v2. The final verifier terminally rejects this +generation, and its protected holdouts remain unopened. + +## Orientation-v2 qualification checkpoint + +The scalar-probe implementation was then exercised through the exact discovery +protocol: five position-balanced rounds, ten measured samples per arm and +round, eight canonical training cases, separate `shadow`, `incumbent`, +`reverse`, and `guarded` artifacts, and a matching two-arm A/A report. All 160 +four-arm records and 80 A/A records were successful, used one binary identity, +matched exact observations, and carried the required Repeatable Read receipts. +The dirty-tree discovery report was emitted, while freeze creation correctly +failed closed because the implementation is not committed. + +The performance decision is nevertheless conclusive before a clean recapture: +all eight training cases failed the immutable selected-arm overhead gate. +Guarded median overhead over the selected exact arm ranged from 156 to 396 +microseconds in the scalar-probe capture, above both the 1.10 ratio and 100 +microsecond absolute limits. A follow-up prototype removed a redundant reverse +gate and state-probe lateral boundary, but a fresh five-round capture still +reported 205-377 microseconds of selected-arm overhead across the cohort. The +prototype was reverted because the matched evidence did not confirm a useful +improvement and the original boundary gives stronger inactive-arm proof. + +Orientation v2 is therefore rejected as a production candidate on its frozen +cohort, not merely blocked on source cleanliness. Its formula, thresholds, and +holdouts must remain unchanged, and the final manifest verifier enforces that +terminal decision. Further fixed-suffix work requires a new policy generation +that avoids paying topology probes plus dual-arm dispatch on every invocation; +the existing exact reverse executor and scalar-probe evidence remain valid +components for that study. + +## Static suffix-reverse guard implementation + +That independent generation is now implemented as the default-off, +tool-only `suffix-reverse-guard-v1` policy. It deliberately removes every +orientation-v2 topology and degree probe. Static enrollment is limited to +complete-path fixed-suffix queries; a 512-row suffix cap and independent +512-row reverse-state cap select either the existing suffix-seeded reverse +executor or the unchanged stepwise-forward statement. Both arms are +marker-gated in one Repeatable Read statement, and the runtime receipt records +the precise reverse, suffix-overflow, or state-overflow branch. + +The diagnostic surface has a separate `suffix_guard` counter family, stable +named CTE attribution, cap+1 observations, complementary marker rows, and +direct executor-loop proof. A real PostgreSQL training-case plan measured +three suffix rows, seven reverse-state rows, three output rows, one candidate +marker/executor loop, and zero fallback marker/executor loops. This verifies +the production-shaped plan boundary, not performance qualification. + +The predeclared feasibility gate is intentionally early and training-only: +the two already-open V3 training path cases are bound by an exact schema-v2 +selection declaration; each uses all six doubled-Williams orders exactly once, +five warmups, and exactly ten samples per arm for exact forward, exact reverse, +and guarded execution, plus matching order-balanced A/A evidence. It rejects +substitute training workloads, diagnostic or protected holdout timing, and +requires invocation-bound timed receipts plus a separate complete plan replay +that proves the inactive executor performed zero work. Guard overhead must be +within `1.10` or `100us` of exact reverse; +regret must be within `1.10` or the A/A floor of the fastest exact arm; and the +guard must materially improve forward p50 (`<=0.95` ratio or `>=100us` +saving) with p95 `<=1.05`. Only a passing stop gate warrants a new sealed +qualification corpus and production-manifest generation. Until then there is +no suffix-guard driver policy, rollback switch, or automatic selection. + +The schedule is physical evidence, not an arm-label convention. Every capture +uses `block == round`; the exact two-case records for each arm/round must share +one nonzero GraphBench process interval. Within a round, those intervals must +be non-overlapping and follow the declared arm positions, and each new round +must start at or after the prior round completes. The gate also requires one +run UUID across all six rounds. Its chronology tamper tests reject relabeled +execution, overlapping arms or rounds, mixed cohort invocations, and missing +timestamps. + +The matching incumbent A/A evidence uses the same fail-closed rule. Its two +processes must physically alternate first position across contiguous rounds, +use `block == round`, share one A/A run UUID, and execute the exact cohort +without arm or round overlap. A/A report schema v4 records artifact-bound +`physical_chronology` provenance, and the suffix gate refuses earlier reports +that lack it. Consequently the original label-balanced `aa.json` cannot be +reused for a compliant recapture. + +The first six-round live training capture could not establish that stop decision. +Both cases matched exact observations, supplied 60 timed samples per arm, and +executed the reverse branch without overflow or fallback, but the recorded +process intervals reveal incumbent, reverse, then guarded execution in every +round. The labels alone claimed the six Williams orders. The hardened gate now +rejects round 2 for contradictory arm chronology before reading its timing into +a decision. + +The legacy `.coverage/suffix-reverse-guard-v1/feasibility.json` remains useful +only as a dirty-tree diagnostic: its estimated guard/reverse ratios (`1.692` +and `1.511`) and overhead intervals are not valid preregistered feasibility +evidence. + +A chronology-valid recapture then reached the stop decision without changing +the cohort, sampling, caps, or thresholds. Both cases again matched exact +observations, supplied 60 timed samples per arm, executed reverse without +overflow or fallback, and materially improved exact forward. Estimated +guard/reverse median ratios were `1.346` and `1.206`; their one-sided upper +bounds were `1.593` and `2.002`, and absolute-overhead upper bounds were +`201us` and `445us`. Both failed the primary `1.10`/`100us` overhead gate, +while regret and forward-improvement gates passed. The authoritative report is +`.coverage/suffix-reverse-guard-v1-chronology/feasibility.json` with +`passed=false`. + +This valid failure does not authorize holdout, manifest, driver-policy, or +automatic-selector work. `suffix-reverse-guard-v1` is terminally stopped; its +thresholds must not be weakened or retuned. The exact reverse executor and its +ordered-ID hydration remain reusable components for a newly preregistered +architecture that removes same-statement guarded-dispatch cost. diff --git a/docs/experiments/sp_bidirectional_p3_preflight_v1.md b/docs/experiments/sp_bidirectional_p3_preflight_v1.md new file mode 100644 index 00000000..8f077edd --- /dev/null +++ b/docs/experiments/sp_bidirectional_p3_preflight_v1.md @@ -0,0 +1,142 @@ +# Compact bidirectional shortest-path P3 preflight V1 + +Status: superseded before direct-floor capture. This is a telemetry and +component-boundary readiness preflight, not a performance qualification. It +does not authorize a selector, a formal tournament, a protected holdout, or +production activation. + +## Disposition + +The primary S3/S4/B1/B2 captures were diagnostic-only and cannot qualify this +generation. The declared direct-floor comparison used the nonexistent +`SP-S4-C-DIRECT` identity, so the frozen schedule could not complete. V1 is +superseded rather than amended. Its raw observations may explain the correction +but must not support a P3 decision. V2 preserves the primary roster and uses +the real `SP-S4-C-D` and `SP-S4-C-WE+MAT-M0` arms in distinct direct-floor +distance and one-path comparisons. + +## Purpose and separation + +P2 is terminal: the `sp-i2-distance-v1`, `sp-i2-distance-v2`, and +`sp-i2-distance-v3-power-study` identities must not be retuned or reused. +P3 evaluates only the existing, tool-forceable compact bidirectional SP +references, with a distinct identity `sp-bidirectional-p3-preflight-v1`. +The candidates remain default-off and reference-only: + +| Arm | Distance identity | One-path identity | +| --- | --- | --- | +| Incumbent | `SP-S4-C-D` | `SP-S4-C-WE+MAT-M0` | +| Single-ended reference | `SP-S3-U-D` | `SP-S3-U-E+MAT-M0` | +| Strict alternating node | `SP-B1-C-ALT-NODE-D` | `SP-B1-C-ALT-NODE-WE+MAT-M0` | +| Smaller current level | `SP-B2-C-MIN-LEVEL-D` | `SP-B2-C-MIN-LEVEL-WE+MAT-M0` | + +`SP-S0-DIRECT` is only a direct one-hop floor; it is not part of the primary +unbounded four-arm comparison. + +## Clean baseline and frozen opportunity selection + +The only selection input is the clean two-round P0 capture from source +`57be1681140a2642639df0c06f7167bc17203e9b`. Its retained artifacts are +`.coverage/p0-clean-57be168-round1.jsonl` +(`5cd14dc4b13008f5e307d44a16c56ff608eb79596b2ecaddf59d1eb70c31c6a1`) and +`.coverage/p0-clean-57be168-round2.jsonl` +(`3bb71d1951b66559677abd4bba5441d844567269e6c1b1694cc95b67b4bc1f4d`). +Each used one PostgreSQL and one Neo4j pool session, one warm-up, and three +timed observations per case per round. The small sample is descriptive only; +it is deliberately insufficient for a P3 performance conclusion. + +Pooling the two round medians identified the following open targets: + +| Case | PostgreSQL median | Neo4j median | PostgreSQL / Neo4j | +| --- | ---: | ---: | ---: | +| `GSP-D08-F001_path_inbound` | 4.219ms | 1.200ms | 3.52x | +| `GSP-D08-F001_distance_inbound` | 3.874ms | 1.472ms | 2.63x | +| `GSP-D64-F1000_path` | 1.939ms | 1.148ms | 1.69x | + +The depth-8 inbound pair is the only material open P3 target. The long +outbound path remains a declared weaker target to prevent an inbound-only +claim. All other selected cases are controls: the same P0 capture already +shows PostgreSQL at or ahead of Neo4j for many of them, so a compact workspace +cannot claim success by moving broad costs into those shapes. + +The frozen primary roster is: + +- Targets: `GSP-D08-F001_distance_inbound`, + `GSP-D08-F001_path_inbound`, and `GSP-D64-F1000_path`. +- Typed controls: `GSP-D16-F016_distance`, `GSP-D16-F016_path`, + `GSP-D04-F128_disconnected`, `GSP-D04-F128_path_disconnected`, + `GSP-D02-F016_distance_cycle`, `GSP-D02-F016_path_cycle`, + `GSP-D02-F016_distance_self_loop`, `GSP-D02-F016_path_self_loop`, + `GSP-D01-F016_distance_parallel`, and `GSP-D01-F016_path_parallel`. +- Untyped controls: `shortest_distance_bound_pair` and + `one_shortest_path_bound_pair`. +- Separate direct-floor probes: `GSP-D01-F001_distance` and + `GSP-D01-F001_path`. + +The roster covers typed and untyped, one and multiple relationship kinds, +inbound and outbound expansion, path and distance observation, shallow and +deep depth bounds, cycle/self-loop, parallel-kind, disconnected, and direct +floor behavior. It deliberately excludes every `generated_shortest_paths_v2`, +SP-I1, and SP-I2 declaration because their existing training/holdout partitions +do not belong to P3. + +## Frozen capture schedule + +Every primary case receives the four corresponding observation arms in four +rounds, using a one-session PostgreSQL pool, Repeatable Read, diagnostic +telemetry, one warm-up, and five timed samples. The arm order is the balanced +four-arm carryover sequence: + +| Round | Order | +| --- | --- | +| 1 | `S4`, `B1`, `S3`, `B2` | +| 2 | `B1`, `B2`, `S4`, `S3` | +| 3 | `B2`, `S3`, `B1`, `S4` | +| 4 | `S3`, `S4`, `B2`, `B1` | + +The direct-floor probes use two counterbalanced S4/S0 comparisons: +`S4,S0` then `S0,S4`. An arm is one separately invoked GraphBench command and +must set matching `round`, `block`, `arm`, and `arm-order` fields. Cap +overrides, reference mode, concurrency measurements, P2 generation options, +and protected corpus tags are forbidden. + +The complete machine-readable contract is +`benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v1.json`. +Changing the roster, arms, order, warm-up count, timed count, or component +requirements creates a different preflight generation. + +## Telemetry and component stop gate + +P3’s first gate is observability, not speed. For every B1/B2 replay, the +invocation-local diagnostic must prove exactly one search call; its scheduler; +the selected runtime branch; per-level side, action, depth, frontier, seen, +queue, predecessor, and meeting counts; aggregate peaks; frozen distance; +witness rows; and workspace high-water bytes. PostgreSQL plan evidence must +attribute shared/local/temp buffers, temporary files and bytes, and WAL records +and bytes. + +For a one-path result, witness recovery, hydration, and decoding must also be +complete and separately attributable. A nested exact S4 fallback currently +marks its hidden traversal work unavailable, and a missing hydration counter +does the same. Those conditions are intentional fail-closed outcomes: the +affected B1/B2 record cannot qualify and must not be compared as a faster +candidate. The preflight also rejects a missing runtime receipt, scheduler +mismatch, non-exact public result, absent workspace measurement, or hidden +inactive-arm work. + +The required component boundaries are workspace reset, temporary-table access, +search, witness recovery, hydration, and result decoding. Existing GraphBench +boundary timings and invocation-local diagnostic replay identify the aggregate +work, but they do not yet separately attribute all six boundaries. Therefore +the preflight cannot become a formal performance tournament without a dedicated +component-boundary implementation and its tests. + +## Next authorization + +No P3 performance threshold or sample count is implied by this preflight. Once +all records are exact and complete, the resulting open trace may calibrate a +new, separately named power simulation. Only a passing simulation may freeze a +formal target/control performance schedule, including its sample counts, +confidence intervals, arm-order strata, median/p95/resource gates, and +component boundaries. A simulation pass still does not authorize a protected +holdout or a production selector. diff --git a/docs/experiments/sp_bidirectional_p3_preflight_v2.md b/docs/experiments/sp_bidirectional_p3_preflight_v2.md new file mode 100644 index 00000000..f9beac69 --- /dev/null +++ b/docs/experiments/sp_bidirectional_p3_preflight_v2.md @@ -0,0 +1,97 @@ +# Compact bidirectional shortest-path P3 preflight V2 + +Status: current B1/B2 function-workspace arms terminally rejected. This was a +telemetry and component-boundary readiness preflight, not a performance +qualification. It supersedes the incomplete V1 schedule and does not authorize +a selector, formal performance tournament, protected holdout, or production +activation. + +## V2 correction and fixed scope + +V1's primary S3/S4/B1/B2 roster is retained only as a diagnostic reference; +its direct-floor lane named a nonexistent `SP-S4-C-DIRECT` identity and never +completed. V2 is a distinct generation and requires a full clean recapture. +Its only schedule change is to run the direct floor separately by observation: +distance compares `SP-S4-C-D` with `SP-S0-DIRECT`, and one-path compares +`SP-S4-C-WE+MAT-M0` with `SP-S0-DIRECT`. + +The baseline is the same clean P0 source `57be1681140a2642639df0c06f7167bc17203e9b` +with retained round hashes `5cd14dc4b13008f5e307d44a16c56ff608eb79596b2ecaddf59d1eb70c31c6a1` +and `3bb71d1951b66559677abd4bba5441d844567269e6c1b1694cc95b67b4bc1f4d`. +Its descriptive target selection remains unchanged: inbound depth-8 one-path +at 3.52x PostgreSQL/Neo4j, inbound depth-8 distance at 2.63x, and long +outbound one-path at 1.69x. + +## Frozen roster and schedule + +The primary targets are `GSP-D08-F001_distance_inbound`, +`GSP-D08-F001_path_inbound`, and `GSP-D64-F1000_path`. The controls are +`GSP-D16-F016_distance`, `GSP-D16-F016_path`, +`GSP-D04-F128_disconnected`, `GSP-D04-F128_path_disconnected`, +`GSP-D02-F016_distance_cycle`, `GSP-D02-F016_path_cycle`, +`GSP-D02-F016_distance_self_loop`, `GSP-D02-F016_path_self_loop`, +`GSP-D01-F016_distance_parallel`, `GSP-D01-F016_path_parallel`, +`shortest_distance_bound_pair`, and `one_shortest_path_bound_pair`. +The old protected V2, SP-I1, and SP-I2 declarations remain excluded. + +Every primary observation receives S4, S3, B1, and B2 with pool size one, +Repeatable Read, diagnostic telemetry, one warm-up, and five timed samples. +The four carryover-balanced orders are `S4,B1,S3,B2`, `B1,B2,S4,S3`, +`B2,S3,B1,S4`, and `S3,S4,B2,B1`. + +The depth-one distance case `GSP-D01-F001_distance` and one-path case +`GSP-D01-F001_path` each receive two separate counterbalanced comparisons: +`S4,S0` followed by `S0,S4`. They must use the observation-specific S4 identity +above; no synthetic direct S4 identity exists. + +## Stop gate + +Every B1/B2 record must have an exact observation, correct runtime identity and +scheduler, one invocation-local search call, complete per-level and aggregate +search counters, measured workspace high water, and attributed plan +buffers/temp/WAL. One-path records additionally require complete hydration and +decode attribution. Any fallback, hidden counter, missing component boundary, +or unexplained inactive work fails closed. The full machine-readable contract +is `benchmark/testdata/scale/protocols/sp_bidirectional_p3_preflight_v2.json`. + +After—and only after—a complete V2 capture, a new separately named power study +may be calibrated. No V2 result can itself authorize formal timing, a holdout, +or a production selector. + +## Clean V2 result + +The complete V2 capture ran from clean source +`d77409674d6da4b00c3e379356955a5678dccbae` with GraphBench binary SHA-256 +`e08dd6d7f95b83421d91e3af19e7462b35d56b3a704b26d11d2e200d44d57ae4`. +The ignored artifact directory `.coverage/p3-preflight-d774096` contains 32 +primary and eight direct-floor JSONL artifacts: 248 exact records and 1,240 +timed observations. The sorted full capture ledger hashes to +`edd64b293ea0bb8a19fac41ee8e58d7042a511090b8043d5f85736fa4a2b567a`. + +All 120 B1/B2 case-round records were exact, had matching candidate runtime +identity, emitted no fallback, and supplied complete invocation-local search, +workspace, hydration, and plan-resource telemetry. The preflight therefore +resolved the earlier validator defects rather than hiding them. It was still +diagnostic-only—its host recorded a `powersave` CPU governor—so it does not +constitute a powered qualification result. + +That limitation cannot rescue either existing B arm: pooling the 20 warm samples +per target/arm gives the following B-to-S4 ratios. + +| Target | B1 median / p95 | B2 median / p95 | +| --- | ---: | ---: | +| `GSP-D08-F001_distance_inbound` | 4.59x / 3.90x | 3.27x / 2.79x | +| `GSP-D08-F001_path_inbound` | 4.65x / 3.92x | 3.74x / 3.18x | +| `GSP-D64-F1000_path` | 8.13x / 8.98x | 4.19x / 4.25x | + +B2 is the faster compact arm but fails the incumbent on every frozen target by +more than threefold. The direct floor is observation-sensitive as expected: +S0 reduces the distance median from 536us to 371us, but raises the one-path +median from 724us to 1,145us. It cannot justify a broad direct policy. + +This stops the existing B1/B2 stored-function/workspace identities before any +component implementation, power simulation, formal performance tournament, +holdout, or selector work. Keep S4 (and the markedly faster S3 references) on +the tested shapes. A future P3 successor requires a distinct executor, +workspace boundary, roster, telemetry contract, and prospective power study; +these V2 observations cannot be repurposed as its qualification evidence. diff --git a/docs/experiments/sp_i2_successor_power_study_v3.md b/docs/experiments/sp_i2_successor_power_study_v3.md new file mode 100644 index 00000000..451e0a99 --- /dev/null +++ b/docs/experiments/sp_i2_successor_power_study_v3.md @@ -0,0 +1,90 @@ +# SP-I2 successor prospective power study V3 + +Status: terminally rejected before implementation. This study was the only +permitted P2 activity after the terminal `suffix-reverse-retry-v1` result. It +does not authorize a hidden-fan-in executor, selector, corpus fixture, +database timing, or protected-holdout access. + +## Separation from terminal identities + +`sp-i2-distance-v1` and `sp-i2-distance-v2` remain terminal. This study has +the distinct identity `sp-i2-distance-v3-power-study`; any later candidate +must receive its own V3 executor, policy, selector, rollback, corpus, and +evidence identities. Archived code and traces may calibrate a study but cannot +be rebound as V3 evidence. + +## Archived calibration inputs + +The sole inputs are the two clean, open V1 discovery traces from source +`3865cbc57758b7b20b7ffe431f27235873422eed`: + +| Arm | Artifact SHA-256 | Structure | +| --- | --- | --- | +| S4 incumbent | `ac3ceb27ee92e3f4e21e3994ff9ee82d483b8081e9d44ddcef8e695ffdb1b6d0` | 20 balanced rounds, six open cases, 10 warm samples per record | +| I2 reference | `f6d79e81bdaafedaa95568d57140c14e0808fbb6fc261387abc916081137785a` | Same rounds, cases, and sample counts | + +The study may use only their within-round timing distribution and empirical +round-drift vectors. It must not treat their old candidate outcome, old corpus, +or terminal V1/V2 gate disposition as a result for a future V3 candidate. + +## Frozen design + +The formal design tested by the study is 800 matched blocks, one pool session, +Repeatable Read, 25 ordinary warm-ups, and 100 timed samples per arm/case/block. +The two arms physically alternate incumbent/candidate then candidate/incumbent +across blocks; every order-stratum has the same number of blocks. The study +therefore provisions 80,000 timed observations per arm/case before separate +fresh-session, cancellation, resource, and holdout requirements. + +The study keeps V2's 97.5% hierarchical interval and nearest-rank p95 +semantics, 100,000 bootstrap draws for a later formal report, and its +95%-Wilson power decision rule. It must simulate at least 20,000 independent +draws for each of the following scenarios: + +- A/A identity and the two 5% equivalence boundaries; +- target power at 0.90 median ratio and 0.97 p95 ratio, plus its boundary; +- control power at 1.00 median ratio and 0.97 p95 ratio, plus its boundary; +- odd and even order-stratum A/A power and their two 5% boundaries. + +The candidate-side labels in the model are placeholders only. A pass requires +the Wilson lower decision-power bound to be at least 0.90 for every power +scenario, calibrated coverage to include 0.975, and false-pass upper bounds +of 0.015 for p95 boundaries and 0.0275 for median/control boundaries. + +## Error model and feasibility threshold + +The V2 calibration at 40 blocks and 100 timed samples estimated log standard +errors of 0.025959 pooled and 0.036712 by order stratum, and absolute standard +errors of 59.338us pooled and 83.917us by order stratum. The V3 simulator must +derive its prospective values by the fixed factor `sqrt(40 / 800)`, yielding +rounded-up bounds of 0.005806 and 0.008210 log units and 13.269us and 18.765us +respectively. It must resample all 800 blocks from the archived 20-round drift +vectors rather than repeat a fixed drift mean. + +This block count is intentional: the 40-block V2 design made a two-sided 5% +A/A interval impossible, while the V3 order-stratum half-width is below the +5% log margin with additional room for estimator variation. Reducing blocks, +changing samples, pooling order strata, or changing thresholds creates a new +study identity. + +## Required disposition + +Implement a reproducible simulator under a V3-specific schema and domain +separator, verify both archive digests and all V3 constants, then commit its +report and test vectors. A failed simulation terminally stops this study before +any P2 executor or corpus work. A passing simulation authorizes only a fresh +V3 corpus and architecture-tournament protocol; it does not authorize a +candidate implementation, database timing, or a holdout. + +## Result + +The deterministic 20,000-run simulation failed the frozen admission-power +gate. The `aa_order_odd_high` and `aa_order_even_high` scenarios reached only +Wilson lower bounds of `0.14201232557116983` and `0.14723913101703448`, versus +the required `0.90`. All other scenario vectors are locked in +`TestSPI2SuccessorPowerStudyV3TerminatesFrozenDesign`; the terminal tombstone is +`benchmark/testdata/scale/protocols/sp_i2_successor_power_study_v3_rejection.json`. + +This 800-block study is terminal. Do not enlarge it, alter its model, reuse its +identity, or use it to authorize a V3 candidate. No candidate implementation, +fresh corpus, database timing, or holdout access occurred. diff --git a/docs/experiments/sql_strategy_routing_preflight_v1.md b/docs/experiments/sql_strategy_routing_preflight_v1.md new file mode 100644 index 00000000..49d80d6b --- /dev/null +++ b/docs/experiments/sql_strategy_routing_preflight_v1.md @@ -0,0 +1,309 @@ +# SQL strategy-routing preflight v1 + +Status: frozen preimplementation feasibility. This is a default-off, +PostgreSQL/driver-only experiment. It is not a retry of +`suffix-reverse-retry-v1`, `suffix-reverse-guard-v1`, or either terminal +orientation-probe identity; it cannot change production routing, cached +translations, schema, or query results. + +## Basis + +The clean two-round P0 recapture at `a4b29f2` recorded 704 successful backend +records. The largest comparable repeated loss was the full-path sparse suffix +case `GFSE-V2-D16-F1000-R1-X1-M1-sparse_path`: PostgreSQL was 91.06 times the +Neo4j median. Its endpoint-only companion was 34.22 times the Neo4j median. +Both execute `EXPANSION-STEPWISE-FORWARD` because the production policy is +`fixed-suffix-static-v1` with `compile_time_fallback`. + +Earlier component evidence establishes that exact suffix-seeded reverse search +and ordered-ID hydration can be fast. It also establishes two hard boundaries: +the same-statement guard paid fixed probe/dispatch cost, and the transaction +retry generation added at least 2.02 ms to its exact-reverse fast path. Neither +execution boundary may be reused here. + +## Question + +Can a separately selected, single exact SQL arm retain the direct reverse +component's benefit without a same-statement probe, an inactive forward body, +or per-query transaction setup? + +The preflight evaluates this question only in an explicit diagnostic mode. A +future automatic policy may be considered only after this preflight passes and +after it separately proves decision-cache staleness, mutation, snapshot, and +rollback behavior. + +## Frozen scope + +The preflight uses the reusable exact executor +`EXPANSION-SUFFIX-SEEDED-REVERSE` with ordered node/edge-ID hydration. The +incumbent is `EXPANSION-STEPWISE-FORWARD`. The temporary diagnostic label is +`suffix-route-component-v1`; it is not a production policy identity. + +The fresh, open training roster is fixed in the accompanying protocol and uses +11 new v3 fixture identities (never a relabelled P0/P1/orientation fixture): + +- sparse endpoint-ID and complete-path fixed suffixes; +- high reverse fan-in, dense suffix, no-path, cap-boundary, cycle, self-loop, + relationship-distinct, and multi-path controls; +- no generated shortest-path, all-shortest, endpoint-seeded, P1 terminal, or + previously protected declaration is eligible for candidate timing. + +The two targets are `GFSE-SRC-V1-TARGET-D16-F1024-sparse_endpoint_ids` and +`GFSE-SRC-V1-TARGET-D17-F1025-sparse_path`. The nine controls cover high +reverse fan-in, dense suffixes, no-path exhaustion, 511/512/513 disconnected +suffix rows, a productive cycle, a productive self-loop, and relationship- +distinct multiple suffix paths. Each declaration carries an exact ID-row or +complete path oracle. The P0 case names remain discovery inputs only and may +not be relabelled as training or holdout evidence. + +## Four-round component comparison + +The comparison is PostgreSQL-only and remains non-promotional. In each round, +capture the complete `suffix-route-component-v1` roster twice: once as the +ordinary `EXPANSION-STEPWISE-FORWARD` incumbent and once with +`-postgres-expansion-suffix-route-component`. Use a fresh JSONL artifact for +each arm/round, one warm-up, five timed iterations, pool size one, caller-owned +Repeatable Read, diagnostic telemetry, and one shared nonempty run UUID. The +four counterbalanced orders are incumbent/component, component/incumbent, +incumbent/component, and component/incumbent. This yields 55 timed samples per +arm/round and 440 timed samples over the full comparison. + +For example, the component half of round one is: + +```bash +go run ./cmd/graphbench \ + -modes postgres_sql -tags suffix-route-component-v1 \ + -warmup-iterations 1 -iterations 5 -pool-size 1 \ + -round 1 -block 1 -run-uuid "$RUN_UUID" -arm reverse_component -arm-order 2 \ + -require-clean-source \ + -postgres-repeatable-read -postgres-traversal-telemetry diagnostic \ + -postgres-expansion-suffix-route-component \ + -jsonl-output .coverage/sql-routing-preflight-v1/round-1-reverse-component.jsonl +``` + +The incumbent command uses the same flags and roster, omits the component +flag, and sets `-arm incumbent -arm-order 1`. Subsequent rounds rotate the +declared order. Do not append a second attempt to an artifact: an incomplete or +non-exact arm stops this generation. + +## Clean-source recapture requirements + +Every arm of a replacement capture must use `-require-clean-source`. GraphBench +checks the tracked diff and untracked source fingerprint before target +validation, fixture loading, or acquisition of the destructive-run lock. The +binary must therefore be built from a clean committed tree, with capture output +kept in an ignored directory or outside the repository. + +Each component record must report diagnostic counter status `complete` with +exact suffix, boundary, reverse-state, one-row receipt, ordered node/edge +hydration loop and row counters, exact public output rows, and the untimed +PostgreSQL planning/execution timings. A missing or ambiguous named CTE, +ordered-hydration alias, receipt, or timing fails closed. + +Before starting the four arms, run the PostgreSQL manual operational test for +`TestPostgreSQLSuffixRouteComponentCancellationReusesPoolSession`. It proves a +statement-timeout cancellation (`57014`), rollback, release/reacquisition of +the size-one pool's same backend PID, and a cardinality-preserving direct +component replay. Store its test output beside the four-round artifact; it is +operational evidence only and must not be mixed into timed samples. + +## First four-round capture (diagnostic only) + +The first capture completed on 2026-08-19 with 88 successful records and 440 +timed samples: five samples for each of 11 cases in both arms across four +counterbalanced rounds. All public row/path observations matched their +declared oracle. Each of the 220 component samples emitted one +`suffix_route_component` receipt with +`EXPANSION-SUFFIX-SEEDED-REVERSE`, no fallback, and no active +`EXPANSION-STEPWISE-FORWARD` SQL body. The ordinary arm remained the +compile-time-forward incumbent. + +The two targets were materially faster as direct components: endpoint IDs had +a four-round median ratio of `0.087x` (about `42.05ms` saved) and the complete +path target had a ratio of `0.050x` (about `74.73ms` saved), both component / +incumbent. The high-fan-in, dense, and 511/512/513 suffix controls were slower +under reverse (`1.33-1.84x`); the no-path and relationship-distinct controls +improved. These are descriptive component results, not a selection rule. + +The raw artifacts are ignored under `.coverage/sql-routing-preflight-v1`. +Their ordered SHA-256 ledger is: + +```text +0bdb199f72751f5a5586f99c599a9fa0c81059da0c244e1f097caa01cc9aa55f round-1-reverse-component.jsonl +6ff9ac81a525f9fcf83891f9d28a50be8424c43b3ef1ede4b3cb58a00f1b329c round-1-incumbent.jsonl +7cc6b084593273ada82245e117597119259774a996f08928ab3c6119bb6b4229 round-2-reverse-component.jsonl +ae943d9616a70e68446f8c29039f150f2371a64f0b8e6b67404e16796ab7d736 round-2-incumbent.jsonl +4051d3908348b069bd246fc656c2d7271d3b8f6f3f61f8c9cdc5e8045e0ecc6d round-3-incumbent.jsonl +eceb58b830c4951001185750facfba45e8489dd4f154435c35231ef4b7381368 round-3-reverse-component.jsonl +4eeb99c02bfd3d9dbde2392eed6e925b98386098d25edcd8791cd802aa6ad5fb round-4-incumbent.jsonl +b9531a518ac4369248fda3ec813a65208375c2828cb312b07e00f2cd9a3d74e1 round-4-reverse-component.jsonl +``` + +This capture has source commit `a4b29f22b81c2191316b54b8383283fc40a1900d`, +dirty diff `ad94dc9497eff73211eef6b6cb519e41286a7bde105745348fc1c0ef6448010e`, +binary `4cdb5d00a5aa8fdb5f4ea8d93537a443312d520cc2864adf50f5534f91af588e`, +and corpus `6255a9495172e0749f5e330b4648631d8d8a8b10cbdfeb88a4f7f2eed5157d60`. +Diagnostic telemetry is present but reports `plan_derived_partial`, so it does +not yet provide all preregistered component counters. The dirty source and +partial telemetry prevent cache work, automatic routing, protected access, or +promotion. A clean-source recapture with complete component telemetry and the +remaining cancellation/pool-reuse evidence is required before the separate +cache-feasibility decision. + +## Clean-source four-round recapture + +The replacement capture completed on 2026-08-19 from committed source +`aaecb745c328128115273b4da7fa71a8de3351b7`, with the clean-tree SHA-256 +(`e3b0...b855`), binary SHA-256 +`4c65ff1f7d642a47bcc4e96b9aee19f57fe8a30c9fe2218cd91514a0bdc71860`, +and corpus SHA-256 +`2aa00d2df9a32e7fbca6e9682058ba30e82a4b2968f87b123c7fa161924cac18`. +All 88 records were exact and successful; all 44 component records carried +complete suffix, boundary, reverse-state, receipt, ordered-hydration, and +planning/execution telemetry. The 220 timed component samples each had one +direct `suffix_route_component` receipt with no fallback or forward SQL body. + +The replacement ledger is: + +```text +2fcd84240a3466facff45e8074b7630491a639d86bd7e532dd42ffd45271389a round-1-incumbent.jsonl +92bee28dc8363ee3165bf4cf3ba9863e1b5254204f8273a15b7b20030ca063ab round-1-reverse-component.jsonl +14b49b15364c6ecf073ab52bb89a6e56710c881eab487e808287ceecf69294b8 round-2-reverse-component.jsonl +e485a599c08863571ebeec847412a8f70628dfbf43c665f074f9872e407b95f3 round-2-incumbent.jsonl +45c853856e30a193649ad097ceb0000464c7e1e5939bee47d5217a5dc6696a5d round-3-reverse-component.jsonl +f95e9fb9c0d489e0f9f5012140b0a9b419f5da221b62454498c323700f5f5c9c round-3-incumbent.jsonl +38bd271656362fb7a850e5137360588a20fa90de5e2a8321801fa809906134fe round-4-reverse-component.jsonl +da5e39fd4e2a4292ef9af5bfac9650787130d6396d19202194f6da7828eaadfe round-4-incumbent.jsonl +86241c76a01bd23c80d21437126f0e55f6b792bbd7d6b9cbb710189c25c4164c cancellation-pool-reuse.log +``` + +The four-round median-of-round-medians component/incumbent ratios were +`0.081x` for sparse endpoint IDs (about `44.86ms` saved) and `0.048x` for the +sparse complete path (about `81.87ms` saved). The high-fan-in, dense-suffix, +and 511/512/513 controls regressed (`1.42-1.87x`); no-path and the three +relationship-distinct controls improved. A timeout cancellation returned +`57014` in `1.159ms`, rolled back successfully, and the size-one pool +reacquired the same backend before an exact replay. This remains descriptive +component evidence only: it does not authorize routing, cache work, protected +access, or promotion. + +## Boundary and workspace closure + +The clean recapture closes exactness, typed component counters, plan-visible +buffers/temp/WAL, cancellation, and pool reuse. It does not yet decompose the +client/raw-PGX boundary into prepared-statement states, nor does it bind the +component telemetry to measured temporary-workspace high water. Those are +separate required observations; the existing `planning_ms` and `execution_ms` +fields must not be treated as a substitute for bind, first-row, decode, drain, +or session-reuse timings. + +[`sql_strategy_routing_component_closure_v1.json`](../../benchmark/testdata/scale/protocols/sql_strategy_routing_component_closure_v1.json) +freezes the only permitted closure. It reuses the same eleven open fixtures, +four counterbalanced incumbent/component rounds, caller-owned Repeatable Read +contract, and size-one PostgreSQL pool. It adds no selector, retry, cache, +schema state, reference arm, or concurrency mode. + +For each arm/case, the closure records one newly opened-session prepared miss, +five same-session prepared hits, one miss on a separate newly opened size-one +raw-PGX pool, and five hits after release/reacquisition of that same pooled +backend. Every raw execution must +match the public row/path observation. The raw-PGX samples separately retain +transaction setup, bind/prepare, first row, complete decode, drain/close, and +total timing. Each sample also records a SHA-256 of its sorted normalized +public rows; the runner rejects disagreement with the primary CySQL observation +or any other prepared-state stratum. Workspace observation runs only after result drain and is +excluded from those timing intervals. The exact client parse/optimize/translate/render +waterfall is retained beside it. + +The command must supply the frozen one-MiB session and pool workspace ceilings +even though direct reverse is expected to allocate no component workspace. The +measurement sums non-diagnostic temporary relations visible in the query +transaction, excluding the runtime-attestation and telemetry scaffolding. The +size-one pool makes pooled-session and pool peaks directly comparable. Direct +component telemetry must then declare both `suffix_component` and `workspace` +families with complete provenance. + +For example, the reverse arm of round one is: + +```bash +graphbench \ + -modes postgres_sql -tags suffix-route-component-v1 \ + -warmup-iterations 1 -iterations 5 -pool-size 1 \ + -round 1 -block 1 -run-uuid "$RUN_UUID" -arm reverse_component -arm-order 2 \ + -require-clean-source \ + -postgres-repeatable-read -postgres-traversal-telemetry diagnostic \ + -postgres-expansion-suffix-route-component \ + -postgres-suffix-route-component-closure \ + -session-memory-ceiling-bytes 1048576 \ + -pool-memory-ceiling-bytes 1048576 \ + -jsonl-output .coverage/sql-routing-component-closure-v1/round-1-reverse-component.jsonl +``` + +The incumbent uses the same closure and ceiling flags but omits +`-postgres-expansion-suffix-route-component`; it remains the exact ordinary +forward statement. A failed row count, absent stage, changed pooled backend, +missing workspace observation, ceiling breach, incomplete component telemetry, +or target performance reversal stops this generation. No closure result is a +cache hit or automatic-selection result. + +### Current closure disposition + +The complete four-round artifact from source `94fe902` is retained as +diagnostic pre-enforcement evidence. It predates the per-sample normalized +observation SHA-256 requirement and cannot assert closure passage. + +The replacement capture from clean commit `c490f3c` is +`.coverage/sql-routing-component-closure-v1-c490f3c`. Its fresh run UUID is +`e4800916-4eee-4dcf-ae99-d68068dcf4d5`; all 88 records are successful, every +record retains all twelve raw-PGX prepared-state samples, each sample has a +normalized-observation SHA-256 matching the primary CySQL observation, pool +reacquisition retained its backend identity, and measured temporary workspace +is zero. The component/incumbent paired-median ratios are `0.0839` for sparse +endpoint IDs and `0.0445` for sparse complete paths. This passes the frozen +closure but remains non-promotional: it authorizes only freezing the separate +transaction-scoped cache-feasibility protocol, not cache code, routing, +holdout access, or a release claim. + +## Required implementation slice + +The first slice is diagnostic only: + +1. Add a GraphBench arm that emits one forced reverse statement with the new + diagnostic identity and records search, ordered-ID hydration, planning, + execution, decode, and first-session timing separately. +2. Add new fixture declarations for the frozen classes and exact path/row + oracles. Do not modify existing terminal-generation fixtures. +3. Capture the incumbent and direct-reverse component in counterbalanced + PostgreSQL-only rounds under the same externally owned Repeatable Read + transaction. No retry, selector, cache hit, or fallback is permitted. +4. Stop before automatic routing unless all exactness, resource, cancellation, + pool-reuse, and direct-component overhead requirements pass. + +The preflight does not measure a cache hit as a candidate result. A later, +separately named cache feasibility protocol must show that its key is scoped to +the graph and transaction/snapshot, that misses preserve the incumbent, and +that stale or absent metadata cannot alter correctness. + +## Stop conditions + +Stop this generation before cache implementation, automatic dispatch, or +protected holdout access if any of the following occur: + +- the direct component is not exact for every frozen target/control; +- component telemetry is absent, contradictory, or attributes work to an + inactive forward arm; +- direct reverse fails its declared resource, cancellation, or pool-reuse + limits; +- direct reverse fails to materially improve both sparse full-path targets; +- a proposed external selection boundary adds more than the predeclared host + A/A floor to an already-fast direct reverse execution; +- the design requires a persistent synopsis, graph epoch, or schema change. + +After the boundary/workspace closure passes, the next decision is a separate +non-native architecture feasibility protocol for a transaction-scoped routing +cache. It must declare cache keys, invalidation, stale-data behavior, +transaction ownership, write/WAL budget, and rollback/removal before code is +added. That contract is now frozen as +[`suffix_route_cache_feasibility_v1.md`](suffix_route_cache_feasibility_v1.md); +it permits no cache implementation or automatic routing until a separately +reviewed feasibility slice is authorized. diff --git a/docs/experiments/suffix_reverse_retry_v1.md b/docs/experiments/suffix_reverse_retry_v1.md new file mode 100644 index 00000000..c2528951 --- /dev/null +++ b/docs/experiments/suffix_reverse_retry_v1.md @@ -0,0 +1,295 @@ +# Suffix reverse transaction retry v1 + +Date: 2026-08-17 + +Status: open training roster frozen; no production selector or protected holdout is authorized + +## Purpose + +`suffix-reverse-retry-v1` tests whether the already-correct suffix-seeded +reverse component retains its sparse-topology advantage when the successful +path contains neither topology probes nor an inactive forward body. It is a +new generation and does not reuse the evidence identity of +`orientation-probe-v1`, `orientation-probe-v2`, or +`suffix-reverse-guard-v1`. + +## Frozen development identity + +- policy: `suffix-reverse-retry-v1`; +- candidate executor: `EXPANSION-SUFFIX-SEEDED-REVERSE`; +- incumbent and retry executor: `EXPANSION-STEPWISE-FORWARD`; +- execution boundary: `transaction_retry`; +- suffix rows: 512 complete rows, with a cap+1 sentinel; +- reverse states: 512 complete rows, with a cap+1 sentinel; +- buffered output rows: 4,096, with a cap+1 sentinel in candidate SQL; +- buffered encoded output: 16 MiB; +- isolation: PostgreSQL Repeatable Read; +- public observation: complete hydrated paths only; +- selection: tool-only exact query/corpus selection; no production policy. + +Changing an identity, cap, observation, or execution boundary creates a new +development generation. Command-line cap overrides are diagnostic and cannot +qualify this frozen identity. + +## P0 descriptive entry snapshot + +On 2026-08-18, two independently reloaded broad GraphBench rounds ran the +complete scale corpus with a fresh binary, `-pool-size 1`, and PostgreSQL +diagnostic telemetry. Both rounds had exact observations for every recorded +row: 173 PostgreSQL and 174 Neo4j records per round. This is descriptive +opportunity accounting only because the source worktree was dirty; it cannot +freeze a corpus, authorize a holdout, or satisfy this generation's stop gate. + +The combined two-round per-case medians confirm the P1 premise. The worst +single loss was hidden-fan-in distance at `63.92x` PostgreSQL/Neo4j, but sparse +fixed-suffix forms occupied the next four positions at `44.17x`, `41.86x`, +`36.43x`, and `31.80x`. Across all 35 generated fixed-suffix cases, the +geometric-mean ratio was `2.74x`; their long-pole concentration makes this the +largest multi-case opportunity. The P1-admissible path targets +`GFSE-D16-F1000-sparse_path` and +`GFSE-V2-D16-F1000-R1-X1-M1-sparse_path` shared incumbent SQL fingerprint +`dc8aab1f84de2cae582bc9252d4b7996653113ed1df785abcd8e4e17b4c32961`. +Their structured plans are retained in the raw captures. + +The ignored raw captures are +`.coverage/p0-20260818-round1.jsonl` +(`248282767bdd041f4b48d4d8c850727b5f27357589ada7790216a8c84832acee`) +and `.coverage/p0-20260818-round2.jsonl` +(`348ae3757f09bd6339a2eef6fcbc073b1621cc22ecc882d986dde60055d5345c`). +Before any qualifying P1 timing, repeat P0 from a clean committed source and +freeze the open training selection. + +## Clean P0 baseline + +On 2026-08-18, P0 was repeated from committed source +`57be1681140a2642639df0c06f7167bc17203e9b` with GraphBench binary SHA-256 +`5f9c5c3b7dcfbb7ffd69554b04b75b899cc6a6f1772e1e952d90a1abd0814c8c`. +Each of two independently reloaded rounds used pool size one, one warm-up, and +three timed iterations in both PostgreSQL and Neo4j modes. Every record was +exact: 176 PostgreSQL and 176 Neo4j records in each round. The raw captures +are `.coverage/p0-clean-57be168-round1.jsonl` +(`5cd14dc4b13008f5e307d44a16c56ff608eb79596b2ecaddf59d1eb70c31c6a1`) +and `.coverage/p0-clean-57be168-round2.jsonl` +(`3bb71d1951b66559677abd4bba5441d844567269e6c1b1694cc95b67b4bc1f4d`). + +The protected hidden-fan-in stress case remains the largest cross-backend +loss, at about `82.21x` PostgreSQL/Neo4j by the two-round case-median ratio. +Among open P1 targets, `GFSE-D16-F1000-sparse_path` and +`GFSE-V2-D16-F1000-R1-X1-M1-sparse_path` remain the leading sparse full-path +opportunities, at about `50.03x` and `39.02x`, respectively. This baseline +authorizes P1 open timing only; it does not authorize a protected holdout. + +## Open transaction smoke + +The initial open transaction smoke ran on 2026-08-18 against the P0 sparse +path target `GFSE-V2-D16-F1000-R1-X1-M1-sparse_path`. It returned the exact +two paths on every observation. With frozen default caps, all three timed warm +samples completed on `EXPANSION-SUFFIX-SEEDED-REVERSE` with +`reverse_complete` receipts (median `6.79ms`). This is directionally faster +than the P0 incumbent descriptive median (`57.52ms`), but it is not a paired +tournament and does not evaluate the stop gate. + +A separate state-limit-one diagnostic forced transaction-local retry. Every +timed sample preserved exact rows and reported the ordered receipt chain +`forward_retry_state_overflow` on the reverse executor followed by +`exact_forward_retry_complete` on `EXPANSION-STEPWISE-FORWARD`; its median was +`62.68ms`. This validates the savepoint rollback, deferred completion receipt, +and exact-incumbent fallback boundary, but cap overrides and dirty source make +it non-qualifying. + +The ignored smoke artifacts are +`.coverage/p1-retry-smoke-20260818.jsonl` +(`9dce1d49f8a8c08e14ddf20d693f61fecb108ceaf99f9a7fd94315cc7c13d5bb`) +and `.coverage/p1-retry-forced-state-20260818.jsonl` +(`cbc9ac50d72688a5d706a4b971a9888b1e1169eaf57f61f7136ad66fc5e117cf`). + +One-iteration comparator smokes on the same target also verified the three +execution surfaces: ordinary exact forward (`58.73ms`), forced exact reverse +(`5.82ms`), and retry (`6.79ms`). They establish neither an overhead bound nor +a performance result. The forward and reverse artifacts are +`.coverage/p1-comparator-forward-smoke-20260818.jsonl` +(`9b1d989b13d6bfdb0f0acf8685fb09cefae325a38dacbf3ced5525c824da2d69`) +and `.coverage/p1-comparator-reverse-smoke-20260818.jsonl` +(`0c17a895ef93496830cfc6ef80d5b2e329a1f284284e071b2e85d8920580f7c9`). + +## Frozen open training roster + +The following exact non-holdout full-path declarations are frozen for P1 open +development. Each is captured independently because retry translation admits +one target per invocation: + +| Role | Cases | +| --- | --- | +| Sparse targets | `GFSE-V2-D16-F1000-R1-X1-M1-sparse_path`; `GFSE-D16-F1000-sparse_path` | +| Shallow and zero-depth controls | `GFSE-D00-F001-none_path`; `GFSE-D01-F010-sparse_path` | +| Suffix density/payload controls | `GFSE-D04-F010-half_payload_path`; `GFSE-D08-F001-all_path` | +| Root multiplicity, cycle, and self-loop control | `GFSE-V3-TRAIN-Q4-C1-S1-productive_cycle_self_loop_path` | +| Multi-root/disconnected control | `GFSE-V3-TRAIN-D05-F008-R4-X3-I0-M1-Q3-path` | +| Candidate suffix-cap retry | `GFSE-BOUNDARY-S513-productive-path` | +| P1-only reverse-fan-in | `GFSE-P1-TRAIN-D09-F017-R0-X2-I1024-M1-Q1-high_reverse_fanin_path` | +| P1-only no-path exhaustion | `GFSE-P1-TRAIN-D09-F513-R0-X512-no_path_exhaustion` | +| P1-only output-byte retry | `GFSE-P1-TRAIN-D00-F001-R0-X0-M4-P2100000-output_byte_retry_path` | + +The three P1-only declarations use fresh V2 fixture identities and the +`suffix-reverse-retry-v1-training` tag; they do not alter the frozen V3 +orientation cohorts or reuse a legacy holdout. The byte case returns four +complete paths whose hydrated root and heads carry 2,100,000-byte payloads, +intentionally exceeding the frozen 16 MiB candidate buffer while remaining an +exact forward-retry control. It is PostgreSQL-only in GraphBench because it +tests that driver's retry buffer; Neo4j does not contribute a comparable +execution boundary and did not complete this hydrated 25 MiB observation +within a 90-second diagnostic deadline. Do not reuse a legacy holdout as a +convenience control. Adding, deleting, or replacing any roster identity +creates a new P1 generation. + +One-iteration PostgreSQL smokes on 2026-08-18 confirmed exact observations for +all three controls. The high-fan-in path naturally produced +`forward_retry_state_overflow` followed by `exact_forward_retry_complete`; the +no-path exhaustion control completed reverse-only; and the payload control +naturally produced `forward_retry_output_bytes` followed by +`exact_forward_retry_complete`. These dirty-source artifacts are diagnostic +only: `.coverage/p1-high-fanin-fixture-smoke-20260818.jsonl` +(`eb47d849fc165c924000169dbdf1b5c75c8eb9d73f7f798f2956afc2deed0cb1`), +`.coverage/p1-no-path-fixture-smoke-20260818.jsonl` +(`b6af2bdf940fc9881082791ab8ae913548e16677ce3f6436d6f9d17418d3cc40`), +and `.coverage/p1-output-byte-fixture-smoke-20260818.jsonl` +(`c33fef4ffecb8d30d06906e3a0c3850531112428ac2362ea604b9acd386cba2a`). + +Before this freeze, one individual PostgreSQL retry capture per roster member +returned its declared exact result. Nine members, including all sparse, +shallow, density, and V3 controls, completed reverse-only. The three required +retry controls emitted the expected ordered chains: suffix overflow for +`GFSE-BOUNDARY-S513-productive-path`, state overflow for the P1 high-fan-in +case, and output-byte overflow for the P1 payload case, each followed by +`exact_forward_retry_complete`. + +## Execution contract + +1. Start an explicit Repeatable Read transaction without initializing unrelated + shortest-path workspaces. +2. Establish a savepoint and execute the reverse-only statement. The statement + computes bounded suffix and reverse-state probes, records one transaction- + local status, and contains no forward CTE or forward executor. +3. Drain and buffer candidate rows. No row is exposed until the SQL status, + row cap, byte cap, and result decoding are complete. +4. On `reverse_complete`, release the savepoint and publish the buffer. +5. On a declared suffix, state, output-row, output-byte, or encoding overflow, + discard the complete candidate buffer, roll back to the savepoint, release + it, and execute the ordinary exact forward translation in the same + transaction. +6. Unknown or missing status, candidate error, cancellation, receipt failure, + or savepoint failure returns an error. It never becomes a performance + fallback. + +Timed invocation receipts distinguish the candidate observation from the +actual forward retry. `exact_forward_retry_complete` is recorded only after +the fallback result drains and validates without error; an error or early close +does not create that completion receipt. The fallback remains the ordinary +independently translated incumbent; the candidate statement cannot initialize +it. + +## Open development command + +Use only open training/control cases: + +```bash +go run ./cmd/graphbench \ + -modes postgres_sql \ + -postgres-expansion-suffix-reverse-retry \ + -postgres-repeatable-read \ + -postgres-traversal-telemetry diagnostic \ + -pool-size 1 \ + -cases \ + -jsonl-output +``` + +The run must use pool size one. Reference and concurrency side measurements +are separate development captures until the retry-aware versions of those +measurement paths are implemented. + +The current retry tool deliberately translates exactly one statically eligible +full-path target per invocation. The retired `orientation-v2-training` cohort +contains multiple endpoint-only and path-observed declarations, so it is not a +valid P1 selector. Build the fresh P1 training/control roster from explicit +single-case captures (or add a dedicated, frozen P1 cohort) before beginning a +multi-case tournament. + +## Frozen P1 capture schedule + +Each frozen roster member receives three separately invoked PostgreSQL arms in +each of six rounds: ordinary exact forward (`F`), forced exact suffix-seeded +reverse (`R`), and transaction retry (`T`). The round orders are the six +permutations `FRT`, `FTR`, `RFT`, `RTF`, `TFR`, and `TRF`, in that order. Each +arm uses the committed binary, pool size one, Repeatable Read, diagnostic +telemetry, one warm-up, and five timed iterations. No cap override, reference, +or concurrency option is permitted. + +An arm is one exact case invocation and writes its own JSONL artifact. Its +GraphBench `round`, `block`, `arm`, and `arm-order` fields must match this +schedule. A retry arm must retain exact rows and timed receipt chains; a +reverse-only arm must never contain a forward retry receipt. The schedule is +prospective: changing cases, binary, arm definitions, counts, warm-ups, or +orders creates a new generation. Only after all open captures are exact and +the early-stop gate passes may a separately authorized holdout step begin. + +## Clean P1 open result (terminal) + +The prospective capture completed on 2026-08-18 from committed source +`3737dd57cb6baeb2bbc21f936adc0049a19ae19e` and GraphBench binary SHA-256 +`6a2c49a6ca903c4794569109491c5c185eecd677801fb0782a70430560db2655`. +It contains all six frozen orders, 216 independently reloaded one-case arm +artifacts, and 1,080 timed PostgreSQL observations (30 per case/arm). Every +artifact was exact, carried the clean source digest, and used the shared run +identity `p1-srr-v1-3737dd5-20260818`. The ignored artifact set is +`.coverage/p1-clean-3737dd5`; its sorted per-artifact SHA-256 ledger hashes to +`3ce552fa1e522333a24c90f655672d203ad209f83beb44785348c6d61fc45028`. + +Every reverse-only arm remained reverse-only. Every retry observation had its +required receipt chain: the boundary, high-fan-in, and byte controls emitted +suffix, state, and output-byte overflow respectively, followed by +`exact_forward_retry_complete`; all other retry targets emitted only +`reverse_complete`. + +Pooling the 30 timed samples per arm makes the early-stop failure +unambiguous. The lowest retry-to-exact-reverse median ratio among all 12 cases +was `1.47x` (an additional `2.02ms`), and the lowest pooled nearest-rank p95 +ratio was `1.31x`. Thus every fast-path target violates both the +`1.10`/`100us` median bound and the `1.05` p95 bound. The two long sparse +targets did improve materially over ordinary forward (retry/forward medians +`0.102x` and `0.117x`), and the no-path control reached `0.646x`, but those +gains cannot offset the required fast-path-overhead failure across the full +roster. + +This generation is terminally stopped. Do not retune it, repeat it as an +authorization attempt, or run a protected holdout. Its artifact is evidence +for a future, separately identified executor or policy generation only. + +## Early stop gate + +The generation stops before a protected holdout unless every open case has +exact observations and complete receipts and satisfies all of: + +- successful retry fast-path overhead versus exact reverse is at most `1.10` + or `100us` at the median, with p95 ratio upper at most `1.05`; +- each target improves over exact forward with median-ratio upper at most + `0.95` or saving lower at least `100us`, with p95 upper at most `1.05`; +- controls and real retries stay within `1.10` or `100us` at the median and + p95 upper at most `1.05`; +- `reverse_complete` performs zero forward work; +- retry exposes zero candidate rows; +- memory, temporary bytes, WAL, cancellation, session reuse, and receipt + attribution pass their declared bounds. + +Only a clean-source open result that passes this gate may create a separately +committed formal corpus, prospective power study, and untouched holdout. A +failure is terminal for this identity. + +## Hidden-fan-in sequencing + +SP-I2 V1 and V2 remain terminal. A hidden-fan-in successor receives a new +executor, selector, corpus, rollback, and evidence identity only after this P1 +generation reaches a passing or terminal disposition. Its first artifact is a +prospectively frozen power study based on archived open V1/V2 traces; candidate +timing and implementation changes are forbidden until that study passes. That +study was frozen and terminally rejected in +[`sp_i2_successor_power_study_v3.md`](sp_i2_successor_power_study_v3.md). diff --git a/docs/experiments/suffix_route_cache_feasibility_v1.md b/docs/experiments/suffix_route_cache_feasibility_v1.md new file mode 100644 index 00000000..4e3a1d6c --- /dev/null +++ b/docs/experiments/suffix_route_cache_feasibility_v1.md @@ -0,0 +1,79 @@ +# Suffix-route cache feasibility v1 + +Status: frozen pre-implementation feasibility contract. The default remains +`EXPANSION-STEPWISE-FORWARD`; this document does not enable a selector, cache, +retry, schema object, or production behavior. + +## Admission + +The direct-component closure from clean commit `c490f3c` passed with 88 exact +records at `.coverage/sql-routing-component-closure-v1-c490f3c`. Every record +has twelve agreeing raw-PGX observation digests, stable pooled backend identity, +and zero measured temporary workspace. Its paired component/incumbent median +ratios are `0.0839` for sparse endpoint IDs and `0.0445` for sparse complete +paths. + +That evidence permits this protocol only. It does not establish that a routing +decision can outlive a transaction or that a cache hit is safe. + +## Cache boundary + +The candidate is an application-memory cache owned by exactly one active, +caller-owned, read-only `REPEATABLE READ` PostgreSQL transaction. It stores an +immutable routing decision, never public results, graph values, translated SQL, +plans, or graph metadata. A cache miss executes the exact ordinary forward +incumbent. A hit may use the already-qualified direct reverse statement only +inside the same owner transaction and snapshot. + +Every key includes an opaque transaction-owner token minted after `BEGIN`, the +graph ID, normalized Cypher shape, canonical parameter names/types/values, +frozen policy identity, and a transaction-local invalidation generation. The +owner token is not a reusable connection, backend PID, pool slot, or process +identity. Missing or unverifiable key data is an incumbent-only bypass. + +There is deliberately no cross-transaction, cross-snapshot, cross-connection, +or retry reuse. The repository has no graph mutation epoch, so permitting any +such reuse would be unsafe. + +## Lifetime and invalidation + +The cache is allocated after `BEGIN` and discarded before the transaction is +returned to its caller. Commit, rollback, cancellation, connection release, +and retry discard every entry. The feasibility scope permits cache use only in +transactions with no graph mutation and no savepoint lifecycle; either boundary +invalidates all entries, increments the local generation, and disables cache +use for the rest of that transaction. + +This restriction intentionally makes rollback removal application-memory-only. +It avoids a cache-maintenance statement whose write, WAL, or cleanup behavior +could contaminate a read-path measurement. + +## Resource and observability contract + +The cache is capped at 64 entries, 64 KiB total, and 4 KiB per entry. It has no +eviction: capacity exhaustion bypasses caching and publishes nothing. Entries +are immutable and must not retain caller-owned mutable buffers. + +The cache may not create database objects or issue data-modifying cache SQL. +It must produce zero cache-attributable WAL and no temp relation, durable +write, catalog change, translation-cache key change, or persistent metadata. +Feasibility evidence must retain redacted key/owner/generation receipts, +bounded-memory high-water, cache state, backend/snapshot provenance, exact +public observations, and cancellation/rollback replay evidence. + +## Required study and stop rules + +Only the eleven open `suffix-route-component-v1` training declarations may be +used. Any separately authorized feasibility implementation must exercise disabled, +miss, hit, capacity-exhausted, invalidated, cancelled, and rolled-back states. +Every state must retain exact public rows/paths. A miss must execute only the +ordinary incumbent; failures, cancellation, timeout, malformed entries, and +capacity exhaustion publish no decision. + +Stop immediately on any cache reuse across an ownership boundary; a candidate +execution on a miss; stale-state divergence; cache-attributable WAL or durable +state; unbounded allocation; incomplete cleanup; protected-fixture access; or +an automatic-routing claim. + +The machine-readable frozen contract is +[`suffix_route_cache_feasibility_v1.json`](../../benchmark/testdata/scale/protocols/suffix_route_cache_feasibility_v1.json). diff --git a/docs/experiments/topology_fixed_suffix_first_use_v1.md b/docs/experiments/topology_fixed_suffix_first_use_v1.md new file mode 100644 index 00000000..d3443ffb --- /dev/null +++ b/docs/experiments/topology_fixed_suffix_first_use_v1.md @@ -0,0 +1,32 @@ +# Fixed-suffix first-use routing protocol v1 + +This protocol is the manifest-v5 successor to the cache-hit-only v4 selector. +It authorizes the fixed-suffix reverse candidate on the first matching query in +a repeatable-read or serializable transaction, but only after the current +topology synopsis passes the frozen sparse-topology estimate. + +The protocol is deliberately separate from v4: + +- candidate: `topology-fixed-suffix-first-use-v1`; +- execution boundary: `first_use_transaction_retry`; +- route-cache protocol: `topology-selected-first-use-routing-v1`; +- estimator: `topology-fixed-suffix-counts-v1` with + `maximum_edge_to_node_ratio_per_mille=1000`; +- fallback: `EXPANSION-STEPWISE-FORWARD` in the same stable transaction. + +Admission requires a v5 manifest with the exact frozen suffix, state, +output-row, and output-byte caps and a structural fixed-suffix bucket. The +driver rejects any changed protocol identity, cap, estimator, synopsis schema, +or threshold. Read-committed transactions, unavailable or stale synopsis data, +unverifiable parameter values, and dense topology remain incumbent. + +The first-use candidate is still correctness-safe because the reverse arm +retains the exact forward fallback. The synopsis influences cost selection only; +it does not alter graph semantics. Its dedicated rollback switch is +`disable_topology_fixed_suffix_first_use`. + +Promotion remains default-off until the v5 manifest has independently recorded +AA, confirmation, performance, resource, reference-closure, and operational +evidence. The v4 capture procedure remains the required evidence format; v5 +must repeat it with first-use transaction samples rather than reusing a v4 +authorization. diff --git a/docs/experiments/topology_fixed_suffix_v4_capture_v1.md b/docs/experiments/topology_fixed_suffix_v4_capture_v1.md new file mode 100644 index 00000000..63c836fb --- /dev/null +++ b/docs/experiments/topology_fixed_suffix_v4_capture_v1.md @@ -0,0 +1,72 @@ +# Topology fixed-suffix v4 capture procedure + +Status: executable qualification procedure; no promotion manifest is granted by this document. + +This procedure produces the six evidence roles required to authorize the +default-off `topology-fixed-suffix-v1` candidate. It must begin from a clean, +committed source tree. An untracked file, generated benchmark output, or a +locally edited manifest makes the capture diagnostic-only. + +## Preconditions + +- Use a disposable PostgreSQL target and the repository destructive-test + guard variables. +- Build one `-trimpath` GraphBench binary into the capture directory and use + that exact binary for every arm. +- Freeze the selected training declarations before running any holdout work. +- Use pool size one, one run UUID, Repeatable Read, and a fresh synopsis for + every fixture load. +- Keep the candidate disabled until `go run ./cmd/graphbench + -promotion-manifest ` verifies all six bound reports. + +The v4 route is intentionally observable only after the incumbent has run for +the same query and parameter values inside one active stable-snapshot +transaction. Every selected candidate record must therefore include an +incumbent-first sample and a same-transaction candidate-hit sample. + +## Required matrix + +Capture both endpoint-ID and complete-path fixed-suffix observations for: + +1. sparse reachable targets; +2. no-path and disconnected controls; +3. high reverse fan-in and dense-suffix controls; +4. output-row and output-byte overflow; +5. missing, stale, incompatible, and refreshed synopses; +6. mutation, savepoint rollback, cancellation, and pool-reuse boundaries. + +Every candidate record must have exact public observations, a complete +candidate/fallback receipt chain, no leaked candidate output on overflow, and +attributable plan/resource telemetry. Every selector failure remains an +incumbent record with its typed reason. + +## Capture order + +1. Capture an order-balanced PostgreSQL A/A artifact. +2. Capture incumbent and v4 candidate training rounds with the frozen corpus. +3. Produce the training discovery report and freeze it before opening holdout. +4. Capture the sealed holdout rounds without changing the binary, host, + corpus, schema, policy caps, estimator, or synopsis version. +5. Produce performance, resource, reference-closure, and operational reports + from the exact bound artifacts. +6. Assemble the manifest only after every report passes, bind each report to + that manifest, and run the independent manifest verifier. + +The relevant GraphBench reports are the standard `-aa-output`, performance +gate, `-resource-output`, `-reference-closure-output`, and operational gate +outputs. The manifest verifier requires exactly these roles: `aa`, +`confirmation`, `performance`, `resource`, `reference_closure`, and +`operational`. + +## Admission and disposition + +The candidate must improve p50 by at least 5 percent or 100 microseconds, keep +p95 at or below 1.05 times the incumbent, and keep selector overhead at or +below 1.10 times or 100 microseconds of the selected exact arm. Refresh cost, +WAL, storage, mutation amplification, and cache capacity are release inputs, +not optional diagnostics. + +If any required record, receipt, chronology proof, or gate is missing or +fails, write a terminal rejection record for that evidence generation. Do not +retune the v4 estimator, thresholds, caps, or first-use behavior. A new design +requires a new selector version and a separately frozen procedure. diff --git a/docs/experiments/topology_fixed_suffix_v4_status_v1.md b/docs/experiments/topology_fixed_suffix_v4_status_v1.md new file mode 100644 index 00000000..a3bfaa96 --- /dev/null +++ b/docs/experiments/topology_fixed_suffix_v4_status_v1.md @@ -0,0 +1,39 @@ +# Topology fixed-suffix v4 status + +Status: implementation complete; promotion unactivated. + +The PostgreSQL V2 driver now implements the manifest-v4 execution boundary +defined by [Topology-selected routing protocol v1](topology_selected_routing_protocol_v1.md). +This record deliberately does not grant a promotion manifest or activate the +candidate. + +## Implemented boundary + +- The driver validates the v4 candidate identity, exact immutable caps, + estimator, synopsis schema, route-cache protocol, fixed-suffix structural + fingerprint, SQL-template fingerprint, and the frozen 1000-per-mille + edge-to-node-density threshold. +- A current v2 synopsis and a read-only Repeatable Read or Serializable + transaction are mandatory. +- Decisions are transaction-owned and parameter-sensitive. A cache miss runs + the incumbent; a cache hit in the identical snapshot may run the one + reverse-only candidate. +- Candidate output is buffered within the v4 output limits. Incompleteness, + overflow, or an unrecognized status discards the candidate and executes the + exact forward fallback in the same transaction. +- The emergency v4 rollback switch and a zero policy immediately restore + incumbent-only SQL. + +## Non-activation decision + +No v4 promotion manifest is committed or installed. A successful implementation +or smoke execution is not performance qualification: the required frozen +training/holdout evidence has not demonstrated selector coverage, regret, +refresh and mutation cost, receipt closure, and the protocol's p50/p95/overhead +gates. The driver therefore remains default-off and any missing, stale, +incompatible, first-seen, mutable, cancelled, or resource-limited route stays +on the incumbent. + +The next promotion action is to capture the frozen v4 cohort with GraphBench, +verify all six evidence roles, and either install its digest-bound manifest or +write a terminal rejection record for that evidence generation. diff --git a/docs/experiments/topology_selected_routing_protocol_v1.md b/docs/experiments/topology_selected_routing_protocol_v1.md new file mode 100644 index 00000000..ea13b0aa --- /dev/null +++ b/docs/experiments/topology_selected_routing_protocol_v1.md @@ -0,0 +1,72 @@ +# Topology-selected routing protocol v1 + +Status: frozen implementation protocol. This supersedes the deferred +pre-schema decision in `traversal_topology_synopsis_adr_v1.md`. It does not +enable a production candidate by itself. + +## Version boundaries + +- Promotion manifest v2 authorizes one exact Cypher query and its rendered SQL. +- Promotion manifest v3 authorizes graph-independent shortest-path structural + buckets. +- Promotion manifest v4 is reserved for topology-dependent fixed-suffix + routing. It must bind the structural shape, estimator version, immutable + thresholds, candidate and fallback template identities, and synopsis schema + compatibility. + +No version reinterprets a previous version. Tool-only and terminal identities +remain ineligible for production activation. + +## Selection and snapshot contract + +Topology selection is permitted only in a caller-owned, read-only Repeatable +Read or Serializable PostgreSQL transaction. The synopsis is read through the +same transaction and snapshot as the graph query. A missing, building, failed, +incompatible, stale, ambiguous, or resource-limited synopsis selects the +incumbent with a query-text-free reason. + +The selector reads no graph values from a synopsis and cannot establish query +correctness. It estimates only candidate cost. Candidate admission caps and an +exact incumbent fallback remain authoritative. + +`topology-fixed-suffix-counts-v1` admits the reverse candidate only when the +synopsis reports `edge_count * 1000 <= node_count * +maximum_edge_to_node_ratio_per_mille`. Version v1 freezes that threshold at +`1000`; it is manifest-bound and therefore part of the route-cache policy +identity. A new estimator or threshold requires a new selector version. + +## Route-decision cache + +A route decision is transaction-owned application memory, never a translation +cache entry. It is keyed by a transaction-owner token, graph ID, structural +shape, canonical parameter fingerprint, policy identity, synopsis generation, +mutation epoch, and local invalidation generation. It is bounded to 64 entries, +64 KiB total, and 4 KiB per entry with no eviction. + +A miss executes the incumbent only. A hit may execute the qualified reverse +candidate only in the same active transaction and snapshot. Writes, savepoints, +rollback, cancellation, retry, pool release, and transaction completion discard +or disable all decisions. + +## Execution contract + +The fixed-suffix candidate is a single arm: it never embeds an inactive +forward arm. Candidate rows are fully buffered within fixed row and byte caps +before they become public. A cap status or candidate incompleteness discards +candidate output and executes the exact forward incumbent in the same snapshot. + +The production selector receives its own identity and emergency rollback +switch. The zero policy and the rollback switch produce an incumbent-specific +translation identity immediately. + +## Evidence and rollout + +GraphBench must independently recompute the structural and SQL-template +identities; report selector coverage, regret, lookup cost, refresh cost, WAL, +storage, mutation amplification, cache state, candidate/fallback receipts, and +all transaction-boundary states. Training is frozen before holdout execution. + +Activation requires exact observations, complete receipts, p50 improvement of +at least 5% or 100 microseconds, p95 no worse than 1.05 times the incumbent, +and selector overhead no worse than 1.10 times or 100 microseconds. A failed +gate produces a terminal rejection record and leaves production selection off. diff --git a/docs/experiments/traversal_priority_implementation_status_v1.md b/docs/experiments/traversal_priority_implementation_status_v1.md new file mode 100644 index 00000000..05a918c8 --- /dev/null +++ b/docs/experiments/traversal_priority_implementation_status_v1.md @@ -0,0 +1,78 @@ +# Traversal priority implementation status v1 + +Date: 2026-08-13 + +Status: canonical-I1 qualified; SP-I2 unqualified; suffix guard rejected; production unchanged + +This record separates repository implementation from empirical promotion for +[`cysql_traversal_priorities.md`](../cysql_traversal_priorities.md). The +candidate algorithms, exact fallbacks, diagnostic surfaces, qualification +corpora, and fail-closed gates are repository code. This change does not claim +new latency results and does not fabricate a clean M0 capture from a modified +working tree. Consequently, no new automatic suffix, SP, ASP, endpoint, +predicate, or `ExpandInto` selector is enabled. + +## Immutable identities + +| Concern | Implemented identity | +| --- | --- | +| Ordinary orientation policies | `orientation-probe-v1`, `orientation-probe-v2` (v2 rejected) | +| Ordinary incumbent | `EXPANSION-STEPWISE-FORWARD` | +| Fixed-suffix candidate | `EXPANSION-SUFFIX-SEEDED-REVERSE` | +| Static fixed-suffix guard | `suffix-reverse-guard-v1` (terminally rejected) | +| Existing endpoint candidate | `EXPANSION-ENDPOINT-SEEDED-REVERSE` | +| SP strict node alternation | `SP-B1-C-ALT-NODE-D`, `SP-B1-C-ALT-NODE-WE+MAT-M0` | +| SP smaller current level | `SP-B2-C-MIN-LEVEL-D`, `SP-B2-C-MIN-LEVEL-WE+MAT-M0` | +| ASP strict node alternation | `ASP-B1-DAG-ALT-NODE` | +| ASP smaller current level | `ASP-B2-DAG-MIN-LEVEL` | +| SP/ASP production controls | `SP-S3-U-D`, `SP-S3-U-E+MAT-M0`, `SP-S4-C-D`, `SP-S4-C-WE+MAT-M0`, `ASP-A1-DAG`, `SP-S0` | +| Inline production canaries | `SP-I1-C-WE+MAT-M0`, `SP-I2-C-D`, `ASP-I1-U-DAG+MAT-M0` | +| Inline tool-only executors | `SP-I1-C-D`, `SP-I1-U-E+MAT-M0` | +| Bounded endpoint analysis | `endpoint-resolution-v1` | +| Traversal predicate analysis | `traversal-predicate-v1` | +| Fixed one-hop study | `expand-into-study-v1` | + +## Milestone disposition + +| Milestone | Repository implementation | Promotion disposition | +| --- | --- | --- | +| M0 | Capture bundle v3 binds source state, patch and untracked payloads, dependency files, executable, the complete sorted corpus declaration and identity, evidence checksums, and sanitized environment metadata. Its independent verifier reconstructs and validates the bundled source and corpus fingerprints. Host-bound A/A schema v4 requires two explicitly executed, order-balanced arms plus artifact-bound physical chronology; frozen training/holdout declarations are enforced. | A fresh clean-source capture is still required. A dirty diagnostic bundle cannot qualify promotion. | +| M1 | Traversal telemetry v2 separates summary identity from untimed diagnostic replay and carries per-field provenance/completeness. PostgreSQL diagnostics fail closed for hidden function work and give guarded SP-I1, SP-I2, and ASP-I1 distinct typed counter families. Neo4j reads use `PROFILE`, preserve ordered children and actual metrics, and explicitly mark opaque SP/ASP internals. Plan-delta v2 uses union pairing and semantic stages. Resource gate v5 enforces attribution, caps, measured memory, spill/WAL policy, fallback, hydration, and inactive-arm work. | Missing, hidden, contradictory, or unattributable counters fail qualification; they are never treated as zero. | +| M2 | The common typed orientation decision records planned/emitted policies, candidates, caps, admission, and fallback separately. Guarded and shadow fixed-suffix statements use bounded root/suffix/directional-degree probes, cap+1 sentinels, strict 3/4 hysteresis, bounded reverse state, and exact forward fallback. The independent static suffix guard removes topology probes, caps suffix/state rows at 512, and marker-gates exact reverse and forward executors. | Production fixed-suffix translation remains the exact forward incumbent. Orientation v2 and `suffix-reverse-guard-v1` both failed their immutable training overhead gates and are terminally rejected. The already-shipped endpoint family retains its established 32/33 endpoint and 4096/4097 state guards. | +| M3 | Compact B1/B2 SP functions retain ID-only two-sided frontier/seen/predecessor state, exact 0/1/2-hop controls, typed schedulers, lower-bound termination, deterministic minimum witnesses, late hydration, invocation-local diagnostics, and exact S4 fallback on cap overflow. GraphBench exposes four full-comparator reference arms on a carryover-balanced three-arm schedule. `SP-I1-C-WE+MAT-M0` has guarded predecessor/witness execution. `SP-I2-C-D` adds reverse-physical hidden-fan-in distance discovery with state/frontier gates, exact S4 fallback, branch receipts, exact-bucket manifest authorization, diagnostic attribution, and an evidence-free rollback switch. Syntax-open singleton SP uses the existing depth-15 policy with explicit provenance. | B1/B2 and legacy unguarded I1 executors remain forceable/reference candidates only. `sp-static-v6` and `sp-static-v8-hidden-fanin` are default-off exact-query canary selectors. SP-I2's dirty training rehearsal won its five targets but missed the frozen cycle-control bounds; the clean-source barrier prevented a freeze and kept all holdouts unopened. | +| M4 | Promotion manifest v2 strictly decodes candidate-specific confirmation and performance evidence and accepts exactly six roles, one query digest, unique buckets, and the canonical training/holdout split. A/A, resource, and reference-closure wrappers embed exact native producer bytes; cross-role validation closes the confirmation/performance/resource candidate artifact, exact performance/resource round and receipt-chain sets, and each reference workload digest against exactly one PostgreSQL A/A workload. Operational gate v2 embeds its canonical 32-record input and digest so final verification recomputes the matrix, cancellation, snapshot, isolation, overflow, and disposition; duplicate input keys fail closed. The independently frozen candidate SQL digest is checked before execution. Orientation v1 remains readable but is rejected for schema insufficiency; v2 remains readable but is terminally rejected because its immutable training overhead gate failed. Default-off driver policies partition the cache by generation and expose evidence-free switches for orientation, endpoint reverse, SP witness, SP distance, and ASP DAG. | The clean `6d56a609` canonical-I1 confirmation passed all four training and three holdout cases with zero fallback and resource-gate v5 passing all 70 case-round records. Automatic production remains unchanged pending exact production-statement, reference-closure, operational, and final manifest evidence. | +| M5 | B1/B2 ASP functions retain all same-minimum-depth predecessors on each side, select one deterministic completed meeting cut, saturate pre-enumeration counts, stage unique ordered edge arrays, and enforce separate discovery, predecessor, enumeration, and output-byte sentinels before exact A1 fallback. Full-multiset references and stress/cap cases are included. `ASP-I1-U-DAG+MAT-M0` has a typed inline emitter, exact bounded one/two-hop preflights, four cap+1 guards, exact A1 same-statement fallback, event-chain runtime receipts, inactive-arm evidence, exact-query manifest buckets, a kill switch, and live driver-policy/isolation/cache/rollback coverage. | B1/B2 ASP remain forceable/reference candidates. `ASP-A1-DAG` remains the automatic production choice. I1 is a default-off, stable-snapshot, exact-query canary; broader activation still requires clean evidence. | +| M6 | Optimizer diagnostics conservatively classify bounded endpoint sources and traversal predicate locality without changing execution. A property name alone is never considered a uniqueness proof; parameterized and literal small sets use the 32/33 contract. Fixed one-hop translation has an optimizer-independent exact dual-bound fallback, recognizes carried and node-valued `UNWIND` endpoints, and preserves directionless self-loops in unbound, single-bound, and dual-bound forms. The corpus and three exact PostgreSQL study arms cover pair join, lower-degree scan, pair reuse, both logical directions, wildcard/multi-kind edges, missing pairs, duplicates, and self-loops. Confirmation now requires material improvement, p95 containment, and one stable winner across separate training and holdout partitions. | Endpoint/predicate broadening remains analysis-only until the SP/ASP candidates it would feed qualify. The `ExpandInto` report is a study and cannot activate a policy. | +| M7 | The versioned topology-synopsis ADR records schema, mutation, refresh, staleness, cache-key, graph-lifecycle, and rollout requirements. | Deferred. Runtime probes remain authoritative; no synopsis schema or cache dependency is introduced. | + +## Qualification invariants + +Release-eligible evidence must satisfy all of the following: + +- complete declared corpus coverage, with diagnostic selections unable to pass; +- checksummed host-matched A/A evidence and balanced rounds at 97.5% confidence; +- a target p50 improvement clearing 5% or 100 microseconds and contained p95; +- independent, nonempty training and frozen-holdout passes for every concrete + prioritized candidate family; +- exact stable observations and SP witness validity or complete ASP/ordinary + result multisets as appropriate; +- complete required search and hydration telemetry with measured, attributable + resource use; +- at-most-once probes, zero work in unselected arms, and an exact, single, + declared fallback before output; +- cancellation, rollback, session reuse, pool isolation, and schema-down + symmetry. + +Stress cases are correctness/resource diagnostics. Their timing cannot tune or +promote a selector, and a stress fallback is accepted only where the case and +candidate declare that exact fallback. + +## Evidence still required for promotion + +Promotion is a later evidence-producing change. It must start from a clean +source checkout and publish credential-free checksums for the baseline and +candidate binaries, corpus declaration, source revision, database versions, +host A/A report, matched plans, discovery, confirmation, frozen holdout, +resource, reference-closure, cancellation/concurrency, and bundle-verification +reports. A passing report then enables only the named runtime-recognizable +topology and observation buckets; all other shapes retain their incumbents. diff --git a/docs/experiments/traversal_topology_synopsis_adr_v1.md b/docs/experiments/traversal_topology_synopsis_adr_v1.md new file mode 100644 index 00000000..f1f1d59f --- /dev/null +++ b/docs/experiments/traversal_topology_synopsis_adr_v1.md @@ -0,0 +1,106 @@ +# Traversal topology synopsis ADR v1 + +Status: **superseded by [topology-selected routing protocol v1](topology_selected_routing_protocol_v1.md)**. The v1 ADR remains the historical pre-schema decision record; the minimal graph epoch and synopsis publication schema now exist, but production translation and execution still do not read a synopsis. + +Decision ID: `traversal-topology-synopsis-v1`. This record defines the design +and qualification boundary requested by M7 of +[`cysql_traversal_priorities.md`](../cysql_traversal_priorities.md). It does not +authorize a schema migration or selector change. Same-statement capped probes +and executor frontier state remain authoritative until a synopsis demonstrates +lower selector regret or lower probe overhead on the frozen holdout and also +passes the mutation, cache, and resource gates below. + +## Decision + +Do not add persistent topology tables yet. First capture the M1 diagnostic +counters and complete the M2--M6 candidate studies. Those artifacts provide the +runtime labels needed to test whether a synopsis predicts anything useful. A +synopsis implementation may proceed only as a separately versioned experiment; +it may influence a candidate score, but it may never prove correctness, bypass +an admission sentinel, or suppress exact fallback. + +If the experiment proceeds, prefer reading the current synopsis at execution +time. Embedding a synopsis value in translated SQL is forbidden until its epoch +is part of `cypherTranslationCacheKey` and an epoch change either invalidates or +misses every affected cached translation. Mutable rollout policy is likewise an +execution input or an explicit cache generation, never unkeyed translator +state. + +## Proposed storage contract + +The candidate schema is graph-scoped and generation-scoped. All rows for a new +generation become visible atomically by advancing one graph metadata row after +the generation is complete. + +| Relation | Key | Candidate values | +| --- | --- | --- | +| `traversal_synopsis_generation` | `(graph_id)` | `epoch`, schema/estimator version, source mutation epoch, build start/end, sampled/full mode, status | +| `traversal_synopsis_node_count` | `(graph_id, epoch, kind_id)` | exact or sampled count and error bound | +| `traversal_synopsis_edge_count` | `(graph_id, epoch, direction, kind_id, endpoint_kind_id)` | count, distinct starts/ends, error bound | +| `traversal_synopsis_degree` | `(graph_id, epoch, direction, kind_id, bucket)` | quantiles, heavy-hitter threshold, sample size | +| `traversal_synopsis_frontier` | `(graph_id, epoch, shape_bucket, depth_bucket)` | survival and reconvergence distributions, sample size | +| `traversal_synopsis_risk` | `(graph_id, epoch, shape_bucket)` | predecessor/output multiplicity buckets and saturation rate | + +Multi-kind node membership is represented by separate overlapping strata; the +reader must not sum them as disjoint populations. Every estimate carries sample +size, method, error bound, build timestamp, source mutation epoch, and estimator +version. Missing, stale, building, failed, or incompatible generations produce +`synopsis_unavailable` and leave the runtime-probe policy unchanged. + +## Refresh and mutation contract + +- Graph creation starts with no usable generation. Graph drop removes or makes + unreachable all generations for that graph. +- Bulk load or fixture replacement builds a fresh generation after the load and + publishes it atomically. Readers never mix epochs. +- Incremental node/edge mutations advance a graph mutation epoch. A published + synopsis whose source epoch differs is stale and advisory-only; the initial + experiment treats it as unavailable rather than estimating staleness. +- Refresh work runs outside query latency measurements, has bounded memory and + temporary storage, and records its own WAL, CPU, elapsed time, and table size. +- Failed or cancelled refresh leaves the previous generation intact but stale. + Cleanup is idempotent and cannot delete the currently published generation. +- The first implementation must include schema-up/schema-down symmetry, + concurrent reader/refresh tests, graph reload/drop tests, and an upgrade test. + +## Shadow comparison + +Shadow mode records a synopsis prediction beside the same-statement runtime +probe decision while executing the incumbent. It must not alter emitted arms. +Each record binds the workload, fixture and holdout identity, source revision, +graph mutation epoch, synopsis epoch/version, runtime policy version, probe +caps, predicted arm/score, observed probe values, actual selected exact arm, +fallback, and measured probe overhead. + +Evaluate normal and envelope tiers on the frozen holdout. Stress remains a +fallback/staleness diagnostic. Report at least: + +- prediction coverage and stale/unavailable frequency; +- selector regret against every exact arm; +- disagreement with capped runtime probes and executor frontier decisions; +- probe latency and buffer work saved after charging synopsis lookup cost; +- refresh latency, WAL, persistent bytes, and mutation write amplification; +- cache hit/miss and invalidation behavior across epoch changes; +- correctness, fallback, cancellation, pool-reuse, and concurrent-writer results. + +## Admission gate + +The synopsis experiment is rejected or remains deferred unless all of these are +shown with checksummed discovery and confirmation artifacts under the standard +97.5% protocol: + +1. The synopsis materially lowers selector regret or probe overhead on both the + declared corpus and frozen holdout after lookup cost. +2. No normal/envelope bucket regresses beyond the host A/A timing floor, and + resource limits pass without unexpected WAL or spill in read execution. +3. Stale, absent, incompatible, or partially refreshed data always selects the + unchanged runtime-probe/incumbent chain with a precise reason. +4. Mutation and refresh overhead passes an independently declared budget; it is + not hidden inside query measurements. +5. Translation-cache tests prove that no SQL can retain an unkeyed embedded + epoch or rollout policy. + +Until those gates pass, `traversal-topology-synopsis-v1` has no database schema, +no cache-key effect, no production feature gate, and no automatic selector +bucket. This is the reversible outcome required by the priority plan: runtime +evidence remains the authority, and lack of a synopsis is normal operation. diff --git a/docs/postgresql_translation.md b/docs/postgresql_translation.md index 34e613d0..2c422007 100644 --- a/docs/postgresql_translation.md +++ b/docs/postgresql_translation.md @@ -28,7 +28,110 @@ Current PostgreSQL optimization coverage includes: `size(relationships(p))`, `startNode`, `endNode`, and `type`. - Recursive traversal optimizations for endpoint kind/property predicates, relationship type predicates, bound-node filters, traversal direction selection, and limit pushdown where ordering and distinct semantics permit it. +- Static shortest-path executor selection for one read-only, uncorrelated, directed traversal with one ID equality per + endpoint and no observed relationship/path predicate. Distance observations use scalar `SP-S3-U-D` state, with deep + physical-inbound searches sent to `SP-S4-C-D`. Bounded directed single-kind one-path witnesses use + `SP-S3-U-E+MAT-M0`; deep inbound and multi-kind or untyped witnesses use + `SP-S4-C-WE+MAT-M0`. Both S4 executors canonicalize + expansion, keep recursive state ID-only, enforce a bounded state + ceiling, and fall back to an exact relationship-trail query in the same statement and snapshot before returning a + row. Singleton ties return one valid minimal trail; physical edge-ID order is not public. See + `docs/shortest_path_tie_policy.md`. +- Default-off compact bidirectional SP candidates preserve that singleton + endpoint and observation envelope. `SP-B1-C-ALT-NODE-D` and + `SP-B1-C-ALT-NODE-WE+MAT-M0` alternate one accepted node per side; + `SP-B2-C-MIN-LEVEL-D` and `SP-B2-C-MIN-LEVEL-WE+MAT-M0` expand the smaller + complete current level. Both use ID-only invocation-local state, a + lower-bound stop condition, late witness hydration, independent + seen/frontier/predecessor caps, and exact S4 fallback before output. They are + reference and explicit-tool arms; the production driver rejects them. + Legacy `SP-I1-C-D` remains tool-only. Its guarded successor `SP-I2-C-D` + performs reverse-physical ID-only distance discovery behind independent + state/frontier gates, exposes same-statement runtime receipts, and invokes + exact `SP-S4-C-D` on overflow. Production authorization is restricted to + exact inbound typed single-kind distance buckets under selector + `sp-static-v8-hidden-fanin` and the preregistered production-form + `state_limit=100000`/`frontier_limit=100000` cap contract. These are + immutable protocol inputs, not qualified values: the dirty-tree rehearsal + stopped before a discovery report or freeze, the cycle-control point + estimates missed the frozen bounds, and no protected holdout was opened. + Tool-forced diagnostic translations may still use smaller caps to exercise + overflow and fallback behavior. Eligible + canaries require SHA-256-allowlisted queries under repeatable-read or + serializable isolation and a schema-v2 promotion manifest whose reports + repeat its complete authorization identity. The ordinary production path + remains unchanged. +- Omitted shortest-path upper bounds retain Cypher's repository-defined + effective maximum of 15. The lowering decision records + `maximum_depth_source=policy_default` and uses selector + `sp-static-v7-contained`; explicit bounds retain `explicit` provenance. +- Static `allShortestPaths` selection through `asp-static-v1` for a single directed, read-only endpoint pair with + minimum depth one. `ASP-A1-DAG` has exact one- and two-hop arms, discovers minimum node-depth layers, retains every + relationship-distinct predecessor at those layers, and enumerates the predecessor DAG. Its returned ordered edge IDs + are hydrated through the inline M0 relation rather than the generic per-row path helper. Open maximum ranges use the + documented depth cap of 15. Unsupported or ambiguous forms retain exact `SP-S0` with a machine-readable reason. +- Default-off `ASP-B1-DAG-ALT-NODE` and `ASP-B2-DAG-MIN-LEVEL` reuse compact + two-sided search while retaining every same-minimum-depth predecessor. They + enumerate at one canonical completed meeting cut and apply separate + discovery, predecessor, saturating path-count, staged-output, and byte gates. + Overflow clears candidate state and invokes exact `ASP-A1-DAG` before output. + Production remains on A1 until independent training, frozen-holdout, + resource, and reference-closure reports pass; the allowlisted canary seam + uses the same explicit stable-snapshot requirement as SP. - Expansion suffix pushdown and `ExpandInto` detection for fixed suffixes and shared-endpoint fanout patterns. +- Typed compound expansion-search planning for directed bounded expansions followed by fixed suffixes. The decision + records its fixed-suffix expansion family, planned candidates, exact eligibility facts, observation mode, suffix + bounds, + selected/fallback strategy, selector version/mode, and stable fallback code separately from the legacy + boolean suffix prefilter. Correlated suffix bindings and predicates spanning the expansion/suffix boundary have + distinct conservative fallback codes. Candidate factored-forward and backward-viability SQL remains + reference-only. `EXPANSION-SUFFIX-SEEDED-REVERSE` has a repository-native, + qualification-only emitter. Explicit tool options select it and fail closed + unless translation records the matching target as applied. Production deliberately + retains the `EXPANSION-STEPWISE-FORWARD` translator and reports + `tournament_unqualified` for otherwise eligible three-hop forms because no + hard suffix-density or reverse-state bound is available before translation. + Full-path reverse rows additionally carry ordered node IDs and hydrate node + and edge composites with graph-partition-scoped aggregate lookups in the + translated statement. Endpoint-only rows omit that state. Guarded candidate + and incumbent branches still expose one identical path-composite column; the + incumbent retains `ordered_edge_ids_to_path` as its exact fallback boundary. +- The default-off `orientation-probe-v1` guarded and shadow statements measure + bounded duplicate-preserving roots, suffix rows/distinct boundaries, and + typed first-hop work from both sides. Every relation has a cap+1 sentinel; + reverse must beat forward by the versioned strict 3/4 hysteresis rule. + Guarded execution also caps reverse state and marker-gates candidate and + incumbent output chains independently. Probe and state overflow select the + exact forward fallback and produce a truthful runtime receipt. Shadow + execution always runs the incumbent, records only `would_select`, and emits + its marker-first receipt even for an empty result. Plan telemetry attributes + work from exact CTE materialization subplans so repeated consumer scans cannot + inflate probe or branch loops. A versioned query-allowlisted + driver canary can emit the guarded form only when it also binds a verified + promotion-manifest SHA-256, while the zero policy and every non-allowlisted + query remain forward. +- The independent `suffix-reverse-guard-v1` statement remains tool-only. It + enrolls complete-path fixed-suffix queries, applies separate 512-row suffix + and reverse-state caps, and marker-gates exact suffix-seeded reverse against + exact stepwise forward in one Repeatable Read statement. Its + chronology-valid training capture failed the immutable guard-overhead gate, + so the generation is terminally stopped: it has no protected-holdout, + manifest, driver-policy, automatic-selector, or rollback path. The exact + reverse executor and ordered-ID hydration remain reusable by a newly + preregistered architecture. +- `suffix-reverse-retry-v1` is that new tool-only development generation. Its + translated statement contains only the bounded reverse arm and a transaction- + local completion status. GraphBench buffers candidate output behind 4,096-row + and 16 MiB caps, uses a savepoint, and runs the independently translated exact + forward incumbent only after a declared overflow in the same Repeatable Read + transaction. Missing status and execution errors fail closed. No production + policy recognizes this identity. +- Guarded endpoint-seeded expansion selection covers a separate + `fixed_prefix_terminal_expansion` family: exactly one directed fixed prefix followed by one terminal, directed, + single-kind variable expansion with minimum depth one and a local selective terminal predicate. Production emits + `EXPANSION-ENDPOINT-SEEDED-REVERSE` with at most 32 terminal seeds and 4096 reverse states. Sentinel rows select an + exact stepwise-forward fallback inside the same statement and snapshot before candidate rows are exposed. Both arms + preserve ordered relationship IDs and enforce relationship uniqueness across the fixed prefix and expansion. - Strict string property equality lowering through `jsonb_typeof(properties -> key) = 'string'` plus `properties ->> key = value`, preserving JSON scalar semantics while allowing existing text expression indexes on selective fields such as `objectid` and `name`. @@ -37,6 +140,22 @@ Current PostgreSQL optimization coverage includes: correlations are sufficient. - Membership-only `collect(entity)` ID-array lowering with `id = any(...)` membership predicates. - Shortest-path strategy and terminal-filter planning for selective endpoint predicates and kind-only terminal filters. +- Analysis-only endpoint resolution metadata classifies ID equality, bounded + nonunique property equality, literal or parameterized small sets, and + correlated pairs with explicit 1/2/32/33 contracts. Property syntax is not a + uniqueness proof. Analysis-only traversal predicate metadata distinguishes + step-local and universal node/relationship forms from whole-path and + unsupported forms. Neither diagnostic broadens execution until the compact + candidates and that semantic class independently qualify. +- The fixed one-hop, bound-pair `ExpandInto` study exposes exact direct-pair, + lower-degree adjacency, and statement-local pair-reuse reference arms. It + covers outbound, inbound, directionless, wildcard/multi-kind, duplicate, + missing, and self-loop behavior but does not select a production policy. + Fixed-hop correctness does not depend on the study marker: dual-bound steps + always retain an exact pair-join fallback, including endpoints carried across + `WITH` or introduced by node-valued `UNWIND`. Directionless fixed hops use + paired endpoint orientations so self-loops are emitted once for unbound, + single-bound, and dual-bound forms. - Exact anonymous directed fixed-range expansion lowering for non-shortest-path `*1..1` and `*2..2` patterns. These shapes use fixed traversal steps instead of recursive CTEs, preserve path projection semantics, and enforce relationship uniqueness across emitted fixed steps. The explicit SQL-size cap is depth 2; broader exact ranges @@ -45,6 +164,187 @@ Current PostgreSQL optimization coverage includes: path edge IDs, avoiding full `edgecomposite[]` materialization when the final projection does not require it. - Dependency-safe clause reordering inside non-optional read regions, using existing selectivity heuristics while preserving stable tie order and pinning clauses with unresolved external dependencies. +- Field-sensitive continuation lowering carries node IDs as scalar columns between eligible fixed or recursive + traversal steps. Property, full-entity, path, cross-pattern, and mutation consumers retain composite bindings; + ID-only expansion endpoints still join the graph-scoped node partition so orphan filtering and multiplicity remain + unchanged. + +## Repeated-query compilation + +Each PostgreSQL driver keeps bounded least-recently-used caches of 256 successfully parsed Cypher ASTs and 256 safe SQL +translations. Parse-cache keys are +the trimmed query text; invalid input is not retained, and queries larger than 64 KiB bypass the cache. Concurrent misses +for the same text are coalesced. Cached ASTs remain immutable: the optimizer copies an AST before applying rules, so +parallel executions cannot mutate shared parser output. + +The cache deliberately retains complete trimmed query text, including literals, until LRU eviction or driver close. +That lifetime is bounded to 256 entries per driver; closing the driver clears all retained keys and AST references and +prevents in-flight misses from repopulating the cache. Queries whose source text exceeds 64 KiB bypass retention. Cache +diagnostics expose aggregate hit, miss, bypass, eviction, coalesced-miss, entry, and pending counts only—never query +text, literals, parameters, or credentials. + +The translation cache is keyed by trimmed query text, graph ID, parameter names, the PostgreSQL data type negotiated +for each parameter, and the exact effective traversal-policy identity. Values are rebound on every hit. This deliberately separates empty untyped lists from typed lists +and separates different graph partitions. A translation containing generated/static fragment parameters is not cached, +because those values cannot be reconstructed safely from caller parameters. Concurrent cacheable misses are coalesced; +waiters rebuild uncacheable translations rather than inheriting the first caller's values. Driver close clears both +caches. `ParseCacheStats` and `TranslationCacheStats` expose aggregate, query-text-free counters. + +### Connection-local translation caching + +The PostgreSQL driver uses connection-local caches for every pool constructed by `pg.NewPool` or +`pg.NewPoolWithRuntimeConfig`. The constructor copies the supplied `pgxpool.Config`, preserves and composes its +lifecycle hooks, and associates the resulting bare `*pgxpool.Pool` with its connection-local runtime. `pg.NewDriver` +accepts that established pool and assumes the required lifecycle handlers are already installed; it never replaces +caller hooks or constructs a second pool. `pg.DefaultRuntimeConfig` selects 64 entries per physical connection; +`RuntimeConfig{TranslationCacheEntries: 0}` executes exact uncached translation and a negative capacity is rejected. + +Each successfully initialized physical connection owns one entry-bounded SIEVE cache. State survives a healthy +`pgxpool` lease release/reacquisition and is removed by `BeforeClose`; it is never keyed by a lease wrapper, backend +PID, transaction, or context. The v2 key adds a cache-format version and monotonic schema generation to the v1 +translation inputs. Successful `AssertSchema` and `RefreshKinds` calls advance that generation, making old entries +immediately unreachable. External schema changes are not detected automatically: reset or recreate the pool when they +change registered types or generated SQL. + +`pg.Driver.TranslationCacheStats` exposes capacities, occupancy, counters, retired/live connection counts, and opaque +diagnostic IDs. It never exposes query text, SQL, parameter names or values, pointer values, connection strings, or +credentials. The driver stores only immutable translation artifacts and fresh bindings; it does not cache result rows, graph +values, transaction/snapshot state, routing decisions, or errors. + +`pg.TraversalPolicy` is default-off and admits one candidate family per +nonzero generation. It requires a nonempty allowlist built with +`pg.TraversalPolicyQuerySHA256`. Operations must first verify the complete +evidence closure with GraphBench and then install those exact manifest bytes +and their digest. The driver revalidates the manifest JSON structure, digest, +candidate, selector, execution boundary, immutable caps, training/holdout +buckets, exact query cohort, exact evidence-role set, and evidence-reference +digests before accepting +the policy. It does not open evidence paths or reproduce the role-specific +reports; acceptance by the driver is therefore not a substitute for the +GraphBench final verifier. Generation and policy contents partition the +translation cache. Setting the zero policy makes older candidate entries +immediately unreachable. B1/B2 candidates are not production-canary eligible. +`DisableEndpointSeededReverse` is an emergency rollback control and +intentionally requires no promotion artifact. Policy forcing never broadens a +lowering's structural correctness envelope. + +If a manifest-backed candidate carries an emergency switch, it may carry +exactly one and it must be the switch dedicated to that candidate: orientation +with `DisableExpansionOrientation`, ASP-I1 with `DisableInlineASPDAG`, canonical +SP-I1 witness with `DisableInlineSPWitness`, or SP-I2 distance with +`DisableInlineSPDistance`. Any unrelated or second switch is rejected. +`DisableEndpointSeededReverse` is standalone-only. Every standalone rollback +policy must disable all manifest candidates and leave the manifest digest, +manifest JSON, and query allowlist empty (`promotion_manifest_sha256`, +`promotion_manifest_json`, and `query_sha256_allowlist`). A matching rollback +retains the installed manifest bytes and candidate anchor unchanged, but derives +an incumbent-only effective policy, clears the effective SQL-anchor comparison, +and uses its new generation as a distinct cache identity. + +Final activation treats the manifest collections as exact sets: it requires +only the six defined evidence roles, one globally unique query digest, unique +bucket names and query entries, and exactly one canonical training/holdout +split declaration per bucket. Duplicate JSON keys, duplicate policy-allowlist +digests, absolute or escaping evidence paths, and extra roles fail closed. The +single operational SQL digest is checked against rendered candidate SQL before +execution; an unanchored GraphBench manifest is permitted only for the +non-promotional preflight that discovers that digest. + +GraphBench final verification also requires each promotion reference case to +match exactly one native PostgreSQL A/A workload by dataset, name, and workload +digest. For every promotion case, the resource report must cover the exact +performance round count and its flattened candidate receipt-chain set must equal +performance's complete set. Reference receipts remain an independently captured +raw-pgx/comparator stream rather than sharing invocation IDs with that set. +Confirmation and performance do not embed raw samples, so their typed decisions +can be recomputed but their bootstrap draws cannot yet be independently replayed. +The operational validator consumes and recomputes an assembled 32-record native +input; the repository does not yet provide a standalone producer for that input. + +The same policy boundary now admits `ASP-I1-U-DAG+MAT-M0` as a default-off, +exact-query canary under Repeatable Read or Serializable isolation. Its +manifest must authorize the query SHA and exact direction/observation/depth/ +relationship-kind bucket, declare positive immutable state, predecessor, +enumeration, and output-byte caps, name `ASP-A1-DAG` as fallback, and use the +`guarded_dual_arm` boundary. Exact one- and two-hop targets bypass recursive +discovery. The inline statement materializes cap+1 preflight, distance, +predecessor, and enumeration relations before opening either output arm. A +version-2 runtime receipt retains the complete ordered event chain and +identifies `inline_predecessor_dag`, `inline_no_path`, or `exact_a1_fallback`; +the unselected arm emits no rows. Read Committed and +queries outside the exact allowlist retain A1. `DisableInlineASPDAG` is the +evidence-free emergency rollback control. + +The default-off fixed-suffix orientation runtime seam recognizes v1 and v2 +selector identities. Either shape must name `EXPANSION-STEPWISE-FORWARD` as +fallback, use `guarded_dual_arm`, and bind the immutable +`root_row_limit=512`, `reverse_seed_row_limit=512`, +`directional_degree_row_limit=16384`, and `state_limit=4096` caps. This is only +structural runtime admission: the final schema-v2 verifier rejects the legacy +`orientation-probe-v1` evidence schema because it cannot bind source, corpus, +and cohort identity. It separately terminally rejects the frozen +`orientation-probe-v2` generation because its immutable training overhead gate +failed. Neither generation is release-authorized; v2 must not be recaptured or +advanced to its unopened holdouts. The guarded statement exposes the production +boundary in traversal telemetry; shadow and forced single-arm statements report +`inline_statement` and cannot stand in for production-boundary evidence. + +Runtime receipt workspaces must exist on the exact PostgreSQL session before +an explicit read-only transaction begins. GraphBench satisfies this by pinning +and preparing one session. Driver callers that intentionally arm receipts from +inside a graph transaction can pass +`pg.OptionInitializeTraversalRuntimeAttestation()`; the driver then prepares +the acquired session immediately before `BEGIN READ ONLY`. + +The driver automatically prepares the production S4 and A1 session-local +workspaces before every explicit Repeatable Read or Serializable read-only +graph transaction. The underlying PostgreSQL transaction uses `READ WRITE` +access because workspace reset mutates session-local temporary tables; graph +data remains non-mutating. This keeps incumbent execution and a guarded +candidate's exact fallback valid on a fresh pooled connection. + +`SP-I1-C-WE+MAT-M0` uses the same guarded production boundary for singleton +one-path observations, with `SP-S4-C-WE+MAT-M0` as its declared fallback. The +manifest must authorize an exact `one_path` bucket and the same four positive +caps. It is admitted only at Repeatable Read or Serializable isolation; +`DisableInlineSPWitness` immediately restores the statically selected S3/S4 +incumbent and changes the cache identity without requiring evidence. + +The shortest-path functions use session-local `ON COMMIT PRESERVE ROWS` +workspace-v2 tables with invocation versions. Calls reset seen, candidate, and +predecessor state once, then derive each frontier from depth-tagged seen rows; +they do not create, drop, swap, or truncate frontier tables at every level. The functions set a +local `recursive_worktable_factor`, declare explicit `COST`/`ROWS` estimates, and carry graph/node/edge IDs until one +outer hydration boundary. Temporary-workspace buffers are expected for S4/ASP; executor temp-file spill and WAL remain +resource-gate failures. + +Raw PostgreSQL graph-composite values are driver implementation details. Use the result value mapper or +`graph.ScanNextResult` for nodes, relationships, paths, and their arrays instead of depending on pgx's historical +`map[string]any` composite representation. + +`Result.Keys()` returns metadata cached once for the result set; callers must treat that slice and its strings as +immutable for the result lifetime. `Result.Values()` remains row-scoped raw driver data. Public graph values produced +through the mapper are owned independently of later row advancement and pooled connection reuse. + +## Structural traversal policy + +Policy manifest v2 remains an exact-query canary: one query digest is bound to +the rendered SQL digest. Manifest v3 is the opt-in production-wide form. Each +bucket binds `traversal-shape-v1`, its query-text-free structural SHA-256, and +a candidate SQL-template SHA-256. The PostgreSQL driver independently +recomputes both digests when a policy is installed. A malformed, ambiguous, or +unmatched structural bucket leaves the query on the incumbent. The V2 driver +reports exact-query, structural-shadow, and structurally-authorized counts +without retaining query text or caller values. + +`RefreshTraversalTopologySynopsis` atomically publishes graph node/edge counts +against the current graph mutation epoch. It is an explicit management action, +not a query-latency operation; a missing or stale synopsis remains +incumbent-only. Manifest v4 can opt in to snapshot-owned fixed-suffix routing: +the first route-cache observation remains incumbent-only and only a later +same-snapshot hit can run the bounded candidate with exact forward retry. +Absent a qualified installed v4 manifest, the production translator remains on +the incumbent. ## Indexing Notes diff --git a/docs/recursive_descent_cost_controls.md b/docs/recursive_descent_cost_controls.md new file mode 100644 index 00000000..ee835342 --- /dev/null +++ b/docs/recursive_descent_cost_controls.md @@ -0,0 +1,102 @@ +# Recursive-descent cost controls + +Date: 2026-08-09 + +This implementation addresses the six recursive-descent findings from the PostgreSQL/Neo4j delta review. It changes +the PostgreSQL execution architecture; it does not claim that the cross-backend latency gap is closed until the same +corpus is recaptured against both supplied backends. + +The next-phase orientation, SP/ASP, topology-evidence, and qualification work is +sequenced in the [CySQL traversal performance priorities](cysql_traversal_priorities.md), +with current candidate and promotion status recorded in +[the implementation status](experiments/traversal_priority_implementation_status_v1.md). + +| Finding | Implemented control | +|---|---| +| 1. `allShortestPaths` retained too much trail state | `ASP-A1-DAG` performs minimum-layer discovery, stores all relationship-distinct predecessors only for minimum layers, then enumerates the predecessor DAG. | +| 2. Small depths paid recursive setup cost | Both production functions have exact one-hop and two-hop SQL arms before workspace allocation. | +| 3. Breadth-first levels churned temporary catalog objects | Session-local workspace v2 is created once per connection and reset once per invocation. A1 and S4 derive each frontier from depth-tagged seen state and share one candidate relation instead of swapping or repeatedly truncating frontier tables. | +| 4. Singleton shortest paths needed a bounded compact search | `SP-S4-C-D` and `SP-S4-C-WE+MAT-M0` use canonical ID-only BFS state, a 100,000-state default ceiling, and exact same-statement fallback. | +| 5. Recursive rows hydrated entities too early | New executors carry node/relationship IDs and perform one ordered path hydration after search. | +| 6. Repeated compilation and unstable recursive estimates added overhead | Functions declare `COST`/`ROWS` and set `recursive_worktable_factor`; the driver has a bounded, coalescing, parameter-shape-aware translation cache. | + +Terminal-selective ordinary expansions also have a guarded reverse lowering. The optimizer only selects it for one +fixed directed prefix hop followed by a terminal directed expansion (`*1..64`) with one relationship kind and a local +terminal ID/property search. The statement probes 33 endpoints and 4097 reverse states: up to 32/4096 uses the reverse +candidate, while either sentinel activates the exact forward incumbent in the same snapshot. Candidate output is +gated until both probes finish, so overflow and cancellation cannot leak partial results. + +## Selection boundaries + +`asp-static-v1` selects `ASP-A1-DAG` only for one read-only, non-optional, directed `allShortestPaths` traversal with one +static ID equality per endpoint, minimum depth one, no path/relationship predicate, and no observed relationship value. +An open maximum uses depth 15. A1's ordered edge IDs are hydrated inline at the outer statement boundary, avoiding a +per-result generic path materializer. Minimum-depth-zero, self-endpoint, directionless, correlated, mutation, and predicate +shapes retain the incumbent exact executor. + +`sp-static-v5-contained` retains `SP-S3-U-D` for qualified distance work, with +`SP-S4-C-D` for deep physical-inbound distance searches. Already-qualified, +bounded, directed, single-kind one-path witnesses use `SP-S3-U-E+MAT-M0`; +deep inbound and multi-kind or untyped witnesses retain +`SP-S4-C-WE+MAT-M0`. This containment avoids paying the S4 workspace boundary +where the relationship-trail executor is the better incumbent. S4 checks a +cap+1 state ceiling before emitting any row and records its exact +`SP-S3-U-E+MAT-M0` fallback in the same statement and snapshot. + +`SP-I1-C-WE+MAT-M0` is a separate default-off canonical-predecessor canary. +Selector `sp-static-v6` restricts it to the qualified inbound, typed, +single-kind singleton one-path envelope with `min=1` and `max=64`; different +directions, kind shapes, or depth bounds fail closed. Its guarded inline statement uses +four cap+1 gates, hydrates only after admission, and falls back through S4. A +state overflow can therefore produce the auditable event chain +`SP-I1-C-WE+MAT-M0 -> SP-S4-C-WE+MAT-M0 -> SP-S3-U-E+MAT-M0` without exposing +rows from an abandoned arm. Stable isolation, an exact manifest bucket, and +positive immutable caps are mandatory; `DisableInlineSPWitness` is the +evidence-free rollback switch. + +`SP-I2-C-D` is the default-off guarded distance canary for hidden-fan-in +shapes. It starts from the public terminal, follows the opposite physical +adjacency direction using `(node_id, depth)` state, and admits results only +when both total-state and per-depth frontier sentinels are complete. Overflow +executes exact `SP-S4-C-D` in the same statement. Selector +`sp-static-v8-hidden-fanin` requires an exact inbound, typed, single-kind, +distance-only bucket with `min=1`, `max<=64`, the qualified immutable +`state_limit=100000` and `frontier_limit=100000` contract, stable isolation, +and query-SHA allowlisting. Final verification, provisional evidence capture, +and driver admission all reject any other production values. +`DisableInlineSPDistance` +removes the candidate without requiring new evidence. + +When the query omits a maximum depth, the optimizer now carries the existing +effective depth-15 policy into contained S3/S4 selection. Diagnostics identify +the source as `policy_default` under `sp-static-v7-contained`, keeping it +distinct from an explicit `..15` query even though emitted execution is +equivalent. + +`ASP-I1-U-DAG+MAT-M0` is also available through the production policy as a +default-off exact-query canary. It is limited to a singleton directed endpoint +pair, `allShortestPaths`, minimum depth one, and an explicit maximum no greater +than 64. Exact one- and two-hop targets are resolved before recursive +discovery. The typed recursive statement bounds distance discovery, +same-minimum-depth predecessor retention, all intermediate enumeration states, +and output bytes with immutable cap+1 sentinels. It exposes candidate and +fallback markers only after every guard is known; any overflow selects exact +`ASP-A1-DAG` before public output. Its canary requires a stable transaction +snapshot and a manifest whose topology bucket matches the optimized target. +Runtime receipts use schema v2 and retain the complete ordered branch-event +chain rather than overwriting nested fallback evidence. The automatic +`asp-static-v1` choice remains A1 until clean confirmation, +holdout, resource, and reference-closure evidence authorizes broader rollout. + +`EXPANSION-SUFFIX-SEEDED-REVERSE` remains tool-only. Existing evidence showed a +fixed-suffix expansion topology crossover that query shape alone does not safely +bound, so this work does not activate the strategy in production. The rejected +bounded-fallback and continuation experiments are retained only as historical +decision records under `docs/experiments`. + +## Qualification contract + +GraphBench recognizes `ASP-A1-DAG`, `ASP-I1-U-DAG+MAT-M0`, `SP-S4-C-D`, and `SP-S4-C-WE+MAT-M0` as applied architectures. Their resource gate +allows the declared local workspace but rejects executor temporary-file reads/writes and WAL for non-mutating queries. +Use the generated depth/fanout corpus, exact path observations, planner modes, concurrency, cancellation/session reuse, +and matched PostgreSQL/Neo4j delta report before treating the implementation as performance-qualified. diff --git a/docs/regression_source_parity.md b/docs/regression_source_parity.md new file mode 100644 index 00000000..13eedbad --- /dev/null +++ b/docs/regression_source_parity.md @@ -0,0 +1,134 @@ +# BloodHound Regression Source Parity + +This workflow keeps the stable query-form manifest synchronized with reviewed +BloodHound Enterprise (BHE) and BloodHound Community Edition (BHCE) source +snapshots. It records query shapes only; DAWGS must not import application +business logic or reproduce complete BloodHound traversal behavior. + +## Dormant tier + +`FUTURE-01` is the outbound tenant reconciliation form: + +```cypher +MATCH (s:AZEntity)-[r:K]->() +WHERE s.tenantid IN $tenant_ids +DELETE r +``` + +At BHE commit `c9f61530f45b`, its callers in +`lib/go/daemons/datapipe/ingest.go` are inside the block labeled "Disabled for +now". The compiled `ReconcileOutboundKindsForTenants` helper does not by itself +make the form production-active. + +Keep `FUTURE-01` in the dormant section of +`regression_coverage_manifest.md`. Do not add it to +`integration/testdata/cases`, `integration/testdata/templates`, or +`benchmark/testdata/scale/cases` while the caller remains disabled. Unit gates +in `cmd/plancorpus` and `cmd/graphbench` reject every `FUTURE-*` ID from those +active corpora. + +When a reviewed source snapshot enables the caller: + +1. Record the enabling entry point and source commit before changing the tier. +2. Move the manifest row from dormant to active and update the corpus gates in + the same change. +3. Add the exact outbound builder composition and the `PG`, `IT`, `PC`, and + `SC` layers required by `regression_coverage_manifest.md`. +4. Cover empty, single-item, 1,000-item, boundary, and stress tenant lists; + include direction, kind, tenant, endpoint, and missing/null decoys. +5. Use exact mutation post-state and rollback/reset isolation. Reuse the + `REC-04` matrix, but do not reuse its inbound query as proof of outbound + orientation. +6. Capture the PostgreSQL plan/runtime baseline with the same source metadata. + +## Audit procedure + +Set source roots to reviewed, immutable checkouts. These sources are audit +inputs and are not copied into DAWGS: + +```bash +export BHE_ROOT=/path/to/bhe +export BHCE_ROOT=/path/to/bhce +git -C "$BHE_ROOT" rev-parse HEAD +git -C "$BHCE_ROOT" rev-parse HEAD +git rev-parse HEAD +``` + +Start with a broad call-site inventory. This intentionally includes helpers and +commented code; activity is classified during the trace step: + +```bash +rg -n --glob '*.go' \ + '\b(Filterf?|Query|First|Count|Fetch[A-Za-z0-9_]*|Create[A-Za-z0-9_]*|Delete[A-Za-z0-9_]*|Update[A-Za-z0-9_]*|BatchOperation)\b' \ + "$BHE_ROOT" "$BHCE_ROOT" +``` + +For each candidate: + +1. Trace the helper to an active reconciliation, post-processing, or changelog + entry point. Label helper-only, test-only, and commented-out forms. +2. Normalize active forms by anchor, pattern, direction, relationship kinds, + predicates, projection, cardinality, mutation target, and execution path. +3. Map the tuple to an existing stable ID or add a new manifest row and source + link. A new operator, grouping, direction, anchor, projection, or mutation + target requires a distinct ID. +4. Treat stepwise traversal evidence as standalone `HOP-*` shapes only. Never + add a test that sequences the application traversal. +5. Recheck projection independently from predicates, and recheck relationship + kind-list and ID-list cardinalities after schema-set changes. +6. Apply the coverage contract from `regression_coverage_manifest.md`, then run + both backend suites and refresh PostgreSQL plan/scale captures when + applicable. + +## Ongoing parity checklist + +For each reviewed BHE/BHCE update: + +- [ ] Search active reconciliation and post-processing entry points for new + `Filter`, `Filterf`, `Query`, `First`, `Count`, `Fetch*`, `Create*`, + `Delete*`, `Update*`, and `BatchOperation` calls. +- [ ] Trace helpers to an active entry point and label helper-only, test-only, + commented-out, or dormant forms accurately. +- [ ] Normalize every active call with the tuple in + `regression_coverage_manifest.md`. +- [ ] Map the tuple to an existing stable ID or add a new ID and source link. +- [ ] If stepwise traversal criteria change, update only the corresponding + standalone `HOP-*` cases; do not sequence the downstream traversal. +- [ ] Recheck projections independently from predicates. +- [ ] Recheck relationship-kind and ID-list cardinalities when schema sets + change. +- [ ] Record the BHE, BHCE, and DAWGS commits used for the audit. + +## Audit record template + +Append one record per reviewed source update: + +```markdown +### YYYY-MM-DD source parity audit + +- BHE commit: `` +- BHCE commit: `` +- DAWGS commit/worktree: `` +- Active entry points reviewed: `` +- Existing IDs confirmed: `` +- IDs added or changed: `` +- Dormant/helper-only forms: `` +- Projection/cardinality changes: `

` +- Validation and captures: `` +``` + +## Seed audit record + +### 2026-08-04 source parity audit + +- BHE commit: `c9f61530f45b` +- BHCE commit: `74dd3daa58a8` +- DAWGS baseline: `v0.6.0-13-g6638cc2`; implementation worktree based on + `8c5fba7` with the dormant-form parity-gate changes +- Active IDs: the `LOGIC-*`, `REC-*`, `TRUST-*`, `PRUNE-*`, `HOP-*`, + `SCAN-*`, `LOOKUP-*`, and `WRITE-*` rows in + `regression_coverage_manifest.md` +- Dormant forms: `FUTURE-01`; both reviewed callers remain in the disabled + Azure reconciliation block +- Validation: PostgreSQL and Neo4j `make test_all`; PostgreSQL scale-plan and + scale captures under `.coverage/` diff --git a/docs/shortest_path_tie_policy.md b/docs/shortest_path_tie_policy.md new file mode 100644 index 00000000..29b60cae --- /dev/null +++ b/docs/shortest_path_tie_policy.md @@ -0,0 +1,23 @@ +# Singleton shortest-path tie policy + +Date: 2026-08-07 + +`shortestPath` promises one valid relationship-unique trail of minimum length. +It does not promise which equally short trail is selected, and PostgreSQL +physical relationship IDs or insertion order are not part of the public +contract. Callers that require every relationship-distinct minimum trail must +use `allShortestPaths`. + +An executor may use a deterministic internal tie breaker for repeatability, +but changing that internal choice is not a semantic change when the returned +trail remains valid and minimal. PostgreSQL/Neo4j compatibility fixtures +therefore compare logical node identities, relationship kinds, and stable +`logical_key` properties. They do not require both backends to select the same +physical relationship ID for singleton output. + +This policy permits a future singleton witness executor to retain one +predecessor per accepted node/depth state. It does not permit deduplication for +`allShortestPaths`, relationship/path predicates, relationship variables, or +other forms whose validity or output multiplicity depends on the complete +trail. Those forms retain their exact incumbent unless independently +qualified. diff --git a/drivers/pg/batch.go b/drivers/pg/batch.go index d7978cc6..89c992f8 100644 --- a/drivers/pg/batch.go +++ b/drivers/pg/batch.go @@ -6,7 +6,6 @@ import ( "fmt" "log/slog" "strconv" - "strings" "github.com/jackc/pgtype" "github.com/jackc/pgx/v5" @@ -18,6 +17,8 @@ import ( ) const ( + // LargeNodeUpdateThreshold is the node count above which batch updates use + // the large-update execution path. LargeNodeUpdateThreshold = 1_000_000 ) @@ -41,21 +42,46 @@ func (s *Int2ArrayEncoder) Encode(values []int16) string { return s.buffer.String() } +// batch buffers graph mutations and applies them through one PostgreSQL transaction in insertion order. type batch struct { - ctx context.Context - innerTransaction *transaction - schemaManager *SchemaManager - nodeDeletionBuffer []graph.ID + // ctx scopes database operations performed while flushing buffered mutations. + ctx context.Context + + // innerTransaction owns the PostgreSQL transaction through which every buffered mutation is applied. + innerTransaction *transaction + + // schemaManager resolves graph metadata and maps graph kinds to their database identifiers. + schemaManager *SchemaManager + + // nodeDeletionBuffer retains node identifiers awaiting a bulk delete. + nodeDeletionBuffer []graph.ID + + // relationshipDeletionBuffer retains relationship identifiers awaiting a bulk delete. relationshipDeletionBuffer []graph.ID - nodeCreateBuffer []*graph.Node - nodeUpdateBuffer []*graph.Node - nodeUpdateByBuffer []graph.NodeUpdate - relationshipCreateBuffer []*graph.Relationship + + // nodeCreateBuffer retains nodes awaiting a bulk insert. + nodeCreateBuffer []*graph.Node + + // nodeUpdateBuffer retains complete node replacements awaiting a bulk update. + nodeUpdateBuffer []*graph.Node + + // nodeUpdateByBuffer retains identity-property node upserts awaiting validation and execution. + nodeUpdateByBuffer []graph.NodeUpdate + + // relationshipCreateBuffer retains relationships awaiting conflict coalescing and insertion. + relationshipCreateBuffer []*graph.Relationship + + // relationshipUpdateByBuffer retains identity-based relationship upserts awaiting validation and execution. relationshipUpdateByBuffer []graph.RelationshipUpdate - batchWriteSize int - kindIDEncoder Int2ArrayEncoder + + // batchWriteSize is the buffer length that triggers an automatic flush. + batchWriteSize int + + // kindIDEncoder reuses one buffer when serializing PostgreSQL int2 arrays for node writes. + kindIDEncoder Int2ArrayEncoder } +// newBatch opens the transaction used by a mutation batch and applies its configured flush threshold. func newBatch(ctx context.Context, conn *pgxpool.Conn, schemaManager *SchemaManager, cfg *Config) (*batch, error) { if tx, err := newTransactionWrapper(ctx, conn, schemaManager, cfg, false); err != nil { return nil, err @@ -288,6 +314,7 @@ func (s *batch) UpdateNodes(nodes []*graph.Node) error { return nil } +// flushNodeDeleteBuffer deletes the buffered node IDs and clears the buffer after a successful execution. func (s *batch) flushNodeDeleteBuffer() error { if _, err := s.innerTransaction.conn.Exec(s.ctx, deleteNodeWithIDStatement, s.nodeDeletionBuffer); err != nil { return err @@ -297,6 +324,7 @@ func (s *batch) flushNodeDeleteBuffer() error { return nil } +// flushRelationshipDeleteBuffer deletes the buffered relationship IDs and clears the buffer after a successful execution. func (s *batch) flushRelationshipDeleteBuffer() error { if _, err := s.innerTransaction.conn.Exec(s.ctx, deleteEdgeWithIDStatement, s.relationshipDeletionBuffer); err != nil { return err @@ -306,6 +334,7 @@ func (s *batch) flushRelationshipDeleteBuffer() error { return nil } +// flushNodeCreateBuffer rejects mixed ID allocation modes and dispatches the buffered nodes to the matching insert path. func (s *batch) flushNodeCreateBuffer() error { var ( withoutIDs = false @@ -331,6 +360,7 @@ func (s *batch) flushNodeCreateBuffer() error { return s.flushNodeCreateBufferWithIDs() } +// flushNodeCreateBufferWithIDs inserts buffered nodes whose IDs were assigned by the caller. func (s *batch) flushNodeCreateBufferWithIDs() error { var ( numCreates = len(s.nodeCreateBuffer) @@ -368,6 +398,7 @@ func (s *batch) flushNodeCreateBufferWithIDs() error { return nil } +// flushNodeCreateBufferWithoutIDs inserts buffered nodes using database-generated IDs. func (s *batch) flushNodeCreateBufferWithoutIDs() error { var ( numCreates = len(s.nodeCreateBuffer) @@ -402,6 +433,7 @@ func (s *batch) flushNodeCreateBufferWithoutIDs() error { return nil } +// flushNodeUpsertBatch validates and executes one identity-based node upsert batch for the target graph. func (s *batch) flushNodeUpsertBatch(updates *sql.NodeUpdateBatch) error { parameters := NewNodeUpsertParameters(len(updates.Updates)) @@ -438,6 +470,7 @@ func (s *batch) flushNodeUpsertBatch(updates *sql.NodeUpdateBatch) error { return nil } +// tryFlushNodeUpdateByBuffer validates, writes, and clears the buffered identity-based node updates. func (s *batch) tryFlushNodeUpdateByBuffer() error { if updates, err := sql.ValidateNodeUpdateByBatch(s.nodeUpdateByBuffer); err != nil { return err @@ -449,6 +482,7 @@ func (s *batch) tryFlushNodeUpdateByBuffer() error { return nil } +// flushNodeUpdateBatch writes complete node replacements for the supplied nodes. func (s *batch) flushNodeUpdateBatch(nodes []*graph.Node) error { parameters := NewNodeUpdateParameters(len(nodes)) @@ -471,6 +505,7 @@ func (s *batch) flushNodeUpdateBatch(nodes []*graph.Node) error { } } +// tryFlushNodeUpdateBuffer writes and clears the buffered complete node updates. func (s *batch) tryFlushNodeUpdateBuffer() error { if err := s.flushNodeUpdateBatch(s.nodeUpdateBuffer); err != nil { return err @@ -648,6 +683,7 @@ func (s *RelationshipUpdateByParameters) AppendAll(ctx context.Context, updates return nil } +// flushRelationshipUpdateByBuffer upserts prerequisite nodes and then applies identity-based relationship updates. func (s *batch) flushRelationshipUpdateByBuffer(updates *sql.RelationshipUpdateBatch) error { if err := s.flushNodeUpsertBatch(updates.NodeUpdates); err != nil { return err @@ -672,6 +708,7 @@ func (s *batch) flushRelationshipUpdateByBuffer(updates *sql.RelationshipUpdateB return nil } +// tryFlushRelationshipUpdateByBuffer validates, writes, and clears the buffered identity-based relationship updates. func (s *batch) tryFlushRelationshipUpdateByBuffer() error { if updateBatch, err := sql.ValidateRelationshipUpdateByBatch(s.relationshipUpdateByBuffer); err != nil { return err @@ -683,13 +720,22 @@ func (s *batch) tryFlushRelationshipUpdateByBuffer() error { return nil } +// relationshipCreateBatch stores column-oriented values for one relationship insert statement. type relationshipCreateBatch struct { - startIDs []uint64 - endIDs []uint64 - edgeKindIDs []int16 + // startIDs contains each relationship's start-node identifier in insert-row order. + startIDs []uint64 + + // endIDs contains each relationship's end-node identifier in insert-row order. + endIDs []uint64 + + // edgeKindIDs contains each relationship's database kind identifier in insert-row order. + edgeKindIDs []int16 + + // edgePropertyBags contains each relationship's JSONB properties in insert-row order. edgePropertyBags []pgtype.JSONB } +// newRelationshipCreateBatch allocates relationship insert columns with capacity for size rows. func newRelationshipCreateBatch(size int) *relationshipCreateBatch { return &relationshipCreateBatch{ startIDs: make([]uint64, 0, size), @@ -717,18 +763,35 @@ func (s *relationshipCreateBatch) EncodeProperties(edgePropertiesBatch []*graph. return nil } +// relationshipCreateBatchBuilder coalesces duplicate relationship keys while retaining their merged properties. type relationshipCreateBatchBuilder struct { - keyToEdgeID map[string]uint64 + // keyToPropertiesIndex locates the property bag associated with each unique relationship key. + keyToPropertiesIndex map[relationshipCreateKey]int + + // relationshipUpdateBatch accumulates the column values emitted for unique relationship keys. relationshipUpdateBatch *relationshipCreateBatch - edgePropertiesIndex map[uint64]int - edgePropertiesBatch []*graph.Properties + + // edgePropertiesBatch retains mergeable properties parallel to relationshipUpdateBatch rows. + edgePropertiesBatch []*graph.Properties +} + +// relationshipCreateKey identifies a relationship by endpoints and kind for conflict coalescing. +type relationshipCreateKey struct { + // startID identifies the relationship's starting node. + startID graph.ID + + // endID identifies the relationship's ending node. + endID graph.ID + + // kind identifies the relationship kind independently of its property bag. + kind string } +// newRelationshipCreateBatchBuilder allocates a conflict index and column buffers for size relationship inputs. func newRelationshipCreateBatchBuilder(size int) *relationshipCreateBatchBuilder { return &relationshipCreateBatchBuilder{ - keyToEdgeID: map[string]uint64{}, + keyToPropertiesIndex: map[relationshipCreateKey]int{}, relationshipUpdateBatch: newRelationshipCreateBatch(size), - edgePropertiesIndex: map[uint64]int{}, } } @@ -736,21 +799,19 @@ func (s *relationshipCreateBatchBuilder) Build() (*relationshipCreateBatch, erro return s.relationshipUpdateBatch, s.relationshipUpdateBatch.EncodeProperties(s.edgePropertiesBatch) } +// Add coalesces edge into the relationship batch, merging properties when its endpoints and kind repeat. func (s *relationshipCreateBatchBuilder) Add(ctx context.Context, kindMapper KindMapper, edge *graph.Relationship) error { - keyBuilder := strings.Builder{} - - keyBuilder.WriteString(edge.StartID.String()) - keyBuilder.WriteString(edge.EndID.String()) - keyBuilder.WriteString(edge.Kind.String()) - - key := keyBuilder.String() + key := relationshipCreateKey{ + startID: edge.StartID, + endID: edge.EndID, + kind: edge.Kind.String(), + } - if existingPropertiesIdx, hasExisting := s.keyToEdgeID[key]; hasExisting { + if existingPropertiesIdx, hasExisting := s.keyToPropertiesIndex[key]; hasExisting { s.edgePropertiesBatch[existingPropertiesIdx].Merge(edge.Properties) } else { var ( startID = edge.StartID.Uint64() - edgeID = edge.ID.Uint64() endID = edge.EndID.Uint64() edgeProperties = edge.Properties.Clone() ) @@ -761,15 +822,14 @@ func (s *relationshipCreateBatchBuilder) Add(ctx context.Context, kindMapper Kin s.relationshipUpdateBatch.Add(startID, endID, edgeKindID) } - s.keyToEdgeID[key] = edgeID - + s.keyToPropertiesIndex[key] = len(s.edgePropertiesBatch) s.edgePropertiesBatch = append(s.edgePropertiesBatch, edgeProperties) - s.edgePropertiesIndex[edgeID] = len(s.edgePropertiesBatch) - 1 } return nil } +// flushRelationshipCreateBuffer coalesces duplicate keys, inserts the resulting relationships, and clears the input buffer. func (s *batch) flushRelationshipCreateBuffer() error { batchBuilder := newRelationshipCreateBatchBuilder(len(s.relationshipCreateBuffer)) @@ -784,7 +844,7 @@ func (s *batch) flushRelationshipCreateBuffer() error { } else if graphTarget, err := s.innerTransaction.getTargetGraph(); err != nil { return err } else if _, err := s.innerTransaction.conn.Exec(s.ctx, createEdgeBatchStatement, graphTarget.ID, createBatch.startIDs, createBatch.endIDs, createBatch.edgeKindIDs, createBatch.edgePropertyBags); err != nil { - slog.Info(fmt.Sprintf("Num merged property bags: %d - Num edge keys: %d - StartID batch size: %d", len(batchBuilder.edgePropertiesIndex), len(batchBuilder.keyToEdgeID), len(batchBuilder.relationshipUpdateBatch.startIDs))) + slog.Info(fmt.Sprintf("Num property bags: %d - Num edge keys: %d - StartID batch size: %d", len(batchBuilder.edgePropertiesBatch), len(batchBuilder.keyToPropertiesIndex), len(batchBuilder.relationshipUpdateBatch.startIDs))) return err } @@ -792,6 +852,7 @@ func (s *batch) flushRelationshipCreateBuffer() error { return nil } +// tryFlush writes any mutation buffer whose length exceeds batchWriteSize. func (s *batch) tryFlush(batchWriteSize int) error { if len(s.nodeUpdateByBuffer) > batchWriteSize { if err := s.tryFlushNodeUpdateByBuffer(); err != nil { diff --git a/drivers/pg/batch_test.go b/drivers/pg/batch_test.go new file mode 100644 index 00000000..8d01de5f --- /dev/null +++ b/drivers/pg/batch_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package pg + +import ( + "context" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// staticKindMapper returns deterministic kind mappings for relationship batch tests. +type staticKindMapper struct { +} + +// MapKindID returns the fixed kind associated with every synthetic ID. +func (s staticKindMapper) MapKindID(context.Context, int16) (graph.Kind, error) { + return graph.StringKind("WriteCreateRelationship"), nil +} + +// MapKindIDs returns the fixed kind set used by the batch fixture. +func (s staticKindMapper) MapKindIDs(context.Context, []int16) (graph.Kinds, error) { + return graph.Kinds{graph.StringKind("WriteCreateRelationship")}, nil +} + +// MapKind returns the fixed database ID associated with every synthetic kind. +func (s staticKindMapper) MapKind(context.Context, graph.Kind) (int16, error) { + return 1, nil +} + +// MapKinds returns the fixed database ID set used by the batch fixture. +func (s staticKindMapper) MapKinds(context.Context, graph.Kinds) ([]int16, error) { + return []int16{1}, nil +} + +// AssertKinds accepts every supplied kind and returns the fixture's fixed database ID. +func (s staticKindMapper) AssertKinds(context.Context, graph.Kinds) ([]int16, error) { + return []int16{1}, nil +} + +// TestRelationshipCreateBatchBuilderMergesPropertiesByConflictKey verifies distinct endpoint tuples cannot collide and duplicate tuples merge properties. +func TestRelationshipCreateBatchBuilderMergesPropertiesByConflictKey(t *testing.T) { + var ( + ctx = context.Background() + kind = graph.StringKind("WriteCreateRelationship") + builder = newRelationshipCreateBatchBuilder(4) + ) + + updates := []*graph.Relationship{ + // These two endpoint pairs had the same concatenated key ("123...") + // before the batch builder used a structured conflict key. + graph.NewRelationship(0, 1, 23, graph.NewProperties().SetAll(map[string]any{"custom": "a-first", "a": true}), kind), + graph.NewRelationship(0, 1, 23, graph.NewProperties().SetAll(map[string]any{"custom": "a-last", "a-last": true}), kind), + graph.NewRelationship(0, 12, 3, graph.NewProperties().SetAll(map[string]any{"custom": "b-first", "b": true}), kind), + graph.NewRelationship(0, 12, 3, graph.NewProperties().SetAll(map[string]any{"custom": "b-last", "b-last": true}), kind), + } + for _, update := range updates { + require.NoError(t, builder.Add(ctx, staticKindMapper{}, update)) + } + + require.Len(t, builder.edgePropertiesBatch, 2) + require.Equal(t, "a-last", builder.edgePropertiesBatch[0].Get("custom").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[0].Get("a").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[0].Get("a-last").Any()) + require.False(t, builder.edgePropertiesBatch[0].Exists("b-last")) + require.Equal(t, "b-last", builder.edgePropertiesBatch[1].Get("custom").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[1].Get("b").Any()) + require.Equal(t, true, builder.edgePropertiesBatch[1].Get("b-last").Any()) + require.False(t, builder.edgePropertiesBatch[1].Exists("a-last")) + + batch, err := builder.Build() + require.NoError(t, err) + require.Len(t, batch.startIDs, 2) + require.Len(t, batch.edgePropertyBags, 2) +} diff --git a/drivers/pg/compiler.go b/drivers/pg/compiler.go index 90157b0d..f68cf90b 100644 --- a/drivers/pg/compiler.go +++ b/drivers/pg/compiler.go @@ -70,7 +70,7 @@ func (s *SchemaManager) compileRegularQuery(ctx context.Context, prepared prepar func (s *SchemaManager) compile(ctx context.Context, source string, parameters map[string]any, graphID int32, parse func() (*cypher.RegularQuery, error)) (string, map[string]any, error) { var ( - translationCache = s.translationCacheProvider.TranslationCache() + translationCache = s.compilationCacheProvider.TranslationCache() optimized = OptimizedTranslationEnabled() translationOptions = translate.Options{ OptimizerMode: translate.OptimizerDisabled, diff --git a/drivers/pg/compiler_test.go b/drivers/pg/compiler_test.go index abf7ea4b..b78af98a 100644 --- a/drivers/pg/compiler_test.go +++ b/drivers/pg/compiler_test.go @@ -87,7 +87,7 @@ func TestBuilderPreparedQueriesShareTranslationCacheKey(t *testing.T) { } key := cache.Key(first.source, 1, first.parameters) - _, bindings, err := cache.GetOrBuild(key, first.parameters, cacheableBuild( + _, bindings, err := cache.GetOrBuild(key, first.parameters, cacheableCompilationBuild( "select @p0", map[string]any{ "p0": uint64(1), @@ -148,7 +148,7 @@ func TestCompileRegularQueryCachesBuilderShapeAndRebindsValues(t *testing.T) { require.Equal(t, uint64(1), firstValue) require.Equal(t, uint64(2), secondBindings[name]) } - stats := manager.translationCache.Stats() + stats := manager.compilationCache.Stats() require.Equal(t, int64(1), stats.Misses) require.Equal(t, int64(1), stats.Hits) require.Equal(t, int64(1), stats.Insertions) @@ -191,8 +191,8 @@ func requireWarmBindingsMatchCold(t *testing.T, first, second preparedRegularQue require.Equal(t, coldSQL, warmSQL) require.Equal(t, coldBindings, warmBindings) require.NotEqual(t, firstBindings, warmBindings, "a cache hit must not retain the first request's values") - require.Equal(t, int64(1), warmManager.translationCache.Stats().Misses) - require.Equal(t, int64(1), warmManager.translationCache.Stats().Hits) + require.Equal(t, int64(1), warmManager.compilationCache.Stats().Misses) + require.Equal(t, int64(1), warmManager.compilationCache.Stats().Hits) } func TestCompileRegularQueryWarmBindingsMatchColdForMultipleParameters(t *testing.T) { @@ -339,7 +339,7 @@ func TestCompileRegularQueryPartitionsParameterTypesAndStructuralLiterals(t *tes intBuilder.Apply(query.Where(query.Equals(query.NodeProperty("value"), int64(1))), query.Returning(query.Node())) intQuery := compilePreparedBuilderQuery(t, intBuilder) require.Equal(t, stringQuery.source, intQuery.source) - require.NotEqual(t, manager.translationCache.Key(stringQuery.source, 7, stringQuery.parameters), manager.translationCache.Key(intQuery.source, 7, intQuery.parameters)) + require.NotEqual(t, manager.compilationCache.Key(stringQuery.source, 7, stringQuery.parameters), manager.compilationCache.Key(intQuery.source, 7, intQuery.parameters)) limitOne := query.NewBuilder(nil) limitOne.Apply(query.Where(query.Equals(query.NodeProperty("value"), "one")), query.Limit(1), query.Returning(query.Node())) @@ -348,13 +348,13 @@ func TestCompileRegularQueryPartitionsParameterTypesAndStructuralLiterals(t *tes limitOneQuery := compilePreparedBuilderQuery(t, limitOne) limitTwoQuery := compilePreparedBuilderQuery(t, limitTwo) require.NotEqual(t, limitOneQuery.source, limitTwoQuery.source) - require.NotEqual(t, manager.translationCache.Key(limitOneQuery.source, 7, limitOneQuery.parameters), manager.translationCache.Key(limitTwoQuery.source, 7, limitTwoQuery.parameters)) + require.NotEqual(t, manager.compilationCache.Key(limitOneQuery.source, 7, limitOneQuery.parameters), manager.compilationCache.Key(limitTwoQuery.source, 7, limitTwoQuery.parameters)) emptyIDs := map[string]any{"ids": []graph.ID{}} populatedIDs := map[string]any{"ids": []graph.ID{1, 2}} strings := map[string]any{"ids": []string{"1", "2"}} - require.Equal(t, manager.translationCache.Key("RETURN $ids", 7, emptyIDs), manager.translationCache.Key("RETURN $ids", 7, populatedIDs)) - require.NotEqual(t, manager.translationCache.Key("RETURN $ids", 7, populatedIDs), manager.translationCache.Key("RETURN $ids", 7, strings)) + require.Equal(t, manager.compilationCache.Key("RETURN $ids", 7, emptyIDs), manager.compilationCache.Key("RETURN $ids", 7, populatedIDs)) + require.NotEqual(t, manager.compilationCache.Key("RETURN $ids", 7, populatedIDs), manager.compilationCache.Key("RETURN $ids", 7, strings)) } func TestPrepareRegularQueryRenamesExplicitAndRepeatedParameterSymbols(t *testing.T) { @@ -392,7 +392,7 @@ func TestCompileRegularQueryDisabledCacheDoesNotRetainTranslation(t *testing.T) require.NoError(t, err) _, _, err = manager.compileRegularQuery(context.Background(), buildPreparedNodeLookup(t, graph.ID(2)), 7) require.NoError(t, err) - stats := manager.translationCache.Stats() + stats := manager.compilationCache.Stats() require.Zero(t, stats.Misses) require.Zero(t, stats.Size) require.Equal(t, int64(2), stats.Bypasses) @@ -407,12 +407,12 @@ func TestCompileRegularQueryUnoptimizedBypassesAndPreservesWarmEntries(t *testin first := buildPreparedNodeLookup(t, graph.ID(1)) _, _, err := manager.compileRegularQuery(context.Background(), first, 7) require.NoError(t, err) - warmStats := manager.translationCache.Stats() + warmStats := manager.compilationCache.Stats() setOptimizedTranslationForTest(t, false) _, _, err = manager.compileRegularQuery(context.Background(), buildPreparedNodeLookup(t, graph.ID(2)), 7) require.NoError(t, err) - disabledStats := manager.translationCache.Stats() + disabledStats := manager.compilationCache.Stats() require.Equal(t, warmStats.Hits, disabledStats.Hits) require.Equal(t, warmStats.Misses, disabledStats.Misses) require.Equal(t, warmStats.Insertions, disabledStats.Insertions) @@ -422,7 +422,7 @@ func TestCompileRegularQueryUnoptimizedBypassesAndPreservesWarmEntries(t *testin setOptimizedTranslationForTest(t, true) _, _, err = manager.compileRegularQuery(context.Background(), buildPreparedNodeLookup(t, graph.ID(3)), 7) require.NoError(t, err) - require.Equal(t, warmStats.Hits+1, manager.translationCache.Stats().Hits) + require.Equal(t, warmStats.Hits+1, manager.compilationCache.Stats().Hits) } func TestCompileTextUnoptimizedBypassesCache(t *testing.T) { @@ -433,12 +433,12 @@ func TestCompileTextUnoptimizedBypassesCache(t *testing.T) { _, _, err := manager.compileText(context.Background(), "MATCH (n) RETURN n", nil, 7) require.NoError(t, err) - warmStats := manager.translationCache.Stats() + warmStats := manager.compilationCache.Stats() setOptimizedTranslationForTest(t, false) _, _, err = manager.compileText(context.Background(), "MATCH (n) RETURN n", nil, 7) require.NoError(t, err) - disabledStats := manager.translationCache.Stats() + disabledStats := manager.compilationCache.Stats() require.Equal(t, warmStats.Hits, disabledStats.Hits) require.Equal(t, warmStats.Misses, disabledStats.Misses) require.Equal(t, warmStats.Bypasses+1, disabledStats.Bypasses) diff --git a/drivers/pg/composite_codec.go b/drivers/pg/composite_codec.go new file mode 100644 index 00000000..237d45da --- /dev/null +++ b/drivers/pg/composite_codec.go @@ -0,0 +1,194 @@ +package pg + +import ( + sqldriver "database/sql/driver" + "fmt" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql" +) + +// ownedComposite is the set of PostgreSQL composites that have a stable, +// driver-owned Go representation. Keeping this set closed makes it difficult +// to accidentally register an unrelated composite with a decoder whose field +// order does not match its PostgreSQL definition. +type ownedComposite interface { + // The closed type set limits optimized decoding to composites whose PostgreSQL field order is owned by this driver. + nodeComposite | edgeComposite | pathComposite +} + +// ownedCompositeCodec retains pgx's encoding and explicit Scan behavior while +// replacing CompositeCodec.DecodeValue's map[string]any result. Rows.Values +// uses DecodeValue, so decoding directly into the concrete representation +// avoids a map and one interface value per field. The field scanners allocate +// their slices and JSON maps, which also makes the returned value independent +// of pgx's reusable wire buffer. +type ownedCompositeCodec[T ownedComposite] struct { + // compositeCodec retains pgx's standard encoding and scan-plan implementation. + compositeCodec *pgtype.CompositeCodec +} + +// ownedCompositeArrayCodec decodes the common, non-null-element case directly +// into []T. PostgreSQL arrays may contain NULL composite elements, so a typed +// scan failure falls back to pgx's []any representation instead of discarding +// that information. +type ownedCompositeArrayCodec[T ownedComposite] struct { + // arrayCodec retains pgx's array metadata and fallback decoding behavior. + arrayCodec *pgtype.ArrayCodec +} + +// FormatSupported reports whether the wrapped composite codec accepts format. +func (s *ownedCompositeCodec[T]) FormatSupported(format int16) bool { + return s.compositeCodec.FormatSupported(format) +} + +// PreferredFormat returns the wire format preferred by the wrapped composite codec. +func (s *ownedCompositeCodec[T]) PreferredFormat() int16 { + return s.compositeCodec.PreferredFormat() +} + +// PlanEncode delegates composite encoding to pgx's registered composite codec. +func (s *ownedCompositeCodec[T]) PlanEncode(m *pgtype.Map, oid uint32, format int16, value any) pgtype.EncodePlan { + return s.compositeCodec.PlanEncode(m, oid, format, value) +} + +// PlanScan preserves pgx's explicit-target composite scanning behavior. +func (s *ownedCompositeCodec[T]) PlanScan(m *pgtype.Map, oid uint32, format int16, target any) pgtype.ScanPlan { + return s.compositeCodec.PlanScan(m, oid, format, target) +} + +// DecodeDatabaseSQLValue delegates database/sql decoding to pgx's composite codec. +func (s *ownedCompositeCodec[T]) DecodeDatabaseSQLValue( + m *pgtype.Map, + oid uint32, + format int16, + src []byte, +) (sqldriver.Value, error) { + return s.compositeCodec.DecodeDatabaseSQLValue(m, oid, format, src) +} + +// DecodeValue decodes non-null composites into their owned Go representation and falls back for nullable fields. +func (s *ownedCompositeCodec[T]) DecodeValue(m *pgtype.Map, oid uint32, format int16, src []byte) (any, error) { + if src == nil { + return nil, nil + } + + var value T + target, typeOK := any(&value).(pgtype.CompositeIndexScanner) + if !typeOK { + return nil, fmt.Errorf("owned composite target %T does not implement pgtype.CompositeIndexScanner", &value) + } + + plan := s.compositeCodec.PlanScan(m, oid, format, target) + if plan == nil { + return nil, fmt.Errorf("unable to scan PostgreSQL composite OID %d in format %d into %T", oid, format, &value) + } + + if err := plan.Scan(src, target); err != nil { + // PostgreSQL permits NULL fields inside a non-NULL composite, while the + // hot-path representation deliberately uses non-nullable scalar fields. + // Preserve the old map representation for those uncommon values rather + // than turning a valid row into a decode error. + return s.compositeCodec.DecodeValue(m, oid, format, src) + } + + return value, nil +} + +// FormatSupported reports whether the wrapped array codec accepts format. +func (s *ownedCompositeArrayCodec[T]) FormatSupported(format int16) bool { + return s.arrayCodec.FormatSupported(format) +} + +// PreferredFormat returns the wire format preferred by the wrapped array codec. +func (s *ownedCompositeArrayCodec[T]) PreferredFormat() int16 { + return s.arrayCodec.PreferredFormat() +} + +// PlanEncode delegates composite-array encoding to pgx's registered array codec. +func (s *ownedCompositeArrayCodec[T]) PlanEncode( + m *pgtype.Map, + oid uint32, + format int16, + value any, +) pgtype.EncodePlan { + return s.arrayCodec.PlanEncode(m, oid, format, value) +} + +// PlanScan preserves pgx's explicit-target composite-array scanning behavior. +func (s *ownedCompositeArrayCodec[T]) PlanScan( + m *pgtype.Map, + oid uint32, + format int16, + target any, +) pgtype.ScanPlan { + return s.arrayCodec.PlanScan(m, oid, format, target) +} + +// DecodeDatabaseSQLValue delegates database/sql decoding to pgx's array codec. +func (s *ownedCompositeArrayCodec[T]) DecodeDatabaseSQLValue( + m *pgtype.Map, + oid uint32, + format int16, + src []byte, +) (sqldriver.Value, error) { + return s.arrayCodec.DecodeDatabaseSQLValue(m, oid, format, src) +} + +// DecodeValue decodes arrays without null elements into []T and otherwise preserves pgx's nullable representation. +func (s *ownedCompositeArrayCodec[T]) DecodeValue(m *pgtype.Map, oid uint32, format int16, src []byte) (any, error) { + if src == nil { + return nil, nil + } + + var values []T + if plan := m.PlanScan(oid, format, &values); plan != nil { + if err := plan.Scan(src, &values); err == nil { + return values, nil + } + } + + // A []T cannot represent a NULL composite array element. Preserve pgx's + // nullable []any behavior for that less common case. + return s.arrayCodec.DecodeValue(m, oid, format, src) +} + +// installOwnedCompositeCodec replaces a supported pgx codec with the matching driver-owned scalar or array decoder. +func installOwnedCompositeCodec(dataType pgsql.DataType, definition *pgtype.Type) error { + switch dataType { + case pgsql.NodeCompositeArray: + arrayCodec, typeOK := definition.Codec.(*pgtype.ArrayCodec) + if !typeOK { + return fmt.Errorf("expected PostgreSQL type %s to use *pgtype.ArrayCodec but received %T", dataType, definition.Codec) + } + + definition.Codec = &ownedCompositeArrayCodec[nodeComposite]{arrayCodec: arrayCodec} + return nil + case pgsql.EdgeCompositeArray: + arrayCodec, typeOK := definition.Codec.(*pgtype.ArrayCodec) + if !typeOK { + return fmt.Errorf("expected PostgreSQL type %s to use *pgtype.ArrayCodec but received %T", dataType, definition.Codec) + } + + definition.Codec = &ownedCompositeArrayCodec[edgeComposite]{arrayCodec: arrayCodec} + return nil + } + + compositeCodec, typeOK := definition.Codec.(*pgtype.CompositeCodec) + if !typeOK { + return fmt.Errorf("expected PostgreSQL type %s to use *pgtype.CompositeCodec but received %T", dataType, definition.Codec) + } + + switch dataType { + case pgsql.NodeComposite: + definition.Codec = &ownedCompositeCodec[nodeComposite]{compositeCodec: compositeCodec} + case pgsql.EdgeComposite: + definition.Codec = &ownedCompositeCodec[edgeComposite]{compositeCodec: compositeCodec} + case pgsql.PathComposite: + definition.Codec = &ownedCompositeCodec[pathComposite]{compositeCodec: compositeCodec} + default: + return fmt.Errorf("PostgreSQL type %s does not have an owned composite decoder", dataType) + } + + return nil +} diff --git a/drivers/pg/composite_codec_integration_test.go b/drivers/pg/composite_codec_integration_test.go new file mode 100644 index 00000000..852e3336 --- /dev/null +++ b/drivers/pg/composite_codec_integration_test.go @@ -0,0 +1,273 @@ +package pg + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/stretchr/testify/require" +) + +// postgresIntegrationConnectionString returns CONNECTION_STRING only for a PostgreSQL target and skips the driver-scoped test otherwise. +func postgresIntegrationConnectionString(t *testing.T) string { + t.Helper() + + connectionString := os.Getenv("CONNECTION_STRING") + if connectionString == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + + normalizedConnectionString := strings.ToLower(connectionString) + if !strings.HasPrefix(normalizedConnectionString, "postgres://") && + !strings.HasPrefix(normalizedConnectionString, "postgresql://") { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + return connectionString +} + +// connectCompositeCodecIntegration opens a timeout-bounded PostgreSQL connection and registers cleanup for composite-codec integration tests. +func connectCompositeCodecIntegration(t *testing.T) (context.Context, *pgx.Conn) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + t.Cleanup(cancel) + + config, err := pgx.ParseConfig(postgresIntegrationConnectionString(t)) + require.NoError(t, err) + + conn, err := pgx.ConnectConfig(ctx, config) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, conn.Close(context.Background())) + }) + + // Keep this driver-scoped test independent of application data and schema + // state. PostgreSQL drops types in pg_temp with the connection. + _, err = conn.Exec(ctx, ` +set search_path = pg_temp, public; +create type pg_temp.nodeComposite as ( + id bigint, + kind_ids smallint[], + properties jsonb +); +create type pg_temp.edgeComposite as ( + id bigint, + start_id bigint, + end_id bigint, + kind_id smallint, + properties jsonb +); +create type pg_temp.pathComposite as ( + nodes nodeComposite[], + edges edgeComposite[] +);`) + require.NoError(t, err) + + require.NoError(t, AfterPooledConnectionEstablished(ctx, conn)) + + return ctx, conn +} + +// TestPostgresOwnedCompositeCodecRegistration verifies pooled connections register optimized codecs for every owned composite type. +func TestPostgresOwnedCompositeCodecRegistration(t *testing.T) { + _, conn := connectCompositeCodecIntegration(t) + typeMap := conn.TypeMap() + + nodeType, typeOK := typeMap.TypeForName(pgsql.NodeComposite.String()) + require.True(t, typeOK) + require.IsType(t, &ownedCompositeCodec[nodeComposite]{}, nodeType.Codec) + + nodeArrayType, typeOK := typeMap.TypeForName(pgsql.NodeCompositeArray.String()) + require.True(t, typeOK) + nodeArrayCodec, typeOK := nodeArrayType.Codec.(*ownedCompositeArrayCodec[nodeComposite]) + require.True(t, typeOK) + require.Same(t, nodeType, nodeArrayCodec.arrayCodec.ElementType) + + edgeType, typeOK := typeMap.TypeForName(pgsql.EdgeComposite.String()) + require.True(t, typeOK) + require.IsType(t, &ownedCompositeCodec[edgeComposite]{}, edgeType.Codec) + + pathType, typeOK := typeMap.TypeForName(pgsql.PathComposite.String()) + require.True(t, typeOK) + require.IsType(t, &ownedCompositeCodec[pathComposite]{}, pathType.Codec) +} + +// TestPostgresOwnedCompositeCodecRowsValues verifies Rows.Values returns driver-owned node and edge composites. +func TestPostgresOwnedCompositeCodecRowsValues(t *testing.T) { + ctx, conn := connectCompositeCodecIntegration(t) + + for _, testCase := range []struct { + // name identifies the wire-format subtest. + name string + + // format selects the pgx result format used by the query. + format int16 + }{ + { + name: "binary", + format: pgtype.BinaryFormatCode, + }, + { + name: "text", + format: pgtype.TextFormatCode, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + rows, err := conn.Query(ctx, ` +select (series.id, array[1::smallint, 2::smallint], jsonb_build_object('id', series.id))::nodeComposite +from generate_series(101::bigint, 102::bigint) series(id) +order by series.id`, pgx.QueryResultFormats{testCase.format}) + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next()) + firstValues, err := rows.Values() + require.NoError(t, err) + require.Len(t, firstValues, 1) + first, typeOK := firstValues[0].(nodeComposite) + require.True(t, typeOK) + require.Equal(t, int64(101), first.ID) + require.Equal(t, []int16{1, 2}, first.KindIDs) + + require.True(t, rows.Next()) + secondValues, err := rows.Values() + require.NoError(t, err) + second, typeOK := secondValues[0].(nodeComposite) + require.True(t, typeOK) + require.Equal(t, int64(102), second.ID) + + // Reading the next row must not overwrite data retained from the + // first Rows.Values call. + require.Equal(t, int64(101), first.ID) + require.Equal(t, []int16{1, 2}, first.KindIDs) + require.Equal(t, float64(101), first.Properties["id"]) + + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} + +// TestPostgresOwnedCompositeCodecArraysAndPaths verifies composite arrays and paths decode into their typed graph representations. +func TestPostgresOwnedCompositeCodecArraysAndPaths(t *testing.T) { + ctx, conn := connectCompositeCodecIntegration(t) + + for _, testCase := range []struct { + // name identifies the wire-format subtest. + name string + + // format selects the pgx result format used by the query. + format int16 + }{ + { + name: "binary", + format: pgtype.BinaryFormatCode, + }, + { + name: "text", + format: pgtype.TextFormatCode, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + rows, err := conn.Query(ctx, ` +select + array[ + (101, array[1::smallint], '{"name":"first"}'::jsonb)::nodeComposite, + null::nodeComposite, + (102, array[2::smallint], '{"name":"second"}'::jsonb)::nodeComposite + ]::nodeComposite[], + ( + array[ + (101, array[1::smallint], '{"name":"first"}'::jsonb)::nodeComposite, + (102, array[2::smallint], '{"name":"second"}'::jsonb)::nodeComposite + ]::nodeComposite[], + array[ + (201, 101, 102, 3::smallint, '{"name":"edge"}'::jsonb)::edgeComposite + ]::edgeComposite[] + )::pathComposite`, pgx.QueryResultFormats{testCase.format}) + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next()) + values, err := rows.Values() + require.NoError(t, err) + require.Len(t, values, 2) + + nodes, typeOK := values[0].([]any) + require.True(t, typeOK) + require.Len(t, nodes, 3) + require.IsType(t, nodeComposite{}, nodes[0]) + require.Nil(t, nodes[1]) + require.IsType(t, nodeComposite{}, nodes[2]) + + path, typeOK := values[1].(pathComposite) + require.True(t, typeOK) + require.Len(t, path.Nodes, 2) + require.Len(t, path.Edges, 1) + require.Equal(t, int64(101), path.Nodes[0].ID) + require.Equal(t, int64(102), path.Nodes[1].ID) + require.Equal(t, int64(201), path.Edges[0].ID) + + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} + +// TestPostgresOwnedCompositeCodecNullInternalFieldFallback verifies nullable composite fields retain pgx's lossless fallback representation. +func TestPostgresOwnedCompositeCodecNullInternalFieldFallback(t *testing.T) { + ctx, conn := connectCompositeCodecIntegration(t) + + for _, testCase := range []struct { + // name identifies the wire-format subtest. + name string + + // format selects the pgx result format used by the query. + format int16 + }{ + { + name: "binary", + format: pgtype.BinaryFormatCode, + }, + { + name: "text", + format: pgtype.TextFormatCode, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + rows, err := conn.Query(ctx, ` +select + (null::bigint, array[1::smallint], '{"name":"nullable"}'::jsonb)::nodeComposite, + array[(null::bigint, array[1::smallint], '{"name":"nullable"}'::jsonb)::nodeComposite]::nodeComposite[]`, + pgx.QueryResultFormats{testCase.format}) + require.NoError(t, err) + defer rows.Close() + + require.True(t, rows.Next()) + values, err := rows.Values() + require.NoError(t, err) + require.Len(t, values, 2) + + node, typeOK := values[0].(map[string]any) + require.True(t, typeOK) + require.Nil(t, node["id"]) + require.Equal(t, []any{int16(1)}, node["kind_ids"]) + + nodes, typeOK := values[1].([]any) + require.True(t, typeOK) + require.Len(t, nodes, 1) + node, typeOK = nodes[0].(map[string]any) + require.True(t, typeOK) + require.Nil(t, node["id"]) + + require.False(t, rows.Next()) + require.NoError(t, rows.Err()) + }) + } +} diff --git a/drivers/pg/composite_codec_test.go b/drivers/pg/composite_codec_test.go new file mode 100644 index 00000000..6e246299 --- /dev/null +++ b/drivers/pg/composite_codec_test.go @@ -0,0 +1,531 @@ +package pg + +import ( + "reflect" + "testing" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/stretchr/testify/require" +) + +const ( + // testNodeCompositeOID is the synthetic scalar node OID registered by codec unit tests. + testNodeCompositeOID uint32 = 91_001 + + // testNodeCompositeArrayOID is the synthetic node-array OID registered by codec unit tests. + testNodeCompositeArrayOID uint32 = 91_002 + + // testEdgeCompositeOID is the synthetic scalar edge OID registered by codec unit tests. + testEdgeCompositeOID uint32 = 91_003 + + // testEdgeCompositeArrayOID is the synthetic edge-array OID registered by codec unit tests. + testEdgeCompositeArrayOID uint32 = 91_004 + + // testPathCompositeOID is the synthetic path OID registered by codec unit tests. + testPathCompositeOID uint32 = 91_005 +) + +// compositeCodecTestTypes provides a controllable test double for PostgreSQL composite decoding; graph values round-trip through pgx without losing identity or properties. +type compositeCodecTestTypes struct { + // node is the registered scalar node type. + node *pgtype.Type + + // nodeArray is the registered node-array type. + nodeArray *pgtype.Type + + // edge is the registered scalar edge type. + edge *pgtype.Type + + // edgeArray is the registered edge-array type. + edgeArray *pgtype.Type + + // path is the registered scalar path type. + path *pgtype.Type +} + +// requirePGType returns the type registered for oid and fails the test when the registration is absent. +func requirePGType(t testing.TB, typeMap *pgtype.Map, oid uint32) *pgtype.Type { + t.Helper() + + dataType, typeOK := typeMap.TypeForOID(oid) + require.True(t, typeOK, "expected PostgreSQL type OID %d", oid) + + return dataType +} + +// newCompositeCodecTestMap registers synthetic node, edge, and path definitions, optionally installing owned codecs. +func newCompositeCodecTestMap(t testing.TB, owned bool) (*pgtype.Map, compositeCodecTestTypes) { + t.Helper() + + typeMap := pgtype.NewMap() + types := compositeCodecTestTypes{} + types.node = &pgtype.Type{ + Name: pgsql.NodeComposite.String(), + OID: testNodeCompositeOID, + Codec: &pgtype.CompositeCodec{ + Fields: []pgtype.CompositeCodecField{ + { + Name: "id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "kind_ids", + Type: requirePGType(t, typeMap, pgtype.Int2ArrayOID), + }, + { + Name: "properties", + Type: requirePGType(t, typeMap, pgtype.JSONBOID), + }, + }, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.NodeComposite, types.node)) + } + typeMap.RegisterType(types.node) + + types.nodeArray = &pgtype.Type{ + Name: pgsql.NodeCompositeArray.String(), + OID: testNodeCompositeArrayOID, + Codec: &pgtype.ArrayCodec{ + ElementType: types.node, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.NodeCompositeArray, types.nodeArray)) + } + typeMap.RegisterType(types.nodeArray) + + types.edge = &pgtype.Type{ + Name: pgsql.EdgeComposite.String(), + OID: testEdgeCompositeOID, + Codec: &pgtype.CompositeCodec{ + Fields: []pgtype.CompositeCodecField{ + { + Name: "id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "start_id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "end_id", + Type: requirePGType(t, typeMap, pgtype.Int8OID), + }, + { + Name: "kind_id", + Type: requirePGType(t, typeMap, pgtype.Int2OID), + }, + { + Name: "properties", + Type: requirePGType(t, typeMap, pgtype.JSONBOID), + }, + }, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.EdgeComposite, types.edge)) + } + typeMap.RegisterType(types.edge) + + types.edgeArray = &pgtype.Type{ + Name: pgsql.EdgeCompositeArray.String(), + OID: testEdgeCompositeArrayOID, + Codec: &pgtype.ArrayCodec{ + ElementType: types.edge, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.EdgeCompositeArray, types.edgeArray)) + } + typeMap.RegisterType(types.edgeArray) + + types.path = &pgtype.Type{ + Name: pgsql.PathComposite.String(), + OID: testPathCompositeOID, + Codec: &pgtype.CompositeCodec{ + Fields: []pgtype.CompositeCodecField{ + { + Name: "nodes", + Type: types.nodeArray, + }, + { + Name: "edges", + Type: types.edgeArray, + }, + }, + }, + } + if owned { + require.NoError(t, installOwnedCompositeCodec(pgsql.PathComposite, types.path)) + } + typeMap.RegisterType(types.path) + + return typeMap, types +} + +// testNodeComposite returns a representative node value with the requested ID. +func testNodeComposite(id int64) nodeComposite { + return nodeComposite{ + ID: id, + KindIDs: []int16{1, 2}, + Properties: map[string]any{"id": float64(id), "name": "node"}, + } +} + +// testEdgeComposite returns a representative edge value with the requested identity and endpoints. +func testEdgeComposite(id, startID, endID int64) edgeComposite { + return edgeComposite{ + ID: id, + StartID: startID, + EndID: endID, + KindID: 3, + Properties: map[string]any{"id": float64(id), "name": "edge"}, + } +} + +// TestOwnedCompositeCodecDecodeValue verifies scalar node, edge, and path values decode into owned concrete types. +func TestOwnedCompositeCodecDecodeValue(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + expectedNode := testNodeComposite(101) + expectedEdge := testEdgeComposite(201, 101, 102) + expectedPath := pathComposite{ + Nodes: []nodeComposite{expectedNode, testNodeComposite(102)}, + Edges: []edgeComposite{expectedEdge}, + } + + for _, testCase := range []struct { + // name identifies the composite type and wire-format subtest. + name string + + // format selects the pgx encoding format. + format int16 + + // dataType supplies the composite codec under test. + dataType *pgtype.Type + + // value is the concrete composite expected after decoding. + value any + }{ + { + name: "node/binary", + format: pgtype.BinaryFormatCode, + dataType: types.node, + value: expectedNode, + }, + { + name: "node/text", + format: pgtype.TextFormatCode, + dataType: types.node, + value: expectedNode, + }, + { + name: "edge/binary", + format: pgtype.BinaryFormatCode, + dataType: types.edge, + value: expectedEdge, + }, + { + name: "edge/text", + format: pgtype.TextFormatCode, + dataType: types.edge, + value: expectedEdge, + }, + { + name: "path/binary", + format: pgtype.BinaryFormatCode, + dataType: types.path, + value: expectedPath, + }, + { + name: "path/text", + format: pgtype.TextFormatCode, + dataType: types.path, + value: expectedPath, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + src, err := typeMap.Encode(testCase.dataType.OID, testCase.format, testCase.value, nil) + require.NoError(t, err) + + decoded, err := testCase.dataType.Codec.DecodeValue(typeMap, testCase.dataType.OID, testCase.format, src) + require.NoError(t, err) + require.IsType(t, testCase.value, decoded) + require.Equal(t, testCase.value, decoded) + + // pgx may reuse its receive buffer after Rows.Values returns. None of + // the concrete composite's slices, strings, or maps may alias it. + clear(src) + require.Equal(t, testCase.value, decoded) + }) + } +} + +// TestOwnedCompositeCodecPreservesExplicitScanAndNull verifies explicit scan targets and null composites keep pgx semantics. +func TestOwnedCompositeCodecPreservesExplicitScanAndNull(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + expected := testNodeComposite(101) + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.node.OID, format, expected, nil) + require.NoError(t, err) + + var decoded nodeComposite + require.NoError(t, typeMap.Scan(types.node.OID, format, src, &decoded)) + require.Equal(t, expected, decoded) + + nullValue, err := types.node.Codec.DecodeValue(typeMap, types.node.OID, format, nil) + require.NoError(t, err) + require.Nil(t, nullValue) + } +} + +// TestOwnedCompositeCodecFallsBackForNullInternalFields verifies nullable internal fields retain pgx's lossless map representation. +func TestOwnedCompositeCodecFallsBackForNullInternalFields(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + value := pgtype.CompositeFields{nil, []int16{1, 2}, map[string]any{"name": "nullable"}} + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.node.OID, format, value, nil) + require.NoError(t, err) + + decoded, err := types.node.Codec.DecodeValue(typeMap, types.node.OID, format, src) + require.NoError(t, err) + require.Equal(t, map[string]any{ + "id": nil, + "kind_ids": []any{int16(1), int16(2)}, + "properties": map[string]any{"name": "nullable"}, + }, decoded) + + arraySource := []pgtype.CompositeFields{value} + src, err = typeMap.Encode(types.nodeArray.OID, format, arraySource, nil) + require.NoError(t, err) + + decoded, err = types.nodeArray.Codec.DecodeValue(typeMap, types.nodeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, []any{map[string]any{ + "id": nil, + "kind_ids": []any{int16(1), int16(2)}, + "properties": map[string]any{"name": "nullable"}, + }}, decoded) + } +} + +// TestOwnedCompositeCodecSupportsArrays verifies non-null composite arrays decode directly into typed slices. +func TestOwnedCompositeCodecSupportsArrays(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + first := testNodeComposite(101) + second := testNodeComposite(102) + expectedNodes := []nodeComposite{first, second} + expectedEdges := []edgeComposite{ + testEdgeComposite(201, 101, 102), + testEdgeComposite(202, 102, 103), + } + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.nodeArray.OID, format, expectedNodes, nil) + require.NoError(t, err) + + decoded, err := types.nodeArray.Codec.DecodeValue(typeMap, types.nodeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, expectedNodes, decoded) + + var typedValues []nodeComposite + require.NoError(t, typeMap.Scan(types.nodeArray.OID, format, src, &typedValues)) + require.Equal(t, expectedNodes, typedValues) + + src, err = typeMap.Encode(types.edgeArray.OID, format, expectedEdges, nil) + require.NoError(t, err) + + decoded, err = types.edgeArray.Codec.DecodeValue(typeMap, types.edgeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, expectedEdges, decoded) + + var typedEdges []edgeComposite + require.NoError(t, typeMap.Scan(types.edgeArray.OID, format, src, &typedEdges)) + require.Equal(t, expectedEdges, typedEdges) + } +} + +// TestOwnedCompositeCodecArrayPreservesNullElements verifies arrays containing null composites retain a nullable representation. +func TestOwnedCompositeCodecArrayPreservesNullElements(t *testing.T) { + typeMap, types := newCompositeCodecTestMap(t, true) + first := testNodeComposite(101) + values := []*nodeComposite{&first, nil} + + for _, format := range []int16{pgtype.BinaryFormatCode, pgtype.TextFormatCode} { + src, err := typeMap.Encode(types.nodeArray.OID, format, values, nil) + require.NoError(t, err) + + decoded, err := types.nodeArray.Codec.DecodeValue(typeMap, types.nodeArray.OID, format, src) + require.NoError(t, err) + require.Equal(t, []any{first, nil}, decoded) + } +} + +// TestInstallOwnedCompositeCodec verifies supported definitions are wrapped and incompatible definitions are rejected. +func TestInstallOwnedCompositeCodec(t *testing.T) { + for _, testCase := range []struct { + // dataType identifies the supported composite definition to install. + dataType pgsql.DataType + + // value selects the concrete owned codec type expected for dataType. + value any + }{ + { + dataType: pgsql.NodeComposite, + value: nodeComposite{}, + }, + { + dataType: pgsql.EdgeComposite, + value: edgeComposite{}, + }, + { + dataType: pgsql.PathComposite, + value: pathComposite{}, + }, + } { + t.Run(testCase.dataType.String(), func(t *testing.T) { + definition := &pgtype.Type{ + Name: testCase.dataType.String(), + OID: testNodeCompositeOID, + Codec: &pgtype.CompositeCodec{}, + } + + require.NoError(t, installOwnedCompositeCodec(testCase.dataType, definition)) + require.NotEqual(t, reflect.TypeOf(&pgtype.CompositeCodec{}), reflect.TypeOf(definition.Codec)) + }) + } + + arrayDefinition := &pgtype.Type{Codec: &pgtype.ArrayCodec{}} + require.NoError(t, installOwnedCompositeCodec(pgsql.NodeCompositeArray, arrayDefinition)) + require.IsType(t, &ownedCompositeArrayCodec[nodeComposite]{}, arrayDefinition.Codec) + + invalidDefinition := &pgtype.Type{Codec: pgtype.TextCodec{}} + require.ErrorContains(t, installOwnedCompositeCodec(pgsql.NodeComposite, invalidDefinition), "*pgtype.CompositeCodec") +} + +// compositeCodecBenchmarkSink retains decoded values so benchmark work cannot be optimized away. +var compositeCodecBenchmarkSink any + +// benchmarkCompositeDecodeValue repeatedly decodes one encoded value through the selected codec implementation. +func benchmarkCompositeDecodeValue( + b *testing.B, + owned bool, + dataType func(compositeCodecTestTypes) *pgtype.Type, + value any, +) { + b.Helper() + + typeMap, types := newCompositeCodecTestMap(b, owned) + selectedType := dataType(types) + src, err := typeMap.Encode(selectedType.OID, pgtype.BinaryFormatCode, value, nil) + require.NoError(b, err) + + b.ReportAllocs() + b.ResetTimer() + for range b.N { + decoded, err := selectedType.Codec.DecodeValue(typeMap, selectedType.OID, pgtype.BinaryFormatCode, src) + if err != nil { + b.Fatal(err) + } + compositeCodecBenchmarkSink = decoded + } +} + +// BenchmarkNodeCompositeDecodeValue compares scalar node decoding through stock and owned codecs. +func BenchmarkNodeCompositeDecodeValue(b *testing.B) { + value := testNodeComposite(101) + for _, testCase := range []struct { + // name identifies whether the benchmark uses stock or owned decoding. + name string + + // owned enables the owned composite codec when true. + owned bool + }{ + { + name: "map", + owned: false, + }, + { + name: "owned", + owned: true, + }, + } { + b.Run(testCase.name, func(b *testing.B) { + benchmarkCompositeDecodeValue(b, testCase.owned, func(types compositeCodecTestTypes) *pgtype.Type { + return types.node + }, value) + }) + } +} + +// BenchmarkNodeCompositeArrayDecodeValue compares node-array decoding through stock and owned codecs. +func BenchmarkNodeCompositeArrayDecodeValue(b *testing.B) { + values := make([]nodeComposite, 128) + for idx := range values { + values[idx] = testNodeComposite(int64(idx + 1)) + } + + for _, testCase := range []struct { + // name identifies whether the benchmark uses stock or owned decoding. + name string + + // owned enables the owned composite codec when true. + owned bool + }{ + { + name: "map", + owned: false, + }, + { + name: "owned", + owned: true, + }, + } { + b.Run(testCase.name, func(b *testing.B) { + benchmarkCompositeDecodeValue(b, testCase.owned, func(types compositeCodecTestTypes) *pgtype.Type { + return types.nodeArray + }, values) + }) + } +} + +// BenchmarkPathCompositeDecodeValue compares path decoding through stock and owned codecs. +func BenchmarkPathCompositeDecodeValue(b *testing.B) { + value := pathComposite{ + Nodes: make([]nodeComposite, 32), + Edges: make([]edgeComposite, 31), + } + for idx := range value.Nodes { + value.Nodes[idx] = testNodeComposite(int64(idx + 1)) + } + for idx := range value.Edges { + value.Edges[idx] = testEdgeComposite(int64(idx+1), int64(idx+1), int64(idx+2)) + } + + for _, testCase := range []struct { + // name identifies whether the benchmark uses stock or owned decoding. + name string + + // owned enables the owned composite codec when true. + owned bool + }{ + { + name: "map", + owned: false, + }, + { + name: "owned", + owned: true, + }, + } { + b.Run(testCase.name, func(b *testing.B) { + benchmarkCompositeDecodeValue(b, testCase.owned, func(types compositeCodecTestTypes) *pgtype.Type { + return types.path + }, value) + }) + } +} diff --git a/drivers/pg/connection_cache_stats.go b/drivers/pg/connection_cache_stats.go new file mode 100644 index 00000000..37866941 --- /dev/null +++ b/drivers/pg/connection_cache_stats.go @@ -0,0 +1,300 @@ +package pg + +import "time" + +// TranslationCacheStats is a query-text-free snapshot of one connection's +// translation cache activity and occupancy. +type TranslationCacheStats struct { + // Hits counts translations served from the connection-local cache. + Hits uint64 `json:"hits"` + + // Misses counts translations not found in the connection-local cache. + Misses uint64 `json:"misses"` + + // Bypasses counts translations intentionally built without retention. + Bypasses uint64 `json:"bypasses"` + + // Insertions counts reusable translations published to the local cache. + Insertions uint64 `json:"insertions"` + + // Evictions counts cached translations displaced at capacity. + Evictions uint64 `json:"evictions"` + + // BindingFailures counts cache hits that cannot bind current caller values. + BindingFailures uint64 `json:"binding_failures"` + + // Entries is the local cache occupancy at snapshot time. + Entries int `json:"entries"` + + // Capacity is the maximum number of retained local translations. + Capacity int `json:"capacity"` +} + +// add accumulates another connection cache snapshot into s. +func (s *TranslationCacheStats) add(other TranslationCacheStats) { + s.Hits += other.Hits + s.Misses += other.Misses + s.Bypasses += other.Bypasses + s.Insertions += other.Insertions + s.Evictions += other.Evictions + s.BindingFailures += other.BindingFailures + s.Entries += other.Entries +} + +// TraversalWorkspaceStats describes setup activity for session-local +// stable-snapshot traversal workspaces without exposing backend identity. +type TraversalWorkspaceStats struct { + // Initializations counts successful workspace setup operations. + Initializations uint64 `json:"initializations"` + + // Reuses counts requests served by a workspace ready in the current generation. + Reuses uint64 `json:"reuses"` + + // Failures counts workspace setup attempts rejected by PostgreSQL. + Failures uint64 `json:"failures"` + + // Ready reports whether the workspace is ready for the current schema generation. + Ready bool `json:"ready"` +} + +// add accumulates setup counts while intentionally excluding connection-local readiness. +func (s *TraversalWorkspaceStats) add(other TraversalWorkspaceStats) { + s.Initializations += other.Initializations + s.Reuses += other.Reuses + s.Failures += other.Failures +} + +// PreparedStatementStats describes opt-in statement warm-up activity. Entries +// counts only SHA-256 statement identities, never SQL text or parameter data. +type PreparedStatementStats struct { + // Attempts counts requests to prepare selected statements. + Attempts uint64 `json:"attempts"` + + // Prepared counts statements successfully prepared on a connection. + Prepared uint64 `json:"prepared"` + + // Reuses counts requests satisfied by an already prepared statement. + Reuses uint64 `json:"reuses"` + + // Failures counts prepare requests rejected by PostgreSQL. + Failures uint64 `json:"failures"` + + // Entries is the number of statement identities retained for the connection. + Entries int `json:"entries"` +} + +// StrategySelectionStats records query-text-free production-routing +// observations. These counters describe selection only; they do not claim an +// emitted candidate executed. +type StrategySelectionStats struct { + // Incumbent counts selections that leave the stable executor active. + Incumbent uint64 `json:"incumbent"` + + // ExactQueryCanary counts candidates selected by exact-query authorization. + ExactQueryCanary uint64 `json:"exact_query_canary"` + + // StructuralShadow counts structural matches observed without authorization. + StructuralShadow uint64 `json:"structural_shadow"` + + // StructuralAuthorized counts candidates selected by structural authorization. + StructuralAuthorized uint64 `json:"structural_authorized"` + + // TopologySelected counts candidates selected by transaction-local topology routing. + TopologySelected uint64 `json:"topology_selected"` + + // ShapeUnavailable counts queries outside the structural classifier scope. + ShapeUnavailable uint64 `json:"shape_unavailable"` +} + +// TraversalShapeCacheStats describes V2's bounded, query-text-free structural +// classification cache. Entries retain only a digest and immutable shape. +type TraversalShapeCacheStats struct { + // Hits counts classifications served from the bounded shape cache. + Hits uint64 `json:"hits"` + + // Misses counts classifications computed without a retained entry. + Misses uint64 `json:"misses"` + + // Entries is the number of retained query-digest classifications. + Entries int `json:"entries"` + + // Capacity bounds retained classifications. + Capacity int `json:"capacity"` +} + +// TraversalRouteDecisionStats aggregates topology-routing shadow states +// without retaining transaction tokens, graph IDs, or caller values. +type TraversalRouteDecisionStats struct { + // Disabled counts queries ineligible for topology routing. + Disabled uint64 `json:"disabled"` + + // SynopsisUnavailable counts queries without a current graph synopsis. + SynopsisUnavailable uint64 `json:"synopsis_unavailable"` + + // ShadowMiss counts first observations that populate route-decision shadow state. + ShadowMiss uint64 `json:"shadow_miss"` + + // ShadowHit counts repeated shadow observations that remain on the incumbent. + ShadowHit uint64 `json:"shadow_hit"` + + // CandidateHit counts repeated observations that select the candidate route. + CandidateHit uint64 `json:"candidate_hit"` + + // FirstUseCandidate counts first-use protocol selections of the candidate route. + FirstUseCandidate uint64 `json:"first_use_candidate"` + + // EstimateRejected counts candidates rejected by the synopsis estimate. + EstimateRejected uint64 `json:"estimate_rejected"` + + // Capacity counts decisions rejected by bounded transaction-local state. + Capacity uint64 `json:"capacity"` + + // ParametersInvalid counts queries whose parameters cannot be fingerprinted. + ParametersInvalid uint64 `json:"parameters_invalid"` +} + +// add accumulates another connection's prepared-statement statistics. +func (s *PreparedStatementStats) add(other PreparedStatementStats) { + s.Attempts += other.Attempts + s.Prepared += other.Prepared + s.Reuses += other.Reuses + s.Failures += other.Failures + s.Entries += other.Entries +} + +// ConnectionCacheStats describes one currently live connection cache. ID is +// an opaque diagnostic identifier; it is not a backend PID or pointer value. +type ConnectionCacheStats struct { + // ID is an opaque provider-assigned identifier for the live connection. + ID uint64 `json:"id"` + + // Translation reports cache activity scoped to this physical connection. + Translation TranslationCacheStats `json:"translation"` + + // TraversalWorkspace reports setup state scoped to this physical connection. + TraversalWorkspace TraversalWorkspaceStats `json:"traversal_workspace"` + + // PreparedStatements reports selected statement warm-up for this connection. + PreparedStatements PreparedStatementStats `json:"prepared_statements"` +} + +// Stats is a query-text-free provider snapshot. Aggregate combines live and +// retired connection counters; its Capacity is the current theoretical bound +// across live connections, not a global retained-entry limit. +type Stats struct { + // SchemaGeneration partitions state invalidated by schema-sensitive changes. + SchemaGeneration uint64 `json:"schema_generation"` + + // CapacityPerConnection bounds retained translations for one live connection. + CapacityPerConnection int `json:"capacity_per_connection"` + + // MinConnections is the configured pgx pool lower connection bound. + MinConnections int32 `json:"min_connections"` + + // MaxConnections is the configured pgx pool upper connection bound. + MaxConnections int32 `json:"max_connections"` + + // LiveConnections counts physical connections currently registered by the provider. + LiveConnections int `json:"live_connections"` + + // RetiredConnections counts connections whose cache state has been closed. + RetiredConnections uint64 `json:"retired_connections"` + + // Aggregate combines active and retired translation-cache statistics. + Aggregate TranslationCacheStats `json:"aggregate"` + + // TraversalWorkspace aggregates reusable workspace setup activity. + TraversalWorkspace TraversalWorkspaceStats `json:"traversal_workspace"` + + // PreparedStatements aggregates selected statement warm-up activity. + PreparedStatements PreparedStatementStats `json:"prepared_statements"` + + // Connections reports cache state for each currently live physical connection. + Connections []ConnectionCacheStats `json:"connections"` + + // SQLGeneration aggregates query-text-free SQL generation timings. + SQLGeneration SQLGenerationStats `json:"sql_generation"` + + // StrategySelection aggregates production routing observations. + StrategySelection StrategySelectionStats `json:"strategy_selection"` + + // TraversalShapeCache reports bounded structural classifier cache activity. + TraversalShapeCache TraversalShapeCacheStats `json:"traversal_shape_cache"` + + // TraversalRouteDecision aggregates topology route-decision outcomes. + TraversalRouteDecision TraversalRouteDecisionStats `json:"traversal_route_decision"` + + // SharedShortestPathTemplates reports the pool-wide immutable template cache. + SharedShortestPathTemplates SharedTemplateStats `json:"shared_shortest_path_templates"` +} + +// SharedTemplateStats reports the bounded V2-wide immutable shortest-path +// template tier. It excludes query text and caller values. +type SharedTemplateStats struct { + // Hits counts templates served from the shared shortest-path tier. + Hits uint64 `json:"hits"` + + // Misses counts shared-tier lookups without a retained template. + Misses uint64 `json:"misses"` + + // Insertions counts templates added to the shared tier. + Insertions uint64 `json:"insertions"` + + // Evictions counts templates displaced from the shared tier at capacity. + Evictions uint64 `json:"evictions"` + + // Entries is the shared-tier occupancy at snapshot time. + Entries int `json:"entries"` + + // Capacity bounds retained templates in the shared tier. + Capacity int `json:"capacity"` +} + +// SQLGenerationTiming contains aggregate V2 SQL-generation durations. Count +// is separate so a zero-cost stage remains observable. +type SQLGenerationTiming struct { + // Count records SQL generation profiles represented by this aggregate. + Count uint64 `json:"count"` + + // Parse accumulates Cypher parsing time. + Parse time.Duration `json:"parse"` + + // Graph accumulates graph-resolution time. + Graph time.Duration `json:"graph"` + + // Policy accumulates traversal-policy selection time. + Policy time.Duration `json:"policy"` + + // Cache accumulates translation-cache lookup time. + Cache time.Duration `json:"cache"` + + // Translate accumulates Cypher-to-SQL lowering time. + Translate time.Duration `json:"translate"` + + // Format accumulates SQL rendering time. + Format time.Duration `json:"format"` + + // Dispatch accumulates pgx query-dispatch time. + Dispatch time.Duration `json:"dispatch"` +} + +// add accumulates one query-text-free SQL-generation profile. +func (s *SQLGenerationTiming) add(profile SQLGenerationProfile) { + s.Count++ + s.Parse += profile.Parse + s.Graph += profile.Graph + s.Policy += profile.Policy + s.Cache += profile.Cache + s.Translate += profile.Translate + s.Format += profile.Format + s.Dispatch += profile.Dispatch +} + +// SQLGenerationStats separates shortest-path timing from other graph work. +type SQLGenerationStats struct { + // ShortestPath aggregates generation timings for shortest-path queries. + ShortestPath SQLGenerationTiming `json:"shortest_path"` + + // Other aggregates generation timings for all other queries. + Other SQLGenerationTiming `json:"other"` +} diff --git a/drivers/pg/connection_runtime_driver_test.go b/drivers/pg/connection_runtime_driver_test.go new file mode 100644 index 00000000..7a852f45 --- /dev/null +++ b/drivers/pg/connection_runtime_driver_test.go @@ -0,0 +1,26 @@ +package pg + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +func TestDriverReportsProviderStatisticsWithoutSensitiveState(t *testing.T) { + provider, err := newConnectionCacheProvider(DefaultRuntimeConfig()) + require.NoError(t, err) + driver := &Driver{runtime: &poolRuntime{provider: provider}} + + stats := driver.TranslationCacheStats() + require.Equal(t, DefaultRuntimeConfig().TranslationCacheEntries, stats.CapacityPerConnection) + require.Equal(t, int32(defaultMinConnections), stats.MinConnections) + require.Equal(t, int32(defaultMaxConnections), stats.MaxConnections) + require.Zero(t, stats.LiveConnections) + require.Empty(t, stats.Connections) +} + +func TestDriverImplementsGraphDatabase(t *testing.T) { + var database graph.Database = &Driver{} + require.NotNil(t, database) +} diff --git a/drivers/pg/connection_runtime_integration_test.go b/drivers/pg/connection_runtime_integration_test.go new file mode 100644 index 00000000..1512cf39 --- /dev/null +++ b/drivers/pg/connection_runtime_integration_test.go @@ -0,0 +1,704 @@ +//go:build manual_integration + +package pg + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "reflect" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +var ( + v2IntegrationNodeKind = graph.StringKind("PGV2IntegrationNode") + v2IntegrationEdgeKind = graph.StringKind("PGV2IntegrationEdge") + v2IntegrationSchema = graph.Schema{ + Graphs: []graph.Graph{{ + Name: "pg_v2_integration", + Nodes: graph.Kinds{v2IntegrationNodeKind}, + Edges: graph.Kinds{v2IntegrationEdgeKind}, + }}, + DefaultGraph: graph.Graph{Name: "pg_v2_integration"}, + } +) + +type v2IntegrationFixture struct { + start graph.ID + end graph.ID +} + +func postgresV2IntegrationConnectionString(t *testing.T) string { + t.Helper() + connectionString := os.Getenv("CONNECTION_STRING") + if connectionString == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + normalized := strings.ToLower(connectionString) + if !strings.HasPrefix(normalized, "postgres://") && !strings.HasPrefix(normalized, "postgresql://") { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + return connectionString +} + +// newV2IntegrationDriver creates a deterministic test pool after exercising +// the same production hook composition used by NewPool. Production continues +// to mirror v1's fixed pool sizing; this test helper uses a bounded pool only +// to prove physical-connection lifecycle behavior. +func newV2IntegrationDriver(t *testing.T, maxConns int32, capacity int, afterRelease func(*pgx.Conn) bool) *Driver { + t.Helper() + ctx := context.Background() + poolConfig, err := pgxpool.ParseConfig(postgresV2IntegrationConnectionString(t)) + require.NoError(t, err) + poolConfig.MinConns = 0 + poolConfig.MaxConns = maxConns + poolConfig.AfterRelease = afterRelease + + config := RuntimeConfig{ + TranslationCacheEntries: capacity, + Pool: &PoolConfig{MinConnections: 0, MaxConnections: maxConns}, + } + pool, err := NewPoolWithRuntimeConfig(ctx, poolConfig, config) + require.NoError(t, err) + driver := NewDriver(0, pool) + t.Cleanup(func() { + require.NoError(t, driver.Close(context.Background())) + }) + return driver +} + +func requireEventually(t *testing.T, condition func() bool, description string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if condition() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", description) +} + +func setUpV2IntegrationGraph(t *testing.T, driver *Driver) v2IntegrationFixture { + t.Helper() + ctx := context.Background() + require.NoError(t, driver.AssertSchema(ctx, v2IntegrationSchema)) + t.Cleanup(func() { + _ = driver.WriteTransaction(context.Background(), func(tx graph.Transaction) error { + if err := deleteV2IntegrationRelationships(tx); err != nil { + return err + } + return tx.Nodes().Delete() + }) + }) + require.NoError(t, driver.WriteTransaction(ctx, func(tx graph.Transaction) error { + if err := deleteV2IntegrationRelationships(tx); err != nil { + return err + } + return tx.Nodes().Delete() + })) + + fixture := v2IntegrationFixture{} + require.NoError(t, driver.WriteTransaction(ctx, func(tx graph.Transaction) error { + start, err := tx.CreateNode(graph.NewProperties().Set("name", "start"), v2IntegrationNodeKind) + if err != nil { + return err + } + end, err := tx.CreateNode(graph.NewProperties().Set("name", "end"), v2IntegrationNodeKind) + if err != nil { + return err + } + if _, err := tx.CreateRelationshipByIDs(start.ID, end.ID, v2IntegrationEdgeKind, graph.NewProperties().Set("name", "edge")); err != nil { + return err + } + fixture.start = start.ID + fixture.end = end.ID + return nil + })) + return fixture +} + +func deleteV2IntegrationRelationships(tx graph.Transaction) error { + result := tx.Raw("delete from edge where graph_id = (select id from graph where name = 'pg_v2_integration')", nil) + defer result.Close() + for result.Next() { + } + return result.Error() +} + +func TestV2RefreshTraversalTopologySynopsisTracksGraphMutation(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + + initial, err := driver.RefreshTraversalTopologySynopsis(ctx, v2IntegrationSchema.DefaultGraph) + require.NoError(t, err) + require.True(t, initial.Available()) + require.Equal(t, "topology-fixed-suffix-counts-v1", initial.EstimatorVersion) + require.GreaterOrEqual(t, initial.NodeCount, int64(2)) + require.GreaterOrEqual(t, initial.EdgeCount, int64(1)) + + require.NoError(t, driver.WriteTransaction(ctx, func(tx graph.Transaction) error { + _, err := tx.CreateNode(graph.NewProperties().Set("name", "after-synopsis"), v2IntegrationNodeKind) + return err + })) + + refreshed, err := driver.RefreshTraversalTopologySynopsis(ctx, v2IntegrationSchema.DefaultGraph) + require.NoError(t, err) + require.True(t, refreshed.Available()) + require.Greater(t, refreshed.Epoch, initial.Epoch) + require.Greater(t, refreshed.SourceMutationEpoch, initial.SourceMutationEpoch) + require.GreaterOrEqual(t, refreshed.NodeCount, initial.NodeCount+1) +} + +func structuralASPV3Policy(t *testing.T, evidenceQuery string) TraversalPolicy { + t.Helper() + shape := TraversalShape{ + Version: TraversalShapeVersion, + Family: "ASP", + Direction: "outbound", + ObservationMode: "all_paths", + MinimumDepth: 1, + MaximumDepth: 4, + RelationshipKindCount: 1, + } + shape.Fingerprint = TraversalShapeFingerprint(shape) + candidate := string(optimize.ShortestPathExecutorASPI1DAG) + selector := "v2-structural-asp-v1" + template := TraversalSQLTemplateSHA256(candidate, selector, "guarded_dual_arm", shape) + digest := TraversalPolicyQuerySHA256(evidenceQuery) + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"path": role + ".json", "sha256": strings.Repeat("0", sha256.Size*2)} + } + raw, err := json.Marshal(map[string]any{ + "version": 3, "candidate": candidate, "selector_version": selector, + "source_commit": "v2-integration", "source_sha256": strings.Repeat("0", sha256.Size*2), + "binary_sha256": strings.Repeat("1", sha256.Size*2), "corpus_sha256": strings.Repeat("2", sha256.Size*2), + "operational_candidate_sql_sha256": strings.Repeat("3", sha256.Size*2), + "execution_boundary": "guarded_dual_arm", "fallback_executor": string(optimize.ShortestPathExecutorASPA1DAG), + "caps": map[string]int64{"state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20}, + "buckets": []map[string]any{{ + "name": "v2-structural-asp", "query_sha256": []string{digest}, "qualification_split": []string{"training", "holdout"}, + "direction": shape.Direction, "observation_mode": shape.ObservationMode, "minimum_depth": shape.MinimumDepth, "maximum_depth": shape.MaximumDepth, + "relationship_kind_count": shape.RelationshipKindCount, "untyped_relationship": shape.UntypedRelationship, + "structural_shape_version": shape.Version, "structural_family": shape.Family, "structural_shape_sha256": shape.Fingerprint, "sql_template_sha256": template, + }}, + "evidence": evidence, + }) + require.NoError(t, err) + sum := sha256.Sum256(raw) + return TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: hex.EncodeToString(sum[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{digest}, + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + } +} + +func TestV2StructuralTraversalPolicyExecutesVerifiedEquivalentShape(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + fixture := setUpV2IntegrationGraph(t, driver) + evidenceQuery := "MATCH p = allShortestPaths((s)-[:PGV2IntegrationEdge*1..4]->(e)) WHERE id(s) = $start AND id(e) = $end RETURN p" + require.NoError(t, driver.SetTraversalPolicy(structuralASPV3Policy(t, evidenceQuery))) + structurallyEquivalent := "MATCH route = allShortestPaths((left)-[:PGV2IntegrationEdge*1..4]->(right)) WHERE id(left) = $source AND id(right) = $target RETURN route" + + ctx := context.Background() + require.NoError(t, driver.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query(structurallyEquivalent, map[string]any{"source": fixture.start, "target": fixture.end}) + defer result.Close() + for result.Next() { + } + return result.Error() + }, OptionSetTransactionIsolation(pgx.RepeatableRead))) + stats := driver.TranslationCacheStats() + require.Equal(t, uint64(1), stats.StrategySelection.StructuralAuthorized) +} + +func TestV2TopologyRouteDecisionShadowsOnlyWithinStableSnapshot(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + _, err := driver.RefreshTraversalTopologySynopsis(ctx, v2IntegrationSchema.DefaultGraph) + require.NoError(t, err) + query := ` +MATCH (root:PGV2IntegrationNode) +WHERE root.name = 'start' +MATCH path = (root)-[:PGV2IntegrationEdge*0..16]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode) +RETURN path` + + require.NoError(t, driver.ReadTransaction(ctx, func(tx graph.Transaction) error { + for range 2 { + result := tx.Query(query, nil) + for result.Next() { + } + result.Close() + if err := result.Error(); err != nil { + return err + } + } + return nil + }, OptionSetTransactionIsolation(pgx.RepeatableRead))) + stats := driver.TranslationCacheStats().TraversalRouteDecision + require.Equal(t, uint64(1), stats.ShadowMiss) + require.Equal(t, uint64(1), stats.ShadowHit) +} + +func v2TopologyFixedSuffixPolicy(t *testing.T, evidenceQuery string, shape TraversalShape) TraversalPolicy { + t.Helper() + digest := TraversalPolicyQuerySHA256(evidenceQuery) + candidate := string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"path": role + ".json", "sha256": strings.Repeat("0", sha256.Size*2)} + } + bucket := map[string]any{ + "name": "v2-topology-fixed-suffix", "query_sha256": []string{digest}, "qualification_split": []string{"training", "holdout"}, + "direction": shape.Direction, "observation_mode": shape.ObservationMode, "minimum_depth": shape.MinimumDepth, "maximum_depth": shape.MaximumDepth, + "suffix_length": shape.SuffixLength, "candidate_strategy": shape.CandidateStrategy, + "structural_shape_version": shape.Version, "structural_family": shape.Family, "structural_shape_sha256": shape.Fingerprint, + "sql_template_sha256": TraversalSQLTemplateSHA256(candidate, candidate, "transaction_retry", shape), + } + raw, err := json.Marshal(map[string]any{ + "version": 4, "candidate": candidate, "selector_version": candidate, "execution_boundary": "transaction_retry", "fallback_executor": string(optimize.ExpansionSearchStepwiseForward), + "source_commit": "v2-integration", "source_sha256": strings.Repeat("1", sha256.Size*2), "binary_sha256": strings.Repeat("2", sha256.Size*2), "corpus_sha256": strings.Repeat("3", sha256.Size*2), + "operational_candidate_sql_sha256": strings.Repeat("4", sha256.Size*2), + "topology_estimator_version": "topology-fixed-suffix-counts-v1", "synopsis_schema_version": "topology-synopsis-schema-v2", "route_cache_protocol": "topology-selected-routing-v1", + "topology_thresholds": map[string]int64{"maximum_edge_to_node_ratio_per_mille": 1000}, + "caps": map[string]int64{ + "suffix_row_limit": optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, "state_limit": optimize.ExpansionSearchSuffixReverseGuardStateLimit, + "output_row_limit": optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, "output_bytes_limit": optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit, + }, + "buckets": []map[string]any{bucket}, "evidence": evidence, + }) + require.NoError(t, err) + sum := sha256.Sum256(raw) + return TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: hex.EncodeToString(sum[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{digest}, + EnableTopologyFixedSuffix: true, + } +} + +func TestV2TopologyFixedSuffixExecutesOnlyAfterSnapshotRouteCacheHit(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 8, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + _, err := driver.RefreshTraversalTopologySynopsis(ctx, v2IntegrationSchema.DefaultGraph) + require.NoError(t, err) + query := ` +MATCH (root:PGV2IntegrationNode) +WHERE root.name = 'start' +MATCH path = (root)-[:PGV2IntegrationEdge*0..16]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode) +RETURN path` + shape := TraversalShape{ + Version: TraversalFixedSuffixShapeVersion, + Family: "fixed_suffix_expansion", + Direction: "outbound", + ObservationMode: "full_path", + MinimumDepth: 0, + MaximumDepth: 16, + SuffixLength: 3, + CandidateStrategy: string(optimize.ExpansionSearchSuffixSeededReverse), + } + shape.Fingerprint = TraversalShapeFingerprint(shape) + require.Equal(t, TraversalFixedSuffixShapeVersion, shape.Version) + require.NoError(t, driver.SetTraversalPolicy(v2TopologyFixedSuffixPolicy(t, query, shape))) + + require.NoError(t, driver.ReadTransaction(ctx, func(tx graph.Transaction) error { + for range 2 { + result := tx.Query(query, nil) + for result.Next() { + } + result.Close() + if err := result.Error(); err != nil { + return err + } + } + return nil + }, OptionSetTransactionIsolation(pgx.RepeatableRead))) + stats := driver.TranslationCacheStats() + require.Equal(t, uint64(1), stats.TraversalRouteDecision.ShadowMiss) + require.Equal(t, uint64(1), stats.TraversalRouteDecision.CandidateHit) + require.Equal(t, uint64(1), stats.StrategySelection.TopologySelected) +} + +func TestV2TopologyFixedSuffixFirstUseExecutesWithinStableSnapshot(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 8, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + _, err := driver.RefreshTraversalTopologySynopsis(ctx, v2IntegrationSchema.DefaultGraph) + require.NoError(t, err) + query := `MATCH (root:PGV2IntegrationNode) WHERE root.name = 'start' MATCH path = (root)-[:PGV2IntegrationEdge*0..16]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(:PGV2IntegrationNode) RETURN path` + shape := TraversalShape{Version: TraversalFixedSuffixShapeVersion, Family: "fixed_suffix_expansion", Direction: "outbound", ObservationMode: "full_path", MinimumDepth: 0, MaximumDepth: 16, SuffixLength: 3, CandidateStrategy: string(optimize.ExpansionSearchSuffixSeededReverse)} + shape.Fingerprint = TraversalShapeFingerprint(shape) + policy := v2TopologyFixedSuffixPolicy(t, query, shape) + policy.EnableTopologyFixedSuffix = false + policy.EnableTopologyFixedSuffixFirstUse = true + var manifest map[string]any + require.NoError(t, json.Unmarshal(policy.PromotionManifestJSON, &manifest)) + firstUse := string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) + manifest["version"], manifest["candidate"], manifest["selector_version"] = 5, firstUse, firstUse + manifest["execution_boundary"], manifest["route_cache_protocol"] = "first_use_transaction_retry", "topology-selected-first-use-routing-v1" + bucket := manifest["buckets"].([]any)[0].(map[string]any) + bucket["sql_template_sha256"] = TraversalSQLTemplateSHA256(firstUse, firstUse, "first_use_transaction_retry", shape) + raw, err := json.Marshal(manifest) + require.NoError(t, err) + sum := sha256.Sum256(raw) + policy.PromotionManifestJSON, policy.PromotionManifestSHA256 = raw, hex.EncodeToString(sum[:]) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + require.NoError(t, driver.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query(query, nil) + for result.Next() { + } + result.Close() + return result.Error() + }, OptionSetTransactionIsolation(pgx.RepeatableRead))) + stats := driver.TranslationCacheStats() + require.Equal(t, uint64(1), stats.TraversalRouteDecision.FirstUseCandidate) + require.Equal(t, uint64(1), stats.StrategySelection.TopologySelected) +} + +func snapshotQuery(ctx context.Context, database graph.Database, query string, parameters map[string]any) ([][]any, error) { + var rows [][]any + err := database.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query(query, parameters) + defer result.Close() + for result.Next() { + rows = append(rows, append([]any(nil), result.Values()...)) + } + return result.Error() + }) + return rows, err +} + +func rawExecutionError(ctx context.Context, database graph.Database) error { + return database.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Raw("select 1 / 0", nil) + defer result.Close() + for result.Next() { + } + return result.Error() + }) +} + +func v2ScalarQuery() string { + return "MATCH (n:PGV2IntegrationNode) RETURN count(n)" +} + +func TestV2NewPoolConstructsAnExplicitOptInDriver(t *testing.T) { + poolConfig, err := pgxpool.ParseConfig(postgresV2IntegrationConnectionString(t)) + require.NoError(t, err) + pool, err := NewPoolWithRuntimeConfig(context.Background(), poolConfig, RuntimeConfig{ + TranslationCacheEntries: 2, + Pool: &PoolConfig{MinConnections: 0, MaxConnections: 1}, + }) + require.NoError(t, err) + driver := NewDriver(0, pool) + t.Cleanup(func() { + require.NoError(t, driver.Close(context.Background())) + }) + setUpV2IntegrationGraph(t, driver) + + _, err = snapshotQuery(context.Background(), driver, v2ScalarQuery(), nil) + require.NoError(t, err) + stats := driver.TranslationCacheStats() + require.Equal(t, 2, stats.CapacityPerConnection) + require.Equal(t, int32(0), stats.MinConnections) + require.Equal(t, int32(1), stats.MaxConnections) + require.NotEmpty(t, stats.Connections) +} + +func TestV2TranslationCacheSurvivesLeaseReleaseAndReacquisition(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + setUpV2IntegrationGraph(t, driver) + + _, err := snapshotQuery(context.Background(), driver, v2ScalarQuery(), nil) + require.NoError(t, err) + first := driver.TranslationCacheStats() + require.Len(t, first.Connections, 1) + require.Equal(t, uint64(1), first.Aggregate.Misses) + connectionID := first.Connections[0].ID + + _, err = snapshotQuery(context.Background(), driver, v2ScalarQuery(), nil) + require.NoError(t, err) + second := driver.TranslationCacheStats() + require.Len(t, second.Connections, 1) + require.Equal(t, connectionID, second.Connections[0].ID) + require.Equal(t, uint64(1), second.Aggregate.Hits) +} + +// TestV2StableSnapshotTraversalWorkspaceReadiness verifies that the v2 +// provider avoids redundant setup only for the same live connection and +// schema generation. +func TestV2StableSnapshotTraversalWorkspaceReadiness(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + stableSnapshot := func() error { + return driver.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query("MATCH p = shortestPath((s)-[*1..]->(e)) RETURN p", nil) + defer result.Close() + return result.Error() + }, OptionSetTransactionIsolation(pgx.RepeatableRead)) + } + + require.NoError(t, stableSnapshot()) + require.NoError(t, stableSnapshot()) + stats := driver.TranslationCacheStats() + require.Equal(t, uint64(1), stats.TraversalWorkspace.Initializations) + require.Equal(t, uint64(1), stats.TraversalWorkspace.Reuses) + require.True(t, stats.Connections[0].TraversalWorkspace.Ready) + + require.NoError(t, driver.RefreshKinds(ctx)) + stats = driver.TranslationCacheStats() + require.False(t, stats.Connections[0].TraversalWorkspace.Ready) + require.NoError(t, stableSnapshot()) + stats = driver.TranslationCacheStats() + require.Equal(t, uint64(2), stats.TraversalWorkspace.Initializations) +} + +// TestV2TraversalWorkspaceReadinessAlsoWarmsReadCommitted verifies the V2 +// physical-connection workspace can serve ordinary one-statement traversal +// reads. The workspace is session-local only, so this does not widen the +// stable-snapshot requirement for manifest-selected candidates. +func TestV2TraversalWorkspaceReadinessAlsoWarmsReadCommitted(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + run := func() error { + return driver.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query("MATCH p = shortestPath((s)-[*1..]->(e)) RETURN p", nil) + defer result.Close() + return result.Error() + }) + } + + require.NoError(t, run()) + require.NoError(t, run()) + stats := driver.TranslationCacheStats() + require.Equal(t, uint64(1), stats.TraversalWorkspace.Initializations) + require.Equal(t, uint64(1), stats.TraversalWorkspace.Reuses) + require.True(t, stats.Connections[0].TraversalWorkspace.Ready) +} + +// TestV2StatementWarmupUsesPooledPGXCacheNames verifies that an opt-in warmup +// creates one server statement per physical connection and leaves only a +// query-text-free identity in V2 lifecycle state. +func TestV2StatementWarmupUsesPooledPGXCacheNames(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 4, nil) + setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + require.NoError(t, driver.WarmStatements(ctx, "select 1", "select 1")) + + connection, err := driver.pool.Acquire(ctx) + require.NoError(t, err) + identity := sha256.Sum256([]byte("select 1")) + var prepared int + require.NoError(t, connection.QueryRow(ctx, "select count(*) from pg_prepared_statements where name = $1", pgxStatementCacheName(identity)).Scan(&prepared)) + require.Equal(t, 1, prepared) + var value int + require.NoError(t, connection.QueryRow(ctx, "select 1", pgx.QueryExecModeCacheStatement).Scan(&value)) + require.Equal(t, 1, value) + + stats := driver.TranslationCacheStats() + require.Equal(t, uint64(1), stats.PreparedStatements.Prepared) + require.Equal(t, 1, stats.PreparedStatements.Entries) + + connection.Release() + driver.pool.Reset() + requireEventually(t, func() bool { + stats := driver.TranslationCacheStats() + return stats.LiveConnections == 0 && stats.PreparedStatements.Entries == 0 + }, "prepared statement retirement") +} + +func TestV2StatementWarmupPolicyAppliesToNewConnections(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 1, nil) + ctx := context.Background() + require.NoError(t, driver.SetStatementWarmupPolicy(ctx, "select 1")) + driver.pool.Reset() + requireEventually(t, func() bool { return driver.TranslationCacheStats().LiveConnections == 0 }, "warm connection retirement") + connection, err := driver.pool.Acquire(ctx) + require.NoError(t, err) + defer connection.Release() + identity := sha256.Sum256([]byte("select 1")) + var prepared int + require.NoError(t, connection.QueryRow(ctx, "select count(*) from pg_prepared_statements where name = $1", pgxStatementCacheName(identity)).Scan(&prepared)) + require.Equal(t, 1, prepared) +} + +func TestV2PhysicalConnectionsHaveIndependentCaches(t *testing.T) { + driver := newV2IntegrationDriver(t, 2, 2, nil) + ctx := context.Background() + first, err := driver.pool.Acquire(ctx) + require.NoError(t, err) + defer first.Release() + second, err := driver.pool.Acquire(ctx) + require.NoError(t, err) + defer second.Release() + require.NotSame(t, first.Conn(), second.Conn()) + + firstCache, ok := driver.runtime.provider.CacheForConnection(first.Conn()).(*connectionTranslationCache) + require.True(t, ok) + secondCache, ok := driver.runtime.provider.CacheForConnection(second.Conn()).(*connectionTranslationCache) + require.True(t, ok) + _, _, err = firstCache.TranslateWithPolicy("RETURN 1", 1, nil, "integration", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + require.Equal(t, 1, firstCache.statsSnapshot().Entries) + require.Zero(t, secondCache.statsSnapshot().Entries) +} + +func TestV2ConnectionCloseAndRejectedReleaseDestroyCaches(t *testing.T) { + t.Run("pool reset retires state and replacement starts empty", func(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 2, nil) + setUpV2IntegrationGraph(t, driver) + _, err := snapshotQuery(context.Background(), driver, v2ScalarQuery(), nil) + require.NoError(t, err) + before := driver.TranslationCacheStats() + require.Len(t, before.Connections, 1) + oldID := before.Connections[0].ID + + driver.pool.Reset() + requireEventually(t, func() bool { + stats := driver.TranslationCacheStats() + return stats.LiveConnections == 0 && stats.RetiredConnections >= 1 + }, "physical connection cache retirement") + + _, err = snapshotQuery(context.Background(), driver, v2ScalarQuery(), nil) + require.NoError(t, err) + after := driver.TranslationCacheStats() + require.Len(t, after.Connections, 1) + require.NotEqual(t, oldID, after.Connections[0].ID) + require.GreaterOrEqual(t, after.Aggregate.Misses, uint64(2)) + }) + + t.Run("rejected release closes registered cache", func(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 2, func(*pgx.Conn) bool { return false }) + setUpV2IntegrationGraph(t, driver) + conn, err := driver.pool.Acquire(context.Background()) + require.NoError(t, err) + physical := conn.Conn() + cache, ok := driver.runtime.provider.CacheForConnection(physical).(*connectionTranslationCache) + require.True(t, ok) + _, _, err = cache.TranslateWithPolicy("RETURN 1", 1, nil, "integration", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + conn.Release() + requireEventually(t, func() bool { + return driver.runtime.provider.CacheForConnection(physical) == nil + }, "rejected-release cache cleanup") + }) +} + +func TestV2SchemaGenerationAndCapacityPreventStaleTranslationReuse(t *testing.T) { + driver := newV2IntegrationDriver(t, 1, 1, nil) + fixture := setUpV2IntegrationGraph(t, driver) + ctx := context.Background() + _, err := snapshotQuery(ctx, driver, v2ScalarQuery(), nil) + require.NoError(t, err) + before := driver.TranslationCacheStats() + require.Len(t, before.Connections, 1) + connectionID := before.Connections[0].ID + generation := before.SchemaGeneration + + require.NoError(t, driver.RefreshKinds(ctx)) + _, err = snapshotQuery(ctx, driver, v2ScalarQuery(), nil) + require.NoError(t, err) + afterGeneration := driver.TranslationCacheStats() + require.Equal(t, connectionID, afterGeneration.Connections[0].ID) + require.Greater(t, afterGeneration.SchemaGeneration, generation) + require.GreaterOrEqual(t, afterGeneration.Aggregate.Misses, uint64(2)) + + _, err = snapshotQuery(ctx, driver, "MATCH (n:PGV2IntegrationNode) WHERE id(n) = $id RETURN n", map[string]any{"id": fixture.start}) + require.NoError(t, err) + stats := driver.TranslationCacheStats() + require.LessOrEqual(t, stats.Connections[0].Translation.Entries, 1) + require.Equal(t, 1, stats.CapacityPerConnection) +} + +func TestFinalizedDriverPreservesCoreResultsAndFailureBoundaries(t *testing.T) { + configuredDriver := newV2IntegrationDriver(t, 1, 4, nil) + fixture := setUpV2IntegrationGraph(t, configuredDriver) + poolConfig, err := pgxpool.ParseConfig(postgresV2IntegrationConnectionString(t)) + require.NoError(t, err) + pool, err := NewPool(poolConfig) + require.NoError(t, err) + defaultDriver := NewDriver(0, pool) + t.Cleanup(func() { require.NoError(t, defaultDriver.Close(context.Background())) }) + require.NoError(t, defaultDriver.AssertSchema(context.Background(), v2IntegrationSchema)) + + for _, testCase := range []struct { + name string + query string + parameters map[string]any + }{ + {"scalar", v2ScalarQuery(), nil}, + {"node", "MATCH (n:PGV2IntegrationNode) WHERE id(n) = $id RETURN n", map[string]any{"id": fixture.start}}, + {"relationship", "MATCH ()-[r:PGV2IntegrationEdge]->() RETURN r", nil}, + {"path", "MATCH p = (a:PGV2IntegrationNode)-[:PGV2IntegrationEdge]->(b:PGV2IntegrationNode) WHERE id(a) = $start_id AND id(b) = $end_id RETURN p", map[string]any{"start_id": fixture.start, "end_id": fixture.end}}, + {"no rows", "MATCH (n:PGV2IntegrationNode) WHERE id(n) = $id RETURN n", map[string]any{"id": graph.ID(0)}}, + } { + t.Run(testCase.name, func(t *testing.T) { + configuredRows, configuredErr := snapshotQuery(context.Background(), configuredDriver, testCase.query, testCase.parameters) + defaultRows, defaultErr := snapshotQuery(context.Background(), defaultDriver, testCase.query, testCase.parameters) + require.NoError(t, configuredErr) + require.NoError(t, defaultErr) + require.True(t, reflect.DeepEqual(configuredRows, defaultRows), "configured rows %v differ from default rows %v", configuredRows, defaultRows) + }) + } + + _, configuredTranslationErr := snapshotQuery(context.Background(), configuredDriver, "MATCH (", nil) + _, defaultTranslationErr := snapshotQuery(context.Background(), defaultDriver, "MATCH (", nil) + require.Error(t, configuredTranslationErr) + require.Error(t, defaultTranslationErr) + require.Error(t, rawExecutionError(context.Background(), configuredDriver)) + require.Error(t, rawExecutionError(context.Background(), defaultDriver)) + + rollback := errors.New("force rollback") + for _, database := range []graph.Database{configuredDriver, defaultDriver} { + err := database.WriteTransaction(context.Background(), func(tx graph.Transaction) error { + if _, err := tx.CreateNode(graph.NewProperties(), v2IntegrationNodeKind); err != nil { + return err + } + return rollback + }) + require.ErrorIs(t, err, rollback) + rows, err := snapshotQuery(context.Background(), database, v2ScalarQuery(), nil) + require.NoError(t, err) + require.Equal(t, [][]any{{int64(2)}}, rows) + } + + cancelCtx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + cancelErr := configuredDriver.ReadTransaction(cancelCtx, func(tx graph.Transaction) error { + result := tx.Raw("select pg_sleep(1)", nil) + defer result.Close() + for result.Next() { + } + return result.Error() + }) + require.Error(t, cancelErr) + rows, err := snapshotQuery(context.Background(), configuredDriver, v2ScalarQuery(), nil) + require.NoError(t, err) + require.Equal(t, [][]any{{int64(2)}}, rows) +} diff --git a/drivers/pg/connection_translation_cache.go b/drivers/pg/connection_translation_cache.go new file mode 100644 index 00000000..358d73fc --- /dev/null +++ b/drivers/pg/connection_translation_cache.go @@ -0,0 +1,778 @@ +package pg + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "sync" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + dawgscache "github.com/specterops/dawgs/cache" + "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// translationEntry contains only immutable SQL and parameter-source metadata. +type translationEntry struct { + // sql is the immutable PostgreSQL statement reused by cache hits. + sql string + + // parameterSources maps generated parameters to caller-provided Cypher values. + parameterSources map[string]string +} + +// bind negotiates current caller values for the immutable cached parameter mapping. +func (s translationEntry) bind(parameters map[string]any) (map[string]any, error) { + // pgx.NamedArgs accepts nil for statements with no placeholders. Returning + // nil avoids allocating an empty map for the common parameter-free cached + // shortest-path shape while preserving its execution semantics. + if len(s.parameterSources) == 0 { + return nil, nil + } + bound := make(map[string]any, len(s.parameterSources)) + for identifier, source := range s.parameterSources { + value, found := parameters[source] + if !found { + return nil, fmt.Errorf("cached translation requires missing parameter source %q", source) + } + + negotiated, err := pgsql.NegotiateValue(value) + if err != nil { + return nil, fmt.Errorf("negotiate cached parameter %s: %w", source, err) + } + bound[identifier] = negotiated + } + return bound, nil +} + +// connectionTranslationCache wraps SIEVE so cache closure and counters remain +// coherent with provider lifecycle operations. +type connectionTranslationCache struct { + // lock serializes cache lifecycle, statistics, and SIEVE operations. + lock sync.Mutex + + // capacity bounds translations retained by this physical connection. + capacity int + + // shared provides the pool-wide tier for immutable shortest-path templates. + shared *sharedTemplateCache + + // generation returns the current schema-sensitive cache generation. + generation func() uint64 + + // sieve stores reusable translations for this physical connection. + sieve dawgscache.Cache[translationKey, translationEntry] + + // closed prevents subsequent cache publication after connection retirement. + closed bool + + // stats records query-text-free local cache activity. + stats TranslationCacheStats +} + +// The CypherTranslationCache assertion ensures the local cache remains compatible with pg transactions. +var _ CypherTranslationCache = (*connectionTranslationCache)(nil) + +// newConnectionTranslationCache initializes one physical connection's bounded translation cache. +func newConnectionTranslationCache(capacity int, generation func() uint64, shared *sharedTemplateCache) *connectionTranslationCache { + cache := &connectionTranslationCache{ + capacity: capacity, + generation: generation, + shared: shared, + stats: TranslationCacheStats{ + Capacity: capacity, + }, + } + if capacity > 0 { + cache.sieve = dawgscache.NewSieve[translationKey, translationEntry](capacity) + } + return cache +} + +// TranslateWithPolicy returns a cached immutable translation with fresh +// bindings, or runs build without retention when this cache is unavailable. +func (s *connectionTranslationCache) TranslateWithPolicy(query string, graphID int32, parameters map[string]any, policyIdentity string, build func() (translate.Result, string, error)) (string, map[string]any, error) { + schemaGeneration := uint64(0) + if s.generation != nil { + schemaGeneration = s.generation() + } + key := newTranslationKey(query, graphID, parameters, policyIdentity, schemaGeneration) + + s.lock.Lock() + if s.closed || s.capacity == 0 || len(query) > MaxCachedCypherQueryBytes { + s.stats.Misses++ + s.stats.Bypasses++ + s.lock.Unlock() + return buildUncached(build) + } + if entry, found := s.sieve.Get(key); found { + s.stats.Hits++ + s.lock.Unlock() + bound, err := entry.bind(parameters) + if err != nil { + s.lock.Lock() + s.stats.BindingFailures++ + s.lock.Unlock() + return "", nil, err + } + return entry.sql, bound, nil + } + s.stats.Misses++ + s.lock.Unlock() + if isShortestPathQuery(query) { + if entry, found := s.shared.get(key); found { + bound, bindErr := entry.bind(parameters) + if bindErr != nil { + s.lock.Lock() + s.stats.BindingFailures++ + s.lock.Unlock() + return "", nil, bindErr + } + s.putL1(key, entry) + return entry.sql, bound, nil + } + } + + result, sql, err := build() + if err != nil { + return "", nil, err + } + if !cacheableTranslation(result, parameters) { + s.lock.Lock() + s.stats.Bypasses++ + s.lock.Unlock() + return sql, result.Parameters, nil + } + + entry := translationEntry{ + sql: strings.Clone(sql), + parameterSources: cloneParameterSources(result.ParameterSources), + } + key.query = strings.Clone(key.query) + + if isShortestPathQuery(query) { + s.shared.put(key, entry) + } + s.putL1(key, entry) + + return sql, result.Parameters, nil +} + +// putL1 retains a local translation unless the connection has closed or already has it. +func (s *connectionTranslationCache) putL1(key translationKey, entry translationEntry) { + s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + s.stats.Bypasses++ + return + } + if _, exists := s.sieve.Get(key); !exists { + if s.sieve.Stats().Size() >= int64(s.capacity) { + s.stats.Evictions++ + } + s.sieve.Put(key, entry) + s.stats.Insertions++ + } +} + +// isShortestPathQuery reports whether Cypher source selects the shortest-path template tier. +func isShortestPathQuery(query string) bool { + return strings.Contains(strings.ToLower(query), "shortestpath") +} + +// buildUncached executes a translation builder and returns its immediate SQL and bindings. +func buildUncached(build func() (translate.Result, string, error)) (string, map[string]any, error) { + result, sql, err := build() + if err != nil { + return "", nil, err + } + return sql, result.Parameters, nil +} + +// cloneParameterSources detaches cached source metadata from translator-owned storage. +func cloneParameterSources(parameterSources map[string]string) map[string]string { + cloned := make(map[string]string, len(parameterSources)) + for identifier, source := range parameterSources { + cloned[strings.Clone(identifier)] = strings.Clone(source) + } + return cloned +} + +// statsSnapshot returns a consistent local cache snapshot with current occupancy. +func (s *connectionTranslationCache) statsSnapshot() TranslationCacheStats { + s.lock.Lock() + defer s.lock.Unlock() + stats := s.stats + if s.sieve != nil { + stats.Entries = int(s.sieve.Stats().Size()) + } + return stats +} + +// close drops retained entries and prevents an already acquired cache handle +// from publishing a translation after its physical connection has closed. +func (s *connectionTranslationCache) close() TranslationCacheStats { + s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + return s.stats + } + + s.closed = true + final := s.stats + if s.sieve != nil { + final.Entries = int(s.sieve.Stats().Size()) + s.sieve = nil + } + s.stats.Entries = 0 + return final +} + +// connectionState is registered only while its physical connection is live. +// It intentionally does not retain the physical connection identity. +type connectionState struct { + // id is the opaque provider-assigned diagnostic identifier. + id uint64 + + // cache owns reusable translations while the physical connection remains live. + cache *connectionTranslationCache + + // workspaceReadyGeneration records the schema generation with initialized workspaces. + workspaceReadyGeneration uint64 + + // workspace records setup and reuse activity for the physical connection. + workspace TraversalWorkspaceStats + + // preparedStatements contains hashes of selected warmed statements. + preparedStatements map[[sha256.Size]byte]struct{} + + // prepared records warm-up outcomes for the physical connection. + prepared PreparedStatementStats +} + +// connectionCacheProvider maps physical connection identities to their cache +// state. Map keys are process-local implementation details and are never +// exposed through diagnostics or cache keys. +type connectionCacheProvider struct { + // lock serializes physical connection state and aggregate diagnostics. + lock sync.RWMutex + + // capacity bounds translations retained on each physical connection. + capacity int + + // minConnections reports the configured pgx pool lower connection bound. + minConnections int32 + + // maxConnections reports the configured pgx pool upper connection bound. + maxConnections int32 + + // generation invalidates schema-sensitive connection state. + generation uint64 + + // nextID allocates opaque identifiers for newly registered connections. + nextID uint64 + + // closed prevents registration and publication after provider teardown. + closed bool + + // states maps live physical connections to their provider-owned state. + states map[*pgx.Conn]*connectionState + + // retiredConnections counts connections removed from states. + retiredConnections uint64 + + // retiredStats accumulates translation-cache activity from retired connections. + retiredStats TranslationCacheStats + + // retiredPrepared accumulates warm-up activity from retired connections. + retiredPrepared PreparedStatementStats + + // sqlGeneration accumulates query-text-free SQL generation timing. + sqlGeneration SQLGenerationStats + + // strategySelection accumulates production traversal selection telemetry. + strategySelection StrategySelectionStats + + // routeDecisions accumulates topology route-decision telemetry. + routeDecisions TraversalRouteDecisionStats + + // shapeCapacity bounds retained query-digest structural classifications. + shapeCapacity int + + // shapeCache maps query digests to immutable structural classifications. + shapeCache map[[sha256.Size]byte]TraversalShape + + // shapeOrder preserves insertion order for bounded shape-cache eviction. + shapeOrder [][sha256.Size]byte + + // shapeStats records query-text-free shape-cache activity. + shapeStats TraversalShapeCacheStats + + // sharedTemplates holds immutable shortest-path templates common to the pool. + sharedTemplates *sharedTemplateCache +} + +// The CypherTranslationCacheProvider assertion preserves pg transaction cache selection. +var _ CypherTranslationCacheProvider = (*connectionCacheProvider)(nil) + +// The StableSnapshotTraversalWorkspaceProvider assertion preserves workspace setup support. +var _ StableSnapshotTraversalWorkspaceProvider = (*connectionCacheProvider)(nil) + +// The LazyStableSnapshotTraversalWorkspaceProvider assertion preserves lazy workspace setup support. +var _ LazyStableSnapshotTraversalWorkspaceProvider = (*connectionCacheProvider)(nil) + +// The SQLGenerationProfileCollector assertion preserves SQL timing telemetry collection. +var _ SQLGenerationProfileCollector = (*connectionCacheProvider)(nil) + +// The TraversalStrategySelectionCollector assertion preserves routing telemetry collection. +var _ TraversalStrategySelectionCollector = (*connectionCacheProvider)(nil) + +// The TraversalShapeCacheProvider assertion preserves bounded structural classification caching. +var _ TraversalShapeCacheProvider = (*connectionCacheProvider)(nil) + +// The TraversalRouteDecisionCollector assertion preserves topology decision telemetry collection. +var _ TraversalRouteDecisionCollector = (*connectionCacheProvider)(nil) + +// TraversalShapeFor caches a bounded classifier result by a query digest. It +// never retains source text, values, parsed ASTs, or database state. +func (s *connectionCacheProvider) TraversalShapeFor(query string, classify func() (TraversalShape, error)) (TraversalShape, error) { + if classify == nil { + return TraversalShape{}, fmt.Errorf("traversal shape classifier is required") + } + if s == nil || len(query) > MaxCachedCypherQueryBytes { + return classify() + } + identity := sha256.Sum256([]byte(strings.TrimSpace(query))) + s.lock.Lock() + if shape, found := s.shapeCache[identity]; found { + s.shapeStats.Hits++ + s.lock.Unlock() + return shape, nil + } + s.shapeStats.Misses++ + s.lock.Unlock() + + shape, err := classify() + if err != nil { + return TraversalShape{}, err + } + if s.shapeCapacity == 0 { + return shape, nil + } + s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + return shape, nil + } + if cached, found := s.shapeCache[identity]; found { + return cached, nil + } + if len(s.shapeOrder) == s.shapeCapacity { + delete(s.shapeCache, s.shapeOrder[0]) + s.shapeOrder = s.shapeOrder[1:] + } + s.shapeCache[identity] = shape + s.shapeOrder = append(s.shapeOrder, identity) + s.shapeStats.Entries = len(s.shapeCache) + return shape, nil +} + +// RecordTraversalStrategySelection records an observation-only routing +// outcome. No per-query, SQL, graph, parameter, or decision data is retained. +func (s *connectionCacheProvider) RecordTraversalStrategySelection(selection TraversalStrategySelection) { + if s == nil { + return + } + s.lock.Lock() + defer s.lock.Unlock() + if selection.Reason == "shape_unavailable" { + s.strategySelection.ShapeUnavailable++ + } + if selection.Mode == "exact_query_canary" { + s.strategySelection.ExactQueryCanary++ + } else if selection.Mode == "structural_authorized" { + s.strategySelection.StructuralAuthorized++ + } else if selection.Mode == "topology_selected" { + s.strategySelection.TopologySelected++ + } else if selection.Mode == "structural_shadow" { + s.strategySelection.StructuralShadow++ + } else { + s.strategySelection.Incumbent++ + } +} + +// RecordTraversalRouteDecision records a query-text-free topology routing outcome. +func (s *connectionCacheProvider) RecordTraversalRouteDecision(decision TraversalRouteDecision) { + if s == nil { + return + } + s.lock.Lock() + defer s.lock.Unlock() + switch decision.Reason { + case "topology_route_disabled": + s.routeDecisions.Disabled++ + case "topology_synopsis_unavailable": + s.routeDecisions.SynopsisUnavailable++ + case "topology_route_shadow_miss": + s.routeDecisions.ShadowMiss++ + case "topology_route_shadow_hit": + s.routeDecisions.ShadowHit++ + case "topology_route_candidate_hit": + s.routeDecisions.CandidateHit++ + case "topology_route_first_use_candidate": + s.routeDecisions.FirstUseCandidate++ + case "topology_estimate_rejected": + s.routeDecisions.EstimateRejected++ + case "topology_route_capacity": + s.routeDecisions.Capacity++ + case "topology_route_parameters_unverifiable": + s.routeDecisions.ParametersInvalid++ + } +} + +// RecordSQLGenerationProfile retains query-text-free timing totals for the +// v2 architecture. A profile is recorded after pgx has returned a row stream, +// not after its rows are consumed. +func (s *connectionCacheProvider) RecordSQLGenerationProfile(profile SQLGenerationProfile) { + if s == nil { + return + } + s.lock.Lock() + defer s.lock.Unlock() + if profile.QueryClass == "shortest_path" { + s.sqlGeneration.ShortestPath.add(profile) + } else { + s.sqlGeneration.Other.add(profile) + } +} + +// DeferStableSnapshotTraversalWorkspaces keeps V2 ordinary repeatable-read +// transactions free of temporary shortest-path workspace initialization. +func (s *connectionCacheProvider) DeferStableSnapshotTraversalWorkspaces() bool { + return true +} + +// newConnectionCacheProvider initializes cache state shared by one v2 pool. +func newConnectionCacheProvider(config RuntimeConfig) (*connectionCacheProvider, error) { + if err := config.validate(); err != nil { + return nil, err + } + poolConfig := config.resolvedPoolConfig() + return &connectionCacheProvider{ + capacity: config.TranslationCacheEntries, + shapeCapacity: config.TranslationCacheEntries, + shapeCache: map[[sha256.Size]byte]TraversalShape{}, + shapeStats: TraversalShapeCacheStats{Capacity: config.TranslationCacheEntries}, + sharedTemplates: newSharedTemplateCache(config.SharedShortestPathTemplateEntries), + minConnections: poolConfig.MinConnections, + maxConnections: poolConfig.MaxConnections, + generation: 1, + states: map[*pgx.Conn]*connectionState{}, + }, nil +} + +// CacheForConnection returns the cache owned by conn's physical connection. +// A connection that was not registered or was already removed bypasses +// retention through the v1 transaction seam. +func (s *connectionCacheProvider) CacheForConnection(conn *pgx.Conn) CypherTranslationCache { + if s == nil || conn == nil { + return nil + } + s.lock.RLock() + state := s.states[conn] + s.lock.RUnlock() + if state == nil { + return nil + } + return state.cache +} + +// EnsureStableSnapshotTraversalWorkspaces initializes a leased connection's +// reusable traversal workspace at most once per schema generation. The setup +// is never marked ready until PostgreSQL accepts it successfully. +func (s *connectionCacheProvider) EnsureStableSnapshotTraversalWorkspaces(ctx context.Context, conn *pgxpool.Conn) error { + if conn == nil { + return fmt.Errorf("PostgreSQL connection is required for traversal workspace setup") + } + return s.ensureWorkspaceForConnection(conn.Conn(), func() error { + return EnsureStableSnapshotTraversalWorkspaces(ctx, conn) + }) +} + +// ensureWorkspaceForConnection initializes a connection workspace once per schema generation. +func (s *connectionCacheProvider) ensureWorkspaceForConnection(conn *pgx.Conn, initialize func() error) error { + if initialize == nil { + return fmt.Errorf("traversal workspace initializer is required") + } + if s == nil || conn == nil { + return initialize() + } + + s.lock.Lock() + state := s.states[conn] + generation := s.generation + if state != nil && !s.closed && state.workspaceReadyGeneration == generation { + state.workspace.Reuses++ + s.lock.Unlock() + return nil + } + s.lock.Unlock() + + err := initialize() + + s.lock.Lock() + defer s.lock.Unlock() + if state != nil && s.states[conn] == state { + if err != nil { + state.workspace.Failures++ + return err + } + state.workspace.Initializations++ + if !s.closed && s.generation == generation { + state.workspaceReadyGeneration = generation + state.workspace.Ready = true + } + } + return err +} + +// registerConnection allocates state only after the pool's earlier +// AfterConnect initialization has completed successfully. +func (s *connectionCacheProvider) registerConnection(conn *pgx.Conn) { + if s == nil || conn == nil { + return + } + s.lock.Lock() + defer s.lock.Unlock() + if s.closed { + return + } + if _, exists := s.states[conn]; exists { + return + } + s.nextID++ + s.states[conn] = &connectionState{ + id: s.nextID, + cache: newConnectionTranslationCache(s.capacity, func() uint64 { + s.lock.RLock() + defer s.lock.RUnlock() + return s.generation + }, s.sharedTemplates), + preparedStatements: map[[sha256.Size]byte]struct{}{}, + } +} + +// preparedStatementWarmup binds normalized SQL to its stable prepared-statement identity. +type preparedStatementWarmup struct { + // identity is the SHA-256 digest used for deduplication and pgx naming. + identity [sha256.Size]byte + + // sql is the normalized statement text prepared on physical connections. + sql string +} + +// normalizePreparedStatementWarmups validates, deduplicates, and clones selected SQL. +func normalizePreparedStatementWarmups(statements []string) ([]preparedStatementWarmup, error) { + warmups := make([]preparedStatementWarmup, 0, len(statements)) + seen := map[[sha256.Size]byte]struct{}{} + for _, statement := range statements { + statement = strings.TrimSpace(statement) + if statement == "" { + return nil, fmt.Errorf("prepared statement SQL must not be empty") + } + if len(statement) > MaxCachedCypherQueryBytes { + return nil, fmt.Errorf("prepared statement SQL exceeds %d bytes", MaxCachedCypherQueryBytes) + } + identity := sha256.Sum256([]byte(statement)) + if _, exists := seen[identity]; exists { + continue + } + seen[identity] = struct{}{} + warmups = append(warmups, preparedStatementWarmup{identity: identity, sql: strings.Clone(statement)}) + } + return warmups, nil +} + +// pgxStatementCacheName derives the name pgx uses for a warmed statement. +func pgxStatementCacheName(identity [sha256.Size]byte) string { + return "stmtcache_" + hex.EncodeToString(identity[:24]) +} + +// warmStatementsForConnection prepares SQL using pgx's CacheStatement naming +// convention. pgx then adopts the already-prepared server statement on its +// first regular CacheStatement execution instead of preparing it twice. +func (s *connectionCacheProvider) warmStatementsForConnection(conn *pgx.Conn, statements []preparedStatementWarmup, prepare func(string, string) error) error { + if len(statements) == 0 { + return nil + } + if prepare == nil { + return fmt.Errorf("prepared statement initializer is required") + } + if s == nil || conn == nil { + return fmt.Errorf("registered PostgreSQL connection is required for statement warm-up") + } + + var errs []error + for _, statement := range statements { + s.lock.Lock() + state := s.states[conn] + if state == nil || s.closed { + s.lock.Unlock() + return fmt.Errorf("PostgreSQL connection is not registered for statement warm-up") + } + if _, prepared := state.preparedStatements[statement.identity]; prepared { + state.prepared.Reuses++ + s.lock.Unlock() + continue + } + state.prepared.Attempts++ + s.lock.Unlock() + + if err := prepare(pgxStatementCacheName(statement.identity), statement.sql); err != nil { + s.lock.Lock() + if s.states[conn] == state { + state.prepared.Failures++ + } + s.lock.Unlock() + errs = append(errs, err) + continue + } + + s.lock.Lock() + if s.states[conn] == state && !s.closed { + state.preparedStatements[statement.identity] = struct{}{} + state.prepared.Prepared++ + } + s.lock.Unlock() + } + return errors.Join(errs...) +} + +// removeConnection unregisters and closes state. It is idempotent so pool +// shutdown and failed initialization paths cannot retain connection state. +func (s *connectionCacheProvider) removeConnection(conn *pgx.Conn) { + if s == nil || conn == nil { + return + } + s.lock.Lock() + state := s.states[conn] + if state != nil { + delete(s.states, conn) + } + s.lock.Unlock() + if state == nil { + return + } + + stats := state.cache.close() + s.lock.Lock() + s.retiredConnections++ + stats.Entries = 0 + stats.Capacity = 0 + s.retiredStats.add(stats) + state.prepared.Entries = 0 + s.retiredPrepared.add(state.prepared) + s.lock.Unlock() +} + +// advanceSchemaGeneration invalidates schema-sensitive shape and workspace state. +func (s *connectionCacheProvider) advanceSchemaGeneration() uint64 { + s.lock.Lock() + defer s.lock.Unlock() + s.generation++ + s.shapeCache = map[[sha256.Size]byte]TraversalShape{} + s.shapeOrder = nil + s.shapeStats.Entries = 0 + for _, state := range s.states { + state.workspace.Ready = false + } + return s.generation +} + +// close retires all provider state after the pool lifecycle has ended. +func (s *connectionCacheProvider) close() { + if s == nil { + return + } + s.lock.Lock() + if s.closed { + s.lock.Unlock() + return + } + s.closed = true + s.shapeCache = nil + s.shapeOrder = nil + s.shapeStats.Entries = 0 + states := make([]*connectionState, 0, len(s.states)) + for _, state := range s.states { + states = append(states, state) + } + s.states = nil + s.lock.Unlock() + + for _, state := range states { + stats := state.cache.close() + s.lock.Lock() + s.retiredConnections++ + stats.Entries = 0 + stats.Capacity = 0 + s.retiredStats.add(stats) + state.prepared.Entries = 0 + s.retiredPrepared.add(state.prepared) + s.lock.Unlock() + } +} + +// stats returns a query-text-free snapshot spanning live and retired connections. +func (s *connectionCacheProvider) stats() Stats { + if s == nil { + return Stats{} + } + s.lock.RLock() + stats := Stats{ + SchemaGeneration: s.generation, + CapacityPerConnection: s.capacity, + MinConnections: s.minConnections, + MaxConnections: s.maxConnections, + LiveConnections: len(s.states), + RetiredConnections: s.retiredConnections, + Aggregate: s.retiredStats, + PreparedStatements: s.retiredPrepared, + SQLGeneration: s.sqlGeneration, + StrategySelection: s.strategySelection, + TraversalShapeCache: s.shapeStats, + TraversalRouteDecision: s.routeDecisions, + SharedShortestPathTemplates: s.sharedTemplates.snapshot(), + Connections: make([]ConnectionCacheStats, 0, len(s.states)), + } + states := make([]*connectionState, 0, len(s.states)) + workspaceStats := make([]TraversalWorkspaceStats, 0, len(s.states)) + preparedStats := make([]PreparedStatementStats, 0, len(s.states)) + for _, state := range s.states { + states = append(states, state) + workspaceStats = append(workspaceStats, state.workspace) + prepared := state.prepared + prepared.Entries = len(state.preparedStatements) + preparedStats = append(preparedStats, prepared) + stats.TraversalWorkspace.add(state.workspace) + stats.PreparedStatements.add(prepared) + } + s.lock.RUnlock() + + for index, state := range states { + connectionStats := state.cache.statsSnapshot() + stats.Connections = append(stats.Connections, ConnectionCacheStats{ + ID: state.id, + Translation: connectionStats, + TraversalWorkspace: workspaceStats[index], + PreparedStatements: preparedStats[index], + }) + stats.Aggregate.add(connectionStats) + } + stats.Aggregate.Capacity = stats.CapacityPerConnection * stats.LiveConnections + return stats +} diff --git a/drivers/pg/connection_translation_cache_test.go b/drivers/pg/connection_translation_cache_test.go new file mode 100644 index 00000000..593d55fd --- /dev/null +++ b/drivers/pg/connection_translation_cache_test.go @@ -0,0 +1,463 @@ +package pg + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/stretchr/testify/require" +) + +func newTestCache(t *testing.T, capacity int) (*connectionCacheProvider, *connectionTranslationCache, *pgx.Conn) { + t.Helper() + provider, err := newConnectionCacheProvider(RuntimeConfig{TranslationCacheEntries: capacity}) + require.NoError(t, err) + conn := &pgx.Conn{} + provider.registerConnection(conn) + cache, ok := provider.CacheForConnection(conn).(*connectionTranslationCache) + require.True(t, ok) + return provider, cache, conn +} + +func cacheableBuild(sql string, value any) func() (translate.Result, string, error) { + return func() (translate.Result, string, error) { + return translate.Result{ + Parameters: map[string]any{"i0": value}, + ParameterSources: map[string]string{"i0": "id"}, + }, sql, nil + } +} + +// TestConnectionTranslationCacheRebindsCurrentValues verifies cached SQL does +// not retain caller parameter values. +func TestConnectionTranslationCacheRebindsCurrentValues(t *testing.T) { + _, cache, _ := newTestCache(t, 2) + + sql, parameters, err := cache.TranslateWithPolicy(" MATCH (n) WHERE id(n) = $id RETURN n ", 1, map[string]any{"id": int64(1)}, "incumbent", cacheableBuild("select @i0", int64(1))) + require.NoError(t, err) + require.Equal(t, "select @i0", sql) + require.Equal(t, int64(1), parameters["i0"]) + + sql, parameters, err = cache.TranslateWithPolicy("MATCH (n) WHERE id(n) = $id RETURN n", 1, map[string]any{"id": int64(2)}, "incumbent", cacheableBuild("wrong", int64(999))) + require.NoError(t, err) + require.Equal(t, "select @i0", sql) + require.Equal(t, int64(2), parameters["i0"]) + require.Equal(t, TranslationCacheStats{Hits: 1, Misses: 1, Insertions: 1, Entries: 1, Capacity: 2}, cache.statsSnapshot()) +} + +func TestStrategySelectionStatsRemainQueryTextFree(t *testing.T) { + provider, _, _ := newTestCache(t, 2) + provider.RecordTraversalStrategySelection(TraversalStrategySelection{Mode: "incumbent", Reason: "shape_unavailable"}) + provider.RecordTraversalStrategySelection(TraversalStrategySelection{Mode: "exact_query_canary", Reason: "exact_query_authorized"}) + provider.RecordTraversalStrategySelection(TraversalStrategySelection{Mode: "structural_shadow", Reason: "structural_bucket-qualified"}) + provider.RecordTraversalStrategySelection(TraversalStrategySelection{Mode: "structural_authorized", Reason: "structural_bucket-qualified"}) + + stats := provider.stats().StrategySelection + require.Equal(t, uint64(1), stats.Incumbent) + require.Equal(t, uint64(1), stats.ExactQueryCanary) + require.Equal(t, uint64(1), stats.StructuralShadow) + require.Equal(t, uint64(1), stats.StructuralAuthorized) + require.Equal(t, uint64(1), stats.ShapeUnavailable) +} + +func TestTraversalShapeCacheRetainsOnlyBoundedClassifications(t *testing.T) { + provider, _, _ := newTestCache(t, 1) + builds := 0 + classify := func() (TraversalShape, error) { + builds++ + return TraversalShape{Version: TraversalShapeVersion, Family: "SP", Fingerprint: "shape"}, nil + } + + first, err := provider.TraversalShapeFor(" MATCH p = shortestPath((a)-[*]->(b)) RETURN p ", classify) + require.NoError(t, err) + second, err := provider.TraversalShapeFor("MATCH p = shortestPath((a)-[*]->(b)) RETURN p", classify) + require.NoError(t, err) + require.Equal(t, first, second) + require.Equal(t, 1, builds) + + _, err = provider.TraversalShapeFor("MATCH p = shortestPath((b)-[*]->(c)) RETURN p", classify) + require.NoError(t, err) + require.Equal(t, 2, builds) + stats := provider.stats().TraversalShapeCache + require.Equal(t, uint64(1), stats.Hits) + require.Equal(t, uint64(2), stats.Misses) + require.Equal(t, 1, stats.Entries) + require.Equal(t, 1, stats.Capacity) + + provider.advanceSchemaGeneration() + stats = provider.stats().TraversalShapeCache + require.Zero(t, stats.Entries) +} + +// TestConnectionTranslationCachePartitionsInputs verifies all inputs that can +// change SQL occupy independent entries. +func TestConnectionTranslationCachePartitionsInputs(t *testing.T) { + provider, cache, _ := newTestCache(t, 8) + builds := 0 + build := func() (translate.Result, string, error) { + builds++ + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + } + + for _, request := range []struct { + query string + graph int32 + params map[string]any + policy string + }{ + {"RETURN $value", 1, map[string]any{"value": int64(1)}, "one"}, + {"RETURN $value", 2, map[string]any{"value": int64(1)}, "one"}, + {"RETURN $value", 1, map[string]any{"value": "1"}, "one"}, + {"RETURN $value", 1, map[string]any{"other": int64(1)}, "one"}, + {"RETURN $value", 1, map[string]any{"value": int64(1)}, "two"}, + } { + _, _, err := cache.TranslateWithPolicy(request.query, request.graph, request.params, request.policy, build) + require.NoError(t, err) + } + provider.advanceSchemaGeneration() + _, _, err := cache.TranslateWithPolicy("RETURN $value", 1, map[string]any{"value": int64(1)}, "one", build) + require.NoError(t, err) + + require.Equal(t, 6, builds) +} + +func TestConnectionTranslationCacheCapacityAndBypasses(t *testing.T) { + t.Run("capacity is enforced", func(t *testing.T) { + _, cache, _ := newTestCache(t, 1) + build := func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + } + _, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "one", build) + require.NoError(t, err) + _, _, err = cache.TranslateWithPolicy("RETURN 2", 1, nil, "one", build) + require.NoError(t, err) + stats := cache.statsSnapshot() + require.Equal(t, 1, stats.Entries) + require.Equal(t, uint64(1), stats.Evictions) + }) + + t.Run("zero capacity bypasses retention", func(t *testing.T) { + _, cache, _ := newTestCache(t, 0) + builds := 0 + build := func() (translate.Result, string, error) { + builds++ + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + } + for range 2 { + _, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "one", build) + require.NoError(t, err) + } + require.Equal(t, 2, builds) + require.Equal(t, TranslationCacheStats{Misses: 2, Bypasses: 2}, cache.statsSnapshot()) + }) +} + +func TestConnectionTranslationCacheRejectsUncacheableResultsAndClonesSources(t *testing.T) { + _, cache, _ := newTestCache(t, 2) + sources := map[string]string{"i0": "id"} + builds := 0 + build := func() (translate.Result, string, error) { + builds++ + return translate.Result{Parameters: map[string]any{"i0": int64(1)}, ParameterSources: sources}, "select @i0", nil + } + + _, _, err := cache.TranslateWithPolicy("RETURN $id", 1, map[string]any{"id": int64(1)}, "one", build) + require.NoError(t, err) + sources["i0"] = "mutated" + _, parameters, err := cache.TranslateWithPolicy("RETURN $id", 1, map[string]any{"id": int64(2)}, "one", build) + require.NoError(t, err) + require.Equal(t, int64(2), parameters["i0"]) + require.Equal(t, 1, builds) + + uncacheable := func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{"i0": int64(1)}}, "select @i0", nil + } + _, _, err = cache.TranslateWithPolicy("RETURN 2", 1, nil, "one", uncacheable) + require.NoError(t, err) + stats := cache.statsSnapshot() + require.Equal(t, 1, stats.Entries) + require.Equal(t, uint64(1), stats.Bypasses) +} + +func TestConnectionTranslationCacheDoesNotRetainFailures(t *testing.T) { + _, cache, _ := newTestCache(t, 2) + expected := errors.New("translation failed") + _, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "one", func() (translate.Result, string, error) { + return translate.Result{}, "partial", expected + }) + require.ErrorIs(t, err, expected) + require.Zero(t, cache.statsSnapshot().Entries) +} + +func TestConnectionCacheProviderSeparatesAndRetiresPhysicalConnections(t *testing.T) { + provider, err := newConnectionCacheProvider(RuntimeConfig{TranslationCacheEntries: 2}) + require.NoError(t, err) + first, second := &pgx.Conn{}, &pgx.Conn{} + provider.registerConnection(first) + provider.registerConnection(second) + firstCache := provider.CacheForConnection(first).(*connectionTranslationCache) + secondCache := provider.CacheForConnection(second).(*connectionTranslationCache) + require.NotSame(t, firstCache, secondCache) + + _, _, err = firstCache.TranslateWithPolicy("RETURN 1", 1, nil, "one", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + require.Zero(t, secondCache.statsSnapshot().Entries) + + provider.removeConnection(first) + provider.removeConnection(first) + require.Nil(t, provider.CacheForConnection(first)) + _, _, err = firstCache.TranslateWithPolicy("RETURN 1", 1, nil, "one", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + + stats := provider.stats() + require.Equal(t, 1, stats.LiveConnections) + require.Equal(t, uint64(1), stats.RetiredConnections) + require.Len(t, stats.Connections, 1) + require.Len(t, provider.states, 1) // first was deleted; the second remains in the registry. +} + +func TestConnectionCacheProviderRejectsNegativeCapacity(t *testing.T) { + provider, err := newConnectionCacheProvider(RuntimeConfig{TranslationCacheEntries: -1}) + require.Nil(t, provider) + require.ErrorContains(t, err, "must not be negative") +} + +func TestConnectionCacheProviderCloseDropsStateAndPreventsResurrection(t *testing.T) { + provider, cache, conn := newTestCache(t, 2) + _, _, err := cache.TranslateWithPolicy("RETURN 1", 1, nil, "one", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + provider.close() + provider.close() + + require.Nil(t, provider.CacheForConnection(conn)) + stats := provider.stats() + require.Zero(t, stats.LiveConnections) + require.Equal(t, uint64(1), stats.RetiredConnections) + require.Empty(t, stats.Connections) + require.Nil(t, provider.states) + + _, _, err = cache.TranslateWithPolicy("RETURN 1", 1, nil, "one", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + require.Zero(t, cache.statsSnapshot().Entries) +} + +func TestConnectionCacheProviderStatsAndCleanupAreRaceSafe(t *testing.T) { + provider, _, conn := newTestCache(t, 2) + const workers = 8 + var group sync.WaitGroup + group.Add(workers + 1) + for range workers { + go func() { + defer group.Done() + for range 100 { + _ = provider.stats() + } + }() + } + go func() { + defer group.Done() + provider.removeConnection(conn) + }() + group.Wait() + + require.Nil(t, provider.CacheForConnection(conn)) +} + +// TestConnectionWorkspaceReadinessTracksGenerationAndFailures verifies that a +// physical connection skips only successfully initialized workspaces and +// becomes unready after a schema-generation change or retirement. +func TestConnectionWorkspaceReadinessTracksGenerationAndFailures(t *testing.T) { + provider, _, conn := newTestCache(t, 2) + var calls int + initialize := func() error { + calls++ + return nil + } + + require.NoError(t, provider.ensureWorkspaceForConnection(conn, initialize)) + require.NoError(t, provider.ensureWorkspaceForConnection(conn, initialize)) + stats := provider.stats() + require.Equal(t, 1, calls) + require.Equal(t, uint64(1), stats.TraversalWorkspace.Initializations) + require.Equal(t, uint64(1), stats.TraversalWorkspace.Reuses) + require.True(t, stats.Connections[0].TraversalWorkspace.Ready) + + provider.advanceSchemaGeneration() + stats = provider.stats() + require.False(t, stats.Connections[0].TraversalWorkspace.Ready) + require.NoError(t, provider.ensureWorkspaceForConnection(conn, initialize)) + require.Equal(t, 2, calls) + + provider.removeConnection(conn) + require.Nil(t, provider.CacheForConnection(conn)) + require.NoError(t, provider.ensureWorkspaceForConnection(conn, initialize)) + require.Equal(t, 3, calls) +} + +func TestConnectionWorkspaceReadinessDoesNotMarkFailuresReady(t *testing.T) { + provider, _, conn := newTestCache(t, 2) + expected := errors.New("workspace setup failed") + require.ErrorIs(t, provider.ensureWorkspaceForConnection(conn, func() error { return expected }), expected) + stats := provider.stats() + require.Equal(t, uint64(1), stats.TraversalWorkspace.Failures) + require.False(t, stats.Connections[0].TraversalWorkspace.Ready) + + require.NoError(t, provider.ensureWorkspaceForConnection(conn, func() error { return nil })) + stats = provider.stats() + require.Equal(t, uint64(1), stats.TraversalWorkspace.Initializations) + require.True(t, stats.Connections[0].TraversalWorkspace.Ready) +} + +// TestPreparedStatementWarmupTracksOnlyStatementIdentities verifies that +// warm-up deduplicates SQL, reuses prepared statements, and drops state when +// its physical connection retires. +func TestPreparedStatementWarmupTracksOnlyStatementIdentities(t *testing.T) { + provider, _, conn := newTestCache(t, 2) + warmups, err := normalizePreparedStatementWarmups([]string{" select 1 ", "select 1"}) + require.NoError(t, err) + require.Len(t, warmups, 1) + + var names []string + require.NoError(t, provider.warmStatementsForConnection(conn, warmups, func(name, sql string) error { + names = append(names, name) + require.Equal(t, "select 1", sql) + return nil + })) + require.Equal(t, []string{pgxStatementCacheName(warmups[0].identity)}, names) + require.NoError(t, provider.warmStatementsForConnection(conn, warmups, func(string, string) error { + t.Fatal("already prepared statement must not be prepared again") + return nil + })) + + stats := provider.stats() + require.Equal(t, uint64(1), stats.PreparedStatements.Attempts) + require.Equal(t, uint64(1), stats.PreparedStatements.Prepared) + require.Equal(t, uint64(1), stats.PreparedStatements.Reuses) + require.Equal(t, 1, stats.PreparedStatements.Entries) + + provider.removeConnection(conn) + stats = provider.stats() + require.Equal(t, uint64(1), stats.PreparedStatements.Prepared) + require.Zero(t, stats.PreparedStatements.Entries) +} + +func TestPreparedStatementWarmupDoesNotRetainFailures(t *testing.T) { + provider, _, conn := newTestCache(t, 2) + warmups, err := normalizePreparedStatementWarmups([]string{"select 1"}) + require.NoError(t, err) + expected := errors.New("prepare failed") + require.ErrorIs(t, provider.warmStatementsForConnection(conn, warmups, func(string, string) error { return expected }), expected) + stats := provider.stats() + require.Equal(t, uint64(1), stats.PreparedStatements.Attempts) + require.Equal(t, uint64(1), stats.PreparedStatements.Failures) + require.Zero(t, stats.PreparedStatements.Entries) + + _, err = normalizePreparedStatementWarmups([]string{""}) + require.ErrorContains(t, err, "must not be empty") +} + +// TestConnectionCacheProviderRecordsSQLGenerationProfiles verifies V2 +// aggregates query-text-free timing samples by shortest-path classification. +func TestConnectionCacheProviderRecordsSQLGenerationProfiles(t *testing.T) { + provider, err := newConnectionCacheProvider(DefaultRuntimeConfig()) + require.NoError(t, err) + + provider.RecordSQLGenerationProfile(SQLGenerationProfile{ + QueryClass: "shortest_path", + Parse: time.Millisecond, + Graph: 2 * time.Millisecond, + Policy: 3 * time.Millisecond, + Cache: 4 * time.Millisecond, + Translate: 5 * time.Millisecond, + Format: 6 * time.Millisecond, + Dispatch: 7 * time.Millisecond, + }) + provider.RecordSQLGenerationProfile(SQLGenerationProfile{QueryClass: "other", Parse: time.Millisecond}) + + stats := provider.stats().SQLGeneration + require.Equal(t, uint64(1), stats.ShortestPath.Count) + require.Equal(t, 5*time.Millisecond, stats.ShortestPath.Translate) + require.Equal(t, 7*time.Millisecond, stats.ShortestPath.Dispatch) + require.Equal(t, uint64(1), stats.Other.Count) + require.Equal(t, time.Millisecond, stats.Other.Parse) +} + +// TestSharedShortestPathTemplateCacheReusesCompilationAcrossConnections +// verifies the V2 L2 retains only immutable templates and still negotiates +// fresh caller values for a different physical connection. +func TestSharedShortestPathTemplateCacheReusesCompilationAcrossConnections(t *testing.T) { + provider, err := newConnectionCacheProvider(RuntimeConfig{TranslationCacheEntries: 2, SharedShortestPathTemplateEntries: 2}) + require.NoError(t, err) + firstConn, secondConn := &pgx.Conn{}, &pgx.Conn{} + provider.registerConnection(firstConn) + provider.registerConnection(secondConn) + first := provider.CacheForConnection(firstConn).(*connectionTranslationCache) + second := provider.CacheForConnection(secondConn).(*connectionTranslationCache) + + builds := 0 + build := func(sql string) func() (translate.Result, string, error) { + return func() (translate.Result, string, error) { + builds++ + return translate.Result{Parameters: map[string]any{"i0": int64(1)}, ParameterSources: map[string]string{"i0": "id"}}, sql, nil + } + } + query := "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $id RETURN p" + _, _, err = first.TranslateWithPolicy(query, 1, map[string]any{"id": int64(1)}, "incumbent", build("select @i0")) + require.NoError(t, err) + sql, params, err := second.TranslateWithPolicy(query, 1, map[string]any{"id": int64(2)}, "incumbent", build("wrong")) + require.NoError(t, err) + require.Equal(t, 1, builds) + require.Equal(t, "select @i0", sql) + require.Equal(t, int64(2), params["i0"]) + stats := provider.stats().SharedShortestPathTemplates + require.Equal(t, uint64(1), stats.Hits) + require.Equal(t, uint64(1), stats.Insertions) + require.Equal(t, 1, stats.Entries) +} + +func TestConnectionTranslationCacheBindingAvoidsEmptyMap(t *testing.T) { + _, cache, _ := newTestCache(t, 1) + _, parameters, err := cache.TranslateWithPolicy("MATCH p = shortestPath((s)-[*]->(e)) RETURN p", 1, nil, "incumbent", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + require.NoError(t, err) + require.Empty(t, parameters) + _, parameters, err = cache.TranslateWithPolicy("MATCH p = shortestPath((s)-[*]->(e)) RETURN p", 1, nil, "incumbent", func() (translate.Result, string, error) { + t.Fatal("cached translation must not rebuild") + return translate.Result{}, "", nil + }) + require.NoError(t, err) + require.Nil(t, parameters) +} + +func BenchmarkConnectionTranslationCacheParameterlessHit(b *testing.B) { + cache := newConnectionTranslationCache(1, nil, nil) + _, _, err := cache.TranslateWithPolicy("MATCH p = shortestPath((s)-[*]->(e)) RETURN p", 1, nil, "incumbent", func() (translate.Result, string, error) { + return translate.Result{Parameters: map[string]any{}, ParameterSources: map[string]string{}}, "select 1", nil + }) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for range b.N { + _, _, err := cache.TranslateWithPolicy("MATCH p = shortestPath((s)-[*]->(e)) RETURN p", 1, nil, "incumbent", func() (translate.Result, string, error) { + b.Fatal("cached translation must not rebuild") + return translate.Result{}, "", nil + }) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/drivers/pg/connection_translation_key.go b/drivers/pg/connection_translation_key.go new file mode 100644 index 00000000..d17520ea --- /dev/null +++ b/drivers/pg/connection_translation_key.go @@ -0,0 +1,40 @@ +package pg + +import "strings" + +// translationCacheKeyVersion partitions retained translations by key schema. +const translationCacheKeyVersion uint8 = 1 + +// translationKey identifies an immutable Cypher-to-SQL translation that is +// safe to reuse on one physical PostgreSQL connection. +type translationKey struct { + // version identifies the key schema used to construct this value. + version uint8 + + // query is normalized Cypher source without caller-owned backing storage. + query string + + // graphID scopes generated SQL to one graph partition. + graphID int32 + + // parameterTypes partitions translations by negotiated parameter types. + parameterTypes string + + // policyIdentity partitions translations by effective traversal policy. + policyIdentity string + + // schemaGeneration prevents reuse after schema-sensitive changes. + schemaGeneration uint64 +} + +// newTranslationKey derives the complete immutable cache identity for one translation. +func newTranslationKey(query string, graphID int32, parameters map[string]any, policyIdentity string, schemaGeneration uint64) translationKey { + return translationKey{ + version: translationCacheKeyVersion, + query: strings.TrimSpace(query), + graphID: graphID, + parameterTypes: TranslationParameterTypeKey(parameters), + policyIdentity: policyIdentity, + schemaGeneration: schemaGeneration, + } +} diff --git a/drivers/pg/driver.go b/drivers/pg/driver.go index 8df09a13..f6e34931 100644 --- a/drivers/pg/driver.go +++ b/drivers/pg/driver.go @@ -12,23 +12,43 @@ import ( ) var ( - batchWriteSize = defaultBatchWriteSize + // batchWriteSize is the process-wide flush threshold used by new batch operations. + batchWriteSize = defaultBatchWriteSize + + // readOnlyTxOptions configures transactions that must not mutate PostgreSQL state. readOnlyTxOptions = pgx.TxOptions{ AccessMode: pgx.ReadOnly, } + // readWriteTxOptions configures transactions that may mutate PostgreSQL state. readWriteTxOptions = pgx.TxOptions{ AccessMode: pgx.ReadWrite, } ) +// Config configures PostgreSQL transaction execution for one graph operation. type Config struct { - Options pgx.TxOptions - QueryExecMode pgx.QueryExecMode + // Options controls PostgreSQL transaction isolation and access mode. + Options pgx.TxOptions + + // QueryExecMode selects pgx's query execution protocol. + QueryExecMode pgx.QueryExecMode + + // QueryResultFormats selects the PostgreSQL wire format for returned columns. QueryResultFormats pgx.QueryResultFormats - BatchWriteSize int + + // BatchWriteSize is the number of mutations accumulated before a batch flushes. + BatchWriteSize int + + // initializeTraversalRuntimeAttestation prepares session-local receipt state before BEGIN. + initializeTraversalRuntimeAttestation bool + + // skipStableSnapshotTraversalWorkspaces prevents ordinary-expansion tool + // studies from paying unrelated SP/ASP temporary-workspace setup. + skipStableSnapshotTraversalWorkspaces bool } +// OptionSetQueryExecMode classifies option set query exec mode for downstream policy decisions. func OptionSetQueryExecMode(queryExecMode pgx.QueryExecMode) graph.TransactionOption { return func(config *graph.TransactionConfig) { if pgCfg, typeOK := config.DriverConfig.(*Config); typeOK { @@ -38,7 +58,8 @@ func OptionSetQueryExecMode(queryExecMode pgx.QueryExecMode) graph.TransactionOp } type Driver struct { - pool *pgxpool.Pool + pool *pgxpool.Pool + runtime *poolRuntime *SchemaManager } @@ -63,9 +84,15 @@ func NewDriver(graphQueryMemoryLimit size.Size, pool *pgxpool.Pool) *Driver { // NewDriverWithOptions constructs a PostgreSQL driver with driver-wide options. func NewDriverWithOptions(graphQueryMemoryLimit size.Size, pool *pgxpool.Pool, options DriverOptions) *Driver { options = normalizeDriverOptions(options) + runtime := poolRuntimeFor(pool) + var provider CypherTranslationCacheProvider + if runtime != nil { + provider = runtime.provider + } return &Driver{ pool: pool, - SchemaManager: NewSchemaManagerWithOptions(pool, graphQueryMemoryLimit, options), + runtime: runtime, + SchemaManager: newSchemaManagerWithOptionsAndProvider(pool, graphQueryMemoryLimit, options, provider), } } @@ -76,22 +103,69 @@ func normalizeDriverOptions(options DriverOptions) DriverOptions { return options } +// OptionSetTransactionIsolation requests an explicit PostgreSQL transaction at +// the supplied isolation level. B traversal candidates are selected only for +// REPEATABLE READ or SERIALIZABLE transactions. The driver prepares the +// production shortest-path and all-shortest-path temporary workspaces on the +// acquired session before beginning either stable-snapshot transaction and +// uses PostgreSQL READ WRITE access so those session-local tables can reset. +func OptionSetTransactionIsolation(isolation pgx.TxIsoLevel) graph.TransactionOption { + return func(config *graph.TransactionConfig) { + if pgCfg, typeOK := config.DriverConfig.(*Config); typeOK { + pgCfg.Options.IsoLevel = isolation + if stableSnapshotIsolation(isolation) { + pgCfg.Options.AccessMode = pgx.ReadWrite + } + } + } +} + +// OptionInitializeTraversalRuntimeAttestation prepares the acquired PostgreSQL +// session before an explicit read-only transaction begins. Callers that arm +// traversal runtime receipts inside a graph transaction need this option +// because PostgreSQL forbids creating the temporary workspace after BEGIN READ +// ONLY. GraphBench normally pins and prepares its session before the timed +// transaction instead. +func OptionInitializeTraversalRuntimeAttestation() graph.TransactionOption { + return func(config *graph.TransactionConfig) { + if pgCfg, typeOK := config.DriverConfig.(*Config); typeOK { + pgCfg.initializeTraversalRuntimeAttestation = true + } + } +} + +// OptionSkipStableSnapshotTraversalWorkspacesForTool keeps a Repeatable Read +// ordinary-expansion measurement free of unrelated shortest-path workspace +// setup. It is intentionally tool-scoped and does not alter production policy. +func OptionSkipStableSnapshotTraversalWorkspacesForTool() graph.TransactionOption { + return func(config *graph.TransactionConfig) { + if pgCfg, typeOK := config.DriverConfig.(*Config); typeOK { + pgCfg.skipStableSnapshotTraversalWorkspaces = true + } + } +} + +// SetDefaultGraph validates and selects graphSchema as the driver's default graph. func (s *Driver) SetDefaultGraph(ctx context.Context, graphSchema graph.Graph) error { return s.SchemaManager.SetDefaultGraph(ctx, graphSchema) } +// KindMapper returns the driver's graph-kind to PostgreSQL-ID mapper. func (s *Driver) KindMapper() KindMapper { return s.SchemaManager } +// SetBatchWriteSize changes the process-wide mutation count used for new batch flushes. func (s *Driver) SetBatchWriteSize(size int) { batchWriteSize = size } +// SetWriteFlushSize is a no-op because PostgreSQL batches do not rotate transactions by size. func (s *Driver) SetWriteFlushSize(size int) { // THis is a no-op function since PostgreSQL does not require transaction rotation like Neo4j does } +// BatchOperation runs batchDelegate in a write batch using the supplied batch options. func (s *Driver) BatchOperation(ctx context.Context, batchDelegate graph.BatchDelegate, options ...graph.BatchOption) error { batchConfig := &graph.BatchConfig{ BatchSize: batchWriteSize, @@ -122,18 +196,47 @@ func (s *Driver) BatchOperation(ctx context.Context, batchDelegate graph.BatchDe } } +// Close stops the driver's query caches before releasing its PostgreSQL pool. func (s *Driver) Close(ctx context.Context) error { - s.translationCache.Close() - s.pool.Close() + if s.SchemaManager != nil { + s.SchemaManager.parseCache.Close() + s.SchemaManager.compilationCache.Close() + } + if s.runtime != nil { + s.runtime.close() + } else if s.pool != nil { + s.pool.Close() + } return nil } -// TranslationCacheStats returns aggregate PostgreSQL translation-cache -// counters. It never exposes cached query text, SQL, or parameter data. -func (s *Driver) TranslationCacheStats() TranslationCacheStats { - return s.translationCache.Stats() +// CompilationCacheStats returns aggregate PostgreSQL compiler-cache counters. +func (s *Driver) CompilationCacheStats() CompilationCacheStats { + if s == nil || s.SchemaManager == nil || s.SchemaManager.compilationCache == nil { + return CompilationCacheStats{} + } + return s.SchemaManager.compilationCache.Stats() +} + +// TranslationCacheStats returns query-text-free counters for this driver's +// bounded Cypher-to-SQL translation cache. +func (s *Driver) TranslationCacheStats() Stats { + if s == nil || s.runtime == nil || s.runtime.provider == nil { + return Stats{} + } + return s.runtime.provider.stats() +} + +// ParseCacheStats returns query-text-free counters for this driver's bounded Cypher parse cache. +func (s *Driver) ParseCacheStats() ParseCacheStats { + if s == nil || s.SchemaManager == nil { + return ParseCacheStats{} + } + return s.SchemaManager.parseCache.Stats() } +// renderConfig applies transaction options to PostgreSQL defaults and rejects +// a driver configuration of the wrong concrete type. func renderConfig(batchWriteSize int, pgxOptions pgx.TxOptions, userOptions []graph.TransactionOption) (*Config, error) { graphCfg := graph.TransactionConfig{ DriverConfig: &Config{ @@ -159,6 +262,7 @@ func renderConfig(batchWriteSize int, pgxOptions pgx.TxOptions, userOptions []gr return nil, fmt.Errorf("driver config is nil") } +// FetchSchema is not implemented because PostgreSQL schema discovery is owned by SchemaManager. func (s *Driver) FetchSchema(ctx context.Context) (graph.Schema, error) { // TODO: This is not required for existing functionality as the SchemaManager type handles most of this negotiation // however, in the future this function would make it easier to make schema management generic and should be @@ -166,6 +270,7 @@ func (s *Driver) FetchSchema(ctx context.Context) (graph.Schema, error) { return graph.Schema{}, fmt.Errorf("not implemented") } +// AssertSchema creates or validates the requested schema and resets pooled type metadata afterward. func (s *Driver) AssertSchema(ctx context.Context, schema graph.Schema) error { // Resetting the pool must be done on every schema assertion as composite types may have changed OIDs defer s.pool.Reset() @@ -181,10 +286,14 @@ func (s *Driver) AssertSchema(ctx context.Context, schema graph.Schema) error { return err } } + if s.runtime != nil && s.runtime.provider != nil { + s.runtime.provider.advanceSchemaGeneration() + } return nil } +// Run executes raw SQL in a write transaction and returns its terminal result error. func (s *Driver) Run(ctx context.Context, query string, parameters map[string]any) error { return s.WriteTransaction(ctx, func(tx graph.Transaction) error { result := tx.Raw(query, parameters) @@ -194,6 +303,7 @@ func (s *Driver) Run(ctx context.Context, query string, parameters map[string]an }) } +// FetchKinds returns the current in-memory graph-kind mapping. func (s *Driver) FetchKinds(_ context.Context) (graph.Kinds, error) { var kinds graph.Kinds for _, kind := range s.SchemaManager.GetKindIDsByKind() { @@ -203,6 +313,7 @@ func (s *Driver) FetchKinds(_ context.Context) (graph.Kinds, error) { return kinds, nil } +// RefreshKinds discards and reloads the driver's in-memory kind mapping. func (s *Driver) RefreshKinds(ctx context.Context) error { s.lock.Lock() defer s.lock.Unlock() @@ -213,10 +324,32 @@ func (s *Driver) RefreshKinds(ctx context.Context) error { return err } - s.translationCache.Invalidate() + s.compilationCache.Invalidate() + if s.runtime != nil && s.runtime.provider != nil { + s.runtime.provider.advanceSchemaGeneration() + } return nil } +// WarmStatements prepares selected SQL on currently idle physical +// connections without executing it. +func (s *Driver) WarmStatements(ctx context.Context, statements ...string) error { + if s == nil || s.runtime == nil { + return nil + } + return s.runtime.warmStatements(ctx, statements...) +} + +// SetStatementWarmupPolicy installs the warm set for current and future +// physical connections. +func (s *Driver) SetStatementWarmupPolicy(ctx context.Context, statements ...string) error { + if s == nil || s.runtime == nil { + return nil + } + return s.runtime.setStatementWarmupPolicy(ctx, statements...) +} + +// OptimizeStorage runs PostgreSQL storage maintenance on a leased pool connection. func (s *Driver) OptimizeStorage(ctx context.Context) error { conn, err := s.pool.Acquire(ctx) if err != nil { diff --git a/drivers/pg/driver_test.go b/drivers/pg/driver_test.go index 65c30285..fd16e6e9 100644 --- a/drivers/pg/driver_test.go +++ b/drivers/pg/driver_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/jackc/pgx/v5" "github.com/specterops/dawgs/graph" "github.com/stretchr/testify/require" ) @@ -64,7 +65,9 @@ func TestBuildNodeDeleteStatement(t *testing.T) { func TestResolveKindIDsDefinedFastPath(t *testing.T) { ctx := context.Background() - driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + driver := &Driver{ + SchemaManager: NewSchemaManager(nil, 0), + } var ( userKind = graph.StringKind("User") @@ -98,3 +101,32 @@ func TestDeleteRelationshipsByKindsEmptyIsNoop(t *testing.T) { require.NoError(t, driver.DeleteRelationshipsByKinds(ctx, nil)) require.NoError(t, driver.DeleteRelationshipsByKinds(ctx, graph.Kinds{})) } + +// TestOptionInitializeTraversalRuntimeAttestation verifies option initialize traversal runtime attestation behavior. +func TestOptionInitializeTraversalRuntimeAttestation(t *testing.T) { + cfg, err := renderConfig(defaultBatchWriteSize, readOnlyTxOptions, []graph.TransactionOption{ + OptionInitializeTraversalRuntimeAttestation(), + }) + require.NoError(t, err) + require.True(t, cfg.initializeTraversalRuntimeAttestation) +} + +// TestStableSnapshotIsolation verifies stable snapshot isolation behavior. +func TestStableSnapshotIsolation(t *testing.T) { + require.False(t, stableSnapshotIsolation("")) + require.False(t, stableSnapshotIsolation(pgx.ReadCommitted)) + require.True(t, stableSnapshotIsolation(pgx.RepeatableRead)) + require.True(t, stableSnapshotIsolation(pgx.Serializable)) +} + +// TestOptionSetStableSnapshotIsolationAllowsTemporaryWorkspaceWrites verifies option set stable snapshot isolation allows temporary workspace writes behavior. +func TestOptionSetStableSnapshotIsolationAllowsTemporaryWorkspaceWrites(t *testing.T) { + for _, isolation := range []pgx.TxIsoLevel{pgx.RepeatableRead, pgx.Serializable} { + cfg, err := renderConfig(defaultBatchWriteSize, readOnlyTxOptions, []graph.TransactionOption{ + OptionSetTransactionIsolation(isolation), + }) + require.NoError(t, err) + require.Equal(t, isolation, cfg.Options.IsoLevel) + require.Equal(t, pgx.ReadWrite, cfg.Options.AccessMode) + } +} diff --git a/drivers/pg/manager.go b/drivers/pg/manager.go index cbd5a0c2..cd11b47c 100644 --- a/drivers/pg/manager.go +++ b/drivers/pg/manager.go @@ -15,38 +15,104 @@ import ( "github.com/specterops/dawgs/util/size" ) +// KindMapper translates graph kind names to and from PostgreSQL kind identifiers. type KindMapper interface { + // MapKindID resolves one PostgreSQL kind identifier to its graph kind. MapKindID(ctx context.Context, kindID int16) (graph.Kind, error) + + // MapKindIDs resolves PostgreSQL kind identifiers to their graph kinds. MapKindIDs(ctx context.Context, kindIDs []int16) (graph.Kinds, error) + + // MapKind resolves one graph kind to its PostgreSQL identifier. MapKind(ctx context.Context, kind graph.Kind) (int16, error) + + // MapKinds resolves graph kinds to their PostgreSQL identifiers. MapKinds(ctx context.Context, kinds graph.Kinds) ([]int16, error) + + // AssertKinds creates missing kinds and returns PostgreSQL identifiers for every input kind. AssertKinds(ctx context.Context, kinds graph.Kinds) ([]int16, error) } +// StableSnapshotTraversalWorkspaceProvider optionally owns the readiness of +// session-local traversal workspaces. Providers must treat a closed or +// replaced physical connection as unready. +type StableSnapshotTraversalWorkspaceProvider interface { + // EnsureStableSnapshotTraversalWorkspaces initializes workspaces for conn when required. + EnsureStableSnapshotTraversalWorkspaces(ctx context.Context, conn *pgxpool.Conn) error +} + +// LazyStableSnapshotTraversalWorkspaceProvider elects to initialize traversal +// workspaces only when a shortest-path query is actually issued. This is safe +// for V2 because readiness remains tied to the leased physical connection and +// schema generation. +type LazyStableSnapshotTraversalWorkspaceProvider interface { + // DeferStableSnapshotTraversalWorkspaces reports whether workspaces may initialize on first use. + DeferStableSnapshotTraversalWorkspaces() bool +} + +// KindMapperFromGraphDatabase returns graphDB's PostgreSQL kind mapper when it exposes one. func KindMapperFromGraphDatabase(graphDB graph.Database) (KindMapper, error) { - switch typedGraphDB := graphDB.(type) { - case *Driver: - return typedGraphDB.SchemaManager, nil - default: - return nil, fmt.Errorf("unsupported graph database type: %T", typedGraphDB) + if kindMapperProvider, supported := graphDB.(interface{ KindMapper() KindMapper }); supported { + return kindMapperProvider.KindMapper(), nil } + return nil, fmt.Errorf("unsupported graph database type: %T", graphDB) } +// SchemaManager coordinates graph and kind metadata with the query caches that depend on that schema state. type SchemaManager struct { - defaultGraph model.Graph - pool *pgxpool.Pool - hasDefaultGraph bool - graphs map[string]model.Graph - kindsByID map[graph.Kind]int16 - kindIDsByKind map[int16]graph.Kind - lock *sync.RWMutex - graphQueryMemoryLimit size.Size - translationCache *translationCache - translationCacheProvider translationCacheProvider + // defaultGraph caches the first graph selected as the schema default. + defaultGraph model.Graph + + // pool supplies PostgreSQL connections for schema operations. + pool *pgxpool.Pool + + // parseCache retains immutable Cypher ASTs keyed by normalized query text. + parseCache *cypherParseCache + + // compilationCache retains upstream compiler results and provenance. + compilationCache *translationCache + + // compilationCacheProvider supplies the upstream compiler cache. + compilationCacheProvider translationCacheProvider + + // translationCacheProvider selects the connection-local translation cache + // for each physical PostgreSQL connection. A missing provider deliberately + // bypasses retention for pools not constructed by this package. + translationCacheProvider CypherTranslationCacheProvider + + // hasDefaultGraph distinguishes a cached default graph from the zero-value graph model. + hasDefaultGraph bool + + // graphs indexes asserted database graph models by schema name. + graphs map[string]model.Graph + + // kindsByID maps graph kind names to their PostgreSQL int2 identifiers. + kindsByID map[graph.Kind]int16 + + // kindIDsByKind maps PostgreSQL int2 identifiers back to graph kind names. + kindIDsByKind map[int16]graph.Kind + + // lock protects cached graph and kind metadata from concurrent access. + lock *sync.RWMutex + + // graphQueryMemoryLimit caps memory available to a graph query transaction. + graphQueryMemoryLimit size.Size + + // traversalPolicyLock protects the versioned default-off production canary policy. + traversalPolicyLock sync.RWMutex + + // traversalPolicy is copied on reads so callers cannot mutate live selection state. + traversalPolicy TraversalPolicy } -func NewSchemaManager(pool *pgxpool.Pool, graphQueryMemoryLimit size.Size) *SchemaManager { - return NewSchemaManagerWithOptions(pool, graphQueryMemoryLimit, DefaultDriverOptions()) +// NewSchemaManager creates an empty metadata manager with bounded parse and +// compilation caches and an optional connection-local translation provider. +func NewSchemaManager(pool *pgxpool.Pool, graphQueryMemoryLimit size.Size, providers ...CypherTranslationCacheProvider) *SchemaManager { + var provider CypherTranslationCacheProvider + if len(providers) > 0 { + provider = providers[0] + } + return newSchemaManagerWithOptionsAndProvider(pool, graphQueryMemoryLimit, DefaultDriverOptions(), provider) } // NewSchemaManagerWithTranslationCache permits an application to disable the @@ -61,24 +127,41 @@ func NewSchemaManagerWithTranslationCache(pool *pgxpool.Pool, graphQueryMemoryLi // NewSchemaManagerWithOptions constructs the shared compilation service with // a bounded translation cache. func NewSchemaManagerWithOptions(pool *pgxpool.Pool, graphQueryMemoryLimit size.Size, options DriverOptions) *SchemaManager { + return newSchemaManagerWithOptionsAndProvider(pool, graphQueryMemoryLimit, options, nil) +} + +func newSchemaManagerWithOptionsAndProvider(pool *pgxpool.Pool, graphQueryMemoryLimit size.Size, options DriverOptions, provider CypherTranslationCacheProvider) *SchemaManager { options = normalizeDriverOptions(options) - translationCache := newTranslationCache(options.TranslationCacheEntries) + compilationCache := newTranslationCache(options.TranslationCacheEntries) return &SchemaManager{ - pool: pool, - hasDefaultGraph: false, - graphs: map[string]model.Graph{}, - kindsByID: map[graph.Kind]int16{}, - kindIDsByKind: map[int16]graph.Kind{}, - lock: &sync.RWMutex{}, - graphQueryMemoryLimit: graphQueryMemoryLimit, - translationCache: translationCache, - translationCacheProvider: sharedTranslationCacheProvider{ - cache: translationCache, + pool: pool, + parseCache: newCypherParseCache(defaultCypherParseCacheEntries), + translationCacheProvider: provider, + hasDefaultGraph: false, + graphs: map[string]model.Graph{}, + kindsByID: map[graph.Kind]int16{}, + kindIDsByKind: map[int16]graph.Kind{}, + lock: &sync.RWMutex{}, + graphQueryMemoryLimit: graphQueryMemoryLimit, + compilationCache: compilationCache, + compilationCacheProvider: sharedTranslationCacheProvider{ + cache: compilationCache, }, } } +// cypherTranslationCacheForConnection selects a cache for conn. A missing +// provider or a nil cache is an intentional uncached fallback. +func (s *SchemaManager) cypherTranslationCacheForConnection(conn *pgx.Conn) CypherTranslationCache { + if s == nil || s.translationCacheProvider == nil { + return nil + } + + return s.translationCacheProvider.CacheForConnection(conn) +} + +// WriteTransaction executes txDelegate in an explicit read-write PostgreSQL transaction. func (s *SchemaManager) WriteTransaction(ctx context.Context, txDelegate graph.TransactionDelegate, options ...graph.TransactionOption) error { if cfg, err := renderConfig(batchWriteSize, readWriteTxOptions, options); err != nil { return err @@ -101,6 +184,7 @@ func (s *SchemaManager) WriteTransaction(ctx context.Context, txDelegate graph.T } } +// fetch replaces both in-memory kind indexes with the kinds visible through tx. func (s *SchemaManager) fetch(tx graph.Transaction) error { if kinds, err := query.On(tx).SelectKinds(); err != nil { return err @@ -115,18 +199,22 @@ func (s *SchemaManager) fetch(tx graph.Transaction) error { return nil } +// GetKindIDsByKind returns the current PostgreSQL-ID-to-kind cache. func (s *SchemaManager) GetKindIDsByKind() map[int16]graph.Kind { s.lock.RLock() defer s.lock.RUnlock() return s.kindIDsByKind } +// Fetch refreshes both in-memory kind indexes from a read transaction against the current schema. func (s *SchemaManager) Fetch(ctx context.Context) error { - return s.WriteTransaction(ctx, func(tx graph.Transaction) error { + return s.ReadTransaction(ctx, func(tx graph.Transaction) error { return s.fetch(tx) }, OptionSetQueryExecMode(pgx.QueryExecModeSimpleProtocol)) } +// defineKinds inserts any missing kinds and records their database IDs in both +// in-memory indexes. func (s *SchemaManager) defineKinds(tx graph.Transaction, kinds graph.Kinds) error { for _, kind := range kinds { if kindID, err := query.On(tx).InsertOrGetKind(kind); err != nil { @@ -140,6 +228,7 @@ func (s *SchemaManager) defineKinds(tx graph.Transaction, kinds graph.Kinds) err return nil } +// mapKinds partitions semantic kinds into cached database IDs and unresolved kinds without refreshing the cache. func (s *SchemaManager) mapKinds(kinds graph.Kinds) ([]int16, graph.Kinds) { var ( missingKinds = make(graph.Kinds, 0, len(kinds)) @@ -157,6 +246,7 @@ func (s *SchemaManager) mapKinds(kinds graph.Kinds) ([]int16, graph.Kinds) { return ids, missingKinds } +// MapKind resolves kind from the cache, refreshing it once on a miss. func (s *SchemaManager) MapKind(ctx context.Context, kind graph.Kind) (int16, error) { s.lock.RLock() @@ -180,6 +270,7 @@ func (s *SchemaManager) MapKind(ctx context.Context, kind graph.Kind) (int16, er } } +// MapKinds resolves kinds from the cache, refreshing it once when any kind is missing. func (s *SchemaManager) MapKinds(ctx context.Context, kinds graph.Kinds) ([]int16, error) { s.lock.RLock() @@ -202,6 +293,8 @@ func (s *SchemaManager) MapKinds(ctx context.Context, kinds graph.Kinds) ([]int1 return nil, fmt.Errorf("unable to map kinds: %s", strings.Join(missingKinds.Strings(), ", ")) } } + +// ReadTransaction executes txDelegate on a leased read-only connection or transaction. func (s *SchemaManager) ReadTransaction(ctx context.Context, txDelegate graph.TransactionDelegate, options ...graph.TransactionOption) error { if cfg, err := renderConfig(batchWriteSize, readOnlyTxOptions, options); err != nil { return err @@ -209,17 +302,64 @@ func (s *SchemaManager) ReadTransaction(ctx context.Context, txDelegate graph.Tr return err } else { defer conn.Release() + if stableSnapshotIsolation(cfg.Options.IsoLevel) && !cfg.skipStableSnapshotTraversalWorkspaces { + workspaceProvider, hasWorkspaceProvider := s.translationCacheProvider.(StableSnapshotTraversalWorkspaceProvider) + lazyProvider, deferWorkspace := s.translationCacheProvider.(LazyStableSnapshotTraversalWorkspaceProvider) + if deferWorkspace && lazyProvider.DeferStableSnapshotTraversalWorkspaces() { + // V2 initializes only when transaction.Query observes a shortest-path + // operation. Ordinary repeatable-read work needs no temporary tables. + } else if hasWorkspaceProvider { + err = workspaceProvider.EnsureStableSnapshotTraversalWorkspaces(ctx, conn) + } else { + err = EnsureStableSnapshotTraversalWorkspaces(ctx, conn) + } + if err != nil { + return err + } + } + if cfg.initializeTraversalRuntimeAttestation { + if _, err := conn.Exec(ctx, "select public.ensure_traversal_runtime_attestation_workspace_v1()"); err != nil { + return fmt.Errorf("initialize traversal runtime attestation workspace: %w", err) + } + } + allocateTransaction := cfg.Options.IsoLevel != "" + wrapper, err := newTransactionWrapper(ctx, conn, s, cfg, allocateTransaction) + if err != nil { + return err + } + defer wrapper.Close() + if err := txDelegate(wrapper); err != nil { + return err + } + if allocateTransaction { + return wrapper.Commit() + } + return nil + } +} - return txDelegate(&transaction{ - schemaManager: s, - queryExecMode: cfg.QueryExecMode, - ctx: ctx, - conn: conn, - targetSchemaSet: false, - }) +// stableSnapshotIsolation reports whether isolation prevents concurrent writes from changing reads. +func stableSnapshotIsolation(isolation pgx.TxIsoLevel) bool { + return isolation == pgx.RepeatableRead || isolation == pgx.Serializable +} + +// EnsureStableSnapshotTraversalWorkspaces initializes the reusable +// session-local workspace required before stable-snapshot traversal queries. +// Drivers with connection-local lifecycle state may call this only when their +// tracked physical connection is not already ready. +func EnsureStableSnapshotTraversalWorkspaces(ctx context.Context, conn *pgxpool.Conn) error { + const initializeSQL = `select + public.ensure_shortest_dag_workspace(), + public.ensure_bidirectional_shortest_path_workspace(), + public.ensure_bidirectional_all_shortest_path_workspace(), + set_config('dawgs.shortest_dag_workspace_ready', 'v2', false)` + if _, err := conn.Exec(ctx, initializeSQL); err != nil { + return fmt.Errorf("initialize stable-snapshot traversal workspaces: %w", err) } + return nil } +// mapKindIDs partitions database kind IDs into cached semantic kinds and unresolved IDs without refreshing the cache. func (s *SchemaManager) mapKindIDs(kindIDs []int16) (graph.Kinds, []int16) { var ( missingIDs = make([]int16, 0, len(kindIDs)) @@ -237,6 +377,7 @@ func (s *SchemaManager) mapKindIDs(kindIDs []int16) (graph.Kinds, []int16) { return kinds, missingIDs } +// MapKindID resolves one PostgreSQL kind identifier from the cache, refreshing it once on a miss. func (s *SchemaManager) MapKindID(ctx context.Context, kindID int16) (graph.Kind, error) { if kindIDs, err := s.MapKindIDs(ctx, []int16{kindID}); err != nil { return nil, err @@ -245,6 +386,7 @@ func (s *SchemaManager) MapKindID(ctx context.Context, kindID int16) (graph.Kind } } +// MapKindIDs resolves PostgreSQL kind identifiers from the cache, refreshing it once on a miss. func (s *SchemaManager) MapKindIDs(ctx context.Context, kindIDs []int16) (graph.Kinds, error) { s.lock.RLock() @@ -268,6 +410,7 @@ func (s *SchemaManager) MapKindIDs(ctx context.Context, kindIDs []int16) (graph. } } +// assertKinds defines any missing kinds while holding the write lock and returns IDs from the refreshed in-memory mapping. func (s *SchemaManager) assertKinds(ctx context.Context, kinds graph.Kinds) ([]int16, error) { // Acquire a write-lock and release on-exit s.lock.Lock() @@ -288,6 +431,7 @@ func (s *SchemaManager) assertKinds(ctx context.Context, kinds graph.Kinds) ([]i return kindIDs, nil } +// AssertKinds ensures every input kind exists and returns its PostgreSQL identifier. func (s *SchemaManager) AssertKinds(ctx context.Context, kinds graph.Kinds) ([]int16, error) { // Acquire a read-lock first to fast-pass validate if we're missing any kind definitions s.lock.RLock() @@ -303,6 +447,7 @@ func (s *SchemaManager) AssertKinds(ctx context.Context, kinds graph.Kinds) ([]i return s.assertKinds(ctx, kinds) } +// setDefaultGraph caches the first successfully resolved default graph and ignores later attempts to replace it. func (s *SchemaManager) setDefaultGraph(defaultGraph model.Graph, schema graph.Graph) { s.lock.Lock() defer s.lock.Unlock() @@ -318,6 +463,7 @@ func (s *SchemaManager) setDefaultGraph(defaultGraph model.Graph, schema graph.G s.hasDefaultGraph = true } +// SetDefaultGraph validates an existing graph and records it as the immutable default target. func (s *SchemaManager) SetDefaultGraph(ctx context.Context, schema graph.Graph) error { return s.ReadTransaction(ctx, func(tx graph.Transaction) error { // Validate the schema if the graph already exists in the database @@ -330,6 +476,7 @@ func (s *SchemaManager) SetDefaultGraph(ctx context.Context, schema graph.Graph) }) } +// AssertDefaultGraph creates or validates schema and records it as the immutable default target. func (s *SchemaManager) AssertDefaultGraph(ctx context.Context, schema graph.Graph) error { return s.WriteTransaction(ctx, func(tx graph.Transaction) error { if graphModel, err := s.AssertGraph(tx, schema); err != nil { @@ -342,6 +489,7 @@ func (s *SchemaManager) AssertDefaultGraph(ctx context.Context, schema graph.Gra }) } +// DefaultGraph returns the cached default graph and whether one has been selected. func (s *SchemaManager) DefaultGraph() (model.Graph, bool) { s.lock.RLock() defer s.lock.RUnlock() @@ -349,6 +497,7 @@ func (s *SchemaManager) DefaultGraph() (model.Graph, bool) { return s.defaultGraph, s.hasDefaultGraph } +// assertGraph creates or validates schema while the caller holds the metadata write lock. func (s *SchemaManager) assertGraph(tx graph.Transaction, schema graph.Graph) (model.Graph, error) { var assertedGraph model.Graph @@ -376,6 +525,7 @@ func (s *SchemaManager) assertGraph(tx graph.Transaction, schema graph.Graph) (m return assertedGraph, nil } +// AssertGraph returns the cached graph or creates and caches schema on the first request. func (s *SchemaManager) AssertGraph(tx graph.Transaction, schema graph.Graph) (model.Graph, error) { // Acquire a read-lock first to fast-pass validate if we're missing the graph definitions s.lock.RLock() @@ -400,6 +550,7 @@ func (s *SchemaManager) AssertGraph(tx graph.Transaction, schema graph.Graph) (m return s.assertGraph(tx, schema) } +// assertSchema creates schema storage and defines every node and relationship kind required by its graphs. func (s *SchemaManager) assertSchema(tx graph.Transaction, schema graph.Schema) error { if err := query.On(tx).CreateSchema(); err != nil { return err @@ -426,6 +577,7 @@ func (s *SchemaManager) assertSchema(tx graph.Transaction, schema graph.Schema) return nil } +// AssertSchema creates base storage and defines kinds required by every graph in schema. func (s *SchemaManager) AssertSchema(ctx context.Context, schema graph.Schema) error { s.lock.Lock() defer s.lock.Unlock() @@ -436,6 +588,6 @@ func (s *SchemaManager) AssertSchema(ctx context.Context, schema graph.Schema) e return err } - s.translationCache.Invalidate() + s.compilationCache.Invalidate() return nil } diff --git a/drivers/pg/mapper.go b/drivers/pg/mapper.go index 0195f2f4..c7fbc4ad 100644 --- a/drivers/pg/mapper.go +++ b/drivers/pg/mapper.go @@ -7,10 +7,14 @@ import ( ) const ( + // minKindID is the smallest integer representable by PostgreSQL's int2 kind column. minKindID = -1 << 15 + + // maxKindID is the largest integer representable by PostgreSQL's int2 kind column. maxKindID = 1<<15 - 1 ) +// mapKindIDs resolves database kind IDs and reports false when the mapper rejects any ID. func mapKindIDs(ctx context.Context, kindMapper KindMapper, kindIDs []int16) (graph.Kinds, bool) { if len(kindIDs) == 0 { return graph.Kinds{}, true @@ -23,6 +27,7 @@ func mapKindIDs(ctx context.Context, kindMapper KindMapper, kindIDs []int16) (gr return nil, false } +// asKindID converts supported integer representations to int16 without truncation. func asKindID(value any) (int16, bool) { switch typedValue := value.(type) { case int: @@ -78,6 +83,7 @@ func asKindID(value any) (int16, bool) { } } +// mapAnyKinds maps a homogeneous list of kind names or numeric IDs and rejects mixed or unsupported values. func mapAnyKinds(ctx context.Context, kindMapper KindMapper, values []any) (graph.Kinds, bool) { if len(values) == 0 { return graph.Kinds{}, true @@ -113,6 +119,7 @@ func mapAnyKinds(ctx context.Context, kindMapper KindMapper, values []any) (grap return mapKindIDs(ctx, kindMapper, kindIDs) } +// mapKinds accepts the slice representations emitted by pgx for graph kind arrays. func mapKinds(ctx context.Context, kindMapper KindMapper, untypedValue any) (graph.Kinds, bool) { switch typedValue := untypedValue.(type) { case []any: @@ -128,6 +135,7 @@ func mapKinds(ctx context.Context, kindMapper KindMapper, untypedValue any) (gra return nil, false } +// mapNodeCompositeArray converts a raw PostgreSQL composite array into graph nodes with resolved kinds. func mapNodeCompositeArray(ctx context.Context, kindMapper KindMapper, value any) ([]*graph.Node, bool) { nodeComposites, err := nodeCompositesFromRaw(value) if err != nil { @@ -147,6 +155,7 @@ func mapNodeCompositeArray(ctx context.Context, kindMapper KindMapper, value any return nodes, true } +// mapEdgeCompositeArray converts a raw PostgreSQL composite array into graph relationships with resolved kinds. func mapEdgeCompositeArray(ctx context.Context, kindMapper KindMapper, value any) ([]*graph.Relationship, bool) { edgeComposites, err := edgeCompositesFromRaw(value) if err != nil { @@ -166,17 +175,14 @@ func mapEdgeCompositeArray(ctx context.Context, kindMapper KindMapper, value any return relationships, true } +// newMapFunc returns the result mapper that recognizes graph composites, arrays, paths, and kind slices. func newMapFunc(ctx context.Context, kindMapper KindMapper) graph.MapFunc { return func(value, target any) bool { switch typedTarget := target.(type) { case *graph.Relationship: - if compositeMap, typeOK := value.(map[string]any); typeOK { - edge := edgeComposite{} - - if edge.TryMap(compositeMap) { - if err := edge.ToRelationship(ctx, kindMapper, typedTarget); err == nil { - return true - } + if edge, typeOK := edgeCompositeFromRaw(value); typeOK { + if err := edge.ToRelationship(ctx, kindMapper, typedTarget); err == nil { + return true } } @@ -200,13 +206,9 @@ func newMapFunc(ctx context.Context, kindMapper KindMapper) graph.MapFunc { } case *graph.Node: - if compositeMap, typeOK := value.(map[string]any); typeOK { - node := nodeComposite{} - - if node.TryMap(compositeMap) { - if err := node.ToNode(ctx, kindMapper, typedTarget); err == nil { - return true - } + if node, typeOK := nodeCompositeFromRaw(value); typeOK { + if err := node.ToNode(ctx, kindMapper, typedTarget); err == nil { + return true } } @@ -230,13 +232,9 @@ func newMapFunc(ctx context.Context, kindMapper KindMapper) graph.MapFunc { } case *graph.Path: - if compositeMap, typeOK := value.(map[string]any); typeOK { - path := pathComposite{} - - if path.TryMap(compositeMap) { - if err := path.ToPath(ctx, kindMapper, typedTarget); err == nil { - return true - } + if path, typeOK := pathCompositeFromRaw(value); typeOK { + if err := path.ToPath(ctx, kindMapper, typedTarget); err == nil { + return true } } diff --git a/drivers/pg/mapper_test.go b/drivers/pg/mapper_test.go index 3145327c..76356eda 100644 --- a/drivers/pg/mapper_test.go +++ b/drivers/pg/mapper_test.go @@ -84,6 +84,7 @@ func TestValueMapperMapsStringArraysByTargetType(t *testing.T) { require.Equal(t, []string{"Alice", "Bob"}, stringTarget) } +// TestValueMapperMapsCompositeArrays verifies typed node and relationship arrays preserve order and graph metadata. func TestValueMapperMapsCompositeArrays(t *testing.T) { ctx := context.Background() mapper := pgutil.NewInMemoryKindMapper() @@ -114,6 +115,28 @@ func TestValueMapperMapsCompositeArrays(t *testing.T) { require.Equal(t, "Alice", nodes[0].Properties.Get("name").Any()) }) + t.Run("typed node array preserves order", func(t *testing.T) { + rawNodes := []any{ + nodeComposite{ + ID: 1, + KindIDs: []int16{userKindID}, + Properties: map[string]any{"name": "Alice"}, + }, + nodeComposite{ + ID: 2, + KindIDs: []int16{userKindID}, + Properties: map[string]any{"name": "Bob"}, + }, + } + + var nodes []*graph.Node + require.True(t, valueMapper.Map(rawNodes, &nodes)) + require.Len(t, nodes, 2) + require.Equal(t, graph.ID(1), nodes[0].ID) + require.Equal(t, graph.ID(2), nodes[1].ID) + require.Equal(t, "Alice", nodes[0].Properties.Get("name").Any()) + }) + t.Run("relationship array preserves order", func(t *testing.T) { rawRelationships := []any{ map[string]any{ @@ -139,6 +162,73 @@ func TestValueMapperMapsCompositeArrays(t *testing.T) { require.Equal(t, graph.ID(11), relationships[1].ID) require.Equal(t, graph.StringKind("MemberOf"), relationships[0].Kind) }) + + t.Run("typed relationship array preserves order", func(t *testing.T) { + rawRelationships := []edgeComposite{ + { + ID: 10, + StartID: 1, + EndID: 2, + KindID: memberOfKindID, + Properties: map[string]any{"ordinal": int64(1)}, + }, + { + ID: 11, + StartID: 2, + EndID: 3, + KindID: memberOfKindID, + Properties: map[string]any{"ordinal": int64(2)}, + }, + } + + var relationships []graph.Relationship + require.True(t, valueMapper.Map(rawRelationships, &relationships)) + require.Len(t, relationships, 2) + require.Equal(t, graph.ID(10), relationships[0].ID) + require.Equal(t, graph.ID(11), relationships[1].ID) + }) +} + +// TestValueMapperMapsTypedComposites verifies owned node, edge, and path composites map to graph-native values. +func TestValueMapperMapsTypedComposites(t *testing.T) { + ctx := context.Background() + mapper := pgutil.NewInMemoryKindMapper() + userKindID := mapper.Put(graph.StringKind("User")) + memberOfKindID := mapper.Put(graph.StringKind("MemberOf")) + valueMapper := NewValueMapper(ctx, mapper) + + rawNode := nodeComposite{ + ID: 1, + KindIDs: []int16{userKindID}, + Properties: map[string]any{"name": "Alice"}, + } + rawEdge := edgeComposite{ + ID: 10, + StartID: 1, + EndID: 2, + KindID: memberOfKindID, + Properties: map[string]any{"ordinal": int64(1)}, + } + + var node graph.Node + require.True(t, valueMapper.Map(rawNode, &node)) + require.Equal(t, graph.ID(1), node.ID) + require.Equal(t, graph.StringKind("User"), node.Kinds[0]) + + var relationship graph.Relationship + require.True(t, valueMapper.Map(&rawEdge, &relationship)) + require.Equal(t, graph.ID(10), relationship.ID) + require.Equal(t, graph.StringKind("MemberOf"), relationship.Kind) + + var path graph.Path + require.True(t, valueMapper.Map(pathComposite{ + Nodes: []nodeComposite{rawNode}, + Edges: []edgeComposite{rawEdge}, + }, &path)) + require.Len(t, path.Nodes, 1) + require.Len(t, path.Edges, 1) + require.Equal(t, graph.ID(1), path.Nodes[0].ID) + require.Equal(t, graph.ID(10), path.Edges[0].ID) } func TestAsKindID(t *testing.T) { diff --git a/drivers/pg/optimize.go b/drivers/pg/optimize.go index dc34f5e3..4b1efe55 100644 --- a/drivers/pg/optimize.go +++ b/drivers/pg/optimize.go @@ -14,6 +14,7 @@ type optimizeStorageConn interface { Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) } +// optimizeStorage vacuums and analyzes the partitioned node and edge parents using a simple-protocol statement. func optimizeStorage(ctx context.Context, conn optimizeStorageConn) error { targets := []string{"node", "edge"} diff --git a/drivers/pg/optimize_test.go b/drivers/pg/optimize_test.go index 6faf9f36..c5a7677c 100644 --- a/drivers/pg/optimize_test.go +++ b/drivers/pg/optimize_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" ) +// TestOptimizeStorage verifies optimization vacuums both graph storage parents in one statement. func TestOptimizeStorage(t *testing.T) { t.Run("always vacuums node and edge", func(t *testing.T) { ctx := context.Background() diff --git a/drivers/pg/pg.go b/drivers/pg/pg.go index 88a5d17e..c9f46010 100644 --- a/drivers/pg/pg.go +++ b/drivers/pg/pg.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "log/slog" - "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -14,20 +13,24 @@ import ( ) const ( + // DriverName is the connection-string scheme registered by the PostgreSQL + // driver. DriverName = "pg" // defaultBatchWriteSize is currently set to 2k. This is meant to strike a balance between the cost of thousands // of round-trips against the cost of locking tables for too long. - defaultBatchWriteSize = 2_000 - poolInitConnectionTimeout = time.Second * 10 + defaultBatchWriteSize = 2_000 ) +// AfterPooledConnectionEstablished loads and registers the driver's owned graph composite types on a new pooled connection. func AfterPooledConnectionEstablished(ctx context.Context, conn *pgx.Conn) error { for _, dataType := range pgsql.CompositeTypes { if definition, err := conn.LoadType(ctx, dataType.String()); err != nil { if !StateObjectDoesNotExist.ErrorMatches(err) { return fmt.Errorf("failed to match composite type %s to database: %w", dataType, err) } + } else if err := installOwnedCompositeCodec(dataType, definition); err != nil { + return fmt.Errorf("failed to configure composite type %s: %w", dataType, err) } else { conn.TypeMap().RegisterType(definition) } @@ -49,27 +52,10 @@ func AfterPooledConnectionRelease(conn *pgx.Conn) bool { return true } -// pgx pool config +// NewPool constructs the default PostgreSQL pool. The returned bare pgx pool +// carries all DAWGS lifecycle hooks and is safe to pass through dawgs.Config. func NewPool(poolCfg *pgxpool.Config) (*pgxpool.Pool, error) { - poolCtx, done := context.WithTimeout(context.Background(), poolInitConnectionTimeout) - defer done() - - // TODO: Min and Max connections for the pool should be configurable - poolCfg.MinConns = 5 - poolCfg.MaxConns = 50 - - // Bind functions to the AfterConnect and AfterRelease hooks to ensure that composite type registration occurs. - // Without composite type registration, the pgx connection type will not be able to marshal PG OIDs to their - // respective Golang structs. - poolCfg.AfterConnect = AfterPooledConnectionEstablished - poolCfg.AfterRelease = AfterPooledConnectionRelease - - pool, err := pgxpool.NewWithConfig(poolCtx, poolCfg) - if err != nil { - return nil, err - } - - return pool, nil + return NewPoolWithRuntimeConfig(context.Background(), poolCfg, DefaultRuntimeConfig()) } func init() { diff --git a/drivers/pg/pool_runtime.go b/drivers/pg/pool_runtime.go new file mode 100644 index 00000000..6dc2ef34 --- /dev/null +++ b/drivers/pg/pool_runtime.go @@ -0,0 +1,308 @@ +package pg + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// poolInitConnectionTimeout bounds initial pgx pool connection setup. +const poolInitConnectionTimeout = 10 * time.Second + +// poolRuntime owns connection-local state for one pool. It is created together +// with the pool by NewPool or NewPoolWithRuntimeConfig. +type poolRuntime struct { + // pool leases and manages physical PostgreSQL connections. + pool *pgxpool.Pool + + // provider owns cache state associated with each physical connection. + provider *connectionCacheProvider + + // warmups supplies the persistent statement warm-up set to new connections. + warmups *statementWarmupPolicy + + // closeOnce prevents duplicate pool and provider teardown. + closeOnce sync.Once +} + +// poolRuntimes associates a pool constructed by this package with the +// connection-local state captured by its lifecycle hooks. The association is +// private: callers continue to exchange the established *pgxpool.Pool API. +var poolRuntimes sync.Map // map[*pgxpool.Pool]*poolRuntime + +func registerPoolRuntime(pool *pgxpool.Pool, runtime *poolRuntime) { + if pool != nil && runtime != nil { + poolRuntimes.Store(pool, runtime) + } +} + +func poolRuntimeFor(pool *pgxpool.Pool) *poolRuntime { + if pool == nil { + return nil + } + if runtime, found := poolRuntimes.Load(pool); found { + return runtime.(*poolRuntime) + } + return nil +} + +// statementWarmupPolicy retains only normalized SQL selected by an operator. +// It is deliberately empty by default and is shared with AfterConnect so new +// physical connections receive the same warming policy as current ones. +type statementWarmupPolicy struct { + // lock serializes snapshots and replacements of the operator-selected warm set. + lock sync.RWMutex + + // statements contains normalized SQL identities and text for future connections. + statements []preparedStatementWarmup + + // generation advances on every replacement, including a clear. Connections + // use it to ensure initialization does not finish with an obsolete policy. + generation uint64 +} + +// snapshot returns the current generation and an independent view of its warm set. +func (s *statementWarmupPolicy) snapshot() (uint64, []preparedStatementWarmup) { + if s == nil { + return 0, nil + } + s.lock.RLock() + defer s.lock.RUnlock() + return s.generation, append([]preparedStatementWarmup(nil), s.statements...) +} + +// replace atomically installs a copy of the supplied warm set. +func (s *statementWarmupPolicy) replace(statements []preparedStatementWarmup) { + s.lock.Lock() + defer s.lock.Unlock() + s.statements = append([]preparedStatementWarmup(nil), statements...) + s.generation++ +} + +// isCurrent reports whether generation is still the published policy. +func (s *statementWarmupPolicy) isCurrent(generation uint64) bool { + if s == nil { + return generation == 0 + } + s.lock.RLock() + defer s.lock.RUnlock() + return s.generation == generation +} + +// warmCurrentStatementPolicy keeps a newly established connection from +// completing initialization against a policy superseded during preparation. +func warmCurrentStatementPolicy(warmups *statementWarmupPolicy, warm func([]preparedStatementWarmup) error) error { + for { + generation, statements := warmups.snapshot() + if err := warm(statements); err != nil { + return err + } + if warmups.isCurrent(generation) { + return nil + } + } +} + +// poolLifecycleHooks groups the driver lifecycle hooks composed into pgx. +type poolLifecycleHooks struct { + // afterConnect initializes a newly established physical connection. + afterConnect func(context.Context, *pgx.Conn) error + + // afterRelease validates a connection before it returns to the pool. + afterRelease func(*pgx.Conn) bool +} + +// productionPoolLifecycleHooks returns the stable PostgreSQL connection lifecycle hooks. +func productionPoolLifecycleHooks() poolLifecycleHooks { + return poolLifecycleHooks{ + afterConnect: AfterPooledConnectionEstablished, + afterRelease: AfterPooledConnectionRelease, + } +} + +// composePoolConfig copies caller configuration and composes v2 cache lifecycle hooks. +func composePoolConfig(poolConfig *pgxpool.Config, runtimeConfig RuntimeConfig, provider *connectionCacheProvider, warmups *statementWarmupPolicy, hooks poolLifecycleHooks) (*pgxpool.Config, error) { + if poolConfig == nil || poolConfig.ConnConfig == nil { + return nil, fmt.Errorf("PostgreSQL pool config is required") + } + if provider == nil { + return nil, fmt.Errorf("connection cache provider is required") + } + if err := runtimeConfig.validate(); err != nil { + return nil, err + } + + configuredPool := poolConfig.Copy() + callerAfterConnect := configuredPool.AfterConnect + callerAfterRelease := configuredPool.AfterRelease + callerBeforeClose := configuredPool.BeforeClose + + poolLimits := runtimeConfig.resolvedPoolConfig() + configuredPool.MinConns = poolLimits.MinConnections + configuredPool.MaxConns = poolLimits.MaxConnections + configuredPool.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error { + if hooks.afterConnect != nil { + if err := hooks.afterConnect(ctx, conn); err != nil { + return err + } + } + if callerAfterConnect != nil { + if err := callerAfterConnect(ctx, conn); err != nil { + return err + } + } + provider.registerConnection(conn) + if err := warmCurrentStatementPolicy(warmups, func(statements []preparedStatementWarmup) error { + return provider.warmStatementsForConnection(conn, statements, func(name, sql string) error { + _, err := conn.Prepare(ctx, name, sql) + return err + }) + }); err != nil { + return err + } + return nil + } + configuredPool.AfterRelease = func(conn *pgx.Conn) bool { + if hooks.afterRelease != nil && !hooks.afterRelease(conn) { + return false + } + if callerAfterRelease != nil && !callerAfterRelease(conn) { + return false + } + return true + } + configuredPool.BeforeClose = func(conn *pgx.Conn) { + provider.removeConnection(conn) + if callerBeforeClose != nil { + callerBeforeClose(conn) + } + } + return configuredPool, nil +} + +// NewPoolWithRuntimeConfig constructs a PostgreSQL pool with connection-local +// cache state. It copies poolConfig before composing required and caller +// lifecycle hooks, leaving the caller's configuration reusable. +func NewPoolWithRuntimeConfig(ctx context.Context, poolConfig *pgxpool.Config, config RuntimeConfig) (*pgxpool.Pool, error) { + if ctx == nil { + return nil, fmt.Errorf("pool context is required") + } + provider, err := newConnectionCacheProvider(config) + if err != nil { + return nil, err + } + warmups := &statementWarmupPolicy{} + configuredPool, err := composePoolConfig(poolConfig, config, provider, warmups, productionPoolLifecycleHooks()) + if err != nil { + provider.close() + return nil, err + } + + poolCtx, cancel := context.WithTimeout(ctx, poolInitConnectionTimeout) + defer cancel() + underlying, err := pgxpool.NewWithConfig(poolCtx, configuredPool) + if err != nil { + provider.close() + return nil, err + } + runtime := &poolRuntime{ + pool: underlying, + provider: provider, + warmups: warmups, + } + registerPoolRuntime(underlying, runtime) + return underlying, nil +} + +// SetStatementWarmupPolicy replaces the opt-in warm set, prepares it on +// currently idle connections, and applies it to every subsequently created +// physical connection. Passing no statements clears the future warm set. +func (s *poolRuntime) setStatementWarmupPolicy(ctx context.Context, statements ...string) error { + if s == nil || s.pool == nil || s.provider == nil || s.warmups == nil { + return fmt.Errorf("PostgreSQL pool runtime is not initialized") + } + warmups, err := normalizePreparedStatementWarmups(statements) + if err != nil { + return err + } + s.warmups.replace(warmups) + return s.warmPreparedStatements(ctx, warmups) +} + +// close retires all runtime state after the owner closes the pool. +func (s *poolRuntime) close() { + if s == nil { + return + } + s.closeOnce.Do(func() { + if s.pool != nil { + s.pool.Close() + poolRuntimes.Delete(s.pool) + } + if s.provider != nil { + s.provider.close() + } + }) +} + +// Reset closes idle physical connections and causes acquired connections to be +// closed when released. BeforeClose retires every affected connection cache. +// Use it after an out-of-band schema change that can affect registered types +// or generated SQL. +func (s *poolRuntime) reset() { + if s != nil && s.pool != nil { + s.pool.Reset() + } +} + +// WarmStatements prepares the supplied PostgreSQL SQL on every currently +// idle physical connection. It never executes the SQL. Call it after schema +// assertion, ideally while the pool is otherwise quiescent; newly created +// connections warm lazily through pgx's normal CacheStatement behavior. +func (s *poolRuntime) warmStatements(ctx context.Context, statements ...string) error { + if s == nil || s.pool == nil || s.provider == nil { + return fmt.Errorf("PostgreSQL pool runtime is not initialized") + } + warmups, err := normalizePreparedStatementWarmups(statements) + if err != nil { + return err + } + return s.warmPreparedStatements(ctx, warmups) +} + +// warmPreparedStatements prepares a normalized warm set on idle connections. +func (s *poolRuntime) warmPreparedStatements(ctx context.Context, warmups []preparedStatementWarmup) error { + if len(warmups) == 0 { + return nil + } + + connections := s.pool.AcquireAllIdle(ctx) + if len(connections) == 0 { + connection, err := s.pool.Acquire(ctx) + if err != nil { + return err + } + connections = []*pgxpool.Conn{connection} + } + defer func() { + for _, connection := range connections { + connection.Release() + } + }() + + var errs []error + for _, connection := range connections { + if err := s.provider.warmStatementsForConnection(connection.Conn(), warmups, func(name, sql string) error { + _, err := connection.Conn().Prepare(ctx, name, sql) + return err + }); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} diff --git a/drivers/pg/pool_runtime_test.go b/drivers/pg/pool_runtime_test.go new file mode 100644 index 00000000..bf130275 --- /dev/null +++ b/drivers/pg/pool_runtime_test.go @@ -0,0 +1,232 @@ +package pg + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" +) + +func testPreparedWarmups(t *testing.T, statements ...string) []preparedStatementWarmup { + t.Helper() + warmups, err := normalizePreparedStatementWarmups(statements) + require.NoError(t, err) + return warmups +} + +func TestStatementWarmupPolicyGenerationsAndSnapshots(t *testing.T) { + policy := &statementWarmupPolicy{} + policy.replace(testPreparedWarmups(t, "select 1")) + + generation, snapshot := policy.snapshot() + require.Equal(t, uint64(1), generation) + require.Len(t, snapshot, 1) + snapshot[0].sql = "select changed" + + _, current := policy.snapshot() + require.Equal(t, "select 1", current[0].sql) + + policy.replace(testPreparedWarmups(t, "select 2")) + generation, current = policy.snapshot() + require.Equal(t, uint64(2), generation) + require.Equal(t, "select 2", current[0].sql) + + policy.replace(nil) + generation, current = policy.snapshot() + require.Equal(t, uint64(3), generation) + require.Empty(t, current) +} + +func TestWarmCurrentStatementPolicyRetriesSupersededGeneration(t *testing.T) { + policy := &statementWarmupPolicy{} + policy.replace(testPreparedWarmups(t, "select 1")) + firstWarmStarted := make(chan struct{}) + allowFirstWarm := make(chan struct{}) + var once sync.Once + var warmed []string + + done := make(chan error, 1) + go func() { + done <- warmCurrentStatementPolicy(policy, func(statements []preparedStatementWarmup) error { + if len(statements) > 0 { + warmed = append(warmed, statements[0].sql) + } + once.Do(func() { + close(firstWarmStarted) + <-allowFirstWarm + }) + return nil + }) + }() + + <-firstWarmStarted + policy.replace(testPreparedWarmups(t, "select 2")) + close(allowFirstWarm) + require.NoError(t, <-done) + require.Equal(t, []string{"select 1", "select 2"}, warmed) +} + +func TestWarmCurrentStatementPolicyObservesClear(t *testing.T) { + policy := &statementWarmupPolicy{} + policy.replace(testPreparedWarmups(t, "select 1")) + firstWarmStarted := make(chan struct{}) + allowFirstWarm := make(chan struct{}) + var once sync.Once + var warmSetSizes []int + + done := make(chan error, 1) + go func() { + done <- warmCurrentStatementPolicy(policy, func(statements []preparedStatementWarmup) error { + warmSetSizes = append(warmSetSizes, len(statements)) + once.Do(func() { + close(firstWarmStarted) + <-allowFirstWarm + }) + return nil + }) + }() + + <-firstWarmStarted + policy.replace(nil) + close(allowFirstWarm) + require.NoError(t, <-done) + require.Equal(t, []int{1, 0}, warmSetSizes) +} + +func testPoolConfig(t *testing.T) *pgxpool.Config { + t.Helper() + config, err := pgxpool.ParseConfig("postgresql://localhost:5432/dawgs") + require.NoError(t, err) + return config +} + +func TestComposePoolConfigCopiesAndOrdersHooks(t *testing.T) { + provider, err := newConnectionCacheProvider(DefaultRuntimeConfig()) + require.NoError(t, err) + config := testPoolConfig(t) + var order []string + config.MinConns = 1 + config.MaxConns = 2 + config.AfterConnect = func(context.Context, *pgx.Conn) error { + order = append(order, "caller-connect") + return nil + } + config.AfterRelease = func(*pgx.Conn) bool { + order = append(order, "caller-release") + return true + } + config.BeforeClose = func(conn *pgx.Conn) { + require.Nil(t, provider.CacheForConnection(conn)) + order = append(order, "caller-close") + } + hooks := poolLifecycleHooks{ + afterConnect: func(context.Context, *pgx.Conn) error { + order = append(order, "dawgs-connect") + return nil + }, + afterRelease: func(*pgx.Conn) bool { + order = append(order, "dawgs-release") + return true + }, + } + + composed, err := composePoolConfig(config, DefaultRuntimeConfig(), provider, &statementWarmupPolicy{}, hooks) + require.NoError(t, err) + require.NotSame(t, config, composed) + require.Equal(t, int32(1), config.MinConns) + require.Equal(t, int32(2), config.MaxConns) + require.Equal(t, int32(5), composed.MinConns) + require.Equal(t, int32(50), composed.MaxConns) + + conn := &pgx.Conn{} + require.NoError(t, composed.AfterConnect(context.Background(), conn)) + require.NotNil(t, provider.CacheForConnection(conn)) + require.True(t, composed.AfterRelease(conn)) + composed.BeforeClose(conn) + require.Equal(t, []string{"dawgs-connect", "caller-connect", "dawgs-release", "caller-release", "caller-close"}, order) +} + +func TestComposePoolConfigPreservesHookFailuresAndRejection(t *testing.T) { + provider, err := newConnectionCacheProvider(DefaultRuntimeConfig()) + require.NoError(t, err) + conn := &pgx.Conn{} + + t.Run("failed required connect does not register state", func(t *testing.T) { + config := testPoolConfig(t) + expected := errors.New("required setup failed") + composed, err := composePoolConfig(config, DefaultRuntimeConfig(), provider, &statementWarmupPolicy{}, poolLifecycleHooks{ + afterConnect: func(context.Context, *pgx.Conn) error { return expected }, + }) + require.NoError(t, err) + require.ErrorIs(t, composed.AfterConnect(context.Background(), conn), expected) + require.Nil(t, provider.CacheForConnection(conn)) + }) + + t.Run("failed caller connect does not register state", func(t *testing.T) { + config := testPoolConfig(t) + expected := errors.New("caller setup failed") + config.AfterConnect = func(context.Context, *pgx.Conn) error { return expected } + composed, err := composePoolConfig(config, DefaultRuntimeConfig(), provider, &statementWarmupPolicy{}, poolLifecycleHooks{ + afterConnect: func(context.Context, *pgx.Conn) error { return nil }, + }) + require.NoError(t, err) + require.ErrorIs(t, composed.AfterConnect(context.Background(), conn), expected) + require.Nil(t, provider.CacheForConnection(conn)) + }) + + t.Run("required release rejection skips caller", func(t *testing.T) { + config := testPoolConfig(t) + called := false + config.AfterRelease = func(*pgx.Conn) bool { + called = true + return true + } + composed, err := composePoolConfig(config, DefaultRuntimeConfig(), provider, &statementWarmupPolicy{}, poolLifecycleHooks{ + afterRelease: func(*pgx.Conn) bool { return false }, + }) + require.NoError(t, err) + require.False(t, composed.AfterRelease(conn)) + require.False(t, called) + }) + + t.Run("caller release rejection is preserved", func(t *testing.T) { + config := testPoolConfig(t) + config.AfterRelease = func(*pgx.Conn) bool { return false } + composed, err := composePoolConfig(config, DefaultRuntimeConfig(), provider, &statementWarmupPolicy{}, poolLifecycleHooks{ + afterRelease: func(*pgx.Conn) bool { return true }, + }) + require.NoError(t, err) + require.False(t, composed.AfterRelease(conn)) + }) +} + +func TestDefaultConfigUsesConservativePerConnectionCapacity(t *testing.T) { + require.Equal(t, defaultTranslationCacheEntries, DefaultRuntimeConfig().TranslationCacheEntries) + require.Equal(t, &PoolConfig{MinConnections: defaultMinConnections, MaxConnections: defaultMaxConnections}, DefaultRuntimeConfig().Pool) +} + +func TestConfigValidatesAndAppliesExplicitPoolLimits(t *testing.T) { + config := RuntimeConfig{ + TranslationCacheEntries: 3, + Pool: &PoolConfig{MinConnections: 0, MaxConnections: 2}, + } + provider, err := newConnectionCacheProvider(config) + require.NoError(t, err) + composed, err := composePoolConfig(testPoolConfig(t), config, provider, &statementWarmupPolicy{}, poolLifecycleHooks{}) + require.NoError(t, err) + require.Equal(t, int32(0), composed.MinConns) + require.Equal(t, int32(2), composed.MaxConns) + + for _, invalid := range []RuntimeConfig{ + {TranslationCacheEntries: -1}, + {Pool: &PoolConfig{MinConnections: -1, MaxConnections: 1}}, + {Pool: &PoolConfig{MinConnections: 0, MaxConnections: 0}}, + {Pool: &PoolConfig{MinConnections: 2, MaxConnections: 1}}, + } { + require.Error(t, invalid.validate()) + } +} diff --git a/drivers/pg/query/p5_adjacency_shadow_integration_test.go b/drivers/pg/query/p5_adjacency_shadow_integration_test.go new file mode 100644 index 00000000..1066fcb7 --- /dev/null +++ b/drivers/pg/query/p5_adjacency_shadow_integration_test.go @@ -0,0 +1,210 @@ +//go:build manual_integration && integration + +package query + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" +) + +// TestP5AdjacencyShadowLifecycle verifies the opt-in P5 relation backfills +// existing edges and remains transactionally synchronized without changing the +// normal graph schema. +func TestP5AdjacencyShadowLifecycle(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + if !isPostgreSQLConnection(connection) { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + + _, err = pool.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + _, err = pool.Exec(ctx, sqlP5AdjacencyShadowDown) + require.NoError(t, err) + t.Cleanup(func() { + _, cleanupErr := pool.Exec(ctx, sqlP5AdjacencyShadowDown) + require.NoError(t, cleanupErr) + }) + + graphName := fmt.Sprintf("p5_adjacency_shadow_%d", time.Now().UnixNano()) + var graphID int32 + require.NoError(t, pool.QueryRow(ctx, `insert into graph(name) values ($1) returning id`, graphName).Scan(&graphID)) + _, err = pool.Exec(ctx, fmt.Sprintf("create table node_%d partition of node for values in (%d)", graphID, graphID)) + require.NoError(t, err) + _, err = pool.Exec(ctx, fmt.Sprintf("create table edge_%d partition of edge for values in (%d)", graphID, graphID)) + require.NoError(t, err) + + insertNode := func() int64 { + var nodeID int64 + require.NoError(t, pool.QueryRow(ctx, + `insert into node(graph_id, kind_ids, properties) values ($1, array[1]::int2[], '{}'::jsonb) returning id`, + graphID, + ).Scan(&nodeID)) + return nodeID + } + insertEdge := func(startID, endID int64) int64 { + var edgeID int64 + require.NoError(t, pool.QueryRow(ctx, + `insert into edge(graph_id, start_id, end_id, kind_id, properties) values ($1, $2, $3, 1, '{}'::jsonb) returning id`, + graphID, startID, endID, + ).Scan(&edgeID)) + return edgeID + } + + nodeOne := insertNode() + nodeTwo := insertNode() + nodeThree := insertNode() + nodeFour := insertNode() + edgeOne := insertEdge(nodeOne, nodeTwo) + edgeTwo := insertEdge(nodeTwo, nodeThree) + + _, err = pool.Exec(ctx, sqlP5AdjacencyShadowUp) + require.NoError(t, err) + + assertExactShadow := func(expectedEdges int64) { + var rows, mismatches int64 + require.NoError(t, pool.QueryRow(ctx, + `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, graphID).Scan(&rows)) + require.Equal(t, expectedEdges*2, rows) + require.NoError(t, pool.QueryRow(ctx, ` + select + (select count(*) + from edge e + where e.graph_id = $1 + and not exists ( + select 1 from public.p5_adjacency_v1 a + where a.graph_id = e.graph_id and a.edge_id = e.id + and a.direction = 1 and a.anchor_id = e.start_id + and a.neighbor_id = e.end_id and a.kind_id = e.kind_id + ) + or not exists ( + select 1 from public.p5_adjacency_v1 a + where a.graph_id = e.graph_id and a.edge_id = e.id + and a.direction = -1 and a.anchor_id = e.end_id + and a.neighbor_id = e.start_id and a.kind_id = e.kind_id + )) + + + (select count(*) + from public.p5_adjacency_v1 a + where a.graph_id = $1 + and not exists ( + select 1 from edge e + where e.graph_id = a.graph_id and e.id = a.edge_id + and ((a.direction = 1 and a.anchor_id = e.start_id and a.neighbor_id = e.end_id) + or (a.direction = -1 and a.anchor_id = e.end_id and a.neighbor_id = e.start_id)) + and a.kind_id = e.kind_id + )) + `, graphID).Scan(&mismatches)) + require.Zero(t, mismatches) + } + + assertExactShadow(2) + + var beforePropertyUpdate string + require.NoError(t, pool.QueryRow(ctx, + `select string_agg(ctid::text, ',' order by direction) from public.p5_adjacency_v1 where graph_id = $1 and edge_id = $2`, + graphID, edgeOne, + ).Scan(&beforePropertyUpdate)) + _, err = pool.Exec(ctx, + `update edge set properties = properties || '{"touch": true}'::jsonb where graph_id = $1 and id = $2`, + graphID, edgeOne, + ) + require.NoError(t, err) + var afterPropertyUpdate string + require.NoError(t, pool.QueryRow(ctx, + `select string_agg(ctid::text, ',' order by direction) from public.p5_adjacency_v1 where graph_id = $1 and edge_id = $2`, + graphID, edgeOne, + ).Scan(&afterPropertyUpdate)) + require.Equal(t, beforePropertyUpdate, afterPropertyUpdate) + + _, err = pool.Exec(ctx, `update edge set start_id = $1 where graph_id = $2 and id = $3`, nodeFour, graphID, edgeOne) + require.NoError(t, err) + assertExactShadow(2) + var updatedOutbound int64 + require.NoError(t, pool.QueryRow(ctx, ` + select count(*) from public.p5_adjacency_v1 + where graph_id = $1 and edge_id = $2 and direction = 1 and anchor_id = $3 and neighbor_id = $4`, + graphID, edgeOne, nodeFour, nodeTwo, + ).Scan(&updatedOutbound)) + require.Equal(t, int64(1), updatedOutbound) + + _, err = pool.Exec(ctx, `delete from node where graph_id = $1 and id = $2`, graphID, nodeThree) + require.NoError(t, err) + assertExactShadow(1) + var deletedEdgeRows int64 + require.NoError(t, pool.QueryRow(ctx, + `select count(*) from public.p5_adjacency_v1 where graph_id = $1 and edge_id = $2`, graphID, edgeTwo, + ).Scan(&deletedEdgeRows)) + require.Zero(t, deletedEdgeRows) + + tx, err := pool.BeginTx(ctx, pgx.TxOptions{}) + require.NoError(t, err) + var rolledBackEdge int64 + require.NoError(t, tx.QueryRow(ctx, + `insert into edge(graph_id, start_id, end_id, kind_id, properties) values ($1, $2, $3, 1, '{}'::jsonb) returning id`, + graphID, nodeFour, nodeOne, + ).Scan(&rolledBackEdge)) + require.NoError(t, tx.Rollback(ctx)) + var rollbackRows int64 + require.NoError(t, pool.QueryRow(ctx, + `select count(*) from public.p5_adjacency_v1 where graph_id = $1 and edge_id = $2`, graphID, rolledBackEdge, + ).Scan(&rollbackRows)) + require.Zero(t, rollbackRows) + assertExactShadow(1) + + cancelledTx, err := pool.BeginTx(ctx, pgx.TxOptions{}) + require.NoError(t, err) + cancelledContext, cancel := context.WithCancel(ctx) + timer := time.AfterFunc(20*time.Millisecond, cancel) + _, err = cancelledTx.Exec(cancelledContext, ` + with delayed as materialized (select pg_sleep(2)) + insert into edge(graph_id, start_id, end_id, kind_id, properties) + select $1, $2, $3, 1, '{}'::jsonb from delayed`, + graphID, nodeOne, nodeFour, + ) + timer.Stop() + cancel() + require.Error(t, err) + _ = cancelledTx.Rollback(ctx) + assertExactShadow(1) + + reusedConnection, err := pool.Acquire(ctx) + require.NoError(t, err) + var reusedRows int64 + require.NoError(t, reusedConnection.QueryRow(ctx, + `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, graphID, + ).Scan(&reusedRows)) + reusedConnection.Release() + require.Equal(t, int64(2), reusedRows) + + _, err = pool.Exec(ctx, `delete from graph where id = $1`, graphID) + require.NoError(t, err) + var graphRows int64 + require.NoError(t, pool.QueryRow(ctx, + `select count(*) from public.p5_adjacency_v1 where graph_id = $1`, graphID, + ).Scan(&graphRows)) + require.Zero(t, graphRows) + + _, err = pool.Exec(ctx, sqlP5AdjacencyShadowDown) + require.NoError(t, err) + var shadowPresent, edgePresent bool + require.NoError(t, pool.QueryRow(ctx, + `select to_regclass('public.p5_adjacency_v1') is not null, to_regclass('public.edge') is not null`, + ).Scan(&shadowPresent, &edgePresent)) + require.False(t, shadowPresent) + require.True(t, edgePresent) +} diff --git a/drivers/pg/query/p5_adjacency_shadow_sql_test.go b/drivers/pg/query/p5_adjacency_shadow_sql_test.go new file mode 100644 index 00000000..c1109407 --- /dev/null +++ b/drivers/pg/query/p5_adjacency_shadow_sql_test.go @@ -0,0 +1,25 @@ +package query + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestP5AdjacencyShadowIsOptIn verifies the experimental materialization has +// its own install/remove boundary and cannot alter normal schema creation. +func TestP5AdjacencyShadowIsOptIn(t *testing.T) { + require.NotContains(t, sqlSchemaUp, "p5_adjacency_v1") + require.NotContains(t, sqlSchemaDown, "p5_adjacency_v1") + require.Contains(t, sqlP5AdjacencyShadowUp, "create table if not exists public.p5_adjacency_v1") + require.Contains(t, sqlP5AdjacencyShadowUp, "partition by list (graph_id)") + require.Contains(t, sqlP5AdjacencyShadowUp, "primary key (graph_id, direction, edge_id)") + require.Contains(t, sqlP5AdjacencyShadowUp, "foreign key (edge_id, graph_id) references edge (id, graph_id) on delete cascade") + require.Contains(t, sqlP5AdjacencyShadowUp, "p5_adjacency_v1_lookup_index") + require.Contains(t, sqlP5AdjacencyShadowUp, "on conflict (graph_id, direction, edge_id) do update") + require.Contains(t, sqlP5AdjacencyShadowUp, "after update of graph_id, start_id, end_id, kind_id") + require.Contains(t, sqlP5AdjacencyShadowUp, "after insert") + require.Contains(t, sqlP5AdjacencyShadowUp, "after delete") + require.Contains(t, sqlP5AdjacencyShadowDown, "drop trigger if exists p5_adjacency_v1_after_insert on edge") + require.Contains(t, sqlP5AdjacencyShadowDown, "drop table if exists public.p5_adjacency_v1") +} diff --git a/drivers/pg/query/query.go b/drivers/pg/query/query.go index c04dc9ac..d4c13e81 100644 --- a/drivers/pg/query/query.go +++ b/drivers/pg/query/query.go @@ -217,6 +217,18 @@ func (s Query) DropSchema() error { return nil } +// InstallP5AdjacencyShadow installs the opt-in P5 adjacency materialization +// used only by the feasibility study. Normal schema creation never calls it. +func (s Query) InstallP5AdjacencyShadow() error { + return s.exec(sqlP5AdjacencyShadowUp, nil) +} + +// DropP5AdjacencyShadow removes the opt-in P5 feasibility schema without +// changing the core graph schema. +func (s Query) DropP5AdjacencyShadow() error { + return s.exec(sqlP5AdjacencyShadowDown, nil) +} + func (s Query) insertGraph(name string) (model.Graph, error) { var ( graphID int32 diff --git a/drivers/pg/query/schema_upgrade_integration_test.go b/drivers/pg/query/schema_upgrade_integration_test.go new file mode 100644 index 00000000..a8af49bc --- /dev/null +++ b/drivers/pg/query/schema_upgrade_integration_test.go @@ -0,0 +1,441 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration && integration + +package query + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" +) + +func isPostgreSQLConnection(connection string) bool { + normalized := strings.ToLower(connection) + return strings.HasPrefix(normalized, "postgres://") || strings.HasPrefix(normalized, "postgresql://") +} + +// TestSchemaUpgradeRemovesLegacyPathMaterializerOverloads verifies an upgrade drops obsolete unscoped path functions while retaining graph-scoped signatures. +func TestSchemaUpgradeRemovesLegacyPathMaterializerOverloads(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + if !isPostgreSQLConnection(connection) { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + + _, err = pool.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + + _, err = pool.Exec(ctx, ` + drop function public.nodes_to_path(int4, int8[]); + drop function public.edges_to_path(int4, int8[]); + drop function public.ordered_edges_to_path(int4, nodeComposite, edgeComposite[], nodeComposite[]); + create function public.nodes_to_path(nodes variadic int8[]) returns pathComposite language sql immutable strict as $$ + select row(array[]::nodeComposite[], array[]::edgeComposite[])::pathComposite + $$; + create function public.edges_to_path(path variadic int8[]) returns pathComposite language sql immutable strict as $$ + select row(array[]::nodeComposite[], array[]::edgeComposite[])::pathComposite + $$; + create function public.ordered_edges_to_path(root nodeComposite, edges edgeComposite[], known_nodes nodeComposite[]) returns pathComposite language sql immutable strict as $$ + select row(array[root]::nodeComposite[], edges)::pathComposite + $$; + `) + require.NoError(t, err) + + _, err = pool.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + + var legacyNodes, legacyEdges, legacyOrdered, scopedNodes, scopedEdges, scopedOrdered bool + err = pool.QueryRow(ctx, `select + to_regprocedure('public.nodes_to_path(bigint[])') is not null, + to_regprocedure('public.edges_to_path(bigint[])') is not null, + to_regprocedure('public.ordered_edges_to_path(nodecomposite,edgecomposite[],nodecomposite[])') is not null, + to_regprocedure('public.nodes_to_path(integer,bigint[])') is not null, + to_regprocedure('public.edges_to_path(integer,bigint[])') is not null, + to_regprocedure('public.ordered_edges_to_path(integer,nodecomposite,edgecomposite[],nodecomposite[])') is not null + `).Scan(&legacyNodes, &legacyEdges, &legacyOrdered, &scopedNodes, &scopedEdges, &scopedOrdered) + require.NoError(t, err) + require.False(t, legacyNodes) + require.False(t, legacyEdges) + require.False(t, legacyOrdered) + require.True(t, scopedNodes) + require.True(t, scopedEdges) + require.True(t, scopedOrdered) +} + +// TestBidirectionalAllShortestPathCapBoundaries proves that every candidate +// admission gate is exact at N, fails closed at N-1, and preserves the full +// ASP-A1 multiset on fallback. The fixture reconverges through two middle +// nodes so equal-depth, relationship-distinct predecessor rows are required +// to produce all six shortest paths. +func TestBidirectionalAllShortestPathCapBoundaries(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + if !isPostgreSQLConnection(connection) { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + connectionHandle, err := pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + + _, err = connectionHandle.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + _, err = connectionHandle.Exec(ctx, ` + create temporary table edge + ( + id int8 not null, + graph_id int4 not null, + start_id int8 not null, + end_id int8 not null, + kind_id int2 not null, + properties jsonb not null + ) on commit preserve rows; + insert into edge(id, graph_id, start_id, end_id, kind_id, properties) values + (101, 1, 1, 2, 1, '{}'), (102, 1, 1, 3, 1, '{}'), (103, 1, 1, 4, 1, '{}'), + (104, 1, 2, 5, 1, '{}'), (105, 1, 2, 6, 1, '{}'), + (106, 1, 3, 5, 1, '{}'), (107, 1, 3, 6, 1, '{}'), + (108, 1, 4, 5, 1, '{}'), (109, 1, 4, 6, 1, '{}'), + (110, 1, 5, 9, 1, '{}'), (111, 1, 6, 9, 1, '{}'); + `) + require.NoError(t, err) + + tx, err := connectionHandle.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + defer func() { _ = tx.Rollback(ctx) }() + + readPaths := func(query string, args ...any) []string { + rows, queryErr := tx.Query(ctx, query, args...) + require.NoError(t, queryErr) + defer rows.Close() + paths := []string{} + for rows.Next() { + var path string + require.NoError(t, rows.Scan(&path)) + paths = append(paths, path) + } + require.NoError(t, rows.Err()) + return paths + } + + exact := readPaths(` + select path::text + from public.all_shortest_paths_dag(1, 1, 9, 1, 8, array[]::int2[], false) + order by path`) + require.Len(t, exact, 6) + + // limits configures each guarded resource dimension exercised by the helper. + type limits struct { + // state retains the state while limits is assembled or evaluated. + state int64 + // frontier retains the frontier while limits is assembled or evaluated. + frontier int64 + // predecessor retains the predecessor while limits is assembled or evaluated. + predecessor int64 + // enumeration retains the enumeration while limits is assembled or evaluated. + enumeration int64 + // outputBytes retains the output bytes while limits is assembled or evaluated. + outputBytes int64 + } + + // diagnostic decodes the runtime receipt returned by the guarded helper. + type diagnostic struct { + // RuntimeBranch supplies the runtime branch input to the diagnostic contract. + RuntimeBranch string `json:"runtime_branch"` + // Overflowed indicates whether overflowed applies. + Overflowed bool `json:"overflowed"` + // FallbackExecuted indicates whether fallback executed applies. + FallbackExecuted bool `json:"fallback_executed"` + // Counters supplies the counters input to the diagnostic contract. + Counters struct { + // SeenPeak supplies the seen peak input to the Counters contract. + SeenPeak int64 `json:"seen_peak"` + // FrontierPeak supplies the frontier peak input to the Counters contract. + FrontierPeak int64 `json:"frontier_peak"` + // PredecessorPeak supplies the predecessor peak input to the Counters contract. + PredecessorPeak int64 `json:"predecessor_peak"` + // OutputPaths identifies the filesystem output paths. + OutputPaths int64 `json:"output_paths"` + // OutputBytes supplies the output bytes input to the Counters contract. + OutputBytes int64 `json:"output_bytes"` + } `json:"counters"` + } + const candidateQuery = ` + select path::text + from public.all_shortest_paths_b1_strict_alternating( + 1, 1, 9, 1, 8, array[]::int2[], false, $1, $2, $3, $4, $5) + order by path` + runCandidate := func(invocationID string, caps limits) ([]string, diagnostic) { + _, execErr := tx.Exec(ctx, "select public.begin_bidirectional_all_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, execErr) + paths := readPaths(candidateQuery, caps.state, caps.frontier, caps.predecessor, caps.enumeration, caps.outputBytes) + var raw string + require.NoError(t, tx.QueryRow(ctx, + "select public.read_bidirectional_all_shortest_path_diagnostic_v1($1)::text", invocationID).Scan(&raw)) + var report diagnostic + require.NoError(t, json.Unmarshal([]byte(raw), &report)) + _, execErr = tx.Exec(ctx, "select public.clear_bidirectional_all_shortest_path_diagnostic_v1($1)", invocationID) + require.NoError(t, execErr) + return paths, report + } + + large := limits{ + state: 1_000_000, + frontier: 1_000_000, + predecessor: 1_000_000, + enumeration: 1_000_000, + outputBytes: 1 << 30, + } + for _, scheduler := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. + query string + }{ + { + name: "B1 strict alternating", + query: candidateQuery, + }, + { + name: "B2 smaller level", + query: ` + select path::text + from public.all_shortest_paths_b2_smaller_current_level( + 1, 1, 9, 1, 8, array[]::int2[], false, $1, $2, $3, $4, $5) + order by path`, + }, + } { + t.Run(scheduler.name+" retains the exact multiset", func(t *testing.T) { + paths := readPaths(scheduler.query, large.state, large.frontier, large.predecessor, large.enumeration, large.outputBytes) + require.Equal(t, exact, paths) + }) + } + + baselinePaths, baseline := runCandidate("asp-cap-baseline", large) + require.Equal(t, exact, baselinePaths) + require.Equal(t, "bidirectional_search", baseline.RuntimeBranch) + require.False(t, baseline.Overflowed) + require.False(t, baseline.FallbackExecuted) + require.Positive(t, baseline.Counters.SeenPeak) + require.Positive(t, baseline.Counters.FrontierPeak) + require.Positive(t, baseline.Counters.PredecessorPeak) + require.Equal(t, int64(len(exact)), baseline.Counters.OutputPaths) + require.Positive(t, baseline.Counters.OutputBytes) + + boundaries := []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // get retains the get while anonymous record is assembled or evaluated. + get func(limits) int64 + // set retains the set while anonymous record is assembled or evaluated. + set func(*limits, int64) + }{ + { + name: "state", + get: func(_ limits) int64 { return baseline.Counters.SeenPeak }, + set: func(value *limits, limit int64) { value.state = limit }, + }, + { + name: "frontier", + get: func(_ limits) int64 { return baseline.Counters.FrontierPeak }, + set: func(value *limits, limit int64) { value.frontier = limit }, + }, + { + name: "predecessor", + get: func(_ limits) int64 { return baseline.Counters.PredecessorPeak }, + set: func(value *limits, limit int64) { value.predecessor = limit }, + }, + { + name: "enumeration", + get: func(_ limits) int64 { return baseline.Counters.OutputPaths }, + set: func(value *limits, limit int64) { value.enumeration = limit }, + }, + { + name: "output bytes", + get: func(_ limits) int64 { return baseline.Counters.OutputBytes }, + set: func(value *limits, limit int64) { value.outputBytes = limit }, + }, + } + for _, boundary := range boundaries { + boundary := boundary + n := boundary.get(large) + for _, delta := range []int64{-1, 0, 1} { + delta := delta + name := boundary.name + map[int64]string{-1: " N-1", 0: " N", 1: " N+1"}[delta] + t.Run(name, func(t *testing.T) { + caps := large + boundary.set(&caps, n+delta) + paths, report := runCandidate("asp-cap-"+boundary.name+map[int64]string{-1: "-minus", 0: "-exact", 1: "-plus"}[delta], caps) + require.Equal(t, exact, paths, "candidate and fallback must preserve the complete ordered multiset") + if delta < 0 { + require.Equal(t, "exact_a1_fallback", report.RuntimeBranch) + require.True(t, report.Overflowed) + require.True(t, report.FallbackExecuted) + } else { + require.Equal(t, "bidirectional_search", report.RuntimeBranch) + require.False(t, report.Overflowed) + require.False(t, report.FallbackExecuted) + } + }) + } + } + + require.NoError(t, tx.Rollback(ctx)) +} + +// TestBidirectionalShortestPathLowerBoundAndWitnesses exercises a graph where +// strict alternation encounters a length-five meeting before the unique +// length-four route. Returning the shorter route proves the queue-head +// lower-bound check continued beyond the first intersection. The tie and +// inbound assertions separately validate the one-witness contract: minimum +// depth, relationship uniqueness, and logical source-to-target edge order. +func TestBidirectionalShortestPathLowerBoundAndWitnesses(t *testing.T) { + connection := os.Getenv("CONNECTION_STRING") + if connection == "" { + t.Skip("CONNECTION_STRING env var is not set") + } + if !isPostgreSQLConnection(connection) { + t.Skip("CONNECTION_STRING is not a PostgreSQL connection string") + } + + ctx := context.Background() + pool, err := pgxpool.New(ctx, connection) + require.NoError(t, err) + t.Cleanup(pool.Close) + connectionHandle, err := pool.Acquire(ctx) + require.NoError(t, err) + defer connectionHandle.Release() + _, err = connectionHandle.Exec(ctx, sqlSchemaUp) + require.NoError(t, err) + _, err = connectionHandle.Exec(ctx, ` + create temporary table edge + ( + id int8 not null, + graph_id int4 not null, + start_id int8 not null, + end_id int8 not null, + kind_id int2 not null, + properties jsonb not null + ) on commit preserve rows; + -- Graph 2: the low-ID length-five branch meets first under B1. Two + -- target-side dead ends delay acceptance of the unique length-four arm. + insert into edge(id, graph_id, start_id, end_id, kind_id, properties) values + (201, 2, 1000, 1001, 1, '{}'), (203, 2, 1001, 1002, 1, '{}'), + (205, 2, 1002, 1003, 1, '{}'), (207, 2, 1003, 1004, 1, '{}'), + (209, 2, 1004, 1999, 1, '{}'), + (202, 2, 1000, 1100, 1, '{}'), (204, 2, 1100, 1101, 1, '{}'), + (206, 2, 1101, 1102, 1, '{}'), (999, 2, 1102, 1999, 1, '{}'), + (210, 2, 1200, 1999, 1, '{}'), (211, 2, 1201, 1999, 1, '{}'), + -- Graph 3: two equally short, relationship-disjoint witnesses. + (301, 3, 2000, 2001, 1, '{}'), (302, 3, 2001, 2002, 1, '{}'), + (303, 3, 2002, 2999, 1, '{}'), + (304, 3, 2000, 2101, 1, '{}'), (305, 3, 2101, 2102, 1, '{}'), + (306, 3, 2102, 2999, 1, '{}'); + `) + require.NoError(t, err) + + tx, err := connectionHandle.BeginTx(ctx, pgx.TxOptions{ + IsoLevel: pgx.RepeatableRead, + AccessMode: pgx.ReadWrite, + }) + require.NoError(t, err) + defer func() { _ = tx.Rollback(ctx) }() + + // result captures the depth and path returned by one helper invocation. + type result struct { + // depth retains the depth while result is assembled or evaluated. + depth int32 + // path retains the path while result is assembled or evaluated. + path []int64 + } + run := func(function string, graphID, sourceID, targetID int64, inbound bool) result { + query := `select depth, path from public.` + function + `( + $1::int4, $2::int8, $3::int8, 1, 8, array[]::int2[], $4, 100000, 100000, 100000)` + var value result + require.NoError(t, tx.QueryRow(ctx, query, graphID, sourceID, targetID, inbound).Scan(&value.depth, &value.path)) + return value + } + + _, err = tx.Exec(ctx, "select public.begin_bidirectional_shortest_path_diagnostic_v1('sp-adversarial-b1')") + require.NoError(t, err) + b1 := run("shortest_path_b1_strict_alternating", 2, 1000, 1999, false) + require.Equal(t, int32(4), b1.depth) + require.Equal(t, []int64{202, 204, 206, 999}, b1.path) + var raw string + require.NoError(t, tx.QueryRow(ctx, + "select public.read_bidirectional_shortest_path_diagnostic_v1('sp-adversarial-b1')::text").Scan(&raw)) + var report struct { + // RuntimeBranch supplies the runtime branch input to the anonymous record contract. + RuntimeBranch string `json:"runtime_branch"` + // Counters supplies the counters input to the anonymous record contract. + Counters struct { + // MeetingCandidates supplies the meeting candidates input to the Counters contract. + MeetingCandidates int64 `json:"meeting_candidates"` + // FrozenDistance supplies the frozen distance input to the Counters contract. + FrozenDistance int32 `json:"frozen_distance"` + // WitnessRows records the number of witness rows. + WitnessRows int64 `json:"witness_rows"` + } `json:"counters"` + } + require.NoError(t, json.Unmarshal([]byte(raw), &report)) + require.Equal(t, "bidirectional_search", report.RuntimeBranch) + require.GreaterOrEqual(t, report.Counters.MeetingCandidates, int64(2), "the longer and shorter intersections must both be observed") + require.Equal(t, int32(4), report.Counters.FrozenDistance) + require.Equal(t, int64(1), report.Counters.WitnessRows) + _, err = tx.Exec(ctx, "select public.clear_bidirectional_shortest_path_diagnostic_v1('sp-adversarial-b1')") + require.NoError(t, err) + + for _, scheduler := range []string{ + "shortest_path_b1_strict_alternating", + "shortest_path_b2_smaller_current_level", + } { + t.Run(scheduler+" unique and inbound witnesses", func(t *testing.T) { + outbound := run(scheduler, 2, 1000, 1999, false) + require.Equal(t, int32(4), outbound.depth) + require.Equal(t, []int64{202, 204, 206, 999}, outbound.path) + require.Len(t, outbound.path, int(outbound.depth)) + + inbound := run(scheduler, 2, 1999, 1000, true) + require.Equal(t, int32(4), inbound.depth) + require.Equal(t, []int64{999, 206, 204, 202}, inbound.path) + require.Len(t, inbound.path, int(inbound.depth)) + + tie := run(scheduler, 3, 2000, 2999, false) + require.Equal(t, int32(3), tie.depth) + require.Len(t, tie.path, int(tie.depth)) + require.Contains(t, [][]int64{{301, 302, 303}, {304, 305, 306}}, tie.path) + relationships := map[int64]struct{}{} + for _, edgeID := range tie.path { + relationships[edgeID] = struct{}{} + } + require.Len(t, relationships, len(tie.path), "a shortest witness may not repeat a relationship") + }) + } + + require.NoError(t, tx.Rollback(ctx)) +} diff --git a/drivers/pg/query/sql.go b/drivers/pg/query/sql.go index bec8b8f9..5ed99d86 100644 --- a/drivers/pg/query/sql.go +++ b/drivers/pg/query/sql.go @@ -43,13 +43,15 @@ func loadSQL(name string) string { } var ( - sqlSchemaUp = loadSQL("schema_up.sql") - sqlSchemaDown = loadSQL("schema_down.sql") - sqlSelectTableIndexes = loadSQL("select_table_indexes.sql") - sqlSelectKindID = loadSQL("select_table_indexes.sql") - sqlSelectGraphs = loadSQL("select_graphs.sql") - sqlInsertGraph = loadSQL("insert_graph.sql") - sqlInsertKind = loadSQL("insert_or_get_kind.sql") - sqlSelectKinds = loadSQL("select_kinds.sql") - sqlSelectGraphByName = loadSQL("select_graph_by_name.sql") + sqlSchemaUp = loadSQL("schema_up.sql") + sqlSchemaDown = loadSQL("schema_down.sql") + sqlP5AdjacencyShadowUp = loadSQL("p5_adjacency_shadow_up.sql") + sqlP5AdjacencyShadowDown = loadSQL("p5_adjacency_shadow_down.sql") + sqlSelectTableIndexes = loadSQL("select_table_indexes.sql") + sqlSelectKindID = loadSQL("select_table_indexes.sql") + sqlSelectGraphs = loadSQL("select_graphs.sql") + sqlInsertGraph = loadSQL("insert_graph.sql") + sqlInsertKind = loadSQL("insert_or_get_kind.sql") + sqlSelectKinds = loadSQL("select_kinds.sql") + sqlSelectGraphByName = loadSQL("select_graph_by_name.sql") ) diff --git a/drivers/pg/query/sql/p5_adjacency_shadow_down.sql b/drivers/pg/query/sql/p5_adjacency_shadow_down.sql new file mode 100644 index 00000000..42994186 --- /dev/null +++ b/drivers/pg/query/sql/p5_adjacency_shadow_down.sql @@ -0,0 +1,7 @@ +-- Remove only the opt-in P5 feasibility schema. Core graph storage remains. + +drop trigger if exists p5_adjacency_v1_after_insert on edge; +drop trigger if exists p5_adjacency_v1_after_delete on edge; +drop trigger if exists p5_adjacency_v1_after_endpoint_update on edge; +drop function if exists public.maintain_p5_adjacency_v1(); +drop table if exists public.p5_adjacency_v1; diff --git a/drivers/pg/query/sql/p5_adjacency_shadow_up.sql b/drivers/pg/query/sql/p5_adjacency_shadow_up.sql new file mode 100644 index 00000000..89138094 --- /dev/null +++ b/drivers/pg/query/sql/p5_adjacency_shadow_up.sql @@ -0,0 +1,89 @@ +-- P5 experimental adjacency materialization. This file is intentionally not +-- part of schema_up.sql: callers must opt in for the feasibility study. + +create table if not exists public.p5_adjacency_v1 +( + graph_id integer not null, + direction smallint not null check (direction in (-1, 1)), + anchor_id bigint not null, + neighbor_id bigint not null, + edge_id bigint not null, + kind_id smallint not null, + + primary key (graph_id, direction, edge_id), + foreign key (graph_id) references graph (id) on delete cascade, + foreign key (edge_id, graph_id) references edge (id, graph_id) on delete cascade +) partition by list (graph_id); + +create index if not exists p5_adjacency_v1_lookup_index + on public.p5_adjacency_v1 (graph_id, direction, anchor_id, kind_id, edge_id) + include (neighbor_id); + +do +$$ +declare + graph_row record; +begin + for graph_row in select id from graph loop + execute format( + 'create table if not exists %I partition of public.p5_adjacency_v1 for values in (%s)', + 'p5_adjacency_v1_' || graph_row.id, + graph_row.id + ); + end loop; +end +$$; + +insert into public.p5_adjacency_v1 (graph_id, direction, anchor_id, neighbor_id, edge_id, kind_id) +select e.graph_id, 1, e.start_id, e.end_id, e.id, e.kind_id +from edge e +union all +select e.graph_id, -1, e.end_id, e.start_id, e.id, e.kind_id +from edge e +on conflict (graph_id, direction, edge_id) do update + set anchor_id = excluded.anchor_id, + neighbor_id = excluded.neighbor_id, + kind_id = excluded.kind_id; + +create or replace function public.maintain_p5_adjacency_v1() returns trigger as +$$ +begin + if tg_op = 'DELETE' or tg_op = 'UPDATE' then + delete from public.p5_adjacency_v1 + where graph_id = old.graph_id + and edge_id = old.id; + end if; + + if tg_op = 'INSERT' or tg_op = 'UPDATE' then + insert into public.p5_adjacency_v1 (graph_id, direction, anchor_id, neighbor_id, edge_id, kind_id) + values + (new.graph_id, 1, new.start_id, new.end_id, new.id, new.kind_id), + (new.graph_id, -1, new.end_id, new.start_id, new.id, new.kind_id); + end if; + + return null; +end +$$ + language plpgsql + volatile; + +drop trigger if exists p5_adjacency_v1_after_insert on edge; +create trigger p5_adjacency_v1_after_insert + after insert + on edge + for each row +execute procedure public.maintain_p5_adjacency_v1(); + +drop trigger if exists p5_adjacency_v1_after_delete on edge; +create trigger p5_adjacency_v1_after_delete + after delete + on edge + for each row +execute procedure public.maintain_p5_adjacency_v1(); + +drop trigger if exists p5_adjacency_v1_after_endpoint_update on edge; +create trigger p5_adjacency_v1_after_endpoint_update + after update of graph_id, start_id, end_id, kind_id + on edge + for each row +execute procedure public.maintain_p5_adjacency_v1(); diff --git a/drivers/pg/query/sql/schema_down.sql b/drivers/pg/query/sql/schema_down.sql index 6e2c0de0..9bb818b3 100644 --- a/drivers/pg/query/sql/schema_down.sql +++ b/drivers/pg/query/sql/schema_down.sql @@ -1,6 +1,19 @@ -- Drop triggers +drop trigger if exists create_graph_traversal_epoch on graph; +drop trigger if exists bump_node_traversal_epoch_insert on node; +drop trigger if exists bump_node_traversal_epoch_update on node; +drop trigger if exists bump_node_traversal_epoch_delete on node; +drop trigger if exists bump_edge_traversal_epoch_insert on edge; +drop trigger if exists bump_edge_traversal_epoch_update on edge; +drop trigger if exists bump_edge_traversal_epoch_delete on edge; +drop trigger if exists bump_node_traversal_epoch_truncate on node; +drop trigger if exists bump_edge_traversal_epoch_truncate on edge; drop trigger if exists delete_node_edges on node; drop function if exists delete_node_edges; +drop function if exists public.create_graph_traversal_epoch; +drop function if exists public.bump_graph_traversal_epoch_new; +drop function if exists public.bump_graph_traversal_epoch_old; +drop function if exists public.bump_all_graph_traversal_epochs; -- Drop functions drop aggregate if exists cypher_min(jsonb); @@ -28,11 +41,60 @@ drop function if exists index_utilization; drop function if exists _format_asp_where_clause; drop function if exists _format_asp_query; drop function if exists asp_harness; -drop function if exists create_traversal_filter_tables; +drop function if exists create_traversal_filter_tables(); drop function if exists create_traversal_filter_tables(text, text, text); drop function if exists create_traversal_filter_tables(text, text); drop function if exists create_traversal_filter_tables(int8[], int8[]); drop function if exists shortest_path_self_endpoint_error(int8, int8); +drop function if exists all_shortest_paths_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8); +drop function if exists all_shortest_paths_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8); +drop function if exists all_shortest_paths_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8, text); +drop function if exists _finish_bidirectional_all_shortest_path_diagnostic_call_v1(text, int8, text, int8, int8, int8, int8, int8, int8, int8, int8, int4, int8, int8, int8, int4, int8, bool, int8, int8, int8, int8, int8, bool, bool); +drop function if exists _record_bidirectional_all_shortest_path_diagnostic_level_v1(text, int8, int8, text, text, int4, int8, int8, int8, int8, int8, int8, int8); +drop function if exists _start_bidirectional_all_shortest_path_diagnostic_call_v1(text, text, int8, int8, int8, int8, int8, int8, int8); +drop function if exists clear_bidirectional_all_shortest_path_diagnostic_v1(text); +drop function if exists read_bidirectional_all_shortest_path_diagnostic_v1(text); +drop function if exists begin_bidirectional_all_shortest_path_diagnostic_v1(text); +drop function if exists ensure_bidirectional_all_shortest_path_telemetry_workspace(); +drop function if exists clear_bidirectional_all_shortest_path_workspace(); +drop function if exists reset_bidirectional_all_shortest_path_workspace(); +drop function if exists ensure_bidirectional_all_shortest_path_workspace(); +drop function if exists shortest_path_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8); +drop function if exists shortest_path_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8); +drop function if exists shortest_path_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, text); +drop function if exists _finish_bidirectional_shortest_path_diagnostic_call_v1(text, int8, text, int8, int8, int8, int8, int8, int8, int8, int8, int4, int8, bool, bool); +drop function if exists _record_bidirectional_shortest_path_diagnostic_level_v1(text, int8, int8, text, text, int4, int8, int8, int8, int8, int8, int8, int8); +drop function if exists _start_bidirectional_shortest_path_diagnostic_call_v1(text, text, int8, int8, int8, int8, int8); +drop function if exists clear_bidirectional_shortest_path_diagnostic_v1(text); +drop function if exists read_bidirectional_shortest_path_diagnostic_v1(text); +drop function if exists begin_bidirectional_shortest_path_diagnostic_v1(text); +drop function if exists ensure_bidirectional_shortest_path_telemetry_workspace(); +drop function if exists reset_bidirectional_shortest_path_workspace(); +drop function if exists ensure_bidirectional_shortest_path_workspace(); +drop function if exists clear_traversal_runtime_attestation_v1(text); +drop function if exists read_traversal_runtime_attestation_v1(text); +drop function if exists record_requested_traversal_runtime_attestation_v1(text, bool, text); +drop function if exists record_traversal_runtime_attestation_v1(text, text, bool); +drop function if exists begin_traversal_runtime_attestation_v1(text, text); +drop function if exists ensure_traversal_runtime_attestation_workspace_v1(); +drop function if exists shortest_path_compact(int4, int8, int8, int4, int4, int2[], bool, int8); +drop function if exists all_shortest_paths_no_path_probe(int4, int8, int8, int4, int4, int2[], bool, int8); +drop function if exists all_shortest_paths_dag(int4, int8, int8, int4, int4, int2[], bool); +drop function if exists clear_all_shortest_paths_a1_diagnostic_v1(text); +drop function if exists read_all_shortest_paths_a1_diagnostic_v1(text); +drop function if exists _finish_all_shortest_paths_a1_diagnostic_v1(text, int4, int8); +drop function if exists _record_all_shortest_paths_a1_diagnostic_level_v1(int4, int8, int8, int8, int8); +drop function if exists _start_all_shortest_paths_a1_diagnostic_v1(int8, int8); +drop function if exists begin_all_shortest_paths_a1_diagnostic_v1(text); +drop function if exists ensure_all_shortest_paths_a1_diagnostic_workspace_v1(); +drop function if exists reset_shortest_dag_workspace(); +drop function if exists ensure_shortest_dag_workspace(); +drop function if exists bsp_workspace_fragment(text); +drop function if exists load_bsp_filter_tables(text, text, text); +drop function if exists reset_bsp_workspace(bool); +drop function if exists ensure_bsp_generic_workspace(); +drop function if exists ensure_bsp_core_workspace(); +drop function if exists graphbench_s1_distance_bfs(int4, int8, int8, int4, int4, int2[], bool, int4); drop function if exists unidirectional_sp_harness(text, text, int4); drop function if exists unidirectional_sp_harness(text, text, int4, int8); drop function if exists unidirectional_sp_harness(text, text, int4, text, text); @@ -53,13 +115,20 @@ drop function if exists bidirectional_asp_harness(text, text, text, text, int4, drop function if exists bidirectional_sp_harness(text, text, text, text, int4); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text, bool, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text, bool); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, text); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, bool, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text, bool); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, text, text); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[]); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[], int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[], bool, int8); +drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[], bool); drop function if exists bidirectional_sp_harness(text, text, text, text, int4, int8[], int8[]); +drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool, bool); drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool); drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], bool); drop function if exists _bidirectional_sp_harness(text, text, text, text, int4, text, text, int8[], int8[], bool); @@ -73,14 +142,24 @@ drop function if exists _format_traversal_query; drop function if exists _format_traversal_initial_query; drop function if exists expand_traversal_step; drop function if exists traverse; +drop function if exists ordered_edges_to_path(int4, nodeComposite, edgeComposite[], nodeComposite[]); drop function if exists ordered_edges_to_path(nodeComposite, edgeComposite[], nodeComposite[]); -drop function if exists edges_to_path; +drop function if exists ordered_edge_ids_to_path(int4, nodeComposite, int8[], nodeComposite[]); +drop function if exists nodes_to_path(int4, int8[]); +drop function if exists nodes_to_path(int8[]); +drop function if exists edges_to_path(int4, int8[]); +drop function if exists edges_to_path(int8[]); drop function if exists traverse_paths; -- Drop all tables in order of dependency. drop table if exists node; drop table if exists edge; drop table if exists kind; +drop table if exists graph_traversal_synopsis_degree; +drop table if exists graph_traversal_synopsis_edge_count; +drop table if exists graph_traversal_synopsis_node_count; +drop table if exists graph_traversal_synopsis_generation; +drop table if exists graph_traversal_epoch; drop table if exists graph; -- Remove custom types diff --git a/drivers/pg/query/sql/schema_up.sql b/drivers/pg/query/sql/schema_up.sql index f112002d..56d5fae5 100644 --- a/drivers/pg/query/sql/schema_up.sql +++ b/drivers/pg/query/sql/schema_up.sql @@ -61,6 +61,108 @@ create table if not exists graph unique (name) ); +-- graph_traversal_epoch is a graph-scoped, transactionally visible generation +-- for topology-aware SQL selection. It is deliberately independent of driver +-- cache generations: a stale or missing epoch is an incumbent-only condition. +create table if not exists graph_traversal_epoch +( + graph_id bigint primary key references graph (id) on delete cascade, + epoch bigint not null default 1, + check (epoch > 0) +); + +-- The latest atomically published topology synopsis for each graph. The +-- initial selector uses only its generation validity; estimator payloads are +-- added in versioned relations as candidate families require them. +create table if not exists graph_traversal_synopsis_generation +( + graph_id bigint primary key references graph (id) on delete cascade, + epoch bigint not null, + source_mutation_epoch bigint not null, + estimator_version text not null, + status text not null, + node_count bigint not null default 0, + edge_count bigint not null default 0, + built_at timestamptz not null default clock_timestamp(), + check (epoch > 0), + check (source_mutation_epoch > 0), + check (status in ('ready', 'building', 'failed')) +); + +alter table graph_traversal_synopsis_generation + add column if not exists schema_version text not null default 'topology-synopsis-v2', + add column if not exists refresh_started_at timestamptz, + add column if not exists refresh_completed_at timestamptz, + add column if not exists refresh_mode text not null default 'full'; + +-- Detail relations are scoped to the atomically published generation. They +-- remain advisory estimates: missing rows are an incumbent-only condition. +create table if not exists graph_traversal_synopsis_node_count +( + graph_id bigint not null references graph (id) on delete cascade, + epoch bigint not null, + kind_id smallint not null, + node_count bigint not null, + primary key (graph_id, epoch, kind_id), + check (epoch > 0), + check (node_count >= 0) +); + +create table if not exists graph_traversal_synopsis_edge_count +( + graph_id bigint not null references graph (id) on delete cascade, + epoch bigint not null, + direction text not null, + kind_id smallint not null, + edge_count bigint not null, + distinct_start_count bigint not null, + distinct_end_count bigint not null, + primary key (graph_id, epoch, direction, kind_id), + check (epoch > 0), + check (direction in ('outbound', 'inbound')), + check (edge_count >= 0), + check (distinct_start_count >= 0), + check (distinct_end_count >= 0) +); + +create table if not exists graph_traversal_synopsis_degree +( + graph_id bigint not null references graph (id) on delete cascade, + epoch bigint not null, + direction text not null, + kind_id smallint not null, + bucket text not null, + node_count bigint not null, + primary key (graph_id, epoch, direction, kind_id, bucket), + check (epoch > 0), + check (direction in ('outbound', 'inbound')), + check (bucket in ('one', 'two_to_four', 'five_to_sixteen', 'seventeen_plus')), + check (node_count >= 0) +); + +insert into graph_traversal_epoch (graph_id) +select id +from graph +on conflict (graph_id) do nothing; + +create or replace function public.create_graph_traversal_epoch() returns trigger as +$$ +begin + insert into graph_traversal_epoch (graph_id) + values (new.id) + on conflict (graph_id) do nothing; + return new; +end +$$ + language plpgsql + volatile; + +drop trigger if exists create_graph_traversal_epoch on graph; +create trigger create_graph_traversal_epoch + after insert on graph + for each row +execute procedure public.create_graph_traversal_epoch(); + -- The kind table contains name to ID mappings for graph kinds. Storage of these types is necessary to maintain search -- capability of a database without the origin application that generated it. -- To support FK in asset_group_tags table, the kind table is now maintained by the stepwise migration files. @@ -173,6 +275,75 @@ create trigger delete_node_edges for each statement execute procedure delete_node_edges(); +-- Each mutating statement advances the affected graph's topology epoch in the +-- same transaction. Multiple statements may advance it more than once; that +-- is conservative and makes every previously read synopsis stale. +create or replace function public.bump_graph_traversal_epoch_new() returns trigger as +$$ +begin + update graph_traversal_epoch + set epoch = epoch + 1 + where graph_id in (select distinct graph_id from new_rows); + return null; +end +$$ + language plpgsql + volatile; + +create or replace function public.bump_graph_traversal_epoch_old() returns trigger as +$$ +begin + update graph_traversal_epoch + set epoch = epoch + 1 + where graph_id in (select distinct graph_id from old_rows); + return null; +end +$$ + language plpgsql + volatile; + +create or replace function public.bump_all_graph_traversal_epochs() returns trigger as +$$ +begin + update graph_traversal_epoch + set epoch = epoch + 1; + return null; +end +$$ + language plpgsql + volatile; + +drop trigger if exists bump_node_traversal_epoch_insert on node; +create trigger bump_node_traversal_epoch_insert after insert on node + referencing new table as new_rows for each statement +execute procedure public.bump_graph_traversal_epoch_new(); +drop trigger if exists bump_node_traversal_epoch_update on node; +create trigger bump_node_traversal_epoch_update after update on node + referencing new table as new_rows for each statement +execute procedure public.bump_graph_traversal_epoch_new(); +drop trigger if exists bump_node_traversal_epoch_delete on node; +create trigger bump_node_traversal_epoch_delete after delete on node + referencing old table as old_rows for each statement +execute procedure public.bump_graph_traversal_epoch_old(); +drop trigger if exists bump_edge_traversal_epoch_insert on edge; +create trigger bump_edge_traversal_epoch_insert after insert on edge + referencing new table as new_rows for each statement +execute procedure public.bump_graph_traversal_epoch_new(); +drop trigger if exists bump_edge_traversal_epoch_update on edge; +create trigger bump_edge_traversal_epoch_update after update on edge + referencing new table as new_rows for each statement +execute procedure public.bump_graph_traversal_epoch_new(); +drop trigger if exists bump_edge_traversal_epoch_delete on edge; +create trigger bump_edge_traversal_epoch_delete after delete on edge + referencing old table as old_rows for each statement +execute procedure public.bump_graph_traversal_epoch_old(); +drop trigger if exists bump_node_traversal_epoch_truncate on node; +create trigger bump_node_traversal_epoch_truncate after truncate on node + for each statement execute procedure public.bump_all_graph_traversal_epochs(); +drop trigger if exists bump_edge_traversal_epoch_truncate on edge; +create trigger bump_edge_traversal_epoch_truncate after truncate on edge + for each statement execute procedure public.bump_all_graph_traversal_epochs(); + -- The storage strategy chosen for the properties JSONB column informs the database of the user's preference to resort -- to creating a TOAST table entry only after there is no other possible way to inline the row attribute in the current @@ -638,31 +809,41 @@ $$ parallel safe strict; -create or replace function public.nodes_to_path(nodes variadic int8[]) returns pathComposite as +-- CREATE OR REPLACE does not replace a function when its argument signature +-- changes. Remove the pre-graph-scope overloads explicitly so upgrades cannot +-- retain helpers that hydrate entities from a different graph partition. +drop function if exists public.nodes_to_path(int8[]); +drop function if exists public.edges_to_path(int8[]); +drop function if exists public.ordered_edges_to_path(nodeComposite, edgeComposite[], nodeComposite[]); + +create or replace function public.nodes_to_path(target_graph_id int4, nodes variadic int8[]) returns pathComposite as $$ select row (array_agg(distinct (n.id, n.kind_ids, n.properties)::nodeComposite)::nodeComposite[], array []::edgeComposite[])::pathComposite from node n -where n.id = any (nodes); +where n.graph_id = target_graph_id + and n.id = any (nodes); $$ language sql immutable parallel safe strict; -create or replace function public.edges_to_path(path variadic int8[]) returns pathComposite as +create or replace function public.edges_to_path(target_graph_id int4, path variadic int8[]) returns pathComposite as $$ select row ( (select array_agg(distinct (n.id, n.kind_ids, n.properties)::nodeComposite) from node n - where n.id in ( - select start_id from edge where id = any(path) + where n.graph_id = target_graph_id + and n.id in ( + select start_id from edge where graph_id = target_graph_id and id = any(path) union - select end_id from edge where id = any(path) + select end_id from edge where graph_id = target_graph_id and id = any(path) )), (select array_agg(distinct (r.id, r.start_id, r.end_id, r.kind_id, r.properties)::edgeComposite) from edge r - where r.id = any(path)) + where r.graph_id = target_graph_id + and r.id = any(path)) )::pathComposite; $$ language sql @@ -670,7 +851,7 @@ $$ parallel safe strict; -create or replace function public.ordered_edges_to_path(root nodeComposite, edges edgeComposite[], known_nodes nodeComposite[]) returns pathComposite as +create or replace function public.ordered_edges_to_path(target_graph_id int4, root nodeComposite, edges edgeComposite[], known_nodes nodeComposite[]) returns pathComposite as $$ with recursive edge_bounds(edge_count) as ( @@ -745,7 +926,7 @@ select row ( where candidate.id = ordered_node.id limit 1 ) known_node on true - left join node n on n.id = ordered_node.id and known_node.node is null + left join node n on n.id = ordered_node.id and n.graph_id = target_graph_id and known_node.node is null ), ( select coalesce( @@ -764,6 +945,87 @@ $$ parallel safe strict; +-- ordered_edge_ids_to_path is the read-expansion materializer. Expansion +-- lowering already knows the edge order, so this helper walks that order once +-- instead of repeatedly searching the remaining edge array. Every persistent +-- lookup is constrained by target_graph_id because entity IDs are only unique +-- within a graph partition. +create or replace function public.ordered_edge_ids_to_path(target_graph_id int4, root nodeComposite, edge_ids int8[], known_nodes nodeComposite[]) returns pathComposite as +$$ +with recursive +edge_count(value) as +( + select coalesce(cardinality(edge_ids), 0) +), +hydrated_edges as materialized +( + select path_edge.ordinality::int4 as ordinality, + (e.id, e.start_id, e.end_id, e.kind_id, e.properties)::edgeComposite as edge + from unnest(edge_ids) with ordinality as path_edge(id, ordinality) + join edge e + on e.id = path_edge.id + and e.graph_id = target_graph_id +), +path_walk(idx, current_node_id, node_ids) as +( + select 0::int4, (root).id, array [(root).id]::int8[] + union all + select path_walk.idx + 1, + case + when path_walk.current_node_id = (next_edge.edge).start_id then (next_edge.edge).end_id + else (next_edge.edge).start_id + end, + path_walk.node_ids || case + when path_walk.current_node_id = (next_edge.edge).start_id then (next_edge.edge).end_id + else (next_edge.edge).start_id + end + from path_walk + join hydrated_edges next_edge + on next_edge.ordinality = path_walk.idx + 1 + and path_walk.current_node_id in ((next_edge.edge).start_id, (next_edge.edge).end_id) +), +final_walk as +( + select path_walk.node_ids + from path_walk + cross join edge_count + where path_walk.idx = edge_count.value +) +select row ( + ( + select coalesce( + array_agg(coalesce(known_node.node, (n.id, n.kind_ids, n.properties)::nodeComposite) order by ordered_node.ordinality)::nodeComposite[], + array []::nodeComposite[] + ) + from final_walk + cross join lateral unnest(final_walk.node_ids) with ordinality as ordered_node(id, ordinality) + left join lateral + ( + select (candidate.id, candidate.kind_ids, candidate.properties)::nodeComposite as node + from unnest(known_nodes) as candidate(id, kind_ids, properties) + where candidate.id = ordered_node.id + limit 1 + ) known_node on true + left join node n + on n.id = ordered_node.id + and n.graph_id = target_graph_id + and known_node.node is null + ), + ( + select coalesce( + array_agg(hydrated_edges.edge order by hydrated_edges.ordinality)::edgeComposite[], + array []::edgeComposite[] + ) + from hydrated_edges + ) +)::pathComposite +from final_walk; +$$ + language sql + stable + parallel safe + strict; + create or replace function public.create_unidirectional_pathspace_tables() returns void as $$ @@ -771,7 +1033,7 @@ begin -- The path column is not used as a primary key. Deduplication is handled by DISTINCT ON clauses in the -- harness functions. Removing the PK on the variable-length int8[] array eliminates O(n)-key B-tree -- maintenance that grows with traversal depth. - create temporary table forward_front + create temporary table if not exists forward_front ( root_id int8 not null, next_id int8 not null, @@ -779,9 +1041,9 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create temporary table next_front + create temporary table if not exists next_front ( root_id int8 not null, next_id int8 not null, @@ -789,15 +1051,17 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; + + create index if not exists forward_front_next_id_index on forward_front using btree (next_id); + create index if not exists forward_front_satisfied_index on forward_front using btree (root_id, next_id, depth) where satisfied; + create index if not exists forward_front_is_cycle_index on forward_front using btree (root_id, next_id) where is_cycle; - create index forward_front_next_id_index on forward_front using btree (next_id); - create index forward_front_satisfied_index on forward_front using btree (root_id, next_id, depth) where satisfied; - create index forward_front_is_cycle_index on forward_front using btree (root_id, next_id) where is_cycle; + create index if not exists next_front_next_id_index on next_front using btree (next_id); + create index if not exists next_front_satisfied_index on next_front using btree (root_id, next_id, depth) where satisfied; + create index if not exists next_front_is_cycle_index on next_front using btree (root_id, next_id) where is_cycle; - create index next_front_next_id_index on next_front using btree (next_id); - create index next_front_satisfied_index on next_front using btree (root_id, next_id, depth) where satisfied; - create index next_front_is_cycle_index on next_front using btree (root_id, next_id) where is_cycle; + truncate table forward_front, next_front; end; $$ language plpgsql @@ -809,14 +1073,14 @@ create or replace function public.create_unidirectional_shortest_path_tables() returns void as $$ begin - create temporary table visited + create temporary table if not exists visited ( root_id int8 not null, id int8 not null, primary key (root_id, id) - ) on commit drop; + ) on commit preserve rows; - create temporary table paths + create temporary table if not exists paths ( root_id int8 not null, next_id int8 not null, @@ -824,19 +1088,21 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create temporary table resolved_roots + create temporary table if not exists resolved_roots ( root_id int8 not null, primary key (root_id) - ) on commit drop; + ) on commit preserve rows; + + truncate table visited, paths, resolved_roots; perform create_unidirectional_pathspace_tables(); - create index forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); - create index next_front_root_id_next_id_index on next_front using btree (root_id, next_id); - create index paths_root_id_next_id_index on paths using btree (root_id, next_id); + create index if not exists forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); + create index if not exists next_front_root_id_next_id_index on next_front using btree (root_id, next_id); + create index if not exists paths_root_id_next_id_index on paths using btree (root_id, next_id); end; $$ language plpgsql @@ -844,9 +1110,8 @@ $$ strict; -- create_traversal_filter_tables materializes the root, terminal and pair filter sets into temporary tables that the --- harness functions join against. The tables use `on commit drop`, so a single transaction can only host one harness --- invocation that depends on these tables; concurrent or sequential expansions in the same transaction will conflict --- on the temporary table names. +-- harness functions join against. Definitions persist for the physical +-- session; each invocation truncates its row state before loading a new filter. create or replace function public.create_traversal_filter_tables() returns void as $$ @@ -855,120 +1120,4095 @@ begin ( id int8 not null, primary key (id) - ) on commit drop; + ) on commit preserve rows; create temporary table if not exists traversal_terminal_filter ( id int8 not null, primary key (id) - ) on commit drop; + ) on commit preserve rows; create temporary table if not exists traversal_pair_filter ( root_id int8 not null, terminal_id int8 not null, primary key (root_id, terminal_id) - ) on commit drop; + ) on commit preserve rows; + + create index if not exists traversal_pair_filter_terminal_id_root_id_index on traversal_pair_filter using btree (terminal_id, root_id); + + truncate table traversal_root_filter; + truncate table traversal_terminal_filter; + truncate table traversal_pair_filter; + + return; +end; +$$ + language plpgsql + volatile; + +create or replace function public.create_traversal_filter_tables(root_ids int8[], terminal_ids int8[]) + returns void as +$$ +begin + perform create_traversal_filter_tables(); + + insert into traversal_root_filter + select distinct root_id + from unnest(root_ids) as root_ids(root_id) + where root_id is not null + on conflict (id) do nothing; + + insert into traversal_terminal_filter + select distinct terminal_id + from unnest(terminal_ids) as terminal_ids(terminal_id) + where terminal_id is not null + on conflict (id) do nothing; + + analyze traversal_root_filter; + analyze traversal_terminal_filter; + + return; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text, pair_filter text) + returns void as +$$ +begin + perform create_traversal_filter_tables(); + + if length(pair_filter) > 0 then + execute pair_filter; + end if; + + if length(root_filter) > 0 then + execute root_filter; + elsif length(pair_filter) > 0 then + insert into traversal_root_filter + select distinct root_id + from traversal_pair_filter + on conflict (id) do nothing; + end if; + + if length(terminal_filter) > 0 then + execute terminal_filter; + elsif length(pair_filter) > 0 then + insert into traversal_terminal_filter + select distinct terminal_id + from traversal_pair_filter + on conflict (id) do nothing; + end if; + + analyze traversal_root_filter; + analyze traversal_terminal_filter; + analyze traversal_pair_filter; + + return; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text) + returns void as +$$ +select public.create_traversal_filter_tables(root_filter, terminal_filter, ''::text); +$$ + language sql + volatile + strict; + +create or replace function public.shortest_path_self_endpoint_error(root_id int8, terminal_id int8) + returns bool as +$$ +begin + raise exception using + errcode = '22023', + message = format('shortest path endpoints must not resolve to the same node: root_id=%s terminal_id=%s', + root_id, + terminal_id); + + return false; +end; +$$ + language plpgsql + volatile + strict; + +-- Compact bound-pair shortest-path searches share a session-local workspace. +-- The tables survive transaction boundaries so their catalog objects and +-- indexes are paid for once per physical connection. Every public executor +-- resets row state before use; an aborted call is therefore harmless to the +-- next invocation on the same pooled connection. +create or replace function public.ensure_shortest_dag_workspace() + returns void as +$$ +declare + expected_version constant int4 := 2; + present_version int4; +begin + if to_regclass('pg_temp.spd_workspace_version') is not null then + select version into present_version from pg_temp.spd_workspace_version limit 1; + end if; + + if to_regclass('pg_temp.spd_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.spd_predecessor; + drop table if exists pg_temp.spd_candidate; + drop table if exists pg_temp.spd_seen; + drop table if exists pg_temp.spd_next; + drop table if exists pg_temp.spd_front; + drop table if exists pg_temp.spd_workspace_version; + end if; + + if to_regclass('pg_temp.spd_workspace_version') is null then + create temporary table spd_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table spd_seen + ( + node_id int8 not null primary key, + depth int4 not null + ) on commit preserve rows; + + create temporary table spd_candidate + ( + node_id int8 not null, + depth int4 not null, + predecessor_id int8 not null, + edge_id int8 not null, + primary key (depth, node_id, predecessor_id, edge_id) + ) on commit preserve rows; + create index spd_candidate_node_id_depth_index + on spd_candidate using btree (node_id, depth); + + create temporary table spd_predecessor + ( + node_id int8 not null, + depth int4 not null, + predecessor_id int8 not null, + edge_id int8 not null, + primary key (node_id, depth, predecessor_id, edge_id) + ) on commit preserve rows; + create index spd_predecessor_predecessor_id_depth_index + on spd_predecessor using btree (predecessor_id, depth); + + insert into spd_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.reset_shortest_dag_workspace() + returns void as +$$ +begin + -- V2 establishes this versioned workspace once per physical connection and + -- schema generation. The trusted marker avoids a catalog lookup and three + -- CREATE TEMP TABLE IF NOT EXISTS checks on every hot traversal. The marker + -- is set only by ensure_shortest_dag_workspace below, while the fallback + -- keeps direct and V1 callers self-contained. + if current_setting('dawgs.shortest_dag_workspace_ready', true) is distinct from 'v2' then + perform public.ensure_shortest_dag_workspace(); + end if; + truncate table pg_temp.spd_seen, pg_temp.spd_candidate, pg_temp.spd_predecessor; +end; +$$ + language plpgsql + volatile; + +-- The A1 diagnostic workspace is armed only by GraphBench's untimed replay. +-- It stays separate from the A1 search workspace so ordinary calls neither +-- allocate telemetry state nor retain a previous invocation's counters. +create or replace function public.ensure_all_shortest_paths_a1_diagnostic_workspace_v1() + returns void as +$$ +begin + if to_regclass('pg_temp.asd_telemetry_invocation') is null then + create temporary table asd_telemetry_invocation + ( + invocation_id text not null primary key, + schema_version int4 not null, + search_calls int8 not null default 0, + source_id int8, + target_id int8, + runtime_branch text, + target_depth int4, + output_paths int8, + fallback_executed bool, + check (btrim(invocation_id) <> ''), + check (search_calls >= 0), + check (output_paths is null or output_paths >= 0) + ) on commit preserve rows; + + create temporary table asd_telemetry_level + ( + invocation_id text not null, + action_index int8 not null, + depth int4 not null, + candidate_edges int8 not null, + distinct_new_nodes int8 not null, + seen_rows int8 not null, + predecessor_rows int8 not null, + primary key (invocation_id, action_index), + check (action_index >= 1), + check (depth >= 0), + check (candidate_edges >= 0), + check (distinct_new_nodes >= 0), + check (seen_rows >= 0), + check (predecessor_rows >= 0) + ) on commit preserve rows; + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.begin_all_shortest_paths_a1_diagnostic_v1(target_invocation_id text) + returns void as +$$ +begin + if target_invocation_id is null or btrim(target_invocation_id) = '' or length(target_invocation_id) > 256 then + raise exception using errcode = '22023', message = 'A1 all-shortest diagnostic invocation ID must contain 1 to 256 characters'; + end if; + perform public.ensure_all_shortest_paths_a1_diagnostic_workspace_v1(); + delete from pg_temp.asd_telemetry_level where invocation_id = target_invocation_id; + delete from pg_temp.asd_telemetry_invocation where invocation_id = target_invocation_id; + insert into pg_temp.asd_telemetry_invocation(invocation_id, schema_version) + values (target_invocation_id, 1); + -- A shallow A1 call does not otherwise touch spd_*, so clear it before every + -- replay and make stale recursive state impossible to report as shallow work. + perform public.reset_shortest_dag_workspace(); + perform set_config('dawgs.asd_diagnostic_invocation_id', target_invocation_id, true); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._start_all_shortest_paths_a1_diagnostic_v1(target_source_id int8, target_target_id int8) + returns void as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.asd_diagnostic_invocation_id', true), ''); +begin + if target_invocation_id is null then + return; + end if; + update pg_temp.asd_telemetry_invocation + set search_calls = search_calls + 1, + source_id = target_source_id, + target_id = target_target_id + where invocation_id = target_invocation_id; + if not found then + raise exception using errcode = '55000', message = 'A1 all-shortest diagnostic invocation is missing'; + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public._record_all_shortest_paths_a1_diagnostic_level_v1( + target_depth int4, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_rows int8, + target_predecessor_rows int8) + returns void as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.asd_diagnostic_invocation_id', true), ''); + next_action_index int8; +begin + if target_invocation_id is null then + return; + end if; + if target_depth < 0 or target_candidate_edges < 0 or target_distinct_new_nodes < 0 or + target_seen_rows < 0 or target_predecessor_rows < 0 then + raise exception using errcode = '22023', message = 'A1 all-shortest diagnostic counters must be non-negative'; + end if; + if not exists (select 1 from pg_temp.asd_telemetry_invocation where invocation_id = target_invocation_id) then + raise exception using errcode = '55000', message = 'A1 all-shortest diagnostic invocation is missing'; + end if; + select coalesce(max(action_index), 0) + 1 into next_action_index + from pg_temp.asd_telemetry_level + where invocation_id = target_invocation_id; + insert into pg_temp.asd_telemetry_level(invocation_id, action_index, depth, candidate_edges, + distinct_new_nodes, seen_rows, predecessor_rows) + values (target_invocation_id, next_action_index, target_depth, target_candidate_edges, + target_distinct_new_nodes, target_seen_rows, target_predecessor_rows); +end; +$$ + language plpgsql + volatile; + +create or replace function public._finish_all_shortest_paths_a1_diagnostic_v1( + target_runtime_branch text, + completed_depth int4, + completed_output_paths int8) + returns void as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.asd_diagnostic_invocation_id', true), ''); +begin + if target_invocation_id is null then + return; + end if; + if target_runtime_branch is null or btrim(target_runtime_branch) = '' or + completed_depth < -1 or completed_output_paths < 0 then + raise exception using errcode = '22023', message = 'A1 all-shortest diagnostic completion is invalid'; + end if; + update pg_temp.asd_telemetry_invocation + set runtime_branch = target_runtime_branch, + target_depth = completed_depth, + output_paths = completed_output_paths, + fallback_executed = false + where invocation_id = target_invocation_id; + if not found then + raise exception using errcode = '55000', message = 'A1 all-shortest diagnostic invocation is missing'; + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.read_all_shortest_paths_a1_diagnostic_v1(target_invocation_id text) + returns jsonb as +$$ +declare + result jsonb; +begin + if target_invocation_id is null or btrim(target_invocation_id) = '' then + return null; + end if; + select jsonb_build_object( + 'schema_version', invocation.schema_version, + 'invocation_id', invocation.invocation_id, + 'scheduler', 'single_ended_level', + 'search_calls', invocation.search_calls, + 'source_id', invocation.source_id, + 'target_id', invocation.target_id, + 'runtime_branch', invocation.runtime_branch, + 'target_depth', invocation.target_depth, + 'output_paths', invocation.output_paths, + 'fallback_executed', invocation.fallback_executed, + 'levels', coalesce(levels.value, '[]'::jsonb) + ) into result + from pg_temp.asd_telemetry_invocation invocation + left join lateral ( + select jsonb_agg(jsonb_build_object( + 'action_index', level.action_index, + 'depth', level.depth, + 'candidate_edges', level.candidate_edges, + 'distinct_new_nodes', level.distinct_new_nodes, + 'seen_rows', level.seen_rows, + 'predecessor_rows', level.predecessor_rows + ) order by level.action_index) as value + from pg_temp.asd_telemetry_level level + where level.invocation_id = invocation.invocation_id + ) levels on true + where invocation.invocation_id = target_invocation_id; + return result; +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_all_shortest_paths_a1_diagnostic_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.asd_telemetry_invocation') is not null then + delete from pg_temp.asd_telemetry_level where invocation_id = target_invocation_id; + delete from pg_temp.asd_telemetry_invocation where invocation_id = target_invocation_id; + end if; + if current_setting('dawgs.asd_diagnostic_invocation_id', true) = target_invocation_id then + perform set_config('dawgs.asd_diagnostic_invocation_id', '', true); + end if; +end; +$$ + language plpgsql + volatile + strict; + +-- all_shortest_paths_dag separates minimum-depth discovery from path +-- enumeration. It retains every relationship-distinct predecessor edge at a +-- node's minimum depth, then enumerates only the resulting predecessor DAG. +-- The min_depth=1/distinct-endpoint contract is enforced by the production +-- selector; the guards below keep direct SQL callers honest as well. +create or replace function public.all_shortest_paths_dag(target_graph_id int4, source_id int8, target_id int8, + min_depth int4, max_depth int4, + edge_kind_ids int2[], inbound bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + search_depth int4; + target_depth int4; + emitted_count int8; + candidate_count int8; + distinct_node_count int8; + seen_count int8; + predecessor_count int8; + diagnostic_enabled bool := nullif(current_setting('dawgs.asd_diagnostic_invocation_id', true), '') is not null; +begin + if source_id is null or target_id is null or max_depth < 1 then + return; + end if; + if min_depth <> 1 then + raise exception using errcode = '22023', message = 'all_shortest_paths_dag requires min_depth = 1'; + end if; + if source_id = target_id then + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + if diagnostic_enabled then + perform public._start_all_shortest_paths_a1_diagnostic_v1(source_id, target_id); + end if; + + -- Exact depth-one fast arm. Every qualifying parallel edge is observable. + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + if diagnostic_enabled then + perform public._record_all_shortest_paths_a1_diagnostic_level_v1(1, emitted_count, 1, 2, emitted_count); + perform public._finish_all_shortest_paths_a1_diagnostic_v1('one_hop_preflight', 1, emitted_count); + end if; + return; + end if; + + -- Exact depth-two fast arm. Relationship uniqueness is explicit so self + -- loops and reciprocal patterns cannot reuse one physical relationship. + if max_depth >= 2 then + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id + and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id + and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + if diagnostic_enabled then + if not inbound then + select count(distinct e1.end_id) into distinct_node_count + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)); + else + select count(distinct e1.start_id) into distinct_node_count + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)); + end if; + perform public._record_all_shortest_paths_a1_diagnostic_level_v1(2, emitted_count * 2, coalesce(distinct_node_count, 0) + 1, coalesce(distinct_node_count, 0) + 2, emitted_count * 2); + perform public._finish_all_shortest_paths_a1_diagnostic_v1('two_hop_preflight', 2, emitted_count); + end if; + return; + end if; + end if; + + if max_depth <= 2 then + if diagnostic_enabled then + perform public._record_all_shortest_paths_a1_diagnostic_level_v1(max_depth, 0, 0, 1, 0); + perform public._finish_all_shortest_paths_a1_diagnostic_v1('preflight_no_path', -1, 0); + end if; + return; + end if; + + perform public.reset_shortest_dag_workspace(); + insert into pg_temp.spd_seen(node_id, depth) values (source_id, 0); + + for search_depth in 1..max_depth loop + if not inbound then + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select e.end_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.start_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.end_id) + on conflict do nothing; + else + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select e.start_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.end_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.start_id) + on conflict do nothing; + end if; + get diagnostics candidate_count = row_count; + + if diagnostic_enabled then + select count(*) into seen_count from pg_temp.spd_seen; + select count(*) into predecessor_count from pg_temp.spd_predecessor; + end if; + + if not exists (select 1 from pg_temp.spd_candidate where depth = search_depth) then + if diagnostic_enabled then + perform public._record_all_shortest_paths_a1_diagnostic_level_v1(search_depth, candidate_count, 0, seen_count, predecessor_count); + end if; + exit; + end if; + + insert into pg_temp.spd_predecessor(node_id, depth, predecessor_id, edge_id) + select node_id, search_depth, predecessor_id, edge_id + from pg_temp.spd_candidate + where depth = search_depth + on conflict do nothing; + + insert into pg_temp.spd_seen(node_id, depth) + select distinct node_id, search_depth from pg_temp.spd_candidate + where depth = search_depth + on conflict do nothing; + get diagnostics distinct_node_count = row_count; + if diagnostic_enabled then + select count(*) into seen_count from pg_temp.spd_seen; + select count(*) into predecessor_count from pg_temp.spd_predecessor; + end if; + if diagnostic_enabled then + perform public._record_all_shortest_paths_a1_diagnostic_level_v1(search_depth, candidate_count, distinct_node_count, seen_count, predecessor_count); + end if; + + if exists (select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = target_id) then + target_depth = search_depth; + exit; + end if; + end loop; + + if target_depth is null then + if diagnostic_enabled then + perform public._finish_all_shortest_paths_a1_diagnostic_v1('search_no_path', -1, 0); + end if; + return; + end if; + + return query + with recursive shortest_paths(node_id, path_depth, edge_ids) as ( + select target_id, target_depth, array []::int8[] + union all + select predecessor.predecessor_id, + shortest_paths.path_depth - 1, + array[predecessor.edge_id]::int8[] || shortest_paths.edge_ids + from shortest_paths + join pg_temp.spd_predecessor predecessor + on predecessor.node_id = shortest_paths.node_id + and predecessor.depth = shortest_paths.path_depth + ) + select source_id, target_id, target_depth, true, false, shortest_paths.edge_ids + from shortest_paths + where shortest_paths.node_id = source_id and shortest_paths.path_depth = 0 + order by shortest_paths.edge_ids; + get diagnostics emitted_count = row_count; + if diagnostic_enabled then + perform public._finish_all_shortest_paths_a1_diagnostic_v1('single_ended_search', target_depth, emitted_count); + end if; +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 100; + +-- all_shortest_paths_no_path_probe is a negative-only optimization boundary. +-- It explores from the target in the reverse physical direction using the +-- existing session-local DAG workspace. Exhaustion proves that no directed +-- path of at most max_depth can exist and may return an empty result. Finding +-- the source or reaching the state sentinel is deliberately inconclusive and +-- delegates to A1 before exposing any rows. +create or replace function public.all_shortest_paths_no_path_probe(target_graph_id int4, source_id int8, target_id int8, + min_depth int4, max_depth int4, + edge_kind_ids int2[], inbound bool, + state_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + search_depth int4; + retained_state int8; + source_reached bool := false; +begin + if source_id is null or target_id is null or max_depth < 1 or state_limit <= 0 then + return query select * from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, min_depth, max_depth, edge_kind_ids, inbound); + return; + end if; + if min_depth <> 1 then + return query select * from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, min_depth, max_depth, edge_kind_ids, inbound); + return; + end if; + -- A1 already has exact one- and two-hop preflights. Avoid adding a probe to + -- those inexpensive cases, where it cannot establish a useful advantage. + if max_depth <= 2 then + return query select * from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, min_depth, max_depth, edge_kind_ids, inbound); + return; + end if; + + -- Most disconnected endpoint pairs have no eligible relationship entering + -- the target. This exact degree-zero proof avoids resetting the shared A1 + -- workspace before returning the empty set. + if source_id <> target_id and ( + (not inbound and not exists ( + select 1 from edge e + where e.graph_id = target_graph_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + )) or + (inbound and not exists ( + select 1 from edge e + where e.graph_id = target_graph_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + )) + ) then + perform public.record_requested_traversal_runtime_attestation_v1('asp_n1_target_degree_zero', false, 'ASP-N1-NEGATIVE-EXHAUSTION'); + return; + end if; + + perform public.reset_shortest_dag_workspace(); + insert into pg_temp.spd_seen(node_id, depth) values (target_id, 0); + + for search_depth in 1..max_depth loop + if not inbound then + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select distinct on (e.start_id) e.start_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.end_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.start_id) + order by e.start_id, e.id, f.node_id + limit greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0) + on conflict do nothing; + else + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select distinct on (e.end_id) e.end_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.start_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.end_id) + order by e.end_id, e.id, f.node_id + limit greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0) + on conflict do nothing; + end if; + + if not exists (select 1 from pg_temp.spd_candidate where depth = search_depth) then + perform public.record_requested_traversal_runtime_attestation_v1('asp_n1_reverse_exhausted', false, 'ASP-N1-NEGATIVE-EXHAUSTION'); + return; + end if; + + select (select count(*) from pg_temp.spd_seen) + + (select count(distinct node_id) from pg_temp.spd_candidate where depth = search_depth) + into retained_state; + if retained_state > state_limit then + exit; + end if; + if exists (select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = source_id) then + source_reached = true; + exit; + end if; + insert into pg_temp.spd_seen(node_id, depth) + select distinct node_id, search_depth from pg_temp.spd_candidate where depth = search_depth + on conflict do nothing; + end loop; + + if source_reached then + perform public.record_requested_traversal_runtime_attestation_v1('asp_n1_source_reached_a1', false, 'ASP-A1-DAG'); + else + perform public.record_requested_traversal_runtime_attestation_v1('asp_n1_state_cap_a1', true, 'ASP-A1-DAG'); + end if; + return query select * from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, min_depth, max_depth, edge_kind_ids, inbound); +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 100; + +-- shortest_path_compact keeps one deterministic predecessor per minimum-depth +-- node. If its bounded state budget is exceeded it restarts an exact +-- relationship-trail recursive search before returning any row, preserving the +-- transaction snapshot and the incumbent relationship-simple semantics. +create or replace function public.shortest_path_compact(target_graph_id int4, source_id int8, target_id int8, + min_depth int4, max_depth int4, + edge_kind_ids int2[], inbound bool, + state_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + search_depth int4; + target_depth int4; + emitted_count int8; + retained_state int8; + overflowed bool := false; +begin + if source_id is null or target_id is null or max_depth < min_depth then + return; + end if; + if min_depth <> 0 and min_depth <> 1 then + raise exception using errcode = '22023', message = 'shortest_path_compact requires min_depth = 0 or 1'; + end if; + if source_id = target_id then + if min_depth = 0 then + return query select source_id, target_id, 0::int4, true, false, array []::int8[]; + return; + end if; + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + if min_depth <= 1 and max_depth >= 1 then + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + perform public.record_requested_traversal_runtime_attestation_v1('one_hop_preflight', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + end if; + + if min_depth <= 2 and max_depth >= 2 then + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + perform public.record_requested_traversal_runtime_attestation_v1('two_hop_preflight', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + end if; + + if max_depth <= 2 then + perform public.record_requested_traversal_runtime_attestation_v1('preflight_no_path', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + + perform public.reset_shortest_dag_workspace(); + insert into pg_temp.spd_seen(node_id, depth) values (source_id, 0); + + for search_depth in 1..max_depth loop + if not inbound then + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select distinct on (e.end_id) e.end_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.start_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.end_id) + order by e.end_id, e.id, f.node_id + limit case when state_limit > 0 then greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0) else 9223372036854775807 end + on conflict do nothing; + else + insert into pg_temp.spd_candidate(node_id, depth, predecessor_id, edge_id) + select distinct on (e.start_id) e.start_id, search_depth, f.node_id, e.id + from pg_temp.spd_seen f + join edge e on e.graph_id = target_graph_id and e.end_id = f.node_id + where f.depth = search_depth - 1 + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spd_seen s where s.node_id = e.start_id) + order by e.start_id, e.id, f.node_id + limit case when state_limit > 0 then greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0) else 9223372036854775807 end + on conflict do nothing; + end if; + + if not exists (select 1 from pg_temp.spd_candidate where depth = search_depth) then + exit; + end if; + + if state_limit > 0 then + select (select count(*) from pg_temp.spd_seen) + + (select count(distinct node_id) from pg_temp.spd_candidate where depth = search_depth) + into retained_state; + if retained_state > state_limit then + overflowed = true; + exit; + end if; + end if; + + insert into pg_temp.spd_predecessor(node_id, depth, predecessor_id, edge_id) + select distinct on (node_id) node_id, search_depth, predecessor_id, edge_id + from pg_temp.spd_candidate + where depth = search_depth + order by node_id, edge_id, predecessor_id + on conflict do nothing; + + insert into pg_temp.spd_seen(node_id, depth) + select distinct node_id, search_depth from pg_temp.spd_candidate + where depth = search_depth + on conflict do nothing; + + if search_depth >= min_depth and exists ( + select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = target_id + ) then + target_depth = search_depth; + exit; + end if; + end loop; + + if overflowed then + perform public.record_requested_traversal_runtime_attestation_v1('exact_relationship_trail_fallback', true, 'SP-S3-U-E+MAT-M0'); + if not inbound then + return query + with recursive trails(node_id, trail_depth, edge_ids) as ( + select source_id, 0::int4, array []::int8[] + union all + select e.end_id, trails.trail_depth + 1, trails.edge_ids || e.id + from trails + join edge e on e.graph_id = target_graph_id and e.start_id = trails.node_id + where trails.trail_depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not e.id = any(trails.edge_ids) + ) + select source_id, target_id, trails.trail_depth, true, false, trails.edge_ids + from trails + where trails.node_id = target_id and trails.trail_depth >= min_depth + order by trails.trail_depth, trails.edge_ids + limit 1; + else + return query + with recursive trails(node_id, trail_depth, edge_ids) as ( + select source_id, 0::int4, array []::int8[] + union all + select e.start_id, trails.trail_depth + 1, trails.edge_ids || e.id + from trails + join edge e on e.graph_id = target_graph_id and e.end_id = trails.node_id + where trails.trail_depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not e.id = any(trails.edge_ids) + ) + select source_id, target_id, trails.trail_depth, true, false, trails.edge_ids + from trails + where trails.node_id = target_id and trails.trail_depth >= min_depth + order by trails.trail_depth, trails.edge_ids + limit 1; + end if; + return; + end if; + + if target_depth is null then + perform public.record_requested_traversal_runtime_attestation_v1('compact_no_path', false, 'SP-S4-C-WE+MAT-M0'); + return; + end if; + + perform public.record_requested_traversal_runtime_attestation_v1('compact_workspace_witness', false, 'SP-S4-C-WE+MAT-M0'); + + return query + with recursive witness(node_id, path_depth, edge_ids) as ( + select target_id, target_depth, array []::int8[] + union all + select predecessor.predecessor_id, + witness.path_depth - 1, + array[predecessor.edge_id]::int8[] || witness.edge_ids + from witness + join pg_temp.spd_predecessor predecessor + on predecessor.node_id = witness.node_id + and predecessor.depth = witness.path_depth + ) + select source_id, target_id, target_depth, true, false, witness.edge_ids + from witness + where witness.node_id = source_id and witness.path_depth = 0 + order by witness.edge_ids + limit 1; +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 1; + +-- Compact bidirectional shortest-path candidates use a workspace that is +-- deliberately disjoint from spd_*. An overflow can therefore invoke the +-- production S4 executor in the same top-level statement without corrupting +-- either search. The version row makes pooled-session reuse fail closed when +-- the typed workspace shape changes. +create or replace function public.ensure_bidirectional_shortest_path_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.spb_workspace_version') is not null then + select version into present_version from pg_temp.spb_workspace_version limit 1; + end if; + + if to_regclass('pg_temp.spb_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.spb_predecessor; + drop table if exists pg_temp.spb_candidate; + drop table if exists pg_temp.spb_active; + drop table if exists pg_temp.spb_seen; + drop table if exists pg_temp.spb_front; + drop table if exists pg_temp.spb_workspace_version; + end if; + + if to_regclass('pg_temp.spb_workspace_version') is null then + create temporary table spb_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + -- side is f for logical source search and b for reverse search from the + -- logical target. queue_order is a stable FIFO order for B1; B2 groups the + -- same ID-only rows by depth into complete levels. + create temporary table spb_front + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + queue_order int8 not null, + primary key (side, node_id), + unique (side, queue_order), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index spb_front_side_depth_index on spb_front using btree (side, depth, queue_order); + + create temporary table spb_seen + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index spb_seen_node_side_index on spb_seen using btree (node_id, side, depth); + + create temporary table spb_active + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + create temporary table spb_candidate + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + -- For f rows adjacent_id is the predecessor toward source. For b rows it + -- is the successor toward target. One stable edge is retained per node. + create temporary table spb_predecessor + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index spb_predecessor_adjacent_side_index on spb_predecessor using btree (adjacent_id, side, depth); + + insert into spb_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.reset_bidirectional_shortest_path_workspace() + returns void as +$$ +begin + perform public.ensure_bidirectional_shortest_path_workspace(); + truncate table pg_temp.spb_front, pg_temp.spb_seen, pg_temp.spb_active, + pg_temp.spb_candidate, pg_temp.spb_predecessor; +end; +$$ + language plpgsql + volatile; + +-- Runtime receipts bind a GraphBench latency sample to the branch executed by +-- that exact statement. The receipt is armed and read outside the timed block +-- on the same session. Instrumentation is inert unless an invocation is armed. +create or replace function public.ensure_traversal_runtime_attestation_workspace_v1() + returns void as +$$ +begin + if to_regclass('pg_temp.traversal_runtime_attestation_v1') is null then + create temporary table traversal_runtime_attestation_v1 + ( + invocation_id text not null primary key, + requested_identity text not null, + runtime_identity text, + runtime_branch text, + fallback_executed bool, + record_count int4 not null default 0, + events jsonb not null default '[]'::jsonb, + check (btrim(invocation_id) <> ''), + check (btrim(requested_identity) <> '') + ) on commit preserve rows; + end if; + -- Avoid issuing even a no-op ALTER in ordinary read-only transactions. + -- The conditional branch is retained for pooled sessions whose temporary + -- v1 receipt table predates the event-chain column. + if not exists ( + select 1 + from pg_attribute + where attrelid = 'pg_temp.traversal_runtime_attestation_v1'::regclass + and attname = 'events' + and not attisdropped + ) then + alter table pg_temp.traversal_runtime_attestation_v1 + add column events jsonb not null default '[]'::jsonb; + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.begin_traversal_runtime_attestation_v1( + target_invocation_id text, + target_requested_identity text) + returns void as +$$ +begin + if target_invocation_id is null or btrim(target_invocation_id) = '' or length(target_invocation_id) > 256 then + raise exception using errcode = '22023', message = 'traversal runtime invocation ID must contain 1 to 256 characters'; + end if; + if target_requested_identity is null or btrim(target_requested_identity) = '' or length(target_requested_identity) > 256 then + raise exception using errcode = '22023', message = 'traversal runtime requested identity must contain 1 to 256 characters'; + end if; + perform public.ensure_traversal_runtime_attestation_workspace_v1(); + delete from pg_temp.traversal_runtime_attestation_v1 where invocation_id = target_invocation_id; + insert into pg_temp.traversal_runtime_attestation_v1(invocation_id, requested_identity) + values (target_invocation_id, target_requested_identity); + -- Session scope deliberately survives the arming autocommit. The matching + -- clear call executes immediately after the timed statement. + perform set_config('dawgs.traversal_runtime_invocation_id', target_invocation_id, false); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.record_traversal_runtime_attestation_v1( + target_runtime_identity text, + target_runtime_branch text, + target_fallback_executed bool) + returns bool as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.traversal_runtime_invocation_id', true), ''); +begin + if target_invocation_id is null then + return true; + end if; + update pg_temp.traversal_runtime_attestation_v1 receipt + set runtime_identity = target_runtime_identity, + runtime_branch = target_runtime_branch, + fallback_executed = coalesce(receipt.fallback_executed, false) or target_fallback_executed, + record_count = receipt.record_count + 1, + events = receipt.events || jsonb_build_array(jsonb_build_object( + 'ordinal', receipt.record_count + 1, + 'runtime_identity', target_runtime_identity, + 'runtime_branch', target_runtime_branch, + 'fallback_executed', target_fallback_executed + )) + where receipt.invocation_id = target_invocation_id; + if not found then + raise exception using errcode = '55000', message = 'traversal runtime receipt is missing'; + end if; + return true; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.record_requested_traversal_runtime_attestation_v1( + target_runtime_branch text, + target_fallback_executed bool, + target_fallback_identity text) + returns bool as +$$ +declare + target_invocation_id text := nullif(current_setting('dawgs.traversal_runtime_invocation_id', true), ''); + target_requested_identity text; +begin + if target_invocation_id is null then + return true; + end if; + select requested_identity into target_requested_identity + from pg_temp.traversal_runtime_attestation_v1 + where invocation_id = target_invocation_id; + if target_requested_identity is null then + raise exception using errcode = '55000', message = 'armed traversal runtime receipt is missing'; + end if; + if target_fallback_executed and target_fallback_identity = 'SP-S4' then + target_fallback_identity = case when target_requested_identity like '%-D' + then 'SP-S4-C-D' else 'SP-S4-C-WE+MAT-M0' end; + end if; + return public.record_traversal_runtime_attestation_v1( + case when target_fallback_executed then target_fallback_identity else target_requested_identity end, + target_runtime_branch, + target_fallback_executed + ); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public.read_traversal_runtime_attestation_v1(target_invocation_id text) + returns jsonb as +$$ +begin + return ( + select jsonb_build_object( + 'schema_version', 2, + 'invocation_id', invocation_id, + 'requested_identity', requested_identity, + 'runtime_identity', runtime_identity, + 'runtime_branch', runtime_branch, + 'fallback_executed', fallback_executed, + 'record_count', record_count, + 'events', events + ) + from pg_temp.traversal_runtime_attestation_v1 + where invocation_id = target_invocation_id + ); +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_traversal_runtime_attestation_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.traversal_runtime_attestation_v1') is not null then + delete from pg_temp.traversal_runtime_attestation_v1 where invocation_id = target_invocation_id; + end if; + if nullif(current_setting('dawgs.traversal_runtime_invocation_id', true), '') = target_invocation_id then + perform set_config('dawgs.traversal_runtime_invocation_id', '', false); + end if; +end; +$$ + language plpgsql + volatile + strict; + +-- Detailed bidirectional SP counters live in a second, independently +-- versioned temporary workspace. GraphBench enables this workspace only for +-- an untimed replay. The transaction-local invocation setting means pooled +-- sessions cannot accidentally attribute a later statement to an earlier +-- replay, while the explicit invocation key keeps every row attributable. +create or replace function public.ensure_bidirectional_shortest_path_telemetry_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.spb_telemetry_workspace_version') is not null then + select version into present_version + from pg_temp.spb_telemetry_workspace_version + limit 1; + end if; + + if to_regclass('pg_temp.spb_telemetry_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.spb_telemetry_level; + drop table if exists pg_temp.spb_telemetry_call; + drop table if exists pg_temp.spb_telemetry_invocation; + drop table if exists pg_temp.spb_telemetry_workspace_version; + end if; + + if to_regclass('pg_temp.spb_telemetry_workspace_version') is null then + create temporary table spb_telemetry_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table spb_telemetry_invocation + ( + invocation_id text not null primary key, + schema_version int4 not null, + scheduler text, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + next_search_id int8 not null default 0, + check (btrim(invocation_id) <> '') + ) on commit preserve rows; + + create temporary table spb_telemetry_call + ( + invocation_id text not null, + search_id int8 not null, + source_id int8 not null, + target_id int8 not null, + runtime_branch text not null default 'started', + scheduler_actions int8 not null default 0, + candidate_edges int8 not null default 0, + distinct_new_nodes int8 not null default 0, + seen_peak int8 not null default 0, + frontier_peak int8 not null default 0, + queue_peak int8 not null default 0, + predecessor_peak int8 not null default 0, + meeting_candidates int8 not null default 0, + frozen_distance int4, + witness_rows int8 not null default 0, + overflowed bool not null default false, + fallback_executed bool not null default false, + primary key (invocation_id, search_id) + ) on commit preserve rows; + + create temporary table spb_telemetry_level + ( + invocation_id text not null, + search_id int8 not null, + action_index int8 not null, + side text not null, + action text not null, + depth int4 not null, + frontier_rows int8 not null, + candidate_edges int8 not null, + distinct_new_nodes int8 not null, + seen_rows int8 not null, + queue_rows int8 not null, + predecessor_rows int8 not null, + meeting_candidates int8 not null, + primary key (invocation_id, search_id, action_index) + ) on commit preserve rows; + + insert into spb_telemetry_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +-- begin_bidirectional_shortest_path_diagnostic_v1 must be called inside the +-- same explicit transaction and on the same PostgreSQL connection as the +-- diagnostic replay. It clears only its own invocation key and enables +-- instrumentation through a transaction-local setting. +create or replace function public.begin_bidirectional_shortest_path_diagnostic_v1(invocation_id text) + returns void as +$$ +begin + if invocation_id is null or btrim(invocation_id) = '' or length(invocation_id) > 256 then + raise exception using errcode = '22023', message = 'bidirectional shortest-path diagnostic invocation ID must contain 1 to 256 characters'; + end if; + + perform public.ensure_bidirectional_shortest_path_telemetry_workspace(); + delete from pg_temp.spb_telemetry_level where spb_telemetry_level.invocation_id = begin_bidirectional_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.spb_telemetry_call where spb_telemetry_call.invocation_id = begin_bidirectional_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.spb_telemetry_invocation where spb_telemetry_invocation.invocation_id = begin_bidirectional_shortest_path_diagnostic_v1.invocation_id; + insert into pg_temp.spb_telemetry_invocation(invocation_id, schema_version) + values (invocation_id, 1); + perform set_config('dawgs.spb_diagnostic_invocation_id', invocation_id, true); +end; +$$ + language plpgsql + volatile; + +-- The reader returns one self-describing document. Aggregate counters support +-- the common single-bound-pair replay, while calls preserve exact per-pair +-- attribution if a translated statement invokes the kernel more than once. +create or replace function public.read_bidirectional_shortest_path_diagnostic_v1(target_invocation_id text) + returns jsonb as +$$ +declare + result jsonb; +begin + select jsonb_build_object( + 'schema_version', invocation.schema_version, + 'invocation_id', invocation.invocation_id, + 'scheduler', invocation.scheduler, + 'state_limit', invocation.state_limit, + 'frontier_limit', invocation.frontier_limit, + 'predecessor_limit', invocation.predecessor_limit, + 'search_calls', coalesce(call_totals.search_calls, 0), + 'runtime_branch', coalesce(call_totals.runtime_branch, 'missing'), + 'overflowed', coalesce(call_totals.overflowed, false), + 'fallback_executed', coalesce(call_totals.fallback_executed, false), + 'counters', jsonb_build_object( + 'scheduler_actions', coalesce(call_totals.scheduler_actions, 0), + 'candidate_edges', coalesce(call_totals.candidate_edges, 0), + 'distinct_new_nodes', coalesce(call_totals.distinct_new_nodes, 0), + 'seen_peak', coalesce(call_totals.seen_peak, 0), + 'frontier_peak', coalesce(call_totals.frontier_peak, 0), + 'queue_peak', coalesce(call_totals.queue_peak, 0), + 'predecessor_peak', coalesce(call_totals.predecessor_peak, 0), + 'meeting_candidates', coalesce(call_totals.meeting_candidates, 0), + -- -1 is the explicit no-frozen-meeting sentinel. Exact values are + -- retained per call below when a statement evaluates many pairs. + 'frozen_distance', coalesce(call_totals.frozen_distance, -1), + 'witness_rows', coalesce(call_totals.witness_rows, 0), + 'levels', coalesce(levels.rows, '[]'::jsonb) + ), + 'calls', coalesce(calls.rows, '[]'::jsonb) + ) +into result +from pg_temp.spb_telemetry_invocation invocation +left join lateral ( + select count(*)::int8 as search_calls, + case when count(distinct call.runtime_branch) = 1 + then min(call.runtime_branch) else 'mixed' end as runtime_branch, + bool_or(call.overflowed) as overflowed, + bool_or(call.fallback_executed) as fallback_executed, + sum(call.scheduler_actions)::int8 as scheduler_actions, + sum(call.candidate_edges)::int8 as candidate_edges, + sum(call.distinct_new_nodes)::int8 as distinct_new_nodes, + max(call.seen_peak)::int8 as seen_peak, + max(call.frontier_peak)::int8 as frontier_peak, + max(call.queue_peak)::int8 as queue_peak, + max(call.predecessor_peak)::int8 as predecessor_peak, + sum(call.meeting_candidates)::int8 as meeting_candidates, + min(call.frozen_distance)::int4 as frozen_distance, + sum(call.witness_rows)::int8 as witness_rows + from pg_temp.spb_telemetry_call call + where call.invocation_id = invocation.invocation_id +) call_totals on true +left join lateral ( + select jsonb_agg(jsonb_build_object( + 'search_id', level.search_id, + 'action_index', level.action_index, + 'side', level.side, + 'action', level.action, + 'depth', level.depth, + 'frontier_rows', level.frontier_rows, + 'candidate_edges', level.candidate_edges, + 'distinct_new_nodes', level.distinct_new_nodes, + 'seen_rows', level.seen_rows, + 'queue_rows', level.queue_rows, + 'predecessor_rows', level.predecessor_rows, + 'meeting_candidates', level.meeting_candidates + ) order by level.search_id, level.action_index) as rows + from pg_temp.spb_telemetry_level level + where level.invocation_id = invocation.invocation_id +) levels on true +left join lateral ( + select jsonb_agg(to_jsonb(call) - 'invocation_id' order by call.search_id) as rows + from pg_temp.spb_telemetry_call call + where call.invocation_id = invocation.invocation_id +) calls on true + where invocation.invocation_id = target_invocation_id; + return result; +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_bidirectional_shortest_path_diagnostic_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.spb_telemetry_invocation') is not null then + delete from pg_temp.spb_telemetry_level where invocation_id = target_invocation_id; + delete from pg_temp.spb_telemetry_call where invocation_id = target_invocation_id; + delete from pg_temp.spb_telemetry_invocation where invocation_id = target_invocation_id; + end if; + if nullif(current_setting('dawgs.spb_diagnostic_invocation_id', true), '') = target_invocation_id then + perform set_config('dawgs.spb_diagnostic_invocation_id', '', true); + end if; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._start_bidirectional_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_scheduler text, + target_state_limit int8, + target_frontier_limit int8, + target_predecessor_limit int8, + target_source_id int8, + target_target_id int8) + returns int8 as +$$ +declare + target_search_id int8; +begin + if target_invocation_id is null then + return null; + end if; + if to_regclass('pg_temp.spb_telemetry_invocation') is null then + raise exception using errcode = '55000', message = 'bidirectional shortest-path diagnostic replay was not initialized on this session'; + end if; + + update pg_temp.spb_telemetry_invocation invocation + set scheduler = coalesce(invocation.scheduler, target_scheduler), + state_limit = coalesce(invocation.state_limit, target_state_limit), + frontier_limit = coalesce(invocation.frontier_limit, target_frontier_limit), + predecessor_limit = coalesce(invocation.predecessor_limit, target_predecessor_limit), + next_search_id = invocation.next_search_id + 1 + where invocation.invocation_id = target_invocation_id + and (invocation.scheduler is null or invocation.scheduler = target_scheduler) + and (invocation.state_limit is null or invocation.state_limit = target_state_limit) + and (invocation.frontier_limit is null or invocation.frontier_limit = target_frontier_limit) + and (invocation.predecessor_limit is null or invocation.predecessor_limit = target_predecessor_limit) + returning invocation.next_search_id into target_search_id; + + if target_search_id is null then + raise exception using + errcode = '55000', + message = 'bidirectional shortest-path diagnostic invocation is missing or mixes scheduler/cap identities'; + end if; + + insert into pg_temp.spb_telemetry_call(invocation_id, search_id, source_id, target_id) + values (target_invocation_id, target_search_id, target_source_id, target_target_id); + return target_search_id; +end; +$$ + language plpgsql + volatile; + +create or replace function public._record_bidirectional_shortest_path_diagnostic_level_v1( + target_invocation_id text, + target_search_id int8, + target_action_index int8, + target_side text, + target_action text, + target_depth int4, + target_frontier_rows int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_rows int8, + target_queue_rows int8, + target_predecessor_rows int8, + target_meeting_candidates int8) + returns void as +$$ +begin + insert into pg_temp.spb_telemetry_level( + invocation_id, search_id, action_index, side, action, depth, + frontier_rows, candidate_edges, distinct_new_nodes, seen_rows, + queue_rows, predecessor_rows, meeting_candidates) + values ( + target_invocation_id, target_search_id, target_action_index, target_side, + target_action, target_depth, target_frontier_rows, target_candidate_edges, + target_distinct_new_nodes, target_seen_rows, target_queue_rows, + target_predecessor_rows, target_meeting_candidates); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._finish_bidirectional_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_search_id int8, + target_runtime_branch text, + target_scheduler_actions int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_peak int8, + target_frontier_peak int8, + target_queue_peak int8, + target_predecessor_peak int8, + target_meeting_candidates int8, + target_frozen_distance int4, + target_witness_rows int8, + target_overflowed bool, + target_fallback_executed bool) + returns void as +$$ +begin + update pg_temp.spb_telemetry_call call + set runtime_branch = target_runtime_branch, + scheduler_actions = target_scheduler_actions, + candidate_edges = target_candidate_edges, + distinct_new_nodes = target_distinct_new_nodes, + seen_peak = target_seen_peak, + frontier_peak = target_frontier_peak, + queue_peak = target_queue_peak, + predecessor_peak = target_predecessor_peak, + meeting_candidates = target_meeting_candidates, + frozen_distance = target_frozen_distance, + witness_rows = target_witness_rows, + overflowed = target_overflowed, + fallback_executed = target_fallback_executed + where call.invocation_id = target_invocation_id + and call.search_id = target_search_id; + + if not found then + raise exception using errcode = '55000', message = 'bidirectional shortest-path diagnostic call is missing'; + end if; +end; +$$ + language plpgsql + volatile; + +-- shortest_path_bidirectional_compact_v1 is the common typed kernel for the +-- B1 and B2 tournament arms. Queue-head depths are lower bounds on every +-- undiscovered source/target distance. Once their sum is at least the best +-- completed meeting distance, no unexpanded pair can produce a shorter path. +-- B1 applies this proof after deterministic one-node alternation; B2 applies it +-- only between complete-level expansions. Merely finding an intersection is +-- never a termination condition. +-- +-- Admission is fail-closed. Candidate state is materialized with LIMIT cap+1 +-- before any seen/front/predecessor mutation. If total seen rows, queued +-- frontier rows, or retained predecessors exceed their independent bound, the +-- function invokes exact S4 before returning any candidate row. VOLATILE +-- PL/pgSQL statements do not provide one transaction snapshot at READ +-- COMMITTED, so the kernel rejects that isolation level. At REPEATABLE READ or +-- SERIALIZABLE, candidate search and nested S4 fallback observe the same +-- transaction snapshot; spb_/spd_ state remains disjoint. +create or replace function public.shortest_path_bidirectional_compact_v1( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + scheduler text) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + chosen_side char(1); + strict_side char(1) := 'f'; + forward_depth int4; + backward_depth int4; + forward_width int8; + backward_width int8; + forward_tail int8 := 0; + backward_tail int8 := 0; + seen_rows int8; + active_rows int8; + frontier_rows int8; + predecessor_rows int8; + candidate_rows int8; + admission_limit int8; + candidate_meeting int8; + candidate_distance int4; + best_meeting int8; + best_distance int4; + emitted_count int8; + overflowed bool := false; + telemetry_invocation_id text := nullif(current_setting('dawgs.spb_diagnostic_invocation_id', true), ''); + telemetry_search_id int8; + telemetry_action_index int8 := 0; + telemetry_action_depth int4 := 0; + telemetry_action_candidate_edges int8 := 0; + telemetry_action_meetings int8 := 0; + telemetry_scheduler_actions int8 := 0; + telemetry_candidate_edges int8 := 0; + telemetry_distinct_new_nodes int8 := 0; + telemetry_seen_peak int8 := 0; + telemetry_frontier_peak int8 := 0; + telemetry_queue_peak int8 := 0; + telemetry_predecessor_peak int8 := 0; + telemetry_meeting_candidates int8 := 0; +begin + if source_id is null or target_id is null or max_depth < min_depth then + return; + end if; + if scheduler <> 'strict_alternating_node' and scheduler <> 'smaller_current_level' then + raise exception using errcode = '22023', message = 'unknown compact bidirectional shortest-path scheduler'; + end if; + if min_depth <> 0 and min_depth <> 1 then + raise exception using errcode = '22023', message = 'compact bidirectional shortest path requires min_depth = 0 or 1'; + end if; + if max_depth > 64 then + raise exception using errcode = '22023', message = 'compact bidirectional shortest path requires max_depth <= 64'; + end if; + if state_limit <= 0 or frontier_limit <= 0 or predecessor_limit <= 0 then + raise exception using errcode = '22023', message = 'compact bidirectional shortest path requires positive state, frontier, and predecessor limits'; + end if; + if current_setting('transaction_isolation') <> 'repeatable read' + and current_setting('transaction_isolation') <> 'serializable' then + raise exception using + errcode = '25001', + message = 'compact bidirectional shortest path requires REPEATABLE READ or SERIALIZABLE transaction isolation'; + end if; + + telemetry_search_id = public._start_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, scheduler, state_limit, frontier_limit, + predecessor_limit, source_id, target_id); + + -- Exact zero-hop preflight precedes workspace allocation. + if source_id = target_id then + if min_depth = 0 then + return query select source_id, target_id, 0::int4, true, false, array []::int8[]; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_zero_hop', 0, 0, 0, 0, 0, 0, 0, emitted_count); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'zero_hop_preflight', + 0, 0, 0, 0, 0, 0, 0, emitted_count, 0, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('zero_hop_preflight', false, 'SP-S4'); + return; + end if; + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + -- Exact one-hop preflight chooses the same deterministic edge ordering as S4. + if min_depth <= 1 and max_depth >= 1 then + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + if telemetry_search_id is not null then + if not inbound then + select count(*) into telemetry_action_candidate_edges + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + else + select count(*) into telemetry_action_candidate_edges + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + end if; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_candidate_edges; + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_one_hop', 1, 0, telemetry_action_candidate_edges, + 0, 0, 0, 0, telemetry_action_candidate_edges); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'one_hop_preflight', + 0, telemetry_candidate_edges, 0, 0, 0, 0, 0, + telemetry_meeting_candidates, 1, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('one_hop_preflight', false, 'SP-S4'); + return; + end if; + end if; + + -- Exact two-hop preflight retains relationship uniqueness and public order. + if min_depth <= 2 and max_depth >= 2 then + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id limit 1; + end if; + get diagnostics emitted_count = row_count; + if emitted_count > 0 then + if telemetry_search_id is not null then + if not inbound then + select count(*) * 2 into telemetry_action_candidate_edges + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)); + else + select count(*) * 2 into telemetry_action_candidate_edges + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)); + end if; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_action_meetings = telemetry_action_candidate_edges / 2; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_two_hop', 2, 0, telemetry_action_candidate_edges, + 0, 0, 0, 0, telemetry_action_meetings); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'two_hop_preflight', + 0, telemetry_candidate_edges, 0, 0, 0, 0, 0, + telemetry_meeting_candidates, 2, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('two_hop_preflight', false, 'SP-S4'); + return; + end if; + end if; + if max_depth <= 2 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_no_path', max_depth, 0, 0, 0, 0, 0, 0, 0); + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_no_path', + 0, 0, 0, 0, 0, 0, 0, 0, null, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_no_path', false, 'SP-S4'); + return; + end if; + + -- Both roots count toward seen and frontier admission. Overflow falls back + -- before allocating or exposing candidate state. + if state_limit < 2 or frontier_limit < 2 then + overflowed = true; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'root_admission', 0, 2, 0, 0, 2, 2, 0, 0); + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + end if; + else + perform public.reset_bidirectional_shortest_path_workspace(); + insert into pg_temp.spb_front(side, node_id, depth, queue_order) + values ('f', source_id, 0, 0), ('b', target_id, 0, 0); + insert into pg_temp.spb_seen(side, node_id, depth) + values ('f', source_id, 0), ('b', target_id, 0); + telemetry_seen_peak = 2; + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + end if; + + while not overflowed loop + select min(depth), count(*) filter (where depth = (select min(depth) from pg_temp.spb_front where side = 'f')) + into forward_depth, forward_width + from pg_temp.spb_front where side = 'f'; + select min(depth), count(*) filter (where depth = (select min(depth) from pg_temp.spb_front where side = 'b')) + into backward_depth, backward_width + from pg_temp.spb_front where side = 'b'; + + if forward_depth is null or backward_depth is null then + exit; + end if; + + -- Dijkstra/BFS lower bound over the two next accepted queue depths. + if best_distance is not null and forward_depth + backward_depth >= best_distance then + exit; + end if; + + truncate table pg_temp.spb_active, pg_temp.spb_candidate; + if scheduler = 'strict_alternating_node' then + chosen_side = strict_side; + if (chosen_side = 'f' and forward_width = 0) or (chosen_side = 'b' and backward_width = 0) then + chosen_side = case chosen_side when 'f' then 'b' else 'f' end; + end if; + strict_side = case chosen_side when 'f' then 'b' else 'f' end; + + insert into pg_temp.spb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.spb_front + where side = chosen_side + order by queue_order + limit 1; + else + -- B2 expands the complete smaller current level. Equality always chooses + -- the forward side, freezing the tie break across artifacts. + chosen_side = case when forward_width <= backward_width then 'f' else 'b' end; + insert into pg_temp.spb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.spb_front + where side = chosen_side + and depth = case chosen_side when 'f' then forward_depth else backward_depth end + order by queue_order; + end if; + + delete from pg_temp.spb_front front + using pg_temp.spb_active active + where front.side = active.side and front.node_id = active.node_id; + + telemetry_scheduler_actions = telemetry_scheduler_actions + 1; + telemetry_action_candidate_edges = 0; + telemetry_action_meetings = 0; + select min(depth) into telemetry_action_depth from pg_temp.spb_active; + + if not exists (select 1 from pg_temp.spb_active where depth < max_depth) then + if telemetry_search_id is not null then + select count(*) into seen_rows from pg_temp.spb_seen; + select count(*) into active_rows from pg_temp.spb_active; + select count(*) into frontier_rows from pg_temp.spb_front; + select count(*) into predecessor_rows from pg_temp.spb_predecessor; + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + continue; + end if; + + select count(*) into seen_rows from pg_temp.spb_seen; + select count(*) into active_rows from pg_temp.spb_active; + select count(*) into frontier_rows from pg_temp.spb_front; + select count(*) into predecessor_rows from pg_temp.spb_predecessor; + admission_limit = least(state_limit - seen_rows, + frontier_limit - active_rows - frontier_rows, + predecessor_limit - predecessor_rows); + if admission_limit < 0 then + overflowed = true; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + exit; + end if; + + -- Candidate selection is graph scoped, ID only, and bounded at cap+1. + -- DISTINCT ON freezes one predecessor/successor before workspace mutation. + if chosen_side = 'f' and not inbound then + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.end_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.end_id) e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.end_id) + order by e.end_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + elsif chosen_side = 'f' and inbound then + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.start_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.start_id) e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'f' and seen.node_id = e.start_id) + order by e.start_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + elsif chosen_side = 'b' and not inbound then + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.start_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.start_id) e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.start_id) + order by e.start_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + else + if telemetry_search_id is not null then + select count(*) into telemetry_action_candidate_edges + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.end_id); + end if; + insert into pg_temp.spb_candidate(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select distinct on (e.end_id) e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.spb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.spb_seen seen where seen.side = 'b' and seen.node_id = e.end_id) + order by e.end_id, e.id, active.node_id + limit admission_limit + 1 + ) candidate; + end if; + + select count(*) into candidate_rows from pg_temp.spb_candidate; + if telemetry_search_id is not null then + select count(*) into telemetry_action_meetings + from pg_temp.spb_candidate candidate + join pg_temp.spb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_distinct_new_nodes = telemetry_distinct_new_nodes + candidate_rows; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows + candidate_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows + candidate_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows + candidate_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows + candidate_rows); + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows + candidate_rows, + telemetry_action_candidate_edges, candidate_rows, seen_rows + candidate_rows, + frontier_rows + candidate_rows, predecessor_rows + candidate_rows, + telemetry_action_meetings); + end if; + if seen_rows + candidate_rows > state_limit + or active_rows + frontier_rows + candidate_rows > frontier_limit + or predecessor_rows + candidate_rows > predecessor_limit then + overflowed = true; + exit; + end if; + + insert into pg_temp.spb_predecessor(side, node_id, depth, adjacent_id, edge_id) + select side, node_id, depth, adjacent_id, edge_id + from pg_temp.spb_candidate + order by side, node_id; + insert into pg_temp.spb_seen(side, node_id, depth) + select side, node_id, depth from pg_temp.spb_candidate order by side, node_id; + + if chosen_side = 'f' then + insert into pg_temp.spb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + forward_tail + row_number() over (order by edge_id, node_id, adjacent_id) + from pg_temp.spb_candidate; + forward_tail = forward_tail + candidate_rows; + else + insert into pg_temp.spb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + backward_tail + row_number() over (order by edge_id, node_id, adjacent_id) + from pg_temp.spb_candidate; + backward_tail = backward_tail + candidate_rows; + end if; + + candidate_meeting = null; + candidate_distance = null; + select candidate.node_id, candidate.depth + opposite.depth + into candidate_meeting, candidate_distance + from pg_temp.spb_candidate candidate + join pg_temp.spb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth + order by candidate.depth + opposite.depth, candidate.node_id + limit 1; + if candidate_distance is not null + and (best_distance is null + or candidate_distance < best_distance + or (candidate_distance = best_distance and candidate_meeting < best_meeting)) then + best_distance = candidate_distance; + best_meeting = candidate_meeting; + end if; + end loop; + + if overflowed then + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.shortest_path_compact(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound, state_limit) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_s4_fallback', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_s4_fallback', true, 'SP-S4'); + return; + end if; + if best_distance is null then + if telemetry_search_id is not null then + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'search_no_path', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + null, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('search_no_path', false, 'SP-S4'); + return; + end if; + + -- Path arrays exist only at the late output boundary. Forward predecessor + -- edges are prepended back to source; backward successor edges are appended + -- toward target, preserving logical source-to-target order for both physical + -- edge orientations. + return query + with recursive + forward_witness(node_id, edge_ids) as ( + select best_meeting, array []::int8[] + union all + select predecessor.adjacent_id, + array[predecessor.edge_id]::int8[] || forward_witness.edge_ids + from forward_witness + join pg_temp.spb_predecessor predecessor + on predecessor.side = 'f' and predecessor.node_id = forward_witness.node_id + ), + backward_witness(node_id, edge_ids) as ( + select best_meeting, array []::int8[] + union all + select successor.adjacent_id, + backward_witness.edge_ids || successor.edge_id + from backward_witness + join pg_temp.spb_predecessor successor + on successor.side = 'b' and successor.node_id = backward_witness.node_id + ) + select source_id, target_id, best_distance, true, false, + forward_witness.edge_ids || backward_witness.edge_ids + from forward_witness + join backward_witness on forward_witness.node_id = source_id + and backward_witness.node_id = target_id + limit 1; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'bidirectional_search', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('bidirectional_search', false, 'SP-S4'); +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 1; + +-- B1 freezes Neo4j-4.4-style strict one-node alternation behind a typed +-- wrapper so scheduler identity is not inferred from generated SQL text. +create or replace function public.shortest_path_b1_strict_alternating( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.shortest_path_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + 'strict_alternating_node'); +$$ + language sql + volatile + strict + cost 100 + rows 1; + +-- B2 expands a complete current level from the smaller side, with a stable +-- forward-side tie break. +create or replace function public.shortest_path_b2_smaller_current_level( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.shortest_path_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + 'smaller_current_level'); +$$ + language sql + volatile + strict + cost 100 + rows 1; + +-- Compact bidirectional all-shortest-path candidates use a workspace that is +-- disjoint from both the production ASP-A1 spd_* state and singleton SP spb_* +-- state. Discovery, relationship-distinct predecessor retention, path-count +-- calculation, and staged output therefore have separately measurable shapes. +create or replace function public.ensure_bidirectional_all_shortest_path_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.asb_workspace_version') is not null then + select version into present_version from pg_temp.asb_workspace_version limit 1; + end if; + + if to_regclass('pg_temp.asb_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.asb_output; + drop table if exists pg_temp.asb_path_count; + drop table if exists pg_temp.asb_predecessor; + drop table if exists pg_temp.asb_candidate_predecessor; + drop table if exists pg_temp.asb_candidate_node; + drop table if exists pg_temp.asb_active; + drop table if exists pg_temp.asb_seen; + drop table if exists pg_temp.asb_front; + drop table if exists pg_temp.asb_workspace_version; + end if; + + if to_regclass('pg_temp.asb_workspace_version') is null then + create temporary table asb_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table asb_front + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + queue_order int8 not null, + primary key (side, node_id), + unique (side, queue_order), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index asb_front_side_depth_order_index + on asb_front using btree (side, depth, queue_order); + + create temporary table asb_seen + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index asb_seen_node_side_depth_index + on asb_seen using btree (node_id, side, depth); + + create temporary table asb_active + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + create temporary table asb_candidate_node + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + primary key (side, node_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + create temporary table asb_candidate_predecessor + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id, depth, adjacent_id, edge_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + + -- Forward adjacent_id points toward the logical source. Backward + -- adjacent_id points toward the logical target. Equal-depth rows are not + -- collapsed: every relationship-distinct shortest predecessor/successor + -- is retained. + create temporary table asb_predecessor + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + adjacent_id int8 not null, + edge_id int8 not null, + primary key (side, node_id, depth, adjacent_id, edge_id), + check (side in ('f', 'b')) + ) on commit preserve rows; + create index asb_predecessor_node_side_depth_index + on asb_predecessor using btree (node_id, side, depth); + create index asb_predecessor_adjacent_side_depth_index + on asb_predecessor using btree (adjacent_id, side, depth); + + create temporary table asb_path_count + ( + side char(1) not null, + node_id int8 not null, + depth int4 not null, + path_count int8 not null, + primary key (side, node_id), + check (side in ('f', 'b')), + check (path_count >= 0) + ) on commit preserve rows; + + create temporary table asb_output + ( + edge_ids int8[] not null primary key, + output_bytes int8 not null, + check (output_bytes >= 0) + ) on commit preserve rows; + + insert into asb_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.reset_bidirectional_all_shortest_path_workspace() + returns void as +$$ +begin + perform public.ensure_bidirectional_all_shortest_path_workspace(); + truncate table pg_temp.asb_front, pg_temp.asb_seen, pg_temp.asb_active, + pg_temp.asb_candidate_node, pg_temp.asb_candidate_predecessor, + pg_temp.asb_predecessor, pg_temp.asb_path_count, + pg_temp.asb_output; +end; +$$ + language plpgsql + volatile; + +-- clear_bidirectional_all_shortest_path_workspace does not allocate state. +-- Overflow paths call it before ASP-A1 so no candidate rows survive into the +-- exact fallback boundary. +create or replace function public.clear_bidirectional_all_shortest_path_workspace() + returns void as +$$ +begin + if to_regclass('pg_temp.asb_workspace_version') is not null then + execute 'truncate table pg_temp.asb_front, pg_temp.asb_seen, pg_temp.asb_active, ' + 'pg_temp.asb_candidate_node, pg_temp.asb_candidate_predecessor, ' + 'pg_temp.asb_predecessor, pg_temp.asb_path_count, pg_temp.asb_output'; + end if; +end; +$$ + language plpgsql + volatile; + +-- Tool-only ASP diagnostic counters use a second versioned, session-local +-- workspace. The transaction-local invocation setting prevents pooled-session +-- reuse from attributing a later call to an earlier replay, while explicit +-- keys make multi-call statements and cleanup independently auditable. +create or replace function public.ensure_bidirectional_all_shortest_path_telemetry_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.asb_telemetry_workspace_version') is not null then + select version into present_version + from pg_temp.asb_telemetry_workspace_version limit 1; + end if; + if to_regclass('pg_temp.asb_telemetry_workspace_version') is not null + and present_version is distinct from expected_version then + drop table if exists pg_temp.asb_telemetry_level; + drop table if exists pg_temp.asb_telemetry_call; + drop table if exists pg_temp.asb_telemetry_invocation; + drop table if exists pg_temp.asb_telemetry_workspace_version; + end if; + if to_regclass('pg_temp.asb_telemetry_workspace_version') is null then + create temporary table asb_telemetry_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + create temporary table asb_telemetry_invocation + ( + invocation_id text not null primary key, + schema_version int4 not null, + scheduler text, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8, + next_search_id int8 not null default 0, + check (btrim(invocation_id) <> '') + ) on commit preserve rows; + create temporary table asb_telemetry_call + ( + invocation_id text not null, + search_id int8 not null, + source_id int8 not null, + target_id int8 not null, + runtime_branch text not null default 'started', + scheduler_actions int8 not null default 0, + candidate_edges int8 not null default 0, + distinct_new_nodes int8 not null default 0, + seen_peak int8 not null default 0, + frontier_peak int8 not null default 0, + queue_peak int8 not null default 0, + predecessor_peak int8 not null default 0, + meeting_candidates int8 not null default 0, + frozen_distance int4, + witness_rows int8 not null default 0, + same_depth_predecessor_additions int8 not null default 0, + meeting_nodes int8 not null default 0, + cut_depth int4, + path_count_estimate int8 not null default 0, + path_count_saturated bool not null default false, + enumerated_candidates int8 not null default 0, + duplicate_rejects int8 not null default 0, + output_paths int8 not null default 0, + output_edge_cells int8 not null default 0, + output_bytes int8 not null default 0, + overflowed bool not null default false, + fallback_executed bool not null default false, + primary key (invocation_id, search_id) + ) on commit preserve rows; + create temporary table asb_telemetry_level + ( + invocation_id text not null, + search_id int8 not null, + action_index int8 not null, + side text not null, + action text not null, + depth int4 not null, + frontier_rows int8 not null, + candidate_edges int8 not null, + distinct_new_nodes int8 not null, + seen_rows int8 not null, + queue_rows int8 not null, + predecessor_rows int8 not null, + meeting_candidates int8 not null, + primary key (invocation_id, search_id, action_index) + ) on commit preserve rows; + insert into asb_telemetry_workspace_version(version) values (expected_version); + end if; +end; +$$ + language plpgsql + volatile; + +create or replace function public.begin_bidirectional_all_shortest_path_diagnostic_v1(invocation_id text) + returns void as +$$ +begin + if invocation_id is null or btrim(invocation_id) = '' or length(invocation_id) > 256 then + raise exception using errcode = '22023', message = 'bidirectional all-shortest-path diagnostic invocation ID must contain 1 to 256 characters'; + end if; + perform public.ensure_bidirectional_all_shortest_path_telemetry_workspace(); + delete from pg_temp.asb_telemetry_level where asb_telemetry_level.invocation_id = begin_bidirectional_all_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.asb_telemetry_call where asb_telemetry_call.invocation_id = begin_bidirectional_all_shortest_path_diagnostic_v1.invocation_id; + delete from pg_temp.asb_telemetry_invocation where asb_telemetry_invocation.invocation_id = begin_bidirectional_all_shortest_path_diagnostic_v1.invocation_id; + insert into pg_temp.asb_telemetry_invocation(invocation_id, schema_version) + values (invocation_id, 1); + perform set_config('dawgs.asb_diagnostic_invocation_id', invocation_id, true); +end; +$$ + language plpgsql + volatile; + +create or replace function public.read_bidirectional_all_shortest_path_diagnostic_v1(target_invocation_id text) + returns jsonb as +$$ +declare + result jsonb; +begin + select jsonb_build_object( + 'schema_version', invocation.schema_version, + 'invocation_id', invocation.invocation_id, + 'scheduler', invocation.scheduler, + 'state_limit', invocation.state_limit, + 'frontier_limit', invocation.frontier_limit, + 'predecessor_limit', invocation.predecessor_limit, + 'enumeration_limit', invocation.enumeration_limit, + 'output_bytes_limit', invocation.output_bytes_limit, + 'search_calls', coalesce(call_totals.search_calls, 0), + 'runtime_branch', coalesce(call_totals.runtime_branch, 'missing'), + 'overflowed', coalesce(call_totals.overflowed, false), + 'fallback_executed', coalesce(call_totals.fallback_executed, false), + 'counters', jsonb_build_object( + 'scheduler_actions', coalesce(call_totals.scheduler_actions, 0), + 'candidate_edges', coalesce(call_totals.candidate_edges, 0), + 'distinct_new_nodes', coalesce(call_totals.distinct_new_nodes, 0), + 'seen_peak', coalesce(call_totals.seen_peak, 0), + 'frontier_peak', coalesce(call_totals.frontier_peak, 0), + 'queue_peak', coalesce(call_totals.queue_peak, 0), + 'predecessor_peak', coalesce(call_totals.predecessor_peak, 0), + 'meeting_candidates', coalesce(call_totals.meeting_candidates, 0), + 'frozen_distance', coalesce(call_totals.frozen_distance, -1), + 'witness_rows', coalesce(call_totals.witness_rows, 0), + 'same_depth_predecessor_additions', coalesce(call_totals.same_depth_predecessor_additions, 0), + 'meeting_nodes', coalesce(call_totals.meeting_nodes, 0), + 'cut_depth', coalesce(call_totals.cut_depth, -1), + 'path_count_estimate', coalesce(call_totals.path_count_estimate, 0), + 'path_count_saturated', coalesce(call_totals.path_count_saturated, false), + 'enumerated_candidates', coalesce(call_totals.enumerated_candidates, 0), + 'duplicate_rejects', coalesce(call_totals.duplicate_rejects, 0), + 'output_paths', coalesce(call_totals.output_paths, 0), + 'output_edge_cells', coalesce(call_totals.output_edge_cells, 0), + 'output_bytes', coalesce(call_totals.output_bytes, 0), + 'levels', coalesce(levels.rows, '[]'::jsonb) + ), + 'calls', coalesce(calls.rows, '[]'::jsonb) + ) into result + from pg_temp.asb_telemetry_invocation invocation + left join lateral ( + select count(*)::int8 as search_calls, + case when count(distinct call.runtime_branch) = 1 + then min(call.runtime_branch) else 'mixed' end as runtime_branch, + bool_or(call.overflowed) as overflowed, + bool_or(call.fallback_executed) as fallback_executed, + sum(call.scheduler_actions)::int8 as scheduler_actions, + sum(call.candidate_edges)::int8 as candidate_edges, + sum(call.distinct_new_nodes)::int8 as distinct_new_nodes, + max(call.seen_peak)::int8 as seen_peak, + max(call.frontier_peak)::int8 as frontier_peak, + max(call.queue_peak)::int8 as queue_peak, + max(call.predecessor_peak)::int8 as predecessor_peak, + sum(call.meeting_candidates)::int8 as meeting_candidates, + min(call.frozen_distance)::int4 as frozen_distance, + sum(call.witness_rows)::int8 as witness_rows, + sum(call.same_depth_predecessor_additions)::int8 as same_depth_predecessor_additions, + sum(call.meeting_nodes)::int8 as meeting_nodes, + min(call.cut_depth)::int4 as cut_depth, + sum(call.path_count_estimate)::int8 as path_count_estimate, + bool_or(call.path_count_saturated) as path_count_saturated, + sum(call.enumerated_candidates)::int8 as enumerated_candidates, + sum(call.duplicate_rejects)::int8 as duplicate_rejects, + sum(call.output_paths)::int8 as output_paths, + sum(call.output_edge_cells)::int8 as output_edge_cells, + sum(call.output_bytes)::int8 as output_bytes + from pg_temp.asb_telemetry_call call + where call.invocation_id = invocation.invocation_id + ) call_totals on true + left join lateral ( + select jsonb_agg(jsonb_build_object( + 'search_id', level.search_id, + 'action_index', level.action_index, + 'side', level.side, + 'action', level.action, + 'depth', level.depth, + 'frontier_rows', level.frontier_rows, + 'candidate_edges', level.candidate_edges, + 'distinct_new_nodes', level.distinct_new_nodes, + 'seen_rows', level.seen_rows, + 'queue_rows', level.queue_rows, + 'predecessor_rows', level.predecessor_rows, + 'meeting_candidates', level.meeting_candidates + ) order by level.search_id, level.action_index) as rows + from pg_temp.asb_telemetry_level level + where level.invocation_id = invocation.invocation_id + ) levels on true + left join lateral ( + select jsonb_agg(to_jsonb(call) - 'invocation_id' order by call.search_id) as rows + from pg_temp.asb_telemetry_call call + where call.invocation_id = invocation.invocation_id + ) calls on true + where invocation.invocation_id = target_invocation_id; + return result; +end; +$$ + language plpgsql + stable + strict; + +create or replace function public.clear_bidirectional_all_shortest_path_diagnostic_v1(target_invocation_id text) + returns void as +$$ +begin + if to_regclass('pg_temp.asb_telemetry_invocation') is not null then + delete from pg_temp.asb_telemetry_level where invocation_id = target_invocation_id; + delete from pg_temp.asb_telemetry_call where invocation_id = target_invocation_id; + delete from pg_temp.asb_telemetry_invocation where invocation_id = target_invocation_id; + end if; + if nullif(current_setting('dawgs.asb_diagnostic_invocation_id', true), '') = target_invocation_id then + perform set_config('dawgs.asb_diagnostic_invocation_id', '', true); + end if; +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._start_bidirectional_all_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_scheduler text, + target_state_limit int8, + target_frontier_limit int8, + target_predecessor_limit int8, + target_enumeration_limit int8, + target_output_bytes_limit int8, + target_source_id int8, + target_target_id int8) + returns int8 as +$$ +declare + target_search_id int8; +begin + if target_invocation_id is null then + return null; + end if; + if to_regclass('pg_temp.asb_telemetry_invocation') is null then + raise exception using errcode = '55000', message = 'bidirectional all-shortest-path diagnostic replay was not initialized on this session'; + end if; + update pg_temp.asb_telemetry_invocation invocation + set scheduler = coalesce(invocation.scheduler, target_scheduler), + state_limit = coalesce(invocation.state_limit, target_state_limit), + frontier_limit = coalesce(invocation.frontier_limit, target_frontier_limit), + predecessor_limit = coalesce(invocation.predecessor_limit, target_predecessor_limit), + enumeration_limit = coalesce(invocation.enumeration_limit, target_enumeration_limit), + output_bytes_limit = coalesce(invocation.output_bytes_limit, target_output_bytes_limit), + next_search_id = invocation.next_search_id + 1 + where invocation.invocation_id = target_invocation_id + and (invocation.scheduler is null or invocation.scheduler = target_scheduler) + and (invocation.state_limit is null or invocation.state_limit = target_state_limit) + and (invocation.frontier_limit is null or invocation.frontier_limit = target_frontier_limit) + and (invocation.predecessor_limit is null or invocation.predecessor_limit = target_predecessor_limit) + and (invocation.enumeration_limit is null or invocation.enumeration_limit = target_enumeration_limit) + and (invocation.output_bytes_limit is null or invocation.output_bytes_limit = target_output_bytes_limit) + returning invocation.next_search_id into target_search_id; + if target_search_id is null then + raise exception using errcode = '55000', message = 'bidirectional all-shortest-path diagnostic invocation is missing or mixes scheduler/cap identities'; + end if; + insert into pg_temp.asb_telemetry_call(invocation_id, search_id, source_id, target_id) + values (target_invocation_id, target_search_id, target_source_id, target_target_id); + return target_search_id; +end; +$$ + language plpgsql + volatile; + +create or replace function public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + target_invocation_id text, + target_search_id int8, + target_action_index int8, + target_side text, + target_action text, + target_depth int4, + target_frontier_rows int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_rows int8, + target_queue_rows int8, + target_predecessor_rows int8, + target_meeting_candidates int8) + returns void as +$$ +begin + insert into pg_temp.asb_telemetry_level( + invocation_id, search_id, action_index, side, action, depth, + frontier_rows, candidate_edges, distinct_new_nodes, seen_rows, + queue_rows, predecessor_rows, meeting_candidates) + values ( + target_invocation_id, target_search_id, target_action_index, target_side, + target_action, target_depth, target_frontier_rows, target_candidate_edges, + target_distinct_new_nodes, target_seen_rows, target_queue_rows, + target_predecessor_rows, target_meeting_candidates); +end; +$$ + language plpgsql + volatile + strict; + +create or replace function public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + target_invocation_id text, + target_search_id int8, + target_runtime_branch text, + target_scheduler_actions int8, + target_candidate_edges int8, + target_distinct_new_nodes int8, + target_seen_peak int8, + target_frontier_peak int8, + target_queue_peak int8, + target_predecessor_peak int8, + target_meeting_candidates int8, + target_frozen_distance int4, + target_witness_rows int8, + target_same_depth_predecessor_additions int8, + target_meeting_nodes int8, + target_cut_depth int4, + target_path_count_estimate int8, + target_path_count_saturated bool, + target_enumerated_candidates int8, + target_duplicate_rejects int8, + target_output_paths int8, + target_output_edge_cells int8, + target_output_bytes int8, + target_overflowed bool, + target_fallback_executed bool) + returns void as +$$ +begin + update pg_temp.asb_telemetry_call call + set runtime_branch = target_runtime_branch, + scheduler_actions = target_scheduler_actions, + candidate_edges = target_candidate_edges, + distinct_new_nodes = target_distinct_new_nodes, + seen_peak = target_seen_peak, + frontier_peak = target_frontier_peak, + queue_peak = target_queue_peak, + predecessor_peak = target_predecessor_peak, + meeting_candidates = target_meeting_candidates, + frozen_distance = target_frozen_distance, + witness_rows = target_witness_rows, + same_depth_predecessor_additions = target_same_depth_predecessor_additions, + meeting_nodes = target_meeting_nodes, + cut_depth = target_cut_depth, + path_count_estimate = target_path_count_estimate, + path_count_saturated = target_path_count_saturated, + enumerated_candidates = target_enumerated_candidates, + duplicate_rejects = target_duplicate_rejects, + output_paths = target_output_paths, + output_edge_cells = target_output_edge_cells, + output_bytes = target_output_bytes, + overflowed = target_overflowed, + fallback_executed = target_fallback_executed + where call.invocation_id = target_invocation_id and call.search_id = target_search_id; + if not found then + raise exception using errcode = '55000', message = 'bidirectional all-shortest-path diagnostic call is missing'; + end if; +end; +$$ + language plpgsql + volatile; + +-- all_shortest_paths_bidirectional_compact_v1 is restricted to one validated, +-- distinct endpoint pair, minimum depth one, directed traversal, and maximum +-- depth 64. Within that envelope a minimum path cannot repeat a node, so two +-- minimum-node-depth predecessor DAGs preserve relationship-simple Cypher +-- semantics. +-- +-- Queue-head depth is a lower bound on every not-yet-completed path from that +-- side. A minimum distance L is proven only when one side is exhausted or the +-- two queue-head depths sum to at least L. The kernel then completes one +-- canonical cut k=floor(L/2): all forward predecessor rows into depth k and +-- all backward successor rows into depth L-k must be complete. Every shortest +-- path crosses exactly one node at this cut and is therefore stitched once, +-- even when the two searches overlap at several depths. +-- +-- Discovery nodes/frontier, relationship-distinct predecessors, enumerated +-- arrays, and materialized array bytes have independent cap+1 admissions. +-- Path counts are evaluated over the completed DAG with saturating arithmetic +-- before enumeration. No candidate row is returned until every gate passes. +-- Overflow clears asb_* and invokes exact ASP-A1 in the same top-level +-- statement. REPEATABLE READ or SERIALIZABLE is mandatory because VOLATILE +-- PL/pgSQL statements at READ COMMITTED do not share one statement snapshot. +create or replace function public.all_shortest_paths_bidirectional_compact_v1( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8, + scheduler text) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +#variable_conflict use_column +declare + chosen_side char(1); + strict_side char(1) := 'f'; + forward_depth int4; + backward_depth int4; + forward_ready_depth int4; + backward_ready_depth int4; + forward_width int8; + backward_width int8; + forward_tail int8 := 0; + backward_tail int8 := 0; + seen_rows int8; + active_rows int8; + frontier_rows int8; + predecessor_rows int8; + candidate_node_rows int8; + candidate_predecessor_rows int8; + discovery_admission_limit int8; + predecessor_admission_limit int8; + candidate_meeting int8; + candidate_distance int4; + best_distance int4; + cut_depth int4; + count_depth int4; + meeting_nodes int8; + path_array_bytes int8; + path_count_limit int8; + path_count_sentinel int8; + path_count_estimate int8; + output_rows int8; + output_bytes int8; + emitted_count int8 := 0; + overflowed bool := false; + telemetry_invocation_id text := nullif(current_setting('dawgs.asb_diagnostic_invocation_id', true), ''); + telemetry_search_id int8; + telemetry_action_index int8 := 0; + telemetry_action_depth int4 := 0; + telemetry_action_candidate_edges int8 := 0; + telemetry_action_meetings int8 := 0; + telemetry_scheduler_actions int8 := 0; + telemetry_candidate_edges int8 := 0; + telemetry_distinct_new_nodes int8 := 0; + telemetry_seen_peak int8 := 0; + telemetry_frontier_peak int8 := 0; + telemetry_queue_peak int8 := 0; + telemetry_predecessor_peak int8 := 0; + telemetry_meeting_candidates int8 := 0; + telemetry_same_depth_predecessors int8 := 0; + telemetry_path_count_saturated bool := false; + telemetry_enumerated_candidates int8 := 0; + telemetry_duplicate_rejects int8 := 0; +begin + if source_id is null or target_id is null or max_depth < 1 then + return; + end if; + if scheduler <> 'strict_alternating_node' and scheduler <> 'smaller_current_level' then + raise exception using errcode = '22023', message = 'unknown compact bidirectional all-shortest-path scheduler'; + end if; + if min_depth <> 1 then + raise exception using errcode = '22023', message = 'compact bidirectional all-shortest paths requires min_depth = 1'; + end if; + if max_depth > 64 then + raise exception using errcode = '22023', message = 'compact bidirectional all-shortest paths requires max_depth <= 64'; + end if; + if state_limit <= 0 or frontier_limit <= 0 or predecessor_limit <= 0 + or enumeration_limit <= 0 or output_bytes_limit <= 0 + or enumeration_limit = 9223372036854775807 + or output_bytes_limit = 9223372036854775807 then + raise exception using errcode = '22023', message = 'compact bidirectional all-shortest paths requires positive bounded limits below int8 maximum'; + end if; + if current_setting('transaction_isolation') <> 'repeatable read' + and current_setting('transaction_isolation') <> 'serializable' then + raise exception using + errcode = '25001', + message = 'compact bidirectional all-shortest paths requires REPEATABLE READ or SERIALIZABLE transaction isolation'; + end if; + if source_id = target_id then + perform public.shortest_path_self_endpoint_error(source_id, target_id); + end if; + + telemetry_search_id = public._start_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, scheduler, state_limit, frontier_limit, + predecessor_limit, enumeration_limit, output_bytes_limit, + source_id, target_id); + -- This non-allocating clear prevents successful shallow preflights, no-path + -- returns, and exact fallback from inheriting an earlier invocation's state. + perform public.clear_bidirectional_all_shortest_path_workspace(); + + -- Exact depth-one preflight remains outside the candidate workspace. It + -- returns every relationship-distinct edge only when enumeration and bytes + -- gates admit the complete multiset. + path_array_bytes = pg_column_size(array_fill(0::int8, array[1])); + if path_array_bytes <= 126 then + path_array_bytes = path_array_bytes - 3; + end if; + path_count_limit = least(enumeration_limit, output_bytes_limit / path_array_bytes); + select count(*) into output_rows + from ( + select 1 + from edge e + where e.graph_id = target_graph_id + and ((not inbound and e.start_id = source_id and e.end_id = target_id) + or (inbound and e.end_id = source_id and e.start_id = target_id)) + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + limit path_count_limit + 1 + ) shallow; + if output_rows > 0 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_one_hop', 1, 0, output_rows, 0, 0, 0, 0, + output_rows); + end if; + if output_rows > path_count_limit then + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + 0, output_rows, 0, 0, 0, 0, 0, output_rows, 1, emitted_count, + 0, 1, 0, output_rows, true, output_rows, 0, + emitted_count, emitted_count, emitted_count * path_array_bytes, + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + if not inbound then + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.start_id = source_id and e.end_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + else + return query + select source_id, target_id, 1::int4, true, false, array[e.id]::int8[] + from edge e + where e.graph_id = target_graph_id + and e.end_id = source_id and e.start_id = target_id + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + order by e.id; + end if; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_one_hop', + 0, output_rows, 0, 0, 0, 0, 0, output_rows, 1, emitted_count, + 0, 1, 0, output_rows, false, output_rows, 0, + emitted_count, emitted_count, emitted_count * path_array_bytes, + false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_one_hop', false, 'ASP-A1-DAG'); + return; + end if; + + -- Exact depth-two preflight similarly stages only a cap+1 scalar count. The + -- full relationship pair multiset is emitted only after both output gates. + if max_depth >= 2 then + path_array_bytes = pg_column_size(array_fill(0::int8, array[2])); + if path_array_bytes <= 126 then + path_array_bytes = path_array_bytes - 3; + end if; + path_count_limit = least(enumeration_limit, output_bytes_limit / path_array_bytes); + if not inbound then + select count(*) into output_rows + from ( + select 1 + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + limit path_count_limit + 1 + ) shallow; + else + select count(*) into output_rows + from ( + select 1 + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + limit path_count_limit + 1 + ) shallow; + end if; + if output_rows > 0 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_two_hop', 2, 0, output_rows * 2, 0, 0, 0, 0, + output_rows); + end if; + if output_rows > path_count_limit then + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + 0, output_rows * 2, 0, 0, 0, 0, 0, output_rows, 2, emitted_count, + 0, 1, 1, output_rows, true, output_rows, 0, + emitted_count, emitted_count * 2, emitted_count * path_array_bytes, + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + if not inbound then + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.start_id = e1.end_id + where e1.graph_id = target_graph_id + and e1.start_id = source_id and e2.end_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + else + return query + select source_id, target_id, 2::int4, true, false, array[e1.id, e2.id]::int8[] + from edge e1 + join edge e2 on e2.graph_id = target_graph_id and e2.end_id = e1.start_id + where e1.graph_id = target_graph_id + and e1.end_id = source_id and e2.start_id = target_id and e1.id <> e2.id + and (cardinality(edge_kind_ids) = 0 or e1.kind_id = any(edge_kind_ids)) + and (cardinality(edge_kind_ids) = 0 or e2.kind_id = any(edge_kind_ids)) + order by e1.id, e2.id; + end if; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_two_hop', + 0, output_rows * 2, 0, 0, 0, 0, 0, output_rows, 2, emitted_count, + 0, 1, 1, output_rows, false, output_rows, 0, + emitted_count, emitted_count * 2, emitted_count * path_array_bytes, + false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_two_hop', false, 'ASP-A1-DAG'); + return; + end if; + end if; + if max_depth <= 2 then + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'preflight_no_path', max_depth, 0, 0, 0, 0, 0, 0, 0); + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'preflight_no_path', + 0, 0, 0, 0, 0, 0, 0, 0, null, 0, + 0, 0, null, 0, false, 0, 0, 0, 0, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('preflight_no_path', false, 'ASP-A1-DAG'); + return; + end if; + + -- The two roots are discovery/frontier state, but not predecessor state. + if state_limit < 2 or frontier_limit < 2 then + overflowed = true; + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + 'none', 'root_admission', 0, 2, 0, 0, 2, 2, 0, 0); + end if; + else + perform public.reset_bidirectional_all_shortest_path_workspace(); + insert into pg_temp.asb_front(side, node_id, depth, queue_order) + values ('f', source_id, 0, 0), ('b', target_id, 0, 0); + insert into pg_temp.asb_seen(side, node_id, depth) + values ('f', source_id, 0), ('b', target_id, 0); + telemetry_seen_peak = 2; + telemetry_frontier_peak = 2; + telemetry_queue_peak = 2; + end if; + + while not overflowed loop + select min(depth) into forward_depth from pg_temp.asb_front where side = 'f'; + select min(depth) into backward_depth from pg_temp.asb_front where side = 'b'; + select count(*) into forward_width from pg_temp.asb_front where side = 'f' and depth = forward_depth; + select count(*) into backward_width from pg_temp.asb_front where side = 'b' and depth = backward_depth; + select coalesce(forward_depth, max(depth), 0) into forward_ready_depth + from pg_temp.asb_seen where side = 'f'; + select coalesce(backward_depth, max(depth), 0) into backward_ready_depth + from pg_temp.asb_seen where side = 'b'; + + if best_distance is null and (forward_depth is null or backward_depth is null) then + exit; + end if; + + if best_distance is not null + and (forward_depth is null or backward_depth is null + or forward_depth + backward_depth >= best_distance) then + cut_depth = best_distance / 2; + if forward_ready_depth >= cut_depth + and backward_ready_depth >= best_distance - cut_depth then + exit; + elsif forward_ready_depth < cut_depth then + chosen_side = 'f'; + else + chosen_side = 'b'; + end if; + elsif scheduler = 'strict_alternating_node' then + chosen_side = strict_side; + if (chosen_side = 'f' and forward_depth is null) + or (chosen_side = 'b' and backward_depth is null) then + chosen_side = case chosen_side when 'f' then 'b' else 'f' end; + end if; + strict_side = case chosen_side when 'f' then 'b' else 'f' end; + else + if forward_depth is null then + chosen_side = 'b'; + elsif backward_depth is null then + chosen_side = 'f'; + else + -- Stable equality tie break: forward. + chosen_side = case when forward_width <= backward_width then 'f' else 'b' end; + end if; + end if; + + truncate table pg_temp.asb_active, pg_temp.asb_candidate_node, + pg_temp.asb_candidate_predecessor; + if scheduler = 'strict_alternating_node' + and not (best_distance is not null + and (forward_depth is null or backward_depth is null + or forward_depth + backward_depth >= best_distance)) then + insert into pg_temp.asb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_front + where side = chosen_side + order by queue_order + limit 1; + elsif scheduler = 'strict_alternating_node' then + -- Cut completion retains node granularity while allowing the incomplete + -- side to advance consecutively after minimum distance is proven. + insert into pg_temp.asb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_front + where side = chosen_side + order by queue_order + limit 1; + else + insert into pg_temp.asb_active(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_front + where side = chosen_side + and depth = case chosen_side when 'f' then forward_depth else backward_depth end + order by queue_order; + end if; + + delete from pg_temp.asb_front front + using pg_temp.asb_active active + where front.side = active.side and front.node_id = active.node_id; + + telemetry_scheduler_actions = telemetry_scheduler_actions + 1; + telemetry_action_candidate_edges = 0; + telemetry_action_meetings = 0; + select min(depth) into telemetry_action_depth from pg_temp.asb_active; + if telemetry_search_id is not null then + if (chosen_side = 'f' and not inbound) or (chosen_side = 'b' and inbound) then + select count(*) into telemetry_action_candidate_edges + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + else + select count(*) into telemetry_action_candidate_edges + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)); + end if; + end if; + + if not exists (select 1 from pg_temp.asb_active where depth < max_depth) then + if telemetry_search_id is not null then + select count(*) into seen_rows from pg_temp.asb_seen; + select count(*) into active_rows from pg_temp.asb_active; + select count(*) into frontier_rows from pg_temp.asb_front; + select count(*) into predecessor_rows from pg_temp.asb_predecessor; + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + continue; + end if; + + select count(*) into seen_rows from pg_temp.asb_seen; + select count(*) into active_rows from pg_temp.asb_active; + select count(*) into frontier_rows from pg_temp.asb_front; + select count(*) into predecessor_rows from pg_temp.asb_predecessor; + discovery_admission_limit = least(state_limit - seen_rows, + frontier_limit - active_rows - frontier_rows); + if discovery_admission_limit < 0 then + overflowed = true; + if telemetry_search_id is not null then + telemetry_action_index = telemetry_action_index + 1; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows); + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows, 0, 0, + seen_rows, frontier_rows, predecessor_rows, 0); + end if; + exit; + end if; + + -- First admit distinct unseen nodes with a discovery cap+1 sentinel. + if chosen_side = 'f' and not inbound then + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'f', candidate.node_id, candidate.depth + from ( + select distinct e.end_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'f' and seen.node_id = e.end_id) + order by e.end_id + limit discovery_admission_limit + 1 + ) candidate; + elsif chosen_side = 'f' and inbound then + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'f', candidate.node_id, candidate.depth + from ( + select distinct e.start_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'f' and seen.node_id = e.start_id) + order by e.start_id + limit discovery_admission_limit + 1 + ) candidate; + elsif chosen_side = 'b' and not inbound then + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'b', candidate.node_id, candidate.depth + from ( + select distinct e.start_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'b' and seen.node_id = e.start_id) + order by e.start_id + limit discovery_admission_limit + 1 + ) candidate; + else + insert into pg_temp.asb_candidate_node(side, node_id, depth) + select 'b', candidate.node_id, candidate.depth + from ( + select distinct e.end_id as node_id, active.depth + 1 as depth + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and not exists (select 1 from pg_temp.asb_seen seen where seen.side = 'b' and seen.node_id = e.end_id) + order by e.end_id + limit discovery_admission_limit + 1 + ) candidate; + end if; + + select count(*) into candidate_node_rows from pg_temp.asb_candidate_node; + if seen_rows + candidate_node_rows > state_limit + or active_rows + frontier_rows + candidate_node_rows > frontier_limit then + overflowed = true; + if telemetry_search_id is not null then + select count(*) into telemetry_action_meetings + from pg_temp.asb_candidate_node candidate + join pg_temp.asb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_distinct_new_nodes = telemetry_distinct_new_nodes + candidate_node_rows; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows + candidate_node_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows + candidate_node_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows + candidate_node_rows); + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows + candidate_node_rows, + telemetry_action_candidate_edges, candidate_node_rows, + seen_rows + candidate_node_rows, frontier_rows + candidate_node_rows, + predecessor_rows, telemetry_action_meetings); + end if; + exit; + end if; + + predecessor_admission_limit = predecessor_limit - predecessor_rows; + if predecessor_admission_limit < 0 then + overflowed = true; + exit; + end if; + + -- Then retain every relationship-distinct edge into a newly discovered or + -- already-seen node at the same minimum depth. This second admission is + -- independent of distinct-node discovery. + if chosen_side = 'f' and not inbound then + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'f' and seen.node_id = e.end_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'f' and admitted.node_id = e.end_id)) + order by e.end_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + elsif chosen_side = 'f' and inbound then + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'f', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'f' and seen.node_id = e.start_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'f' and admitted.node_id = e.start_id)) + order by e.start_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + elsif chosen_side = 'b' and not inbound then + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.start_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.end_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'b' and seen.node_id = e.start_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'b' and admitted.node_id = e.start_id)) + order by e.start_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + else + insert into pg_temp.asb_candidate_predecessor(side, node_id, depth, adjacent_id, edge_id) + select 'b', candidate.node_id, candidate.depth, candidate.adjacent_id, candidate.edge_id + from ( + select e.end_id as node_id, active.depth + 1 as depth, + active.node_id as adjacent_id, e.id as edge_id + from pg_temp.asb_active active + join edge e on e.graph_id = target_graph_id and e.start_id = active.node_id + left join pg_temp.asb_seen seen on seen.side = 'b' and seen.node_id = e.end_id + where active.depth < max_depth + and (cardinality(edge_kind_ids) = 0 or e.kind_id = any(edge_kind_ids)) + and (seen.node_id is null or seen.depth = active.depth + 1) + and (seen.node_id is not null or exists ( + select 1 from pg_temp.asb_candidate_node admitted + where admitted.side = 'b' and admitted.node_id = e.end_id)) + order by e.end_id, e.id, active.node_id + limit predecessor_admission_limit + 1 + ) candidate; + end if; + + select count(*) into candidate_predecessor_rows + from pg_temp.asb_candidate_predecessor; + if telemetry_search_id is not null then + select count(*) into telemetry_action_meetings + from pg_temp.asb_candidate_node candidate + join pg_temp.asb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth; + telemetry_candidate_edges = telemetry_candidate_edges + telemetry_action_candidate_edges; + telemetry_distinct_new_nodes = telemetry_distinct_new_nodes + candidate_node_rows; + telemetry_meeting_candidates = telemetry_meeting_candidates + telemetry_action_meetings; + telemetry_same_depth_predecessors = telemetry_same_depth_predecessors + + greatest(candidate_predecessor_rows - candidate_node_rows, 0); + telemetry_seen_peak = greatest(telemetry_seen_peak, seen_rows + candidate_node_rows); + telemetry_frontier_peak = greatest(telemetry_frontier_peak, active_rows + frontier_rows + candidate_node_rows); + telemetry_queue_peak = greatest(telemetry_queue_peak, frontier_rows + candidate_node_rows); + telemetry_predecessor_peak = greatest(telemetry_predecessor_peak, predecessor_rows + candidate_predecessor_rows); + telemetry_action_index = telemetry_action_index + 1; + perform public._record_bidirectional_all_shortest_path_diagnostic_level_v1( + telemetry_invocation_id, telemetry_search_id, telemetry_action_index, + chosen_side::text, + case scheduler when 'strict_alternating_node' then 'dequeue_node' else 'expand_level' end, + telemetry_action_depth, active_rows + frontier_rows + candidate_node_rows, + telemetry_action_candidate_edges, candidate_node_rows, + seen_rows + candidate_node_rows, frontier_rows + candidate_node_rows, + predecessor_rows + candidate_predecessor_rows, + telemetry_action_meetings); + end if; + if predecessor_rows + candidate_predecessor_rows > predecessor_limit then + overflowed = true; + exit; + end if; + + insert into pg_temp.asb_predecessor(side, node_id, depth, adjacent_id, edge_id) + select side, node_id, depth, adjacent_id, edge_id + from pg_temp.asb_candidate_predecessor + order by side, node_id, edge_id, adjacent_id + on conflict do nothing; + insert into pg_temp.asb_seen(side, node_id, depth) + select side, node_id, depth + from pg_temp.asb_candidate_node + order by side, node_id + on conflict do nothing; + + if chosen_side = 'f' then + insert into pg_temp.asb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + forward_tail + row_number() over (order by node_id) + from pg_temp.asb_candidate_node; + forward_tail = forward_tail + candidate_node_rows; + else + insert into pg_temp.asb_front(side, node_id, depth, queue_order) + select side, node_id, depth, + backward_tail + row_number() over (order by node_id) + from pg_temp.asb_candidate_node; + backward_tail = backward_tail + candidate_node_rows; + end if; + + candidate_meeting = null; + candidate_distance = null; + select candidate.node_id, candidate.depth + opposite.depth + into candidate_meeting, candidate_distance + from pg_temp.asb_candidate_node candidate + join pg_temp.asb_seen opposite + on opposite.node_id = candidate.node_id and opposite.side <> candidate.side + where candidate.depth + opposite.depth between min_depth and max_depth + order by candidate.depth + opposite.depth, candidate.node_id + limit 1; + if candidate_distance is not null + and (best_distance is null or candidate_distance < best_distance) then + best_distance = candidate_distance; + end if; + end loop; + + if overflowed then + perform public.clear_bidirectional_all_shortest_path_workspace(); + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, telemetry_same_depth_predecessors, + coalesce(meeting_nodes, 0), cut_depth, coalesce(path_count_estimate, 0), + telemetry_path_count_saturated, telemetry_enumerated_candidates, + telemetry_duplicate_rejects, emitted_count, + emitted_count * coalesce(best_distance, 0), coalesce(output_bytes, 0), + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + if best_distance is null then + perform public.clear_bidirectional_all_shortest_path_workspace(); + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'search_no_path', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + null, 0, telemetry_same_depth_predecessors, 0, null, 0, false, + 0, 0, 0, 0, 0, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('search_no_path', false, 'ASP-A1-DAG'); + return; + end if; + + cut_depth = best_distance / 2; + select count(*) into meeting_nodes + from pg_temp.asb_seen forward_seen + join pg_temp.asb_seen backward_seen + on backward_seen.node_id = forward_seen.node_id and backward_seen.side = 'b' + where forward_seen.side = 'f' and forward_seen.depth = cut_depth + and backward_seen.depth = best_distance - cut_depth; + if meeting_nodes = 0 then + overflowed = true; + end if; + + -- Saturating dynamic programming over each half-DAG bounds enumeration and + -- bytes before any edge array is materialized. + if not overflowed then + path_array_bytes = pg_column_size(array_fill(0::int8, array[best_distance])); + if path_array_bytes <= 126 then + path_array_bytes = path_array_bytes - 3; + end if; + path_count_limit = least(enumeration_limit, output_bytes_limit / path_array_bytes); + path_count_sentinel = path_count_limit + 1; + truncate table pg_temp.asb_path_count, pg_temp.asb_output; + insert into pg_temp.asb_path_count(side, node_id, depth, path_count) + values ('f', source_id, 0, 1), ('b', target_id, 0, 1); + + for count_depth in 1..cut_depth loop + insert into pg_temp.asb_path_count(side, node_id, depth, path_count) + select 'f', predecessor.node_id, count_depth, + least(path_count_sentinel::numeric, + sum(adjacent.path_count::numeric))::int8 + from pg_temp.asb_predecessor predecessor + join pg_temp.asb_path_count adjacent + on adjacent.side = 'f' and adjacent.node_id = predecessor.adjacent_id + and adjacent.depth = count_depth - 1 + where predecessor.side = 'f' and predecessor.depth = count_depth + group by predecessor.node_id; + end loop; + for count_depth in 1..(best_distance - cut_depth) loop + insert into pg_temp.asb_path_count(side, node_id, depth, path_count) + select 'b', predecessor.node_id, count_depth, + least(path_count_sentinel::numeric, + sum(adjacent.path_count::numeric))::int8 + from pg_temp.asb_predecessor predecessor + join pg_temp.asb_path_count adjacent + on adjacent.side = 'b' and adjacent.node_id = predecessor.adjacent_id + and adjacent.depth = count_depth - 1 + where predecessor.side = 'b' and predecessor.depth = count_depth + group by predecessor.node_id; + end loop; + + select least(path_count_sentinel::numeric, + coalesce(sum(least(path_count_sentinel::numeric, + forward_count.path_count::numeric + * backward_count.path_count::numeric)), 0))::int8 + into path_count_estimate + from pg_temp.asb_path_count forward_count + join pg_temp.asb_path_count backward_count + on backward_count.side = 'b' and backward_count.node_id = forward_count.node_id + and backward_count.depth = best_distance - cut_depth + where forward_count.side = 'f' and forward_count.depth = cut_depth; + telemetry_path_count_saturated = path_count_estimate >= path_count_sentinel; + if path_count_estimate > path_count_limit or path_count_estimate = 0 then + overflowed = true; + end if; + end if; + + if not overflowed then + insert into pg_temp.asb_output(edge_ids, output_bytes) + with recursive + meeting(node_id) as materialized ( + select forward_seen.node_id + from pg_temp.asb_seen forward_seen + join pg_temp.asb_seen backward_seen + on backward_seen.node_id = forward_seen.node_id and backward_seen.side = 'b' + where forward_seen.side = 'f' and forward_seen.depth = cut_depth + and backward_seen.depth = best_distance - cut_depth + ), + forward_paths(meeting_id, node_id, path_depth, edge_ids) as ( + select meeting.node_id, meeting.node_id, cut_depth, array []::int8[] + from meeting + union all + select forward_paths.meeting_id, predecessor.adjacent_id, + forward_paths.path_depth - 1, + array[predecessor.edge_id]::int8[] || forward_paths.edge_ids + from forward_paths + join pg_temp.asb_predecessor predecessor + on predecessor.side = 'f' and predecessor.node_id = forward_paths.node_id + and predecessor.depth = forward_paths.path_depth + ), + backward_paths(meeting_id, node_id, path_depth, edge_ids) as ( + select meeting.node_id, meeting.node_id, best_distance - cut_depth, + array []::int8[] + from meeting + union all + select backward_paths.meeting_id, successor.adjacent_id, + backward_paths.path_depth - 1, + backward_paths.edge_ids || successor.edge_id + from backward_paths + join pg_temp.asb_predecessor successor + on successor.side = 'b' and successor.node_id = backward_paths.node_id + and successor.depth = backward_paths.path_depth + ), + stitched(edge_ids) as ( + select forward_paths.edge_ids || backward_paths.edge_ids + from forward_paths + join backward_paths using (meeting_id) + where forward_paths.node_id = source_id and forward_paths.path_depth = 0 + and backward_paths.node_id = target_id and backward_paths.path_depth = 0 + ) + select staged.edge_ids, pg_column_size(staged.edge_ids)::int8 + from ( + select distinct stitched.edge_ids + from stitched + where cardinality(stitched.edge_ids) = best_distance + and cardinality(stitched.edge_ids) = ( + select count(distinct path_edge.edge_id) + from unnest(stitched.edge_ids) path_edge(edge_id)) + order by stitched.edge_ids + limit enumeration_limit + 1 + ) staged; + + select count(*), coalesce(sum(asb_output.output_bytes), 0) + into output_rows, output_bytes + from pg_temp.asb_output; + telemetry_enumerated_candidates = output_rows; + telemetry_duplicate_rejects = greatest(coalesce(path_count_estimate, 0) - output_rows, 0); + if output_rows > enumeration_limit or output_bytes > output_bytes_limit + or output_rows <> path_count_estimate then + overflowed = true; + end if; + end if; + + if overflowed then + perform public.clear_bidirectional_all_shortest_path_workspace(); + return query + select fallback.root_id, fallback.next_id, fallback.depth, + fallback.satisfied, fallback.is_cycle, fallback.path + from public.all_shortest_paths_dag(target_graph_id, source_id, target_id, + min_depth, max_depth, edge_kind_ids, + inbound) fallback; + get diagnostics emitted_count = row_count; + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'exact_a1_fallback', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, telemetry_same_depth_predecessors, + coalesce(meeting_nodes, 0), cut_depth, coalesce(path_count_estimate, 0), + telemetry_path_count_saturated, telemetry_enumerated_candidates, + telemetry_duplicate_rejects, emitted_count, + emitted_count * coalesce(best_distance, 0), coalesce(output_bytes, 0), + true, true); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('exact_a1_fallback', true, 'ASP-A1-DAG'); + return; + end if; + + return query + select source_id, target_id, best_distance, true, false, output.edge_ids + from pg_temp.asb_output output + order by output.edge_ids; + get diagnostics emitted_count = row_count; + perform public.clear_bidirectional_all_shortest_path_workspace(); + if telemetry_search_id is not null then + perform public._finish_bidirectional_all_shortest_path_diagnostic_call_v1( + telemetry_invocation_id, telemetry_search_id, 'bidirectional_search', + telemetry_scheduler_actions, telemetry_candidate_edges, + telemetry_distinct_new_nodes, telemetry_seen_peak, + telemetry_frontier_peak, telemetry_queue_peak, + telemetry_predecessor_peak, telemetry_meeting_candidates, + best_distance, emitted_count, telemetry_same_depth_predecessors, + meeting_nodes, cut_depth, path_count_estimate, + telemetry_path_count_saturated, telemetry_enumerated_candidates, + telemetry_duplicate_rejects, emitted_count, + emitted_count * best_distance, output_bytes, false, false); + end if; + perform public.record_requested_traversal_runtime_attestation_v1('bidirectional_search', false, 'ASP-A1-DAG'); +end; +$$ + language plpgsql + volatile + strict + cost 100 + set recursive_worktable_factor = 1 + rows 100; + +create or replace function public.all_shortest_paths_b1_strict_alternating( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.all_shortest_paths_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + enumeration_limit, output_bytes_limit, 'strict_alternating_node'); +$$ + language sql + volatile + strict + cost 100 + rows 100; + +create or replace function public.all_shortest_paths_b2_smaller_current_level( + target_graph_id int4, + source_id int8, + target_id int8, + min_depth int4, + max_depth int4, + edge_kind_ids int2[], + inbound bool, + state_limit int8, + frontier_limit int8, + predecessor_limit int8, + enumeration_limit int8, + output_bytes_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.all_shortest_paths_bidirectional_compact_v1( + target_graph_id, source_id, target_id, min_depth, max_depth, + edge_kind_ids, inbound, state_limit, frontier_limit, predecessor_limit, + enumeration_limit, output_bytes_limit, 'smaller_current_level'); +$$ + language sql + volatile + strict + cost 100 + rows 100; + +create or replace function public.bsp_workspace_fragment(fragment text) + returns text as +$$ +select replace( + replace( + replace( + case + when position('pg_temp.bsp_' in fragment) > 0 then fragment + else replace( + replace( + replace( + replace( + replace( + replace( + replace(fragment, + 'on conflict on constraint forward_visited_pkey', 'on conflict on constraint bsp_forward_visited_pkey'), + 'on conflict on constraint backward_visited_pkey', 'on conflict on constraint bsp_backward_visited_pkey'), + 'forward_visited', 'pg_temp.bsp_forward_visited'), + 'backward_visited', 'pg_temp.bsp_backward_visited'), + 'forward_front', 'pg_temp.bsp_forward_front'), + 'backward_front', 'pg_temp.bsp_backward_front'), + 'next_front', 'pg_temp.bsp_next_front') + end, + 'traversal_root_filter', 'pg_temp.bsp_root_filter'), + 'traversal_terminal_filter', 'pg_temp.bsp_terminal_filter'), + 'traversal_pair_filter', 'pg_temp.bsp_pair_filter'); +$$ + language sql + immutable + parallel safe + strict; + +-- The bidirectional shortest-path workspace is session-local and survives +-- transaction boundaries. Warm calls retain the table and index OIDs and only +-- clear row state. The version marker lets upgrades rebuild the known object +-- set without touching unrelated temporary objects in the session. +create or replace function public.ensure_bsp_core_workspace() + returns void as +$$ +declare + expected_version constant int4 := 1; + present_version int4; +begin + if to_regclass('pg_temp.bsp_workspace_version') is not null then + select version into present_version from pg_temp.bsp_workspace_version limit 1; + end if; + + if present_version is not null and present_version is distinct from expected_version then + drop table if exists pg_temp.bsp_resolved_pairs; + drop table if exists pg_temp.bsp_unresolved_pairs; + drop table if exists pg_temp.bsp_pair_filter; + drop table if exists pg_temp.bsp_terminal_filter; + drop table if exists pg_temp.bsp_root_filter; + drop table if exists pg_temp.bsp_backward_visited; + drop table if exists pg_temp.bsp_forward_visited; + drop table if exists pg_temp.bsp_backward_front; + drop table if exists pg_temp.bsp_next_front; + drop table if exists pg_temp.bsp_forward_front; + drop table if exists pg_temp.bsp_workspace_version; + end if; + + if to_regclass('pg_temp.bsp_workspace_version') is null then + create temporary table bsp_workspace_version + ( + version int4 not null primary key + ) on commit preserve rows; + + create temporary table bsp_forward_front + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null + ) on commit preserve rows; + create index bsp_forward_front_next_id_index on bsp_forward_front using btree (next_id); + create index bsp_forward_front_root_id_next_id_index on bsp_forward_front using btree (root_id, next_id); + + create temporary table bsp_backward_front + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null + ) on commit preserve rows; + create index bsp_backward_front_next_id_index on bsp_backward_front using btree (next_id); + create index bsp_backward_front_root_id_next_id_index on bsp_backward_front using btree (root_id, next_id); + + create temporary table bsp_next_front + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null + ) on commit preserve rows; + create index bsp_next_front_next_id_index on bsp_next_front using btree (next_id); + create index bsp_next_front_root_id_next_id_index on bsp_next_front using btree (root_id, next_id); - create index if not exists traversal_pair_filter_terminal_id_root_id_index on traversal_pair_filter using btree (terminal_id, root_id); + create temporary table bsp_forward_visited + ( + root_id int8 not null, + id int8 not null, + constraint bsp_forward_visited_pkey primary key (root_id, id) + ) on commit preserve rows; - truncate table traversal_root_filter; - truncate table traversal_terminal_filter; - truncate table traversal_pair_filter; + create temporary table bsp_backward_visited + ( + root_id int8 not null, + id int8 not null, + constraint bsp_backward_visited_pkey primary key (root_id, id) + ) on commit preserve rows; - return; + insert into bsp_workspace_version(version) values (expected_version); + end if; end; $$ language plpgsql volatile; -create or replace function public.create_traversal_filter_tables(root_ids int8[], terminal_ids int8[]) +create or replace function public.ensure_bsp_generic_workspace() returns void as $$ begin - perform create_traversal_filter_tables(); + perform public.ensure_bsp_core_workspace(); - insert into traversal_root_filter - select distinct root_id - from unnest(root_ids) as root_ids(root_id) - where root_id is not null - on conflict (id) do nothing; + if to_regclass('pg_temp.bsp_root_filter') is null then + create temporary table bsp_root_filter + ( + id int8 not null primary key + ) on commit preserve rows; + create temporary table bsp_terminal_filter + ( + id int8 not null primary key + ) on commit preserve rows; + create temporary table bsp_pair_filter + ( + root_id int8 not null, + terminal_id int8 not null, + primary key (root_id, terminal_id) + ) on commit preserve rows; + create index bsp_pair_filter_terminal_id_root_id_index on bsp_pair_filter using btree (terminal_id, root_id); - insert into traversal_terminal_filter - select distinct terminal_id - from unnest(terminal_ids) as terminal_ids(terminal_id) - where terminal_id is not null - on conflict (id) do nothing; + create temporary table bsp_unresolved_pairs + ( + root_id int8 not null, + terminal_id int8 not null, + constraint bsp_unresolved_pairs_pkey primary key (root_id, terminal_id) + ) on commit preserve rows; + create index bsp_unresolved_pairs_terminal_id_root_id_index on bsp_unresolved_pairs using btree (terminal_id, root_id); - analyze traversal_root_filter; - analyze traversal_terminal_filter; + create temporary table bsp_resolved_pairs + ( + root_id int8 not null, next_id int8 not null, depth int4 not null, + satisfied bool, is_cycle bool not null, path int8[] not null, + constraint bsp_resolved_pairs_pkey primary key (root_id, next_id) + ) on commit preserve rows; + end if; +end; +$$ + language plpgsql + volatile; - return; +create or replace function public.reset_bsp_workspace(include_generic bool) + returns void as +$$ +begin + if include_generic then + perform public.ensure_bsp_generic_workspace(); + truncate table pg_temp.bsp_forward_front, pg_temp.bsp_backward_front, pg_temp.bsp_next_front, + pg_temp.bsp_forward_visited, pg_temp.bsp_backward_visited, + pg_temp.bsp_root_filter, pg_temp.bsp_terminal_filter, pg_temp.bsp_pair_filter, + pg_temp.bsp_unresolved_pairs, pg_temp.bsp_resolved_pairs; + else + perform public.ensure_bsp_core_workspace(); + truncate table pg_temp.bsp_forward_front, pg_temp.bsp_backward_front, pg_temp.bsp_next_front, + pg_temp.bsp_forward_visited, pg_temp.bsp_backward_visited; + end if; end; $$ language plpgsql volatile strict; -create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text, pair_filter text) +create or replace function public.load_bsp_filter_tables(root_filter text, terminal_filter text, pair_filter text) returns void as $$ begin - perform create_traversal_filter_tables(); - if length(pair_filter) > 0 then - execute pair_filter; + execute replace(pair_filter, 'traversal_pair_filter', 'pg_temp.bsp_pair_filter'); end if; - if length(root_filter) > 0 then - execute root_filter; + execute replace(root_filter, 'traversal_root_filter', 'pg_temp.bsp_root_filter'); elsif length(pair_filter) > 0 then - insert into traversal_root_filter - select distinct root_id - from traversal_pair_filter + insert into pg_temp.bsp_root_filter + select distinct root_id from pg_temp.bsp_pair_filter on conflict (id) do nothing; end if; - if length(terminal_filter) > 0 then - execute terminal_filter; + execute replace(terminal_filter, 'traversal_terminal_filter', 'pg_temp.bsp_terminal_filter'); elsif length(pair_filter) > 0 then - insert into traversal_terminal_filter - select distinct terminal_id - from traversal_pair_filter + insert into pg_temp.bsp_terminal_filter + select distinct terminal_id from pg_temp.bsp_pair_filter on conflict (id) do nothing; end if; - analyze traversal_root_filter; - analyze traversal_terminal_filter; - analyze traversal_pair_filter; - - return; -end; -$$ - language plpgsql - volatile - strict; - -create or replace function public.create_traversal_filter_tables(root_filter text, terminal_filter text) - returns void as -$$ -select public.create_traversal_filter_tables(root_filter, terminal_filter, ''::text); -$$ - language sql - volatile - strict; - -create or replace function public.shortest_path_self_endpoint_error(root_id int8, terminal_id int8) - returns bool as -$$ -begin - raise exception using - errcode = '22023', - message = format('shortest path endpoints must not resolve to the same node: root_id=%s terminal_id=%s', - root_id, - terminal_id); - - return false; + analyze pg_temp.bsp_root_filter; + analyze pg_temp.bsp_terminal_filter; + analyze pg_temp.bsp_pair_filter; end; $$ language plpgsql @@ -981,7 +5221,7 @@ $$ begin perform create_unidirectional_pathspace_tables(); - create temporary table backward_front + create temporary table if not exists backward_front ( root_id int8 not null, next_id int8 not null, @@ -989,11 +5229,13 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; - create index backward_front_next_id_index on backward_front using btree (next_id); - create index backward_front_satisfied_index on backward_front using btree (root_id, next_id, depth) where satisfied; - create index backward_front_is_cycle_index on backward_front using btree (root_id, next_id) where is_cycle; + create index if not exists backward_front_next_id_index on backward_front using btree (next_id); + create index if not exists backward_front_satisfied_index on backward_front using btree (root_id, next_id, depth) where satisfied; + create index if not exists backward_front_is_cycle_index on backward_front using btree (root_id, next_id) where is_cycle; + + truncate table backward_front; end; $$ language plpgsql @@ -1004,9 +5246,9 @@ create or replace function public.create_bidirectional_pair_pathspace_indexes() returns void as $$ begin - create index forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); - create index backward_front_root_id_next_id_index on backward_front using btree (root_id, next_id); - create index next_front_root_id_next_id_index on next_front using btree (root_id, next_id); + create index if not exists forward_front_root_id_next_id_index on forward_front using btree (root_id, next_id); + create index if not exists backward_front_root_id_next_id_index on backward_front using btree (root_id, next_id); + create index if not exists next_front_root_id_next_id_index on next_front using btree (root_id, next_id); end; $$ language plpgsql @@ -1017,19 +5259,21 @@ create or replace function public.create_bidirectional_shortest_path_tables() returns void as $$ begin - create temporary table forward_visited + create temporary table if not exists forward_visited ( root_id int8 not null, id int8 not null, primary key (root_id, id) - ) on commit drop; + ) on commit preserve rows; - create temporary table backward_visited + create temporary table if not exists backward_visited ( root_id int8 not null, id int8 not null, primary key (root_id, id) - ) on commit drop; + ) on commit preserve rows; + + truncate table forward_visited, backward_visited; perform create_bidirectional_pathspace_tables(); perform create_bidirectional_pair_pathspace_indexes(); @@ -1043,13 +5287,9 @@ create or replace function public.swap_forward_front() returns void as $$ begin - alter table forward_front - rename to forward_front_old; - alter table next_front - rename to forward_front; - alter table forward_front_old - rename to next_front; + truncate table forward_front; + insert into forward_front select * from next_front; truncate table next_front; delete from forward_front r where r.is_cycle; @@ -1067,13 +5307,9 @@ create or replace function public.swap_backward_front() returns void as $$ begin - alter table backward_front - rename to backward_front_old; - alter table next_front - rename to backward_front; - alter table backward_front_old - rename to next_front; + truncate table backward_front; + insert into backward_front select * from next_front; truncate table next_front; delete from backward_front r where r.is_cycle; @@ -1718,24 +5954,24 @@ begin perform create_bidirectional_pair_pathspace_indexes(); end if; - create temporary table unresolved_pairs + create temporary table if not exists unresolved_pairs ( root_id int8 not null, terminal_id int8 not null, primary key (root_id, terminal_id) - ) on commit drop; + ) on commit preserve rows; - create index unresolved_pairs_terminal_id_root_id_index on unresolved_pairs using btree (terminal_id, root_id); + create index if not exists unresolved_pairs_terminal_id_root_id_index on unresolved_pairs using btree (terminal_id, root_id); - create temporary table resolved_pair_depths + create temporary table if not exists resolved_pair_depths ( root_id int8 not null, terminal_id int8 not null, depth int4 not null, primary key (root_id, terminal_id) - ) on commit drop; + ) on commit preserve rows; - create temporary table resolved_paths + create temporary table if not exists resolved_paths ( root_id int8 not null, next_id int8 not null, @@ -1743,7 +5979,9 @@ begin satisfied bool, is_cycle bool not null, path int8[] not null - ) on commit drop; + ) on commit preserve rows; + + truncate table unresolved_pairs, resolved_pair_depths, resolved_paths; if use_pair_filter then insert into unresolved_pairs (root_id, terminal_id) @@ -2034,6 +6272,7 @@ $$ drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, int8[], int8[], bool); drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], bool); drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool); +drop function if exists public._bidirectional_sp_harness(text, text, text, text, int4, text, text, text, int8[], int8[], int8, bool, bool); -- _bidirectional_sp_harness implements the shortest-path bidirectional BFS in two control paths selected by -- `use_array_parameters`: @@ -2051,6 +6290,7 @@ create or replace function public._bidirectional_sp_harness(forward_primer text, root_filter text, terminal_filter text, pair_filter text, root_ids int8[], terminal_ids int8[], path_limit int8, + allow_zero_depth bool, use_array_parameters bool) returns table ( @@ -2074,42 +6314,56 @@ declare use_pair_filter bool := not use_array_parameters and length(pair_filter) > 0; matched_count int8 := 0; resolved_pairs_count int8 := 0; + unresolved_pairs_remaining bool := true; begin raise debug 'bidirectional_sp_harness start'; - perform create_bidirectional_shortest_path_tables(); + -- Validate the lean array mode before allocating its session workspace. + -- NULL endpoints represent an empty endpoint relation. Equal singleton IDs + -- retain the existing shortest-path error contract. if use_array_parameters then - perform create_traversal_filter_tables(root_ids, terminal_ids); - else - perform create_traversal_filter_tables(root_filter, terminal_filter, pair_filter); + if cardinality(root_ids) = 0 or cardinality(terminal_ids) = 0 or + root_ids[1] is null or terminal_ids[1] is null then + return; + end if; + if cardinality(root_ids) = 1 and cardinality(terminal_ids) = 1 and root_ids[1] = terminal_ids[1] then + if allow_zero_depth then + return query select root_ids[1], terminal_ids[1], 0::int4, true, false, array []::int8[]; + return; + else + perform public.shortest_path_self_endpoint_error(root_ids[1], terminal_ids[1]); + end if; + end if; end if; - create temporary table unresolved_pairs - ( - root_id int8 not null, - terminal_id int8 not null, - primary key (root_id, terminal_id) - ) on commit drop; - create index unresolved_pairs_terminal_id_root_id_index on unresolved_pairs using btree (terminal_id, root_id); + -- Array-parameter calls (including the proven singleton lowering) need only + -- the frontier/visited core. Text-filter calls lazily add pair/filter state. + perform public.reset_bsp_workspace(not use_array_parameters); - create temporary table resolved_pairs - ( - root_id int8 not null, - next_id int8 not null, - depth int4 not null, - satisfied bool, - is_cycle bool not null, - path int8[] not null, - primary key (root_id, next_id) - ) on commit drop; + if not use_array_parameters then + perform public.load_bsp_filter_tables(root_filter, terminal_filter, pair_filter); + end if; if use_pair_filter then - insert into unresolved_pairs (root_id, terminal_id) + insert into pg_temp.bsp_unresolved_pairs (root_id, terminal_id) select distinct root_id, terminal_id - from traversal_pair_filter - on conflict on constraint unresolved_pairs_pkey do nothing; + from pg_temp.bsp_pair_filter + on conflict on constraint bsp_unresolved_pairs_pkey do nothing; + + if allow_zero_depth then + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + select root_id, terminal_id, 0::int4, true, false, array []::int8[] + from pg_temp.bsp_unresolved_pairs + where root_id = terminal_id + on conflict on constraint bsp_resolved_pairs_pkey do nothing; + get diagnostics resolved_pairs_count = row_count; + + delete from pg_temp.bsp_unresolved_pairs where root_id = terminal_id; + end if; + + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; end if; -- Pair-filter mode keeps expanding until each requested pair is resolved or @@ -2117,29 +6371,29 @@ begin -- current BFS depth produces results. while forward_front_depth + backward_front_depth < max_depth and (path_limit <= 0 or resolved_pairs_count < path_limit) and - (not use_pair_filter or exists(select 1 from unresolved_pairs)) and + unresolved_pairs_remaining and (forward_front_depth = 0 or forward_front_count > 0) and (backward_front_depth = 0 or backward_front_count > 0) loop if forward_front_depth = 0 or (backward_front_depth > 0 and forward_front_count <= backward_front_count) then if forward_front_depth = 0 then if use_array_parameters then - execute forward_primer using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(forward_primer) using root_ids, terminal_ids; else - execute forward_primer; + execute public.bsp_workspace_fragment(forward_primer); end if; get diagnostics next_front_count = row_count; - insert into forward_visited (root_id, id) + insert into pg_temp.bsp_forward_visited (root_id, id) select distinct f.root_id, f.root_id - from next_front f - on conflict on constraint forward_visited_pkey do nothing; + from pg_temp.bsp_next_front f + on conflict on constraint bsp_forward_visited_pkey do nothing; else if use_array_parameters then - execute forward_recursive using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(forward_recursive) using root_ids, terminal_ids; else - execute forward_recursive; + execute public.bsp_workspace_fragment(forward_recursive); end if; get diagnostics next_front_count = row_count; @@ -2147,65 +6401,66 @@ begin forward_front_depth = forward_front_depth + 1; - delete from next_front f where f.is_cycle; + delete from pg_temp.bsp_next_front f where f.is_cycle; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f where f.satisfied is null; + delete from pg_temp.bsp_next_front f where f.satisfied is null; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f using forward_visited v where f.root_id = v.root_id and f.next_id = v.id; + delete from pg_temp.bsp_next_front f using pg_temp.bsp_forward_visited v where f.root_id = v.root_id and f.next_id = v.id; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; raise debug 'Forward shortest expansion as step % - Available Root Paths %', forward_front_depth + backward_front_depth, next_front_count; - truncate table forward_front; + truncate table pg_temp.bsp_forward_front; - insert into forward_front + insert into pg_temp.bsp_forward_front select distinct on (f.root_id, f.next_id) f.root_id, f.next_id, f.depth, f.satisfied, f.is_cycle, f.path - from next_front f + from pg_temp.bsp_next_front f order by f.root_id, f.next_id, f.depth; get diagnostics forward_front_count = row_count; - truncate table next_front; + truncate table pg_temp.bsp_next_front; - insert into forward_visited (root_id, id) + insert into pg_temp.bsp_forward_visited (root_id, id) select f.root_id, f.next_id - from forward_front f - on conflict on constraint forward_visited_pkey do nothing; + from pg_temp.bsp_forward_front f + on conflict on constraint bsp_forward_visited_pkey do nothing; - if exists(select 1 from forward_front r where r.satisfied) then + if exists(select 1 from pg_temp.bsp_forward_front r where r.satisfied) then if use_pair_filter then -- A direct forward hit resolves only the requested pairs it satisfies. -- Frontiers for completed roots/terminals are pruned below. - insert into resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) select distinct on (r.root_id, r.next_id) r.root_id, r.next_id, r.depth, r.satisfied, r.is_cycle, r.path - from forward_front r - join unresolved_pairs p on p.root_id = r.root_id and p.terminal_id = r.next_id + from pg_temp.bsp_forward_front r + join pg_temp.bsp_unresolved_pairs p on p.root_id = r.root_id and p.terminal_id = r.next_id where r.satisfied order by r.root_id, r.next_id, r.depth - on conflict on constraint resolved_pairs_pkey do nothing; + on conflict on constraint bsp_resolved_pairs_pkey do nothing; get diagnostics matched_count = row_count; resolved_pairs_count = resolved_pairs_count + matched_count; delete - from unresolved_pairs p - using resolved_pairs r + from pg_temp.bsp_unresolved_pairs p + using pg_temp.bsp_resolved_pairs r where p.root_id = r.root_id and p.terminal_id = r.next_id; + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; - delete from forward_front f where not exists(select 1 from unresolved_pairs p where p.root_id = f.root_id); + delete from pg_temp.bsp_forward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.root_id = f.root_id); get diagnostics deleted_count = row_count; forward_front_count = forward_front_count - deleted_count; - delete from backward_front b where not exists(select 1 from unresolved_pairs p where p.terminal_id = b.root_id); + delete from pg_temp.bsp_backward_front b where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.terminal_id = b.root_id); get diagnostics deleted_count = row_count; backward_front_count = backward_front_count - deleted_count; else @@ -2217,7 +6472,7 @@ begin r.satisfied, r.is_cycle, r.path - from forward_front r + from pg_temp.bsp_forward_front r where r.satisfied order by r.root_id, r.next_id, r.depth limit case when path_limit > 0 then path_limit else null end; @@ -2227,22 +6482,22 @@ begin else if backward_front_depth = 0 then if use_array_parameters then - execute backward_primer using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(backward_primer) using root_ids, terminal_ids; else - execute backward_primer; + execute public.bsp_workspace_fragment(backward_primer); end if; get diagnostics next_front_count = row_count; - insert into backward_visited (root_id, id) + insert into pg_temp.bsp_backward_visited (root_id, id) select distinct f.root_id, f.root_id - from next_front f - on conflict on constraint backward_visited_pkey do nothing; + from pg_temp.bsp_next_front f + on conflict on constraint bsp_backward_visited_pkey do nothing; else if use_array_parameters then - execute backward_recursive using root_ids, terminal_ids; + execute public.bsp_workspace_fragment(backward_recursive) using root_ids, terminal_ids; else - execute backward_recursive; + execute public.bsp_workspace_fragment(backward_recursive); end if; get diagnostics next_front_count = row_count; @@ -2250,65 +6505,66 @@ begin backward_front_depth = backward_front_depth + 1; - delete from next_front f where f.is_cycle; + delete from pg_temp.bsp_next_front f where f.is_cycle; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f where f.satisfied is null; + delete from pg_temp.bsp_next_front f where f.satisfied is null; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; - delete from next_front f using backward_visited v where f.root_id = v.root_id and f.next_id = v.id; + delete from pg_temp.bsp_next_front f using pg_temp.bsp_backward_visited v where f.root_id = v.root_id and f.next_id = v.id; get diagnostics deleted_count = row_count; next_front_count = next_front_count - deleted_count; raise debug 'Backward shortest expansion as step % - Available Terminal Paths %', forward_front_depth + backward_front_depth, next_front_count; - truncate table backward_front; + truncate table pg_temp.bsp_backward_front; - insert into backward_front + insert into pg_temp.bsp_backward_front select distinct on (f.root_id, f.next_id) f.root_id, f.next_id, f.depth, f.satisfied, f.is_cycle, f.path - from next_front f + from pg_temp.bsp_next_front f order by f.root_id, f.next_id, f.depth; get diagnostics backward_front_count = row_count; - truncate table next_front; + truncate table pg_temp.bsp_next_front; - insert into backward_visited (root_id, id) + insert into pg_temp.bsp_backward_visited (root_id, id) select f.root_id, f.next_id - from backward_front f - on conflict on constraint backward_visited_pkey do nothing; + from pg_temp.bsp_backward_front f + on conflict on constraint bsp_backward_visited_pkey do nothing; - if exists(select 1 from backward_front r where r.satisfied) then + if exists(select 1 from pg_temp.bsp_backward_front r where r.satisfied) then if use_pair_filter then -- Symmetric direct hit from the terminal side; swap root/terminal -- columns back into the function's result shape. - insert into resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) select distinct on (r.next_id, r.root_id) r.next_id, r.root_id, r.depth, r.satisfied, r.is_cycle, r.path - from backward_front r - join unresolved_pairs p on p.root_id = r.next_id and p.terminal_id = r.root_id + from pg_temp.bsp_backward_front r + join pg_temp.bsp_unresolved_pairs p on p.root_id = r.next_id and p.terminal_id = r.root_id where r.satisfied order by r.next_id, r.root_id, r.depth - on conflict on constraint resolved_pairs_pkey do nothing; + on conflict on constraint bsp_resolved_pairs_pkey do nothing; get diagnostics matched_count = row_count; resolved_pairs_count = resolved_pairs_count + matched_count; delete - from unresolved_pairs p - using resolved_pairs r + from pg_temp.bsp_unresolved_pairs p + using pg_temp.bsp_resolved_pairs r where p.root_id = r.root_id and p.terminal_id = r.next_id; + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; - delete from backward_front f where not exists(select 1 from unresolved_pairs p where p.terminal_id = f.root_id); + delete from pg_temp.bsp_backward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.terminal_id = f.root_id); get diagnostics deleted_count = row_count; backward_front_count = backward_front_count - deleted_count; - delete from forward_front f where not exists(select 1 from unresolved_pairs p where p.root_id = f.root_id); + delete from pg_temp.bsp_forward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.root_id = f.root_id); get diagnostics deleted_count = row_count; forward_front_count = forward_front_count - deleted_count; else @@ -2318,7 +6574,7 @@ begin r.satisfied, r.is_cycle, r.path - from backward_front r + from pg_temp.bsp_backward_front r where r.satisfied order by r.next_id, r.root_id, r.depth limit case when path_limit > 0 then path_limit else null end; @@ -2330,39 +6586,40 @@ begin if use_pair_filter then -- For unresolved pairs that meet in the middle, keep one shortest -- stitched path per pair and leave already-resolved pairs untouched. - insert into resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) + insert into pg_temp.bsp_resolved_pairs (root_id, next_id, depth, satisfied, is_cycle, path) select p.root_id, p.terminal_id, midpoint.depth, true, false, midpoint.path - from unresolved_pairs p + from pg_temp.bsp_unresolved_pairs p join lateral ( select f.depth + b.depth as depth, f.path || b.path as path - from forward_front f - join backward_front b on b.root_id = p.terminal_id and b.next_id = f.next_id + from pg_temp.bsp_forward_front f + join pg_temp.bsp_backward_front b on b.root_id = p.terminal_id and b.next_id = f.next_id where f.root_id = p.root_id order by f.depth + b.depth limit 1 ) midpoint on true - on conflict on constraint resolved_pairs_pkey do nothing; + on conflict on constraint bsp_resolved_pairs_pkey do nothing; get diagnostics matched_count = row_count; resolved_pairs_count = resolved_pairs_count + matched_count; if matched_count > 0 then delete - from unresolved_pairs p - using resolved_pairs r + from pg_temp.bsp_unresolved_pairs p + using pg_temp.bsp_resolved_pairs r where p.root_id = r.root_id and p.terminal_id = r.next_id; + select exists(select 1 from pg_temp.bsp_unresolved_pairs) into unresolved_pairs_remaining; - delete from forward_front f where not exists(select 1 from unresolved_pairs p where p.root_id = f.root_id); + delete from pg_temp.bsp_forward_front f where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.root_id = f.root_id); get diagnostics deleted_count = row_count; forward_front_count = forward_front_count - deleted_count; - delete from backward_front b where not exists(select 1 from unresolved_pairs p where p.terminal_id = b.root_id); + delete from pg_temp.bsp_backward_front b where not exists(select 1 from pg_temp.bsp_unresolved_pairs p where p.terminal_id = b.root_id); get diagnostics deleted_count = row_count; backward_front_count = backward_front_count - deleted_count; end if; @@ -2373,8 +6630,8 @@ begin true, false, f.path || b.path - from forward_front f - join backward_front b on f.next_id = b.next_id + from pg_temp.bsp_forward_front f + join pg_temp.bsp_backward_front b on f.next_id = b.next_id order by f.root_id, b.root_id, f.depth + b.depth limit case when path_limit > 0 then path_limit else null end; get diagnostics matched_count = row_count; @@ -2390,12 +6647,12 @@ begin -- for unresolved pairs after the first frontier-level success. if path_limit > 0 then return query select * - from resolved_pairs + from pg_temp.bsp_resolved_pairs order by root_id, next_id, depth limit path_limit; else return query select * - from resolved_pairs + from pg_temp.bsp_resolved_pairs order by root_id, next_id, depth; end if; end if; @@ -2422,7 +6679,51 @@ create or replace function public.bidirectional_sp_harness(forward_primer text, as $$ select * -from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, ''::text, ''::text, ''::text, root_ids, terminal_ids, path_limit, true); +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, ''::text, ''::text, ''::text, root_ids, terminal_ids, path_limit, false, true); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_ids int8[], terminal_ids int8[], + allow_zero_depth bool, path_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, ''::text, ''::text, ''::text, root_ids, terminal_ids, path_limit, allow_zero_depth, true); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_ids int8[], terminal_ids int8[], + allow_zero_depth bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_ids, terminal_ids, allow_zero_depth, 0::int8); $$ language sql volatile strict; @@ -2464,7 +6765,51 @@ create or replace function public.bidirectional_sp_harness(forward_primer text, as $$ select * -from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, ''::text, array []::int8[], array []::int8[], path_limit, false); +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, ''::text, array []::int8[], array []::int8[], path_limit, false, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, + allow_zero_depth bool, path_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, ''::text, array []::int8[], array []::int8[], path_limit, allow_zero_depth, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, + allow_zero_depth bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, allow_zero_depth, 0::int8); $$ language sql volatile strict; @@ -2507,7 +6852,51 @@ create or replace function public.bidirectional_sp_harness(forward_primer text, as $$ select * -from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, array []::int8[], array []::int8[], path_limit, false); +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, array []::int8[], array []::int8[], path_limit, false, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, pair_filter text, + allow_zero_depth bool, path_limit int8) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public._bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, array []::int8[], array []::int8[], path_limit, allow_zero_depth, false); +$$ + language sql volatile + strict; + +create or replace function public.bidirectional_sp_harness(forward_primer text, forward_recursive text, + backward_primer text, + backward_recursive text, max_depth int4, + root_filter text, terminal_filter text, pair_filter text, + allow_zero_depth bool) + returns table + ( + root_id int8, + next_id int8, + depth int4, + satisfied bool, + is_cycle bool, + path int8[] + ) +as +$$ +select * +from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward_primer, backward_recursive, max_depth, root_filter, terminal_filter, pair_filter, allow_zero_depth, 0::int8); $$ language sql volatile strict; @@ -2615,3 +7004,92 @@ from public.bidirectional_sp_harness(forward_primer, forward_recursive, backward $$ language sql volatile strict; + +-- graphbench_s1_distance_bfs is the typed, array-resident SP-S1 distance +-- prototype. It is additive and benchmark-only: production translation does +-- not call it. The caller must transparently restart a correct fallback when +-- overflow is true. +create or replace function public.graphbench_s1_distance_bfs(target_graph_id int4, start_id int8, terminal_id int8, + min_depth int4, max_depth int4, edge_kind_ids int2[], + inbound bool, state_limit int4) + returns table + ( + depth int4, + matched bool, + overflow bool, + examined_edges int8, + retained_nodes int4 + ) +as +$$ +#variable_conflict use_variable +declare + current_depth int4 := 0; + frontier int8[] := array[start_id]::int8[]; + next_frontier int8[]; + visited int8[] := array[start_id]::int8[]; + edge_count int8; +begin + depth := null; + matched := false; + overflow := false; + examined_edges := 0; + retained_nodes := 1; + + if state_limit < 1 then + overflow := true; + return next; + return; + end if; + + if start_id = terminal_id and min_depth = 0 then + depth := 0; + matched := true; + return next; + return; + end if; + + while current_depth < max_depth and cardinality(frontier) > 0 loop + select + coalesce(array_agg(distinct candidate.next_id order by candidate.next_id) + filter (where not candidate.next_id = any(visited)), array[]::int8[]), + count(*) + into next_frontier, edge_count + from ( + select case when inbound then edge.start_id else edge.end_id end as next_id + from unnest(frontier) as active(node_id) + join edge on edge.graph_id = target_graph_id + and ((not inbound and edge.start_id = active.node_id) + or (inbound and edge.end_id = active.node_id)) + where cardinality(edge_kind_ids) = 0 or edge.kind_id = any(edge_kind_ids) + ) candidate; + + examined_edges := examined_edges + edge_count; + current_depth := current_depth + 1; + + if terminal_id = any(next_frontier) and current_depth >= min_depth then + depth := current_depth; + matched := true; + retained_nodes := cardinality(visited) + cardinality(next_frontier); + return next; + return; + end if; + + if cardinality(visited) + cardinality(next_frontier) > state_limit then + overflow := true; + retained_nodes := cardinality(visited); + return next; + return; + end if; + + visited := visited || next_frontier; + frontier := next_frontier; + retained_nodes := cardinality(visited); + end loop; + + return next; +end; +$$ + language plpgsql + volatile + strict; diff --git a/drivers/pg/query/sql_workspace_test.go b/drivers/pg/query/sql_workspace_test.go new file mode 100644 index 00000000..23da35c0 --- /dev/null +++ b/drivers/pg/query/sql_workspace_test.go @@ -0,0 +1,539 @@ +package query + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestBidirectionalShortestPathWorkspaceIsReusable verifies shortest-path SQL creates reusable session-scoped workspace tables. +func TestBidirectionalShortestPathWorkspaceIsReusable(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public._bidirectional_sp_harness") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.bidirectional_sp_harness") + require.NotEqual(t, -1, end) + harness := sqlSchemaUp[start : start+end] + + require.Contains(t, sqlSchemaUp, "create or replace function public.ensure_bsp_core_workspace()") + require.Contains(t, sqlSchemaUp, "if present_version is not null and present_version is distinct from expected_version then") + require.Contains(t, sqlSchemaUp, "on commit preserve rows") + require.Contains(t, harness, "perform public.reset_bsp_workspace(not use_array_parameters)") + require.Contains(t, harness, "pg_temp.bsp_forward_front") + require.Contains(t, harness, "pg_temp.bsp_backward_front") + require.Contains(t, harness, "pg_temp.bsp_next_front") + require.NotContains(t, harness, "create temporary table") + require.NotContains(t, harness, "create index") + require.Contains(t, harness, "truncate table pg_temp.bsp_forward_front") + require.Contains(t, harness, "truncate table pg_temp.bsp_backward_front") + require.Contains(t, harness, "truncate table pg_temp.bsp_next_front") +} + +// TestBidirectionalShortestPathWarmWorkspaceUsesTruncate verifies repeated shortest-path execution clears existing workspace instead of recreating it. +func TestBidirectionalShortestPathWarmWorkspaceUsesTruncate(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.reset_bsp_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.load_bsp_filter_tables") + require.NotEqual(t, -1, end) + reset := sqlSchemaUp[start : start+end] + + require.Contains(t, reset, "truncate table pg_temp.bsp_forward_front") + require.Contains(t, reset, "pg_temp.bsp_resolved_pairs") + require.NotContains(t, reset, "delete from pg_temp.bsp_") + require.NotContains(t, sqlSchemaUp, "current_setting('transaction_read_only')") +} + +// TestBidirectionalShortestPathArrayModeSkipsGenericWorkspace verifies array-backed execution does not initialize table-backed workspace. +func TestBidirectionalShortestPathArrayModeSkipsGenericWorkspace(t *testing.T) { + require.Contains(t, sqlSchemaUp, "if not use_array_parameters then\nperform public.load_bsp_filter_tables") + require.Contains(t, sqlSchemaUp, "perform public.reset_bsp_workspace(not use_array_parameters)") +} + +// TestBidirectionalShortestPathFragmentsRewriteLegacyFilterTables verifies generated fragments target the current workspace filter tables. +func TestBidirectionalShortestPathFragmentsRewriteLegacyFilterTables(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.bsp_workspace_fragment") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.reset_bsp_workspace") + require.NotEqual(t, -1, end) + rewriter := sqlSchemaUp[start : start+end] + + require.Contains(t, rewriter, "'traversal_root_filter', 'pg_temp.bsp_root_filter'") + require.Contains(t, rewriter, "'traversal_terminal_filter', 'pg_temp.bsp_terminal_filter'") + require.Contains(t, rewriter, "'traversal_pair_filter', 'pg_temp.bsp_pair_filter'") +} + +// TestLinearPathMaterializerScopesPersistentLookups verifies persistent node and edge lookups include the selected graph ID. +func TestLinearPathMaterializerScopesPersistentLookups(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ordered_edge_ids_to_path") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.create_unidirectional_pathspace_tables") + require.NotEqual(t, -1, end) + materializer := sqlSchemaUp[start : start+end] + + require.Contains(t, materializer, "e.graph_id = target_graph_id") + require.Contains(t, materializer, "n.graph_id = target_graph_id") + require.Contains(t, materializer, "next_edge.ordinality = path_walk.idx + 1") + require.NotContains(t, materializer, "order by case when") +} + +// TestLegacyPathMaterializersRequireTargetGraph verifies legacy materializer signatures cannot bypass graph scoping. +func TestLegacyPathMaterializersRequireTargetGraph(t *testing.T) { + require.Contains(t, sqlSchemaUp, "drop function if exists public.nodes_to_path(int8[])") + require.Contains(t, sqlSchemaUp, "drop function if exists public.edges_to_path(int8[])") + require.Contains(t, sqlSchemaUp, "drop function if exists public.ordered_edges_to_path(nodeComposite, edgeComposite[], nodeComposite[])") + require.Contains(t, sqlSchemaUp, "nodes_to_path(target_graph_id int4") + require.Contains(t, sqlSchemaUp, "edges_to_path(target_graph_id int4") + require.Contains(t, sqlSchemaUp, "ordered_edges_to_path(target_graph_id int4") + require.Contains(t, sqlSchemaUp, "n.graph_id = target_graph_id") + require.Contains(t, sqlSchemaUp, "r.graph_id = target_graph_id") + require.Contains(t, sqlSchemaDown, "drop function if exists nodes_to_path(int4, int8[])") + require.Contains(t, sqlSchemaDown, "drop function if exists nodes_to_path(int8[])") + require.Contains(t, sqlSchemaDown, "drop function if exists edges_to_path(int4, int8[])") + require.Contains(t, sqlSchemaDown, "drop function if exists edges_to_path(int8[])") + require.NotContains(t, sqlSchemaDown, "drop function if exists nodes_to_path;") + require.NotContains(t, sqlSchemaDown, "drop function if exists edges_to_path;") +} + +// TestGraphBenchS1DistancePrototypeIsBoundedAndGraphScoped verifies the benchmark prototype constrains depth and graph identity. +func TestGraphBenchS1DistancePrototypeIsBoundedAndGraphScoped(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.graphbench_s1_distance_bfs") + require.NotEqual(t, -1, start) + prototype := sqlSchemaUp[start:] + + require.Contains(t, prototype, "edge.graph_id = target_graph_id") + require.Contains(t, prototype, "cardinality(visited) + cardinality(next_frontier) > state_limit") + require.Contains(t, prototype, "overflow := true") + require.NotContains(t, prototype, "create temporary table") + require.NotContains(t, prototype, "insert into") + require.Contains(t, sqlSchemaDown, "drop function if exists graphbench_s1_distance_bfs") +} + +// TestCompactShortestExecutorsUseReusableTypedWorkspace verifies compact executors use typed, reusable workspace structures. +func TestCompactShortestExecutorsUseReusableTypedWorkspace(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create or replace function public.ensure_shortest_dag_workspace()") + require.Contains(t, sqlSchemaUp, "create or replace function public.reset_shortest_dag_workspace()") + require.Contains(t, sqlSchemaUp, "on commit preserve rows") + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_dag(") + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_no_path_probe(") + require.Contains(t, sqlSchemaUp, "create or replace function public.shortest_path_compact(") + require.Contains(t, sqlSchemaUp, "rows 100") + require.Contains(t, sqlSchemaUp, "rows 1") + require.Contains(t, sqlSchemaDown, "drop function if exists all_shortest_paths_dag") + require.Contains(t, sqlSchemaDown, "drop function if exists all_shortest_paths_no_path_probe") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_compact") +} + +// TestAllShortestNoPathProbeFailsClosed verifies the candidate can return an +// empty set only after a reverse exhaustion proof and otherwise delegates to A1. +func TestAllShortestNoPathProbeFailsClosed(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_no_path_probe") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_compact") + require.NotEqual(t, -1, end) + probe := sqlSchemaUp[start : start+end] + + require.Contains(t, probe, "asp_n1_target_degree_zero") + require.Contains(t, probe, "asp_n1_reverse_exhausted") + require.Contains(t, probe, "asp_n1_source_reached_a1") + require.Contains(t, probe, "asp_n1_state_cap_a1") + require.Contains(t, probe, "return query select * from public.all_shortest_paths_dag") + require.Contains(t, probe, "limit greatest(state_limit - (select count(*) from pg_temp.spd_seen) + 1, 0)") +} + +// TestCompactShortestWorkspaceHotResetPreservesV1Fallback verifies the V2 +// connection marker only skips repeated workspace catalog checks after setup; +// direct and V1 callers retain self-contained initialization. +func TestCompactShortestWorkspaceHotResetPreservesV1Fallback(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.reset_shortest_dag_workspace()") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.ensure_all_shortest_paths_a1_diagnostic_workspace_v1()") + require.NotEqual(t, -1, end) + reset := sqlSchemaUp[start : start+end] + + require.Contains(t, reset, "current_setting('dawgs.shortest_dag_workspace_ready', true) is distinct from 'v2'") + require.Contains(t, reset, "perform public.ensure_shortest_dag_workspace()") + require.Contains(t, reset, "truncate table pg_temp.spd_seen, pg_temp.spd_candidate, pg_temp.spd_predecessor") +} + +// TestAllShortestDAGHasExactSmallDepthArmsAndLateEnumeration verifies shallow-depth specializations precede deferred path enumeration. +func TestAllShortestDAGHasExactSmallDepthArmsAndLateEnumeration(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_dag") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_compact") + require.NotEqual(t, -1, end) + executor := sqlSchemaUp[start : start+end] + + require.Contains(t, executor, "array[e.id]::int8[]") + require.Contains(t, executor, "array[e1.id, e2.id]::int8[]") + require.Contains(t, executor, "e1.id <> e2.id") + require.Contains(t, executor, "perform public.reset_shortest_dag_workspace()") + require.Contains(t, executor, "insert into pg_temp.spd_predecessor") + require.Contains(t, executor, "with recursive shortest_paths") + require.Contains(t, executor, "if exists (select 1 from pg_temp.spd_candidate where depth = search_depth and node_id = target_id) then") + require.NotContains(t, executor, "execute ") +} + +// TestAllShortestDAGA1DiagnosticIsSessionLocalAndOptIn verifies the A1 +// counter reader clears and reads only the existing predecessor-DAG workspace, +// while ordinary executor work retains no telemetry calls after its one GUC +// check. +func TestAllShortestDAGA1DiagnosticIsSessionLocalAndOptIn(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_all_shortest_paths_a1_diagnostic_workspace_v1") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_dag") + require.NotEqual(t, -1, end) + telemetry := sqlSchemaUp[start : start+end] + + for _, table := range []string{"asd_telemetry_invocation", "asd_telemetry_level"} { + require.Contains(t, telemetry, "create temporary table "+table) + } + require.Contains(t, telemetry, "perform public.reset_shortest_dag_workspace()") + require.Contains(t, telemetry, "set_config('dawgs.asd_diagnostic_invocation_id', target_invocation_id, true)") + require.Contains(t, telemetry, "'single_ended_level'") + require.NotContains(t, telemetry, "create table public.asd_telemetry") + + executorStart := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_dag") + executorEnd := strings.Index(sqlSchemaUp[executorStart:], "create or replace function public.shortest_path_compact") + require.NotEqual(t, executorStart, -1) + require.NotEqual(t, executorEnd, -1) + executor := sqlSchemaUp[executorStart : executorStart+executorEnd] + require.Contains(t, executor, "diagnostic_enabled bool := nullif(current_setting('dawgs.asd_diagnostic_invocation_id', true), '') is not null") + require.Contains(t, executor, "if diagnostic_enabled then") + require.Contains(t, executor, "_record_all_shortest_paths_a1_diagnostic_level_v1") + require.Contains(t, executor, "_finish_all_shortest_paths_a1_diagnostic_v1('single_ended_search'") + + for _, signature := range []string{ + "clear_all_shortest_paths_a1_diagnostic_v1(text)", + "read_all_shortest_paths_a1_diagnostic_v1(text)", + "_finish_all_shortest_paths_a1_diagnostic_v1(text, int4, int8)", + "_record_all_shortest_paths_a1_diagnostic_level_v1(int4, int8, int8, int8, int8)", + "_start_all_shortest_paths_a1_diagnostic_v1(int8, int8)", + "begin_all_shortest_paths_a1_diagnostic_v1(text)", + "ensure_all_shortest_paths_a1_diagnostic_workspace_v1()", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+signature) + } +} + +// TestCompactSingletonOverflowFallsBackBeforeReturning verifies compact overflow takes the safe fallback before emitting a result. +func TestCompactSingletonOverflowFallsBackBeforeReturning(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.shortest_path_compact") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.ensure_bidirectional_shortest_path_workspace") + require.NotEqual(t, -1, end) + executor := sqlSchemaUp[start : start+end] + + require.Contains(t, executor, "retained_state > state_limit") + require.Contains(t, executor, "if overflowed then") + require.Contains(t, executor, "with recursive trails") + require.Contains(t, executor, "not e.id = any(trails.edge_ids)") + require.NotContains(t, executor, "execute ") +} + +// TestCompactBidirectionalWorkspaceIsVersionedAndDisjoint verifies candidate +// state can coexist with the S4 fallback workspace on a pooled session. +func TestCompactBidirectionalWorkspaceIsVersionedAndDisjoint(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_shortest_path_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + workspace := sqlSchemaUp[start : start+end] + + require.Contains(t, workspace, "expected_version constant int4 := 1") + require.Contains(t, workspace, "pg_temp.spb_workspace_version") + require.Contains(t, workspace, "create temporary table spb_front") + require.Contains(t, workspace, "create temporary table spb_seen") + require.Contains(t, workspace, "create temporary table spb_active") + require.Contains(t, workspace, "create temporary table spb_candidate") + require.Contains(t, workspace, "create temporary table spb_predecessor") + require.Contains(t, workspace, "queue_order int8 not null") + require.Contains(t, workspace, "truncate table pg_temp.spb_front") + require.NotContains(t, workspace, "spd_front") + require.NotContains(t, workspace, "path int8[]") +} + +// TestTraversalRuntimeAttestationIsSessionLocalAndSymmetric verifies the +// timed-invocation receipt cannot persist data or survive schema teardown. +func TestTraversalRuntimeAttestationIsSessionLocalAndSymmetric(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create temporary table traversal_runtime_attestation_v1") + require.Contains(t, sqlSchemaUp, "on commit preserve rows") + require.Contains(t, sqlSchemaUp, "current_setting('dawgs.traversal_runtime_invocation_id', true)") + require.Contains(t, sqlSchemaUp, "record_count = receipt.record_count + 1") + require.Contains(t, sqlSchemaUp, "events = receipt.events || jsonb_build_array") + require.Contains(t, sqlSchemaUp, "'schema_version', 2") + require.Contains(t, sqlSchemaUp, "if not exists (\nselect 1\nfrom pg_attribute") + require.Contains(t, sqlSchemaUp, "create or replace function public.read_traversal_runtime_attestation_v1") + require.Contains(t, sqlSchemaUp, "create or replace function public.clear_traversal_runtime_attestation_v1") + for _, function := range []string{ + "clear_traversal_runtime_attestation_v1(text)", + "read_traversal_runtime_attestation_v1(text)", + "record_requested_traversal_runtime_attestation_v1(text, bool, text)", + "record_traversal_runtime_attestation_v1(text, text, bool)", + "begin_traversal_runtime_attestation_v1(text, text)", + "ensure_traversal_runtime_attestation_workspace_v1()", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+function) + } +} + +// TestCompactBidirectionalKernelHasExactPreflightBoundsAndFallback verifies all +// candidate gates run before output and overflow delegates to exact S4 state. +func TestCompactBidirectionalKernelHasExactPreflightBoundsAndFallback(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_b1_strict_alternating") + require.NotEqual(t, -1, end) + kernel := sqlSchemaUp[start : start+end] + + zeroHop := strings.Index(kernel, "if source_id = target_id then") + oneHop := strings.Index(kernel, "if min_depth <= 1 and max_depth >= 1 then") + twoHop := strings.Index(kernel, "if min_depth <= 2 and max_depth >= 2 then") + workspaceReset := strings.Index(kernel, "reset_bidirectional_shortest_path_workspace") + require.Greater(t, zeroHop, -1) + require.Greater(t, oneHop, zeroHop) + require.Greater(t, twoHop, oneHop) + require.Greater(t, workspaceReset, twoHop) + + require.Contains(t, kernel, "forward_depth + backward_depth >= best_distance") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'repeatable read'") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'serializable'") + require.Contains(t, kernel, "limit admission_limit + 1") + require.Contains(t, kernel, "seen_rows + candidate_rows > state_limit") + require.Contains(t, kernel, "active_rows + frontier_rows + candidate_rows > frontier_limit") + require.Contains(t, kernel, "predecessor_rows + candidate_rows > predecessor_limit") + require.Contains(t, kernel, "from public.shortest_path_compact(") + require.Contains(t, kernel, "with recursive\nforward_witness") + require.Less(t, strings.Index(kernel, "from public.shortest_path_compact("), strings.Index(kernel, "with recursive\nforward_witness")) + require.NotContains(t, kernel, "nodeComposite") + require.NotContains(t, kernel, "edgeComposite") +} + +// TestCompactBidirectionalWrappersFreezeSchedulersAndDownMigration verifies the +// two scheduler identities have typed wrappers and symmetric teardown. +func TestCompactBidirectionalWrappersFreezeSchedulersAndDownMigration(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create or replace function public.shortest_path_b1_strict_alternating(") + require.Contains(t, sqlSchemaUp, "'strict_alternating_node'") + require.Contains(t, sqlSchemaUp, "create or replace function public.shortest_path_b2_smaller_current_level(") + require.Contains(t, sqlSchemaUp, "'smaller_current_level'") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8)") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8)") + require.Contains(t, sqlSchemaDown, "drop function if exists shortest_path_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, text)") + require.Contains(t, sqlSchemaDown, "drop function if exists reset_bidirectional_shortest_path_workspace()") + require.Contains(t, sqlSchemaDown, "drop function if exists ensure_bidirectional_shortest_path_workspace()") +} + +// TestCompactBidirectionalDiagnosticTelemetryIsInvocationScoped verifies the +// untimed replay API records explicit internal counters in a distinct, +// session-local workspace and has symmetric teardown. +func TestCompactBidirectionalDiagnosticTelemetryIsInvocationScoped(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_shortest_path_telemetry_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + telemetry := sqlSchemaUp[start : start+end] + + require.Contains(t, telemetry, "expected_version constant int4 := 1") + require.Contains(t, telemetry, "create temporary table spb_telemetry_invocation") + require.Contains(t, telemetry, "create temporary table spb_telemetry_call") + require.Contains(t, telemetry, "create temporary table spb_telemetry_level") + require.Contains(t, telemetry, "on commit preserve rows") + require.Contains(t, telemetry, "set_config('dawgs.spb_diagnostic_invocation_id', invocation_id, true)") + require.Contains(t, telemetry, "where invocation.invocation_id = target_invocation_id") + require.Contains(t, telemetry, "'scheduler_actions'") + require.Contains(t, telemetry, "'candidate_edges'") + require.Contains(t, telemetry, "'seen_peak'") + require.Contains(t, telemetry, "'frontier_peak'") + require.Contains(t, telemetry, "'queue_peak'") + require.Contains(t, telemetry, "'predecessor_peak'") + require.Contains(t, telemetry, "'meeting_candidates'") + require.Contains(t, telemetry, "'fallback_executed'") + require.NotContains(t, telemetry, "create unlogged table") + require.NotContains(t, telemetry, "create table public.spb_telemetry") + + kernelStart := strings.Index(sqlSchemaUp, "create or replace function public.shortest_path_bidirectional_compact_v1") + require.NotEqual(t, -1, kernelStart) + wrapperStart := strings.Index(sqlSchemaUp[kernelStart:], "create or replace function public.shortest_path_b1_strict_alternating") + require.NotEqual(t, -1, wrapperStart) + kernel := sqlSchemaUp[kernelStart : kernelStart+wrapperStart] + require.Contains(t, kernel, "_start_bidirectional_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "_record_bidirectional_shortest_path_diagnostic_level_v1") + require.Contains(t, kernel, "_finish_bidirectional_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "if telemetry_search_id is not null then") + require.Contains(t, kernel, "select count(*) into telemetry_action_candidate_edges") + require.Contains(t, kernel, "'exact_s4_fallback'") + require.Contains(t, kernel, "'preflight_zero_hop'") + require.Contains(t, kernel, "'preflight_one_hop'") + require.Contains(t, kernel, "'preflight_two_hop'") + + for _, function := range []string{ + "_finish_bidirectional_shortest_path_diagnostic_call_v1", + "_record_bidirectional_shortest_path_diagnostic_level_v1", + "_start_bidirectional_shortest_path_diagnostic_call_v1", + "clear_bidirectional_shortest_path_diagnostic_v1", + "read_bidirectional_shortest_path_diagnostic_v1", + "begin_bidirectional_shortest_path_diagnostic_v1", + "ensure_bidirectional_shortest_path_telemetry_workspace", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+function) + } +} + +// TestBidirectionalAllShortestWorkspaceSeparatesDiscoveryPredecessorAndOutput +// verifies reusable candidate state is ID-only until the staged output boundary +// and remains disjoint from the exact ASP-A1 fallback workspace. +func TestBidirectionalAllShortestWorkspaceSeparatesDiscoveryPredecessorAndOutput(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_all_shortest_path_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + workspace := sqlSchemaUp[start : start+end] + + require.Contains(t, workspace, "expected_version constant int4 := 1") + for _, table := range []string{ + "asb_front", "asb_seen", "asb_active", "asb_candidate_node", + "asb_candidate_predecessor", "asb_predecessor", "asb_path_count", "asb_output", + } { + require.Contains(t, workspace, "temporary table "+table) + require.Contains(t, workspace, "pg_temp."+table) + } + require.Contains(t, workspace, "primary key (side, node_id, depth, adjacent_id, edge_id)") + require.Contains(t, workspace, "edge_ids int8[] not null primary key") + require.Contains(t, workspace, "on commit preserve rows") + require.NotContains(t, workspace, "spd_") + require.NotContains(t, workspace, "spb_") + // Discovery/frontier tables carry scalar IDs only; arrays are confined to + // asb_output after path-count admission. + discoveryEnd := strings.Index(workspace, "create temporary table asb_output") + require.Greater(t, discoveryEnd, -1) + require.NotContains(t, workspace[:discoveryEnd], "int8[]") +} + +// TestBidirectionalAllShortestKernelProvesOneCutAndGatesBeforeOutput verifies +// scheduler termination, complete equal-depth predecessor retention, and all +// independent cap+1/fallback boundaries are explicit in the SQL kernel. +func TestBidirectionalAllShortestKernelProvesOneCutAndGatesBeforeOutput(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_b1_strict_alternating") + require.NotEqual(t, -1, end) + kernel := sqlSchemaUp[start : start+end] + + require.Contains(t, kernel, "if min_depth <> 1 then") + require.Contains(t, kernel, "if max_depth > 64 then") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'repeatable read'") + require.Contains(t, kernel, "current_setting('transaction_isolation') <> 'serializable'") + require.Contains(t, kernel, "forward_depth + backward_depth >= best_distance") + require.Contains(t, kernel, "cut_depth = best_distance / 2") + require.Contains(t, kernel, "forward_ready_depth >= cut_depth") + require.Contains(t, kernel, "backward_ready_depth >= best_distance - cut_depth") + require.Contains(t, kernel, "scheduler = 'strict_alternating_node'") + require.Contains(t, kernel, "scheduler <> 'smaller_current_level'") + require.Contains(t, kernel, "seen.depth = active.depth + 1") + require.Contains(t, kernel, "limit discovery_admission_limit + 1") + require.Contains(t, kernel, "limit predecessor_admission_limit + 1") + require.Contains(t, kernel, "path_count_sentinel = path_count_limit + 1") + require.Contains(t, kernel, "least(path_count_sentinel::numeric") + require.Contains(t, kernel, "limit enumeration_limit + 1") + require.Contains(t, kernel, "output_bytes > output_bytes_limit") + require.Contains(t, kernel, "select distinct stitched.edge_ids") + require.Contains(t, kernel, "count(distinct path_edge.edge_id)") + require.Contains(t, kernel, "join backward_paths using (meeting_id)") + + firstFallback := strings.Index(kernel, "perform public.clear_bidirectional_all_shortest_path_workspace();") + firstPublicOutput := strings.LastIndex(kernel, "from pg_temp.asb_output output") + require.Greater(t, firstFallback, -1) + require.Greater(t, firstPublicOutput, firstFallback) + require.Contains(t, kernel, "from public.all_shortest_paths_dag(") + require.NotContains(t, kernel, "nodeComposite") + require.NotContains(t, kernel, "edgeComposite") +} + +// TestBidirectionalAllShortestWrappersAndDownMigrationAreSymmetric verifies +// both frozen scheduler identities and every new helper have exact teardown. +func TestBidirectionalAllShortestWrappersAndDownMigrationAreSymmetric(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_b1_strict_alternating(") + require.Contains(t, sqlSchemaUp, "create or replace function public.all_shortest_paths_b2_smaller_current_level(") + require.Contains(t, sqlSchemaUp, "enumeration_limit, output_bytes_limit, 'strict_alternating_node'") + require.Contains(t, sqlSchemaUp, "enumeration_limit, output_bytes_limit, 'smaller_current_level'") + for _, signature := range []string{ + "all_shortest_paths_b1_strict_alternating(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8)", + "all_shortest_paths_b2_smaller_current_level(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8)", + "all_shortest_paths_bidirectional_compact_v1(int4, int8, int8, int4, int4, int2[], bool, int8, int8, int8, int8, int8, text)", + "clear_bidirectional_all_shortest_path_workspace()", + "reset_bidirectional_all_shortest_path_workspace()", + "ensure_bidirectional_all_shortest_path_workspace()", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+signature) + } +} + +// TestBidirectionalAllShortestDiagnosticTelemetryIsInvocationScoped verifies +// the ASP replay API carries every required search, predecessor, cut, count, +// and output counter in session-local keyed state with symmetric teardown. +func TestBidirectionalAllShortestDiagnosticTelemetryIsInvocationScoped(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.ensure_bidirectional_all_shortest_path_telemetry_workspace") + require.NotEqual(t, -1, start) + end := strings.Index(sqlSchemaUp[start:], "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + require.NotEqual(t, -1, end) + telemetry := sqlSchemaUp[start : start+end] + + require.Contains(t, telemetry, "expected_version constant int4 := 1") + for _, table := range []string{"asb_telemetry_invocation", "asb_telemetry_call", "asb_telemetry_level"} { + require.Contains(t, telemetry, "create temporary table "+table) + } + require.Contains(t, telemetry, "on commit preserve rows") + require.Contains(t, telemetry, "set_config('dawgs.asb_diagnostic_invocation_id', invocation_id, true)") + require.Contains(t, telemetry, "where invocation.invocation_id = target_invocation_id") + for _, counter := range []string{ + "scheduler_actions", "candidate_edges", "distinct_new_nodes", "seen_peak", + "frontier_peak", "queue_peak", "predecessor_peak", "meeting_candidates", + "frozen_distance", "witness_rows", "same_depth_predecessor_additions", + "meeting_nodes", "cut_depth", "path_count_estimate", "path_count_saturated", + "enumerated_candidates", "duplicate_rejects", "output_paths", + "output_edge_cells", "output_bytes", + } { + require.Contains(t, telemetry, "'"+counter+"'") + } + require.NotContains(t, telemetry, "create table public.asb_telemetry") + + kernelStart := strings.Index(sqlSchemaUp, "create or replace function public.all_shortest_paths_bidirectional_compact_v1") + wrapperStart := strings.Index(sqlSchemaUp[kernelStart:], "create or replace function public.all_shortest_paths_b1_strict_alternating") + require.NotEqual(t, kernelStart, -1) + require.NotEqual(t, wrapperStart, -1) + kernel := sqlSchemaUp[kernelStart : kernelStart+wrapperStart] + require.Contains(t, kernel, "_start_bidirectional_all_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "_record_bidirectional_all_shortest_path_diagnostic_level_v1") + require.Contains(t, kernel, "_finish_bidirectional_all_shortest_path_diagnostic_call_v1") + require.Contains(t, kernel, "'exact_a1_fallback'") + require.Contains(t, kernel, "'preflight_one_hop'") + require.Contains(t, kernel, "'preflight_two_hop'") + require.Contains(t, kernel, "perform public.clear_bidirectional_all_shortest_path_workspace();") + + for _, function := range []string{ + "_finish_bidirectional_all_shortest_path_diagnostic_call_v1", + "_record_bidirectional_all_shortest_path_diagnostic_level_v1", + "_start_bidirectional_all_shortest_path_diagnostic_call_v1", + "clear_bidirectional_all_shortest_path_diagnostic_v1", + "read_bidirectional_all_shortest_path_diagnostic_v1", + "begin_bidirectional_all_shortest_path_diagnostic_v1", + "ensure_bidirectional_all_shortest_path_telemetry_workspace", + } { + require.Contains(t, sqlSchemaDown, "drop function if exists "+function) + } +} + +// TestLegacyASPFallbackReusesWorkspaceWithoutCatalogSwaps verifies legacy all-shortest fallback reuses workspace without replacing catalog objects. +func TestLegacyASPFallbackReusesWorkspaceWithoutCatalogSwaps(t *testing.T) { + start := strings.Index(sqlSchemaUp, "create or replace function public.create_unidirectional_pathspace_tables") + require.NotEqual(t, -1, start) + legacyWorkspace := sqlSchemaUp[start:] + + require.Contains(t, legacyWorkspace, "create temporary table if not exists forward_front") + require.Contains(t, legacyWorkspace, "create temporary table if not exists backward_front") + require.Contains(t, legacyWorkspace, "on commit preserve rows") + require.Contains(t, legacyWorkspace, "truncate table forward_front, next_front") + require.Contains(t, legacyWorkspace, "insert into forward_front select * from next_front") + require.Contains(t, legacyWorkspace, "insert into backward_front select * from next_front") + require.NotContains(t, legacyWorkspace, "alter table forward_front") + require.NotContains(t, legacyWorkspace, "alter table backward_front") +} diff --git a/drivers/pg/query/topology_epoch_sql_test.go b/drivers/pg/query/topology_epoch_sql_test.go new file mode 100644 index 00000000..1b280397 --- /dev/null +++ b/drivers/pg/query/topology_epoch_sql_test.go @@ -0,0 +1,40 @@ +package query + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSchemaDefinesTransactionallyVisibleTraversalEpochs(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create table if not exists graph_traversal_epoch") + require.Contains(t, sqlSchemaUp, "create_graph_traversal_epoch") + require.Contains(t, sqlSchemaUp, "bump_graph_traversal_epoch_new") + require.Contains(t, sqlSchemaUp, "bump_graph_traversal_epoch_old") + require.Contains(t, sqlSchemaUp, "bump_all_graph_traversal_epochs") + require.Contains(t, sqlSchemaUp, "referencing new table as new_rows") + require.Contains(t, sqlSchemaUp, "referencing old table as old_rows") + require.Contains(t, sqlSchemaDown, "drop table if exists graph_traversal_epoch") + + for _, table := range []string{"node", "edge"} { + require.True(t, strings.Contains(sqlSchemaUp, "bump_"+table+"_traversal_epoch_insert")) + require.True(t, strings.Contains(sqlSchemaUp, "bump_"+table+"_traversal_epoch_delete")) + require.True(t, strings.Contains(sqlSchemaUp, "bump_"+table+"_traversal_epoch_truncate")) + } +} + +func TestSchemaDefinesVersionedTraversalTopologySynopsis(t *testing.T) { + require.Contains(t, sqlSchemaUp, "create table if not exists graph_traversal_synopsis_generation") + require.Contains(t, sqlSchemaUp, "source_mutation_epoch") + require.Contains(t, sqlSchemaUp, "estimator_version") + require.Contains(t, sqlSchemaUp, "status in ('ready', 'building', 'failed')") + require.Contains(t, sqlSchemaUp, "graph_traversal_synopsis_node_count") + require.Contains(t, sqlSchemaUp, "graph_traversal_synopsis_edge_count") + require.Contains(t, sqlSchemaUp, "graph_traversal_synopsis_degree") + require.Contains(t, sqlSchemaUp, "schema_version") + require.Contains(t, sqlSchemaDown, "drop table if exists graph_traversal_synopsis_node_count") + require.Contains(t, sqlSchemaDown, "drop table if exists graph_traversal_synopsis_edge_count") + require.Contains(t, sqlSchemaDown, "drop table if exists graph_traversal_synopsis_degree") + require.Contains(t, sqlSchemaDown, "drop table if exists graph_traversal_synopsis_generation") +} diff --git a/drivers/pg/query_cache.go b/drivers/pg/query_cache.go new file mode 100644 index 00000000..631dfa3a --- /dev/null +++ b/drivers/pg/query_cache.go @@ -0,0 +1,272 @@ +package pg + +import ( + "container/list" + "strings" + "sync" + "sync/atomic" + + "github.com/cespare/xxhash/v2" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/cypher" +) + +const ( + // defaultCypherParseCacheEntries is the maximum number of parsed ASTs + // retained when no cache capacity is configured. + defaultCypherParseCacheEntries = 256 + + // targetCypherParseCacheEntriesPerShard keeps unrelated query shapes from + // contending on one LRU lock while retaining a bounded shared AST cache. + targetCypherParseCacheEntriesPerShard = 16 + + // MaxCachedCypherQueryBytes excludes oversized query strings from reusable + // parse and translation caches while still allowing them to execute. + MaxCachedCypherQueryBytes = 64 * 1024 +) + +// cypherParseCacheEntry pairs an immutable parsed AST with the normalized query text used as its LRU key. +type cypherParseCacheEntry struct { + // query is the normalized, cloned cache key. + query string + + // parsed is the immutable parser result shared by cache hits. + parsed *cypher.RegularQuery +} + +// cypherParseCall publishes one in-flight parse result to callers waiting on the same query. +type cypherParseCall struct { + // done closes after parsed and err have been published. + done chan struct{} + + // parsed is the AST produced by the coalesced parse. + parsed *cypher.RegularQuery + + // err is the parser failure, if any, shared with waiters. + err error +} + +// cypherParseCacheShard owns one bounded LRU and its single-flight parse +// calls. A query hashes to exactly one shard, so identical query text still +// shares one immutable AST while unrelated text can proceed independently. +type cypherParseCacheShard struct { + // lock serializes access to this shard's LRU, pending parses, and counters. + lock sync.Mutex + + // capacity is the maximum number of completed ASTs this shard may retain. + capacity int + + // entries maps normalized query text to its LRU entry. + entries map[string]*list.Element + + // lru orders completed entries from most to least recently used. + lru *list.List + + // pending coalesces callers parsing the same normalized query. + pending map[string]*cypherParseCall + + // stats holds this shard's counters while lock is held. + stats ParseCacheStats +} + +// cypherParseCache retains immutable parser output in independently locked +// shards. Translation is safe to run concurrently against a cached query +// because the optimizer copies the Cypher AST before applying rules or +// lowering it. +type cypherParseCache struct { + // capacity is the exact aggregate maximum number of completed parses. + capacity int + + // shards partition query text and each contain a local LRU. + shards []cypherParseCacheShard + + // closed prevents completed or future parses from being retained. It is + // atomic so hot cache hits do not need a driver-wide lifecycle lock. + closed atomic.Bool +} + +// ParseCacheStats contains aggregate, query-text-free diagnostics. It is a +// snapshot; counters are scoped to one driver instance and reset only when the +// driver is reconstructed. +type ParseCacheStats struct { + // Hits counts lookups served from completed cache entries. + Hits uint64 `json:"hits"` + + // Misses counts queries parsed by the caller that established a pending entry. + Misses uint64 `json:"misses"` + + // Bypasses counts queries parsed without retention because caching was unavailable or disallowed. + Bypasses uint64 `json:"bypasses"` + + // Evictions counts least-recently-used translations removed at capacity. + Evictions uint64 `json:"evictions"` + + // CoalescedMisses counts callers that waited for an existing parse of the same query. + CoalescedMisses uint64 `json:"coalesced_misses"` + + // Entries is the number of completed parses retained when the snapshot was taken. + Entries int `json:"entries"` + + // Pending is the number of in-flight parses when the snapshot was taken. + Pending int `json:"pending"` + + // Shards is the number of independently locked cache partitions. + Shards int `json:"shards"` +} + +// newCypherParseCache initializes a bounded shared AST cache with independently locked shards. +func newCypherParseCache(capacity int) *cypherParseCache { + shardCount := parseCacheShardCount(capacity) + cache := &cypherParseCache{ + capacity: capacity, + shards: make([]cypherParseCacheShard, shardCount), + } + for index := range cache.shards { + shardCapacity := capacity / shardCount + if index < capacity%shardCount { + shardCapacity++ + } + cache.shards[index] = cypherParseCacheShard{ + capacity: shardCapacity, + entries: make(map[string]*list.Element, shardCapacity), + lru: list.New(), + pending: map[string]*cypherParseCall{}, + } + } + return cache +} + +// parseCacheShardCount chooses a bounded number of locks for an aggregate cache capacity. +func parseCacheShardCount(capacity int) int { + if capacity <= 0 { + return 1 + } + count := capacity / targetCypherParseCacheEntriesPerShard + if count < 1 { + return 1 + } + if count > targetCypherParseCacheEntriesPerShard { + return targetCypherParseCacheEntriesPerShard + } + return count +} + +// shardForQuery returns the deterministic shard that owns query's cache entry. +func (s *cypherParseCache) shardForQuery(query string) *cypherParseCacheShard { + return &s.shards[xxhash.Sum64String(query)%uint64(len(s.shards))] +} + +// Parse returns an immutable Cypher AST and reports whether it came from a completed or coalesced cache hit. +func (s *cypherParseCache) Parse(input string) (*cypher.RegularQuery, bool, error) { + query := strings.TrimSpace(input) + // Bound the caller-owned input rather than only the trimmed view. A short + // query padded with a very large amount of whitespace must not retain that + // backing allocation through an LRU key. + if s == nil { + parsed, err := parseCypher(query) + return parsed, false, err + } + + shard := s.shardForQuery(query) + shard.lock.Lock() + if s.closed.Load() || s.capacity <= 0 || len(input) > MaxCachedCypherQueryBytes { + shard.stats.Bypasses++ + shard.lock.Unlock() + parsed, err := parseCypher(query) + return parsed, false, err + } + if element, found := shard.entries[query]; found { + shard.stats.Hits++ + shard.lru.MoveToFront(element) + parsed := element.Value.(cypherParseCacheEntry).parsed + shard.lock.Unlock() + return parsed, true, nil + } + if call, found := shard.pending[query]; found { + shard.stats.CoalescedMisses++ + shard.lock.Unlock() + <-call.done + if call.err != nil { + return nil, false, call.err + } + return call.parsed, true, nil + } + + // Lookups do not retain the caller's string. Clone only a true miss before + // using it as a pending/cache key so the zero-allocation hit path remains + // intact. + query = strings.Clone(query) + shard.stats.Misses++ + call := &cypherParseCall{done: make(chan struct{})} + shard.pending[query] = call + shard.lock.Unlock() + + parsed, err := parseCypher(query) + + shard.lock.Lock() + call.parsed = parsed + call.err = err + if err == nil && !s.closed.Load() { + element := shard.lru.PushFront(cypherParseCacheEntry{ + query: query, + parsed: parsed, + }) + shard.entries[query] = element + if shard.lru.Len() > shard.capacity { + evicted := shard.lru.Back() + shard.lru.Remove(evicted) + delete(shard.entries, evicted.Value.(cypherParseCacheEntry).query) + shard.stats.Evictions++ + } + } + delete(shard.pending, query) + close(call.done) + shard.lock.Unlock() + + if err != nil { + return nil, false, err + } + return parsed, false, nil +} + +// parseCypher parses query in a fresh frontend context for cache misses and bypasses. +func parseCypher(query string) (*cypher.RegularQuery, error) { + return frontend.ParseCypher(frontend.NewContext(), query) +} + +// Stats returns a consistent aggregate snapshot of all shard counters and occupancy. +func (s *cypherParseCache) Stats() ParseCacheStats { + if s == nil { + return ParseCacheStats{} + } + stats := ParseCacheStats{Shards: len(s.shards)} + for index := range s.shards { + shard := &s.shards[index] + shard.lock.Lock() + stats.Hits += shard.stats.Hits + stats.Misses += shard.stats.Misses + stats.Bypasses += shard.stats.Bypasses + stats.Evictions += shard.stats.Evictions + stats.CoalescedMisses += shard.stats.CoalescedMisses + stats.Entries += len(shard.entries) + stats.Pending += len(shard.pending) + shard.lock.Unlock() + } + return stats +} + +// Close prevents future retention and releases every cached query/AST +// reference. In-flight parses wake their waiters normally but do not repopulate +// the cache after closure. +func (s *cypherParseCache) Close() { + if s == nil || s.closed.Swap(true) { + return + } + for index := range s.shards { + shard := &s.shards[index] + shard.lock.Lock() + shard.entries = nil + shard.lru.Init() + shard.lock.Unlock() + } +} diff --git a/drivers/pg/query_cache_test.go b/drivers/pg/query_cache_test.go new file mode 100644 index 00000000..0d6fa799 --- /dev/null +++ b/drivers/pg/query_cache_test.go @@ -0,0 +1,195 @@ +package pg + +import ( + "strings" + "sync" + "testing" + + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +// TestCypherParseCacheReusesTrimmedQuery verifies whitespace-equivalent queries share one immutable AST entry. +func TestCypherParseCacheReusesTrimmedQuery(t *testing.T) { + cache := newCypherParseCache(2) + + first, hit, err := cache.Parse(" MATCH (n) RETURN n ") + require.NoError(t, err) + require.False(t, hit) + + second, hit, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + require.True(t, hit) + require.Same(t, first, second) +} + +// TestCypherParseCacheEvictsLeastRecentlyUsedQuery verifies capacity pressure removes the coldest completed parse. +func TestCypherParseCacheEvictsLeastRecentlyUsedQuery(t *testing.T) { + cache := newCypherParseCache(2) + + _, _, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + second, _, err := cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + _, hit, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + require.True(t, hit) + _, _, err = cache.Parse("MATCH (n) RETURN count(n)") + require.NoError(t, err) + + reparsed, hit, err := cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + require.False(t, hit) + require.NotSame(t, second, reparsed) +} + +// TestCypherParseCacheDoesNotRetainErrorsOrOversizedQueries verifies failed and over-limit parses always bypass retention. +func TestCypherParseCacheDoesNotRetainErrorsOrOversizedQueries(t *testing.T) { + cache := newCypherParseCache(2) + + parsed, hit, err := cache.Parse("MATCH (") + require.Error(t, err) + require.Nil(t, parsed) + require.False(t, hit) + parsed, hit, err = cache.Parse("MATCH (") + require.Error(t, err) + require.Nil(t, parsed) + require.False(t, hit) + require.Zero(t, cache.Stats().Entries) + + oversized := "MATCH (n) RETURN n // " + strings.Repeat("x", MaxCachedCypherQueryBytes) + _, hit, err = cache.Parse(oversized) + require.NoError(t, err) + require.False(t, hit) + require.Zero(t, cache.Stats().Entries) + + padded := strings.Repeat(" ", MaxCachedCypherQueryBytes) + "MATCH (n) RETURN n" + _, hit, err = cache.Parse(padded) + require.NoError(t, err) + require.False(t, hit) + require.Zero(t, cache.Stats().Entries) + require.Equal(t, uint64(2), cache.Stats().Bypasses) +} + +// TestCypherParseCacheCoalescesConcurrentMissesAndSupportsConcurrentOptimization verifies one parse can safely serve simultaneous callers. +func TestCypherParseCacheCoalescesConcurrentMissesAndSupportsConcurrentOptimization(t *testing.T) { + cache := newCypherParseCache(2) + const workers = 32 + + queries := make([]any, workers) + errors := make([]error, workers) + var waitGroup sync.WaitGroup + waitGroup.Add(workers) + for idx := 0; idx < workers; idx++ { + go func(index int) { + defer waitGroup.Done() + query, _, err := cache.Parse("MATCH (n) WHERE id(n) = $id RETURN n") + if err == nil { + _, err = optimize.Optimize(query) + } + errors[index] = err + queries[index] = query + }(idx) + } + waitGroup.Wait() + + for _, err := range errors { + require.NoError(t, err) + } + for idx := 1; idx < len(queries); idx++ { + require.Same(t, queries[0], queries[idx]) + } + require.Equal(t, 1, cache.Stats().Entries) + require.Equal(t, uint64(workers-1), cache.Stats().Hits+cache.Stats().CoalescedMisses) +} + +// TestCypherParseCacheSupportsConcurrentDifferentKeys verifies independent queries can populate the cache concurrently. +func TestCypherParseCacheSupportsConcurrentDifferentKeys(t *testing.T) { + cache := newCypherParseCache(64) + require.Equal(t, 4, cache.Stats().Shards) + const workers = 32 + var waitGroup sync.WaitGroup + errors := make([]error, workers) + waitGroup.Add(workers) + for idx := 0; idx < workers; idx++ { + go func(index int) { + defer waitGroup.Done() + _, _, errors[index] = cache.Parse("MATCH (n) RETURN n // key " + strings.Repeat("x", index)) + }(idx) + } + waitGroup.Wait() + for _, err := range errors { + require.NoError(t, err) + } + require.Equal(t, uint64(workers), cache.Stats().Misses) + require.Equal(t, workers, cache.Stats().Entries) +} + +// TestCypherParseCachePartitionsUnrelatedShapes verifies that the shared AST +// cache routes different query strings to independently locked shards while +// retaining one shared shard for an identical shape. +func TestCypherParseCachePartitionsUnrelatedShapes(t *testing.T) { + cache := newCypherParseCache(64) + first := cache.shardForQuery("MATCH (n) RETURN n") + require.Same(t, first, cache.shardForQuery("MATCH (n) RETURN n")) + + var second *cypherParseCacheShard + for index := 0; index < 128; index++ { + candidate := cache.shardForQuery("MATCH (n) RETURN n // shard " + strings.Repeat("x", index)) + if candidate != first { + second = candidate + break + } + } + require.NotNil(t, second) +} + +// TestCypherParseCacheStatsAndCloseReleaseEntries verifies snapshots reflect activity and Close releases retained ASTs. +func TestCypherParseCacheStatsAndCloseReleaseEntries(t *testing.T) { + cache := newCypherParseCache(1) + _, _, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + _, hit, err := cache.Parse("MATCH (n) RETURN n") + require.NoError(t, err) + require.True(t, hit) + _, _, err = cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + require.Equal(t, ParseCacheStats{ + Hits: 1, + Misses: 2, + Evictions: 1, + Entries: 1, + Shards: 1, + }, cache.Stats()) + + cache.Close() + require.Zero(t, cache.Stats().Entries) + require.Zero(t, cache.Stats().Entries) + _, hit, err = cache.Parse("MATCH (n) RETURN id(n)") + require.NoError(t, err) + require.False(t, hit) + require.Equal(t, uint64(1), cache.Stats().Bypasses) +} + +// BenchmarkCypherParseCache measures repeated lookup of a normalized cached query. +func BenchmarkCypherParseCache(b *testing.B) { + const query = "MATCH (n) WHERE id(n) = $id RETURN n" + b.Run("uncached", func(b *testing.B) { + for idx := 0; idx < b.N; idx++ { + cache := newCypherParseCache(0) + _, _, err := cache.Parse(query) + require.NoError(b, err) + } + }) + b.Run("cached", func(b *testing.B) { + cache := newCypherParseCache(1) + _, _, err := cache.Parse(query) + require.NoError(b, err) + b.ResetTimer() + for idx := 0; idx < b.N; idx++ { + _, hit, err := cache.Parse(query) + require.NoError(b, err) + require.True(b, hit) + } + }) +} diff --git a/drivers/pg/result.go b/drivers/pg/result.go index 1927dfa0..226bbd2c 100644 --- a/drivers/pg/result.go +++ b/drivers/pg/result.go @@ -11,11 +11,21 @@ import ( "github.com/specterops/dawgs/graph" ) +// queryResult adapts pgx rows to graph.Result, caching column names and decoding JSON values for each current row. type queryResult struct { - ctx context.Context - rows pgx.Rows - values []any - keys []string + // ctx supplies cancellation and request scope when decoded graph values require kind mapping. + ctx context.Context + + // rows is the pgx result set being adapted. + rows pgx.Rows + + // values contains the decoded values for the current row. + values []any + + // keys caches immutable column names shared by every row in the result set. + keys []string + + // kindMapper resolves database kind identifiers while scanning graph values. kindMapper KindMapper } @@ -27,18 +37,17 @@ func (s *queryResult) Keys() []string { return s.keys } +// Next advances to the next row, caching its column names and decoding JSON values before exposing it. func (s *queryResult) Next() bool { if s.rows.Next() { - s.keys = []string{} - for _, desc := range s.rows.FieldDescriptions() { - s.keys = append(s.keys, desc.Name) - } + fields := s.rows.FieldDescriptions() + s.cacheKeys(fields) // This error check exists just as a guard for a successful return of this function. The expectation is that // the pgx type will have error information attached to it which is reflected by the Error receiver function // of this type if values, err := s.rows.Values(); err == nil { - s.values = decodeJSONValues(values, s.rows.FieldDescriptions()) + s.values = decodeJSONValues(values, fields) return true } } @@ -46,6 +55,21 @@ func (s *queryResult) Next() bool { return false } +// cacheKeys records immutable column names once for the lifetime of the result set. +func (s *queryResult) cacheKeys(fields []pgconn.FieldDescription) { + if s.keys != nil { + return + } + + // A pgx Rows value represents one result set, whose field descriptions do + // not change between rows. Retain the names once instead of rebuilding the + // same slice for every row. + s.keys = make([]string, len(fields)) + for idx, field := range fields { + s.keys[idx] = field.Name + } +} + func (s *queryResult) Mapper() graph.ValueMapper { return NewValueMapper(s.ctx, s.kindMapper) } @@ -62,22 +86,26 @@ func (s *queryResult) Close() { s.rows.Close() } +// decodeJSONValues replaces raw JSON and JSONB fields in the caller-owned row slice with decoded Go values. func decodeJSONValues(values []any, fields []pgconn.FieldDescription) []any { - decodedValues := make([]any, len(values)) - copy(decodedValues, values) - + // pgx Rows.Values returns a decoded value slice for the current row. The old + // implementation made a shallow copy before replacing JSON scalars, but its + // nested values were still shared. Updating this otherwise-unexposed slice + // in place therefore preserves ownership while avoiding one allocation and + // copy per row. for idx, field := range fields { switch field.DataTypeOID { case pgtype.JSONOID, pgtype.JSONBOID: if decoded, ok := decodeJSONValue(values[idx]); ok { - decodedValues[idx] = decoded + values[idx] = decoded } } } - return decodedValues + return values } +// decodeJSONValue decodes byte JSON and structured string JSON while preserving already-decoded scalar strings. func decodeJSONValue(value any) (any, bool) { switch typedValue := value.(type) { case []byte: diff --git a/drivers/pg/result_test.go b/drivers/pg/result_test.go index a637976b..35bd498a 100644 --- a/drivers/pg/result_test.go +++ b/drivers/pg/result_test.go @@ -1,13 +1,23 @@ package pg import ( + "context" "testing" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgtype" + "github.com/pashagolub/pgxmock/v5" "github.com/stretchr/testify/require" ) +var ( + // benchmarkDecodedJSONValues retains decoded rows so benchmark work cannot be optimized away. + benchmarkDecodedJSONValues []any + + // benchmarkResultKeys retains cached column names so benchmark work cannot be optimized away. + benchmarkResultKeys []string +) + func TestDecodeJSONValue(t *testing.T) { t.Run("number", func(t *testing.T) { value, ok := decodeJSONValue([]byte("42")) @@ -46,6 +56,7 @@ func TestDecodeJSONValue(t *testing.T) { }) } +// TestDecodeJSONValuesPreservesDecodedStringScalars verifies JSON-typed strings already decoded by pgx are not reinterpreted as JSON tokens. func TestDecodeJSONValuesPreservesDecodedStringScalars(t *testing.T) { var ( values = []any{ @@ -60,7 +71,164 @@ func TestDecodeJSONValuesPreservesDecodedStringScalars(t *testing.T) { {DataTypeOID: pgtype.JSONBOID}, {DataTypeOID: pgtype.JSONBOID}, } + expected = append([]any(nil), values...) + ) + + decoded := decodeJSONValues(values, fields) + require.Equal(t, expected, decoded) + require.Same(t, &values[0], &decoded[0]) +} + +// TestDecodeJSONValuesReusesInputSlice verifies JSON replacement occurs in the pgx-owned row slice without an extra copy. +func TestDecodeJSONValuesReusesInputSlice(t *testing.T) { + var ( + values = []any{ + []byte(`{"name":"alpha"}`), + int64(42), + } + fields = []pgconn.FieldDescription{ + {DataTypeOID: pgtype.JSONBOID}, + {DataTypeOID: pgtype.Int8OID}, + } + ) + + decoded := decodeJSONValues(values, fields) + + require.Same(t, &values[0], &decoded[0]) + require.Equal(t, map[string]any{"name": "alpha"}, decoded[0]) + require.Equal(t, int64(42), decoded[1]) +} + +// TestDecodeJSONValuesDoesNotAllocateForDecodedFields verifies already-decoded fields follow the zero-allocation path. +func TestDecodeJSONValuesDoesNotAllocateForDecodedFields(t *testing.T) { + var ( + values = []any{ + map[string]any{"name": "alpha"}, + int64(42), + } + fields = []pgconn.FieldDescription{ + {DataTypeOID: pgtype.JSONBOID}, + {DataTypeOID: pgtype.Int8OID}, + } ) - require.Equal(t, values, decodeJSONValues(values, fields)) + require.Zero(t, testing.AllocsPerRun(100, func() { + decodeJSONValues(values, fields) + })) +} + +// TestQueryResultCachesKeysAcrossRows verifies column-name storage is reused while row values remain independently owned. +func TestQueryResultCachesKeysAcrossRows(t *testing.T) { + mock, err := pgxmock.NewConn() + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, mock.Close(context.Background())) + require.NoError(t, mock.ExpectationsWereMet()) + }) + + mock.ExpectQuery("select values").WillReturnRows( + pgxmock.NewRows([]string{"name", "count"}). + AddRow("alpha", int64(1)). + AddRow("beta", int64(2)), + ) + mock.ExpectClose() + + rows, err := mock.Query(context.Background(), "select values") + require.NoError(t, err) + + result := &queryResult{ + rows: rows, + } + require.True(t, result.Next()) + require.Equal(t, []string{"name", "count"}, result.Keys()) + firstKey := &result.Keys()[0] + firstValues := result.Values() + require.Equal(t, []any{"alpha", int64(1)}, firstValues) + + require.True(t, result.Next()) + require.Same(t, firstKey, &result.Keys()[0]) + require.Equal(t, []any{"beta", int64(2)}, result.Values()) + // Rows.Values owns each returned row slice. Advancing the cursor must not + // mutate values retained by a caller or mapper from the previous row. + require.Equal(t, []any{"alpha", int64(1)}, firstValues) + require.False(t, result.Next()) + require.NoError(t, result.Error()) +} + +// TestQueryResultCacheKeysDoesNotAllocateAfterInitialization verifies repeated key access performs no allocation. +func TestQueryResultCacheKeysDoesNotAllocateAfterInitialization(t *testing.T) { + var ( + result = &queryResult{} + fields = []pgconn.FieldDescription{ + {Name: "name"}, + {Name: "count"}, + } + ) + result.cacheKeys(fields) + + require.Zero(t, testing.AllocsPerRun(100, func() { + result.cacheKeys(fields) + })) +} + +// BenchmarkDecodeJSONValuesDecodedFields compares in-place decoding with the previous shallow-copy approach. +func BenchmarkDecodeJSONValuesDecodedFields(b *testing.B) { + var ( + values = []any{ + map[string]any{"name": "alpha"}, + int64(42), + } + fields = []pgconn.FieldDescription{ + {DataTypeOID: pgtype.JSONBOID}, + {DataTypeOID: pgtype.Int8OID}, + } + ) + + b.Run("in_place", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + benchmarkDecodedJSONValues = decodeJSONValues(values, fields) + } + }) + + b.Run("shallow_copy_reference", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + copiedValues := make([]any, len(values)) + copy(copiedValues, values) + benchmarkDecodedJSONValues = decodeJSONValues(copiedValues, fields) + } + }) +} + +// BenchmarkQueryResultCacheKeys compares cached column names with rebuilding them for every row. +func BenchmarkQueryResultCacheKeys(b *testing.B) { + fields := []pgconn.FieldDescription{ + {Name: "name"}, + {Name: "count"}, + } + + b.Run("cached", func(b *testing.B) { + result := &queryResult{} + result.cacheKeys(fields) + b.ReportAllocs() + b.ResetTimer() + + for b.Loop() { + result.cacheKeys(fields) + benchmarkResultKeys = result.keys + } + }) + + b.Run("rebuild_reference", func(b *testing.B) { + result := &queryResult{} + b.ReportAllocs() + for b.Loop() { + result.keys = make([]string, len(fields)) + for idx, field := range fields { + result.keys[idx] = field.Name + } + benchmarkResultKeys = result.keys + } + }) } diff --git a/drivers/pg/runtime_config.go b/drivers/pg/runtime_config.go new file mode 100644 index 00000000..54b0ccfd --- /dev/null +++ b/drivers/pg/runtime_config.go @@ -0,0 +1,88 @@ +package pg + +import "fmt" + +const ( + // defaultTranslationCacheEntries bounds retained translations on one physical connection. + defaultTranslationCacheEntries = 64 + + // defaultSharedShortestPathTemplateEntries bounds templates shared across pool connections. + defaultSharedShortestPathTemplateEntries = 128 + + // defaultMinConnections preserves the v1-compatible lower pool bound. + defaultMinConnections = 5 + + // defaultMaxConnections preserves the v1-compatible upper pool bound. + defaultMaxConnections = 50 +) + +// PoolConfig controls the pgx pool size owned by a PostgreSQL driver. It is optional +// in RuntimeConfig so existing callers retain the 5-50 connection +// defaults. Supplying PoolConfig permits a minimum of zero connections. +type PoolConfig struct { + // MinConnections keeps this many idle physical connections available when possible. + MinConnections int32 + + // MaxConnections caps the number of physical PostgreSQL connections. + MaxConnections int32 +} + +// RuntimeConfig configures the connection-resident translation cache. +type RuntimeConfig struct { + // TranslationCacheEntries is the exact SIEVE entry capacity for each live + // physical PostgreSQL connection. Zero disables retention. + TranslationCacheEntries int + + // SharedShortestPathTemplateEntries bounds immutable shortest-path SQL + // templates shared by V2 physical connections. Zero disables this L2 tier. + SharedShortestPathTemplateEntries int + + // Pool optionally overrides the v1-compatible connection limits. Nil uses + // DefaultConfig's limits; a non-nil value must have MaxConnections >= 1 + // and MinConnections <= MaxConnections. + Pool *PoolConfig +} + +// DefaultRuntimeConfig returns the conservative PostgreSQL defaults. The aggregate upper +// bound is TranslationCacheEntries multiplied by the number of live physical +// PostgreSQL connections. +func DefaultRuntimeConfig() RuntimeConfig { + return RuntimeConfig{ + TranslationCacheEntries: defaultTranslationCacheEntries, + SharedShortestPathTemplateEntries: defaultSharedShortestPathTemplateEntries, + Pool: &PoolConfig{ + MinConnections: defaultMinConnections, + MaxConnections: defaultMaxConnections, + }, + } +} + +// validate reports whether cache and pool limits can be safely applied to pgx. +func (s RuntimeConfig) validate() error { + if s.TranslationCacheEntries < 0 { + return fmt.Errorf("translation cache entries must not be negative: %d", s.TranslationCacheEntries) + } + if s.SharedShortestPathTemplateEntries < 0 { + return fmt.Errorf("shared shortest-path template entries must not be negative: %d", s.SharedShortestPathTemplateEntries) + } + if s.Pool != nil { + if s.Pool.MinConnections < 0 { + return fmt.Errorf("pool minimum connections must not be negative: %d", s.Pool.MinConnections) + } + if s.Pool.MaxConnections < 1 { + return fmt.Errorf("pool maximum connections must be at least 1: %d", s.Pool.MaxConnections) + } + if s.Pool.MinConnections > s.Pool.MaxConnections { + return fmt.Errorf("pool minimum connections %d exceeds maximum connections %d", s.Pool.MinConnections, s.Pool.MaxConnections) + } + } + return nil +} + +// resolvedPoolConfig returns either the explicit limits or the v1-compatible defaults. +func (s RuntimeConfig) resolvedPoolConfig() PoolConfig { + if s.Pool != nil { + return *s.Pool + } + return PoolConfig{MinConnections: defaultMinConnections, MaxConnections: defaultMaxConnections} +} diff --git a/drivers/pg/shared_template_cache.go b/drivers/pg/shared_template_cache.go new file mode 100644 index 00000000..75526ba3 --- /dev/null +++ b/drivers/pg/shared_template_cache.go @@ -0,0 +1,83 @@ +package pg + +import ( + "strings" + "sync" + + dawgscache "github.com/specterops/dawgs/cache" +) + +// sharedTemplateCache is a bounded V2-driver-wide L2 containing only +// immutable shortest-path SQL templates and source metadata. L1 remains the +// connection-local cache; the shared tier removes duplicate compilation when +// a pool expands or rotates connections. +type sharedTemplateCache struct { + // lock serializes shared template lookups, insertion, and statistics. + lock sync.Mutex + + // capacity bounds retained immutable shortest-path templates. + capacity int + + // sieve stores the shared immutable templates by translation key. + sieve dawgscache.Cache[translationKey, translationEntry] + + // stats records query-text-free shared-cache activity. + stats SharedTemplateStats +} + +// newSharedTemplateCache creates a bounded shared tier when capacity is positive. +func newSharedTemplateCache(capacity int) *sharedTemplateCache { + cache := &sharedTemplateCache{capacity: capacity, stats: SharedTemplateStats{Capacity: capacity}} + if capacity > 0 { + cache.sieve = dawgscache.NewSieve[translationKey, translationEntry](capacity) + } + return cache +} + +// get returns a retained template and records the lookup outcome. +func (s *sharedTemplateCache) get(key translationKey) (translationEntry, bool) { + if s == nil || s.capacity == 0 { + return translationEntry{}, false + } + s.lock.Lock() + defer s.lock.Unlock() + entry, found := s.sieve.Get(key) + if found { + s.stats.Hits++ + } else { + s.stats.Misses++ + } + return entry, found +} + +// put retains a new immutable template unless the bounded tier already has it. +func (s *sharedTemplateCache) put(key translationKey, entry translationEntry) { + if s == nil || s.capacity == 0 { + return + } + s.lock.Lock() + defer s.lock.Unlock() + if _, exists := s.sieve.Get(key); exists { + return + } + if s.sieve.Stats().Size() >= int64(s.capacity) { + s.stats.Evictions++ + } + key.query = strings.Clone(key.query) + s.sieve.Put(key, entry) + s.stats.Insertions++ +} + +// snapshot returns a consistent query-text-free view of shared-cache activity. +func (s *sharedTemplateCache) snapshot() SharedTemplateStats { + if s == nil { + return SharedTemplateStats{} + } + s.lock.Lock() + defer s.lock.Unlock() + stats := s.stats + if s.sieve != nil { + stats.Entries = int(s.sieve.Stats().Size()) + } + return stats +} diff --git a/drivers/pg/sql_generation_profile.go b/drivers/pg/sql_generation_profile.go new file mode 100644 index 00000000..9e86a12d --- /dev/null +++ b/drivers/pg/sql_generation_profile.go @@ -0,0 +1,43 @@ +package pg + +import "time" + +// SQLGenerationProfile is a query-text-free timing sample for the PostgreSQL +// Cypher-to-SQL execution boundary. Durations end once pgx has accepted the +// query and returned its row stream; server planning and execution are +// captured separately through PostgreSQL EXPLAIN diagnostics. +// +// The profile intentionally contains neither Cypher nor SQL text, parameter +// values, backend identifiers, nor result data. +type SQLGenerationProfile struct { + // QueryClass is a low-cardinality category that omits query text and values. + QueryClass string + + // Parse measures Cypher parsing and parse-cache lookup time. + Parse time.Duration + + // Graph measures graph-target resolution time. + Graph time.Duration + + // Policy measures traversal-shape classification and policy selection time. + Policy time.Duration + + // Cache measures translation-cache lookup and binding time. + Cache time.Duration + + // Translate measures Cypher-to-SQL translation time on cache misses. + Translate time.Duration + + // Format measures SQL rendering time after translation. + Format time.Duration + + // Dispatch measures client-side PostgreSQL query dispatch time. + Dispatch time.Duration +} + +// SQLGenerationProfileCollector receives completed query-text-free timing +// samples. Implementations must be safe for concurrent transactions. +type SQLGenerationProfileCollector interface { + // RecordSQLGenerationProfile receives a completed query-text-free timing sample. + RecordSQLGenerationProfile(profile SQLGenerationProfile) +} diff --git a/drivers/pg/strategy_selection.go b/drivers/pg/strategy_selection.go new file mode 100644 index 00000000..58ee2ae7 --- /dev/null +++ b/drivers/pg/strategy_selection.go @@ -0,0 +1,247 @@ +package pg + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + + "github.com/specterops/dawgs/cypher/models/cypher" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" +) + +// TraversalShapeVersion identifies the stable structural identity used by +// production-wide selection. It deliberately excludes query text, identifiers, +// caller values, graph contents, and transaction state. +const TraversalShapeVersion = "traversal-shape-v1" + +// TraversalFixedSuffixShapeVersion identifies the independently versioned +// structural identity for a variable expansion with a fixed terminal suffix. +// It intentionally does not reinterpret existing shortest-path v1 hashes. +const TraversalFixedSuffixShapeVersion = "fixed-suffix-shape-v1" + +// TraversalShape describes one statically classified shortest-path target. +// A zero value means the query is not in the initial structural selector scope. +type TraversalShape struct { + // Version identifies the structural-classification schema used for this shape. + Version string `json:"version"` + + // Family identifies the eligible traversal family. + Family string `json:"family"` + + // Direction records the traversal's logical direction. + Direction string `json:"direction"` + + // ObservationMode identifies the semantic observation mode used by the plan. + ObservationMode string `json:"observation_mode"` + + // MinimumDepth is the inclusive lower bound of the variable expansion. + MinimumDepth int64 `json:"minimum_depth"` + + // MaximumDepth is the inclusive upper bound of the variable expansion. + MaximumDepth int64 `json:"maximum_depth"` + + // RelationshipKindCount is the number of relationship kinds constrained by the query. + RelationshipKindCount int `json:"relationship_kind_count"` + + // UntypedRelationship reports whether the expansion permits any relationship kind. + UntypedRelationship bool `json:"untyped_relationship"` + + // SuffixLength is the fixed terminal suffix length for fixed-suffix expansions. + SuffixLength int `json:"suffix_length,omitempty"` + + // CandidateStrategy is the eligible optimizer strategy for this shape. + CandidateStrategy string `json:"candidate_strategy,omitempty"` + + // Fingerprint is the stable digest of the fields that define this shape. + Fingerprint string `json:"fingerprint"` +} + +// Available reports whether the classifier found exactly one initial-scope +// traversal target. Multiple targets remain on the incumbent until a later +// selector version defines their joint semantics. +func (s TraversalShape) Available() bool { + return s.Version != "" && s.Family != "" && s.Fingerprint != "" +} + +// TraversalStrategySelection is query-text-free selection telemetry. It +// observes policy routing without retaining a decision outside the query. +type TraversalStrategySelection struct { + // Shape is the classified traversal structure considered for selection. + Shape TraversalShape `json:"shape"` + + // PolicyGeneration identifies the active traversal-policy generation. + PolicyGeneration uint64 `json:"policy_generation"` + + // SelectorVersion identifies the selection protocol evaluated for the query. + SelectorVersion string `json:"selector_version"` + + // Candidate names the strategy that could replace the incumbent. + Candidate string `json:"candidate"` + + // SelectedArm identifies the candidate or incumbent path that was selected. + SelectedArm string `json:"selected_arm"` + + // Fallback identifies the incumbent strategy used when the candidate is not selected. + Fallback string `json:"fallback"` + + // Bucket identifies the manifest bucket that authorized the selection, when any. + Bucket string `json:"bucket,omitempty"` + + // TemplateSHA256 identifies the approved SQL template used by the selected bucket. + TemplateSHA256 string `json:"template_sha256,omitempty"` + + // Mode identifies the selection mechanism that produced the decision. + Mode string `json:"mode"` + + // Reason records the query-text-free rationale for the decision. + Reason string `json:"reason"` +} + +// TraversalStrategySelectionCollector is an optional diagnostic seam. It is +// intentionally separate from translation caching and cannot alter routing. +type TraversalStrategySelectionCollector interface { + // RecordTraversalStrategySelection receives query-text-free routing telemetry. + RecordTraversalStrategySelection(TraversalStrategySelection) +} + +// TraversalShapeCacheProvider optionally retains bounded, query-text-free +// structural classifications. It must invalidate every entry with schema +// generation changes and may always bypass retention safely. +type TraversalShapeCacheProvider interface { + // TraversalShapeFor returns a cached classification or calls classify to produce one. + TraversalShapeFor(query string, classify func() (TraversalShape, error)) (TraversalShape, error) +} + +// traversalShapeForQuery optimizes query and classifies its sole eligible traversal target. +func traversalShapeForQuery(query *cypher.RegularQuery) (TraversalShape, error) { + plan, err := optimize.Optimize(query) + if err != nil { + return TraversalShape{}, err + } + return traversalShapeForPlan(plan), nil +} + +// traversalShapeForPlan derives a cacheable structural shape from a lowered plan. +func traversalShapeForPlan(plan optimize.Plan) TraversalShape { + if len(plan.LoweringPlan.ShortestPathExecutor) != 1 { + if len(plan.LoweringPlan.ExpansionSearchStrategy) != 1 { + return TraversalShape{} + } + decision := plan.LoweringPlan.ExpansionSearchStrategy[0] + if decision.Family != "fixed_suffix_expansion" || !decision.StructurallyEligible || decision.SuffixLength != 3 { + return TraversalShape{} + } + shape := TraversalShape{ + Version: TraversalFixedSuffixShapeVersion, + Family: decision.Family, + Direction: decision.LogicalDirection, + ObservationMode: string(decision.ObservationMode), + MinimumDepth: decision.MinimumDepth, + MaximumDepth: decision.MaximumDepth, + SuffixLength: decision.SuffixLength, + CandidateStrategy: string(decision.CandidateStrategy), + } + shape.Fingerprint = TraversalShapeFingerprint(shape) + return shape + } + decision := plan.LoweringPlan.ShortestPathExecutor[0] + if !decision.StructurallyEligible { + return TraversalShape{} + } + shape := TraversalShape{ + Version: TraversalShapeVersion, + Family: decision.Family, + Direction: decision.Direction.String(), + ObservationMode: string(decision.ObservationMode), + MinimumDepth: decision.MinimumDepth, + MaximumDepth: decision.MaximumDepth, + RelationshipKindCount: decision.RelationshipKindCount, + UntypedRelationship: decision.UntypedRelationship, + } + shape.Fingerprint = TraversalShapeFingerprint(shape) + return shape +} + +// TraversalShapeFingerprint returns the immutable digest that a structural +// promotion bucket must bind. It never includes query text or runtime values. +func TraversalShapeFingerprint(shape TraversalShape) string { + if shape.Version == TraversalFixedSuffixShapeVersion { + canonical := fmt.Sprintf("%s|%s|%s|%s|%d|%d|%d|%s", shape.Version, shape.Family, shape.Direction, shape.ObservationMode, shape.MinimumDepth, shape.MaximumDepth, shape.SuffixLength, shape.CandidateStrategy) + digest := sha256.Sum256([]byte(canonical)) + return hex.EncodeToString(digest[:]) + } + canonical := fmt.Sprintf("%s|%s|%s|%s|%d|%d|%d|%t", shape.Version, shape.Family, shape.Direction, shape.ObservationMode, shape.MinimumDepth, shape.MaximumDepth, shape.RelationshipKindCount, shape.UntypedRelationship) + digest := sha256.Sum256([]byte(canonical)) + return hex.EncodeToString(digest[:]) +} + +// shouldClassifyTraversal reports whether policy routing or telemetry needs a traversal shape. +func (s *SchemaManager) shouldClassifyTraversal() bool { + if _, observed := s.translationCacheProvider.(TraversalStrategySelectionCollector); observed { + return true + } + return s.hasStructuralTraversalPolicy() +} + +// classifyTraversalShape classifies parsed and uses the provider's bounded cache when available. +func (s *SchemaManager) classifyTraversalShape(query string, parsed *cypher.RegularQuery) (TraversalShape, error) { + classify := func() (TraversalShape, error) { + return traversalShapeForQuery(parsed) + } + if cache, found := s.translationCacheProvider.(TraversalShapeCacheProvider); found { + return cache.TraversalShapeFor(query, classify) + } + return classify() +} + +// observeTraversalStrategySelection emits query-text-free routing telemetry to an optional collector. +func (s *SchemaManager) observeTraversalStrategySelection(query string, shape TraversalShape, policy TraversalPolicy) { + collector, ok := s.translationCacheProvider.(TraversalStrategySelectionCollector) + if !ok { + return + } + selection := TraversalStrategySelection{Shape: shape, SelectedArm: "incumbent", Fallback: "incumbent", Mode: "incumbent", Reason: "policy_inactive"} + if !shape.Available() { + selection.Reason = "shape_unavailable" + } else if policy.enabled() { + selection.PolicyGeneration = policy.Generation + selection.SelectorVersion = policy.compiledManifest.SelectorVersion + selection.Candidate = string(policy.ShortestPathExecutor) + selection.Fallback = policy.compiledManifest.FallbackExecutor + if policy.EnableTopologyFixedSuffix || policy.EnableTopologyFixedSuffixFirstUse { + selection.Candidate = string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) + if policy.EnableTopologyFixedSuffixFirstUse { + selection.Candidate = string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) + } + if bucket, authorized := policy.authorizedStructuralBucketForShape(shape); authorized { + selection.SelectedArm = "candidate" + selection.Bucket = bucket.Name + selection.TemplateSHA256 = bucket.SQLTemplateSHA256 + selection.Mode = "topology_selected" + selection.Reason = "topology_route_candidate_hit" + } + } + selectCandidate := func(bucket traversalPromotionBucket, mode, reason string) { + selection.SelectedArm = "candidate" + selection.Bucket = bucket.Name + selection.TemplateSHA256 = bucket.SQLTemplateSHA256 + selection.Mode = mode + selection.Reason = reason + } + if policy.EnableTopologyFixedSuffix || policy.EnableTopologyFixedSuffixFirstUse { + // The topology branch above is selected only by a transaction-local + // route-cache hit, never by an evidence-query allowlist. + } else if _, authorized := policy.compiledBuckets[TraversalPolicyQuerySHA256(strings.TrimSpace(query))]; authorized { + selectCandidate(policy.compiledBuckets[TraversalPolicyQuerySHA256(strings.TrimSpace(query))], "exact_query_canary", "exact_query_authorized") + } else if bucket, authorized := policy.authorizedStructuralBucketForShape(shape); authorized { + selectCandidate(bucket, "structural_authorized", "structural_bucket_"+bucket.Name) + } else if bucket, matched := policy.structuralBucketForShape(shape); matched { + selection.Mode = "structural_shadow" + selection.Reason = "structural_bucket_" + bucket.Name + } else { + selection.Reason = "exact_query_not_authorized" + } + } + collector.RecordTraversalStrategySelection(selection) +} diff --git a/drivers/pg/strategy_selection_test.go b/drivers/pg/strategy_selection_test.go new file mode 100644 index 00000000..311c1d5e --- /dev/null +++ b/drivers/pg/strategy_selection_test.go @@ -0,0 +1,111 @@ +package pg + +import ( + "testing" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/stretchr/testify/require" +) + +func TestTraversalShapeUsesOptimizerFactsWithoutIdentifiersOrValues(t *testing.T) { + first, err := frontend.ParseCypher(frontend.NewContext(), "MATCH p = allShortestPaths((s)-[:Edge*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p") + require.NoError(t, err) + second, err := frontend.ParseCypher(frontend.NewContext(), "MATCH route = allShortestPaths((x)-[:Edge*1..4]->(y)) WHERE id(x) = $left AND id(y) = $right RETURN route") + require.NoError(t, err) + + firstShape, err := traversalShapeForQuery(first) + require.NoError(t, err) + secondShape, err := traversalShapeForQuery(second) + require.NoError(t, err) + require.True(t, firstShape.Available()) + require.Equal(t, TraversalShapeVersion, firstShape.Version) + require.Equal(t, "ASP", firstShape.Family) + require.Equal(t, "outbound", firstShape.Direction) + require.Equal(t, "all_paths", firstShape.ObservationMode) + require.Equal(t, int64(1), firstShape.MinimumDepth) + require.Equal(t, int64(4), firstShape.MaximumDepth) + require.Equal(t, 1, firstShape.RelationshipKindCount) + require.False(t, firstShape.UntypedRelationship) + require.Equal(t, firstShape.Fingerprint, secondShape.Fingerprint) +} + +func TestTraversalPolicyStructuralBucketIsObservationOnlyAndUnambiguous(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[:Edge*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + shape, err := traversalShapeForQuery(parsed) + require.NoError(t, err) + + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorASPI1DAG, false) + manifest, err := decodeTraversalPromotionManifest(policy.PromotionManifestJSON) + require.NoError(t, err) + policy.compiledManifest = manifest + bucket, matched := policy.structuralBucketForShape(shape) + require.True(t, matched) + require.Equal(t, "qualified-query", bucket.Name) + + policy.compiledManifest.Buckets = append(policy.compiledManifest.Buckets, bucket) + policy.compiledManifest.Buckets[1].Name = "ambiguous" + _, matched = policy.structuralBucketForShape(shape) + require.False(t, matched) +} + +func TestTraversalShapeRejectsMultipleTraversalTargets(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), "MATCH p = shortestPath((a)-[:Edge*1..4]->(b)), q = shortestPath((c)-[:Edge*1..4]->(d)) RETURN p, q") + require.NoError(t, err) + + shape, err := traversalShapeForQuery(query) + require.NoError(t, err) + require.False(t, shape.Available()) +} + +func TestTraversalShapeClassifiesQualifiedFixedSuffixExpansion(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), ` +MATCH (root:ExpansionRoot) +WHERE root.root_key = $root_key +MATCH route = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) +RETURN route`) + require.NoError(t, err) + + shape, err := traversalShapeForQuery(query) + require.NoError(t, err) + require.True(t, shape.Available()) + require.Equal(t, TraversalFixedSuffixShapeVersion, shape.Version) + require.Equal(t, "fixed_suffix_expansion", shape.Family) + require.Equal(t, 3, shape.SuffixLength) + require.Equal(t, "EXPANSION-SUFFIX-SEEDED-REVERSE", shape.CandidateStrategy) + require.NotEmpty(t, shape.Fingerprint) +} + +func TestTraversalPolicyAuthorizesVerifiedStructuralBucket(t *testing.T) { + query := "MATCH p = allShortestPaths((s)-[:Edge*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + otherQuery := "MATCH route = allShortestPaths((left)-[:Edge*1..4]->(right)) WHERE id(left) = $a AND id(right) = $b RETURN route" + parsed, err := frontend.ParseCypher(frontend.NewContext(), otherQuery) + require.NoError(t, err) + shape, err := traversalShapeForQuery(parsed) + require.NoError(t, err) + + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorASPI1DAG, false) + manifest, err := decodeTraversalPromotionManifest(policy.PromotionManifestJSON) + require.NoError(t, err) + manifest.Version = 3 + manifest.Buckets[0].StructuralShapeVersion = shape.Version + manifest.Buckets[0].StructuralFamily = shape.Family + manifest.Buckets[0].StructuralShapeSHA256 = shape.Fingerprint + manifest.Buckets[0].SQLTemplateSHA256 = structuralSQLTemplateSHA256(manifest, manifest.Buckets[0]) + policy = rewriteTestTraversalPolicyManifest(t, policy, func(current *traversalPromotionManifest) { + *current = manifest + }) + + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(policy)) + effective, identity := driver.SchemaManager.effectiveTraversalPolicyForShape(otherQuery, shape, pgx.RepeatableRead) + require.True(t, effective.enabled()) + require.NotEqual(t, "production-incumbent-v1", identity) + options, err := effective.productionOptionsForShape(otherQuery, shape) + require.NoError(t, err) + require.NotNil(t, options.AuthorizedBucket) + require.Equal(t, "outbound", options.AuthorizedBucket.Direction) +} diff --git a/drivers/pg/suffix_reverse_retry.go b/drivers/pg/suffix_reverse_retry.go new file mode 100644 index 00000000..b60d1a4c --- /dev/null +++ b/drivers/pg/suffix_reverse_retry.go @@ -0,0 +1,238 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package pg + +import ( + "encoding/json" + "fmt" + + "github.com/specterops/dawgs/graph" +) + +// suffixReverseRetrySavepoint isolates a candidate attempt from its exact fallback. +const suffixReverseRetrySavepoint = "dawgs_suffix_reverse_retry_v1" + +// SuffixReverseRetryLimits freezes the development candidate's public buffer +// boundary independently of SQL translation. Values must match the lowering +// metadata used to render the candidate statement. +type SuffixReverseRetryLimits struct { + // OutputRows caps the number of candidate rows buffered before publication. + OutputRows int64 + + // OutputBytes caps the JSON-encoded size of buffered candidate values. + OutputBytes int64 +} + +// SuffixReverseRetryTransaction is a tool-only PostgreSQL execution surface. +// It is intentionally absent from graph.Transaction and cannot affect ordinary +// production queries without an explicit type assertion by repository tooling. +type SuffixReverseRetryTransaction interface { + // RawSuffixReverseRetry runs a bounded candidate and, if needed, its exact fallback. + RawSuffixReverseRetry(candidateSQL, fallbackSQL string, candidateParameters, fallbackParameters map[string]any, limits SuffixReverseRetryLimits) graph.Result +} + +// bufferedResult owns a completely drained candidate result. No database rows +// remain live when it is returned to a caller. +type bufferedResult struct { + // keys names the columns reported by every buffered row. + keys []string + + // rows contains the complete candidate output before it is exposed. + rows [][]any + + // mapper translates database values for consumers of the result. + mapper graph.ValueMapper + + // index is the next buffered row position returned by Next. + index int + + // err stores a terminal buffering error. + err error +} + +// Next advances to the next buffered row. +func (s *bufferedResult) Next() bool { + if s.index >= len(s.rows) { + return false + } + s.index++ + return true +} + +// Keys returns the buffered result's column names. +func (s *bufferedResult) Keys() []string { return s.keys } + +// Values returns the current buffered row's values. +func (s *bufferedResult) Values() []any { + if s.index == 0 || s.index > len(s.rows) { + return nil + } + return s.rows[s.index-1] +} + +// Mapper returns the database value mapper captured from the source result. +func (s *bufferedResult) Mapper() graph.ValueMapper { return s.mapper } + +// Scan maps the current row into targets using graph's standard result helper. +func (s *bufferedResult) Scan(targets ...any) error { return graph.ScanNextResult(s, targets...) } + +// Error returns the terminal buffering error, if any. +func (s *bufferedResult) Error() error { return s.err } + +// Close is a no-op because the source result was drained and closed before publication. +func (s *bufferedResult) Close() {} + +// suffixReverseRetryFallbackResult records completion only after the exact +// forward retry has drained without error. Closing early intentionally does +// not create a completion receipt: an interrupted retry is not evidence of a +// complete incumbent execution. +type suffixReverseRetryFallbackResult struct { + // Result supplies rows from the exact incumbent fallback. + graph.Result + + // complete records a successful complete fallback drain. + complete func() error + + // completed reports whether the fallback has drained and been attested. + completed bool + + // err stores an error from the fallback or completion attestation. + err error +} + +// Next advances the fallback and records its completion after the final row. +func (s *suffixReverseRetryFallbackResult) Next() bool { + if s.err != nil || s.completed { + return false + } + if s.Result.Next() { + return true + } + if err := s.Result.Error(); err != nil { + s.err = err + return false + } + if err := s.complete(); err != nil { + s.err = err + return false + } + s.completed = true + return false +} + +// Error returns the fallback or completion error before the embedded result's error. +func (s *suffixReverseRetryFallbackResult) Error() error { + if s.err != nil { + return s.err + } + return s.Result.Error() +} + +// bufferGraphResult drains result and enforces both public candidate caps. The +// returned branch is empty only when the complete buffer is publishable. +func bufferGraphResult(result graph.Result, limits SuffixReverseRetryLimits) (*bufferedResult, string, error) { + defer result.Close() + buffered := &bufferedResult{mapper: result.Mapper()} + var encodedBytes int64 + for result.Next() { + values := append([]any(nil), result.Values()...) + if buffered.keys == nil { + buffered.keys = append([]string(nil), result.Keys()...) + } + encoded, err := json.Marshal(values) + if err != nil { + return nil, "forward_retry_output_encoding", nil + } + encodedBytes += int64(len(encoded)) + buffered.rows = append(buffered.rows, values) + if int64(len(buffered.rows)) > limits.OutputRows { + return nil, "forward_retry_output_rows", nil + } + if encodedBytes > limits.OutputBytes { + return nil, "forward_retry_output_bytes", nil + } + } + if err := result.Error(); err != nil { + return nil, "", err + } + return buffered, "", nil +} + +// suffixReverseRetryExec executes a control statement within the active transaction. +func (s *transaction) suffixReverseRetryExec(statement string, arguments ...any) error { + _, err := s.driver().Exec(s.ctx, statement, arguments...) + return err +} + +// suffixReverseRetryAbort restores the savepoint and returns err as a graph result. +func (s *transaction) suffixReverseRetryAbort(err error) graph.Result { + _ = s.suffixReverseRetryExec("rollback to savepoint " + suffixReverseRetrySavepoint) + _ = s.suffixReverseRetryExec("release savepoint " + suffixReverseRetrySavepoint) + return graph.NewErrorResult(err) +} + +// RawSuffixReverseRetry executes a reverse-only candidate and exact incumbent +// fallback in one stable snapshot. Candidate rows are fully buffered and +// validated before they can be observed. +func (s *transaction) RawSuffixReverseRetry(candidateSQL, fallbackSQL string, candidateParameters, fallbackParameters map[string]any, limits SuffixReverseRetryLimits) graph.Result { + if s.tx == nil || !stableSnapshotIsolation(s.isolation) { + return graph.NewErrorResult(fmt.Errorf("suffix reverse retry requires an explicit Repeatable Read or Serializable transaction")) + } + if limits.OutputRows <= 0 || limits.OutputBytes <= 0 { + return graph.NewErrorResult(fmt.Errorf("suffix reverse retry requires positive output row and byte limits")) + } + if err := s.suffixReverseRetryExec("savepoint " + suffixReverseRetrySavepoint); err != nil { + return graph.NewErrorResult(err) + } + if err := s.suffixReverseRetryExec("select set_config('dawgs.suffix_reverse_retry_status', '', true)"); err != nil { + return s.suffixReverseRetryAbort(err) + } + + candidate, bufferBranch, err := bufferGraphResult(s.raw(candidateSQL, candidateParameters), limits) + if err != nil { + return s.suffixReverseRetryAbort(err) + } + var sqlBranch string + if err := s.driver().QueryRow(s.ctx, "select current_setting('dawgs.suffix_reverse_retry_status', true)").Scan(&sqlBranch); err != nil { + return s.suffixReverseRetryAbort(err) + } + branch := sqlBranch + if bufferBranch != "" { + branch = bufferBranch + } + if branch == "reverse_complete" { + if err := s.suffixReverseRetryExec("release savepoint " + suffixReverseRetrySavepoint); err != nil { + return graph.NewErrorResult(err) + } + return candidate + } + if branch != "forward_retry_suffix_overflow" && branch != "forward_retry_state_overflow" && + branch != "forward_retry_output_rows" && branch != "forward_retry_output_bytes" && + branch != "forward_retry_output_encoding" { + return s.suffixReverseRetryAbort(fmt.Errorf("suffix reverse retry returned unknown or empty status %q", branch)) + } + if err := s.suffixReverseRetryExec("rollback to savepoint " + suffixReverseRetrySavepoint); err != nil { + return graph.NewErrorResult(err) + } + if err := s.suffixReverseRetryExec("release savepoint " + suffixReverseRetrySavepoint); err != nil { + return graph.NewErrorResult(err) + } + if err := s.suffixReverseRetryExec( + "select public.record_requested_traversal_runtime_attestation_v1($1, false, $2)", + branch, + "EXPANSION-STEPWISE-FORWARD", + ); err != nil { + return graph.NewErrorResult(err) + } + return &suffixReverseRetryFallbackResult{ + Result: s.raw(fallbackSQL, fallbackParameters), + complete: func() error { + return s.suffixReverseRetryExec( + "select public.record_requested_traversal_runtime_attestation_v1($1, true, $2)", + "exact_forward_retry_complete", + "EXPANSION-STEPWISE-FORWARD", + ) + }, + } +} diff --git a/drivers/pg/suffix_reverse_retry_test.go b/drivers/pg/suffix_reverse_retry_test.go new file mode 100644 index 00000000..948efcf8 --- /dev/null +++ b/drivers/pg/suffix_reverse_retry_test.go @@ -0,0 +1,180 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +package pg + +import ( + "context" + "errors" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/pashagolub/pgxmock/v5" + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +func newSuffixRetryMockTransaction(t *testing.T) (*transaction, pgxmock.PgxConnIface) { + t.Helper() + ctx := context.Background() + mock, err := pgxmock.NewConn(pgxmock.QueryMatcherOption(pgxmock.QueryMatcherEqual)) + require.NoError(t, err) + mock.ExpectBeginTx(pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + pgxTx, err := mock.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.RepeatableRead}) + require.NoError(t, err) + t.Cleanup(func() { + mock.ExpectRollback() + require.NoError(t, pgxTx.Rollback(ctx)) + mock.ExpectClose() + require.NoError(t, mock.Close(ctx)) + require.NoError(t, mock.ExpectationsWereMet()) + }) + return &transaction{ + schemaManager: &SchemaManager{}, + ctx: ctx, + tx: pgxTx, + isolation: pgx.RepeatableRead, + }, mock +} + +type suffixRetryTestResult struct { + rows [][]any + index int + err error +} + +func (s *suffixRetryTestResult) Next() bool { + if s.index >= len(s.rows) { + return false + } + s.index++ + return true +} +func (s *suffixRetryTestResult) Keys() []string { return []string{"value"} } +func (s *suffixRetryTestResult) Values() []any { return s.rows[s.index-1] } +func (s *suffixRetryTestResult) Mapper() graph.ValueMapper { return graph.ValueMapper{} } +func (s *suffixRetryTestResult) Scan(...any) error { return nil } +func (s *suffixRetryTestResult) Error() error { return s.err } +func (s *suffixRetryTestResult) Close() {} + +func TestBufferGraphResultPublishesOnlyCompleteBoundedRows(t *testing.T) { + buffered, branch, err := bufferGraphResult(&suffixRetryTestResult{ + rows: [][]any{{"a"}, {"b"}}, + }, SuffixReverseRetryLimits{OutputRows: 2, OutputBytes: 64}) + require.NoError(t, err) + require.Empty(t, branch) + require.Equal(t, []string{"value"}, buffered.Keys()) + require.True(t, buffered.Next()) + require.Equal(t, []any{"a"}, buffered.Values()) + require.True(t, buffered.Next()) + require.Equal(t, []any{"b"}, buffered.Values()) + require.False(t, buffered.Next()) +} + +func TestBufferGraphResultFailsClosedToRetryCaps(t *testing.T) { + tests := []struct { + name string + limits SuffixReverseRetryLimits + branch string + }{ + {name: "rows", limits: SuffixReverseRetryLimits{OutputRows: 1, OutputBytes: 64}, branch: "forward_retry_output_rows"}, + {name: "bytes", limits: SuffixReverseRetryLimits{OutputRows: 2, OutputBytes: 3}, branch: "forward_retry_output_bytes"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + buffered, branch, err := bufferGraphResult(&suffixRetryTestResult{ + rows: [][]any{{"alpha"}, {"beta"}}, + }, test.limits) + require.NoError(t, err) + require.Nil(t, buffered) + require.Equal(t, test.branch, branch) + }) + } +} + +func TestBufferGraphResultPropagatesCandidateFailure(t *testing.T) { + expected := errors.New("candidate failed") + buffered, branch, err := bufferGraphResult(&suffixRetryTestResult{err: expected}, SuffixReverseRetryLimits{OutputRows: 1, OutputBytes: 1}) + require.ErrorIs(t, err, expected) + require.Nil(t, buffered) + require.Empty(t, branch) +} + +func TestSuffixReverseRetryFallbackResultRecordsCompletionOnlyAfterDrain(t *testing.T) { + completed := 0 + result := &suffixReverseRetryFallbackResult{ + Result: &suffixRetryTestResult{rows: [][]any{{"fallback-row"}}}, + complete: func() error { completed++; return nil }, + } + + require.NoError(t, result.Error()) + require.Equal(t, 0, completed) + require.True(t, result.Next()) + require.Equal(t, []any{"fallback-row"}, result.Values()) + require.Equal(t, 0, completed) + require.False(t, result.Next()) + require.Equal(t, 1, completed) + require.False(t, result.Next()) + require.Equal(t, 1, completed) +} + +func TestSuffixReverseRetryFallbackResultDoesNotRecordCompletionAfterFailure(t *testing.T) { + completed := 0 + expected := errors.New("fallback failed") + result := &suffixReverseRetryFallbackResult{ + Result: &suffixRetryTestResult{err: expected}, + complete: func() error { completed++; return nil }, + } + + require.False(t, result.Next()) + require.ErrorIs(t, result.Error(), expected) + require.Zero(t, completed) +} + +func TestRawSuffixReverseRetryPublishesCompletedCandidate(t *testing.T) { + tx, mock := newSuffixRetryMockTransaction(t) + mock.ExpectExec("savepoint " + suffixReverseRetrySavepoint).WillReturnResult(pgxmock.NewResult("SAVEPOINT", 0)) + mock.ExpectExec("select set_config('dawgs.suffix_reverse_retry_status', '', true)").WillReturnResult(pgxmock.NewResult("SELECT", 1)) + mock.ExpectQuery("candidate").WithArgs(pgxmock.AnyArg(), pgxmock.AnyArg()).WillReturnRows( + pgxmock.NewRows([]string{"value"}).AddRow("candidate-row"), + ) + mock.ExpectQuery("select current_setting('dawgs.suffix_reverse_retry_status', true)").WillReturnRows( + pgxmock.NewRows([]string{"current_setting"}).AddRow("reverse_complete"), + ) + mock.ExpectExec("release savepoint " + suffixReverseRetrySavepoint).WillReturnResult(pgxmock.NewResult("RELEASE", 0)) + + result := tx.RawSuffixReverseRetry("candidate", "fallback", nil, nil, SuffixReverseRetryLimits{OutputRows: 2, OutputBytes: 64}) + require.NoError(t, result.Error()) + require.True(t, result.Next()) + require.Equal(t, []any{"candidate-row"}, result.Values()) + require.False(t, result.Next()) +} + +func TestRawSuffixReverseRetryRollsBackCandidateBeforeExactForward(t *testing.T) { + tx, mock := newSuffixRetryMockTransaction(t) + mock.ExpectExec("savepoint " + suffixReverseRetrySavepoint).WillReturnResult(pgxmock.NewResult("SAVEPOINT", 0)) + mock.ExpectExec("select set_config('dawgs.suffix_reverse_retry_status', '', true)").WillReturnResult(pgxmock.NewResult("SELECT", 1)) + mock.ExpectQuery("candidate").WithArgs(pgxmock.AnyArg(), pgxmock.AnyArg()).WillReturnRows(pgxmock.NewRows([]string{"value"})) + mock.ExpectQuery("select current_setting('dawgs.suffix_reverse_retry_status', true)").WillReturnRows( + pgxmock.NewRows([]string{"current_setting"}).AddRow("forward_retry_state_overflow"), + ) + mock.ExpectExec("rollback to savepoint " + suffixReverseRetrySavepoint).WillReturnResult(pgxmock.NewResult("ROLLBACK", 0)) + mock.ExpectExec("release savepoint " + suffixReverseRetrySavepoint).WillReturnResult(pgxmock.NewResult("RELEASE", 0)) + mock.ExpectExec("select public.record_requested_traversal_runtime_attestation_v1($1, false, $2)"). + WithArgs("forward_retry_state_overflow", "EXPANSION-STEPWISE-FORWARD"). + WillReturnResult(pgxmock.NewResult("SELECT", 1)) + mock.ExpectQuery("fallback").WithArgs(pgxmock.AnyArg(), pgxmock.AnyArg()).WillReturnRows( + pgxmock.NewRows([]string{"value"}).AddRow("fallback-row"), + ) + mock.ExpectExec("select public.record_requested_traversal_runtime_attestation_v1($1, true, $2)"). + WithArgs("exact_forward_retry_complete", "EXPANSION-STEPWISE-FORWARD"). + WillReturnResult(pgxmock.NewResult("SELECT", 1)) + + result := tx.RawSuffixReverseRetry("candidate", "fallback", nil, nil, SuffixReverseRetryLimits{OutputRows: 2, OutputBytes: 64}) + defer result.Close() + require.NoError(t, result.Error()) + require.True(t, result.Next()) + require.Equal(t, []any{"fallback-row"}, result.Values()) + require.False(t, result.Next()) + require.NoError(t, result.Error()) +} diff --git a/drivers/pg/topology_route_decision.go b/drivers/pg/topology_route_decision.go new file mode 100644 index 00000000..f140dbd5 --- /dev/null +++ b/drivers/pg/topology_route_decision.go @@ -0,0 +1,162 @@ +package pg + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "sort" + "strings" + "sync/atomic" +) + +const ( + // topologyRouteDecisionMaximumEntries bounds cached observations per transaction. + topologyRouteDecisionMaximumEntries = 64 + + // topologyRouteDecisionMaximumBytes bounds aggregate cache-key memory per transaction. + topologyRouteDecisionMaximumBytes = 64 * 1024 + + // topologyRouteDecisionMaximumEntry rejects unusually large individual cache keys. + topologyRouteDecisionMaximumEntry = 4 * 1024 +) + +// topologyRouteDecisionOwner assigns distinct identities to transaction-local route caches. +var topologyRouteDecisionOwner uint64 + +// TraversalRouteDecision is query-text-free shadow telemetry for a +// transaction-owned topology decision. Shadow decisions never alter emitted +// SQL and never retain caller values. +type TraversalRouteDecision struct { + // Mode identifies the selected candidate or incumbent arm. + Mode string `json:"mode"` + + // Reason records the query-text-free rationale for the route decision. + Reason string `json:"reason"` +} + +// TraversalRouteDecisionCollector optionally records aggregate decision +// states. Implementations must not retain the per-transaction cache key. +type TraversalRouteDecisionCollector interface { + // RecordTraversalRouteDecision receives query-text-free topology routing telemetry. + RecordTraversalRouteDecision(TraversalRouteDecision) +} + +// topologyRouteDecisionCache retains bounded repeated route observations for one transaction. +type topologyRouteDecisionCache struct { + // owner distinguishes this cache from every other transaction cache. + owner uint64 + + // generation changes after invalidation so prior observations cannot match. + generation uint64 + + // disabled prevents route reuse after a mutation invalidates the synopsis. + disabled bool + + // entries contains fingerprints of repeated safe observations. + entries map[string]struct{} + + // bytes tracks the approximate storage used by entries. + bytes int +} + +// newTopologyRouteDecisionCache initializes an empty cache with a unique owner identity. +func newTopologyRouteDecisionCache() *topologyRouteDecisionCache { + return &topologyRouteDecisionCache{ + owner: atomic.AddUint64(&topologyRouteDecisionOwner, 1), + entries: map[string]struct{}{}, + } +} + +// topologyRouteParameterFingerprint produces a deterministic digest of JSON-encodable parameters. +func topologyRouteParameterFingerprint(parameters map[string]any) (string, bool) { + keys := make([]string, 0, len(parameters)) + for key := range parameters { + keys = append(keys, key) + } + sort.Strings(keys) + var canonical strings.Builder + for _, key := range keys { + encoded, err := json.Marshal(parameters[key]) + if err != nil { + return "", false + } + fmt.Fprintf(&canonical, "%d:%s:%T:%d:%s|", len(key), key, parameters[key], len(encoded), encoded) + } + digest := sha256.Sum256([]byte(canonical.String())) + return hex.EncodeToString(digest[:]), true +} + +// invalidateTopologyRouteDecisions permanently disables the transaction's route cache after mutation. +func (s *transaction) invalidateTopologyRouteDecisions() { + if s.topologyRouteDecisions == nil { + return + } + s.topologyRouteDecisions.entries = map[string]struct{}{} + s.topologyRouteDecisions.bytes = 0 + s.topologyRouteDecisions.generation++ + s.topologyRouteDecisions.disabled = true +} + +// recordTopologyRouteDecision sends a topology decision to an optional telemetry collector. +func (s *transaction) recordTopologyRouteDecision(decision TraversalRouteDecision) { + if collector, found := s.schemaManager.translationCacheProvider.(TraversalRouteDecisionCollector); found { + collector.RecordTraversalRouteDecision(decision) + } +} + +// topologyRouteDecision selects a snapshot-bound fixed-suffix candidate. V4 +// selects only on a repeated exact observation; v5 is a separately versioned +// first-use protocol and may select after the same synopsis checks. The +// returned instruction is valid only for the current transaction. +func (s *transaction) topologyRouteDecision(graphID int32, shape TraversalShape, parameters map[string]any, policyIdentity, estimatorVersion string, maximumEdgeToNodeRatioPerMille int64, candidateAuthorized, firstUseAuthorized bool) bool { + if shape.Version != TraversalFixedSuffixShapeVersion { + return false + } + cache := s.topologyRouteDecisions + if cache == nil || cache.disabled || s.tx == nil || !stableSnapshotIsolation(s.isolation) { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_route_disabled"}) + return false + } + parametersFingerprint, valid := topologyRouteParameterFingerprint(parameters) + if !valid { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_route_parameters_unverifiable"}) + return false + } + synopsis, err := s.traversalTopologySynopsis(graphID) + if err != nil || !synopsis.Available() || synopsis.SchemaVersion != "topology-synopsis-schema-v2" || synopsis.NodeCount == 0 || synopsis.EdgeCount == 0 { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_synopsis_unavailable"}) + return false + } + // v1 freezes a 1000-per-mille limit, so this overflow-safe comparison is + // exactly edge_count <= node_count. Validation rejects any other threshold + // until a separately versioned estimator defines its arithmetic. + if candidateAuthorized && (synopsis.EstimatorVersion != estimatorVersion || maximumEdgeToNodeRatioPerMille != 1000 || synopsis.EdgeCount > synopsis.NodeCount) { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_estimate_rejected"}) + return false + } + keyMaterial := fmt.Sprintf("%d|%d|%d|%s|%s|%s|%d|%d", cache.owner, graphID, cache.generation, shape.Fingerprint, parametersFingerprint, policyIdentity, synopsis.Epoch, synopsis.CurrentMutationEpoch) + digest := sha256.Sum256([]byte(keyMaterial)) + key := hex.EncodeToString(digest[:]) + if _, found := cache.entries[key]; found { + if candidateAuthorized { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "candidate", Reason: "topology_route_candidate_hit"}) + return true + } + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_route_shadow_hit"}) + return false + } + if firstUseAuthorized { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "candidate", Reason: "topology_route_first_use_candidate"}) + return true + } + entryBytes := len(key) + len(shape.Fingerprint) + len(policyIdentity) + 64 + if entryBytes > topologyRouteDecisionMaximumEntry || len(cache.entries) == topologyRouteDecisionMaximumEntries || cache.bytes+entryBytes > topologyRouteDecisionMaximumBytes { + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_route_capacity"}) + return false + } + cache.entries[key] = struct{}{} + cache.bytes += entryBytes + s.recordTopologyRouteDecision(TraversalRouteDecision{Mode: "incumbent", Reason: "topology_route_shadow_miss"}) + return false +} diff --git a/drivers/pg/topology_synopsis.go b/drivers/pg/topology_synopsis.go new file mode 100644 index 00000000..32165db0 --- /dev/null +++ b/drivers/pg/topology_synopsis.go @@ -0,0 +1,172 @@ +package pg + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/graph" +) + +// TraversalTopologySynopsis is the current graph-scoped, atomically published +// topology summary. It is advisory only: callers must treat unavailable or +// stale state as an incumbent-only routing outcome. +type TraversalTopologySynopsis struct { + // GraphID identifies the graph summarized by this record. + GraphID int32 `json:"graph_id"` + + // Epoch identifies this atomically published synopsis generation. + Epoch uint64 `json:"epoch"` + + // SourceMutationEpoch is the graph mutation epoch observed while building the synopsis. + SourceMutationEpoch uint64 `json:"source_mutation_epoch"` + + // CurrentMutationEpoch is the graph mutation epoch visible when the synopsis was read. + CurrentMutationEpoch uint64 `json:"current_mutation_epoch"` + + // EstimatorVersion identifies the fixed estimator whose assumptions produced the synopsis. + EstimatorVersion string `json:"estimator_version"` + + // SchemaVersion identifies the synopsis storage schema used to publish this record. + SchemaVersion string `json:"schema_version"` + + // Status reports whether publication completed successfully. + Status string `json:"status"` + + // NodeCount is the graph's node count recorded by the refresh. + NodeCount int64 `json:"node_count"` + + // EdgeCount is the graph's edge count recorded by the refresh. + EdgeCount int64 `json:"edge_count"` +} + +// Available reports whether this synopsis was atomically published for the +// current visible graph mutation epoch. +func (s TraversalTopologySynopsis) Available() bool { + return s.Epoch != 0 && s.Status == "ready" && s.SourceMutationEpoch == s.CurrentMutationEpoch +} + +// RefreshTraversalTopologySynopsis publishes a versioned graph topology +// synopsis. The graph epoch row is locked before collection, so the synopsis +// is current at its commit boundary; a mutation committed afterwards can +// immediately make it stale. +func (s *Driver) RefreshTraversalTopologySynopsis(ctx context.Context, target graph.Graph) (TraversalTopologySynopsis, error) { + if s == nil || s.pool == nil { + return TraversalTopologySynopsis{}, fmt.Errorf("PostgreSQL driver is not initialized") + } + if target.Name == "" { + return TraversalTopologySynopsis{}, fmt.Errorf("topology synopsis requires a named graph") + } + conn, err := s.pool.Acquire(ctx) + if err != nil { + return TraversalTopologySynopsis{}, fmt.Errorf("acquire connection for topology synopsis: %w", err) + } + defer conn.Release() + + tx, err := conn.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return TraversalTopologySynopsis{}, fmt.Errorf("begin topology synopsis refresh: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var graphID int32 + var sourceEpoch, nodeCount, edgeCount, nextEpoch int64 + if err := tx.QueryRow(ctx, `select graph.id::int4, graph_traversal_epoch.epoch::bigint from graph join graph_traversal_epoch on graph_traversal_epoch.graph_id = graph.id where graph.name = $1 for update of graph_traversal_epoch`, target.Name).Scan(&graphID, &sourceEpoch); err != nil { + if err == pgx.ErrNoRows { + return TraversalTopologySynopsis{}, fmt.Errorf("topology synopsis graph %q does not exist", target.Name) + } + return TraversalTopologySynopsis{}, fmt.Errorf("read topology synopsis graph: %w", err) + } + if err := tx.QueryRow(ctx, `select count(*)::bigint, (select count(*)::bigint from edge where graph_id = $1) from node where graph_id = $1`, graphID).Scan(&nodeCount, &edgeCount); err != nil { + return TraversalTopologySynopsis{}, fmt.Errorf("count topology synopsis graph: %w", err) + } + if err := tx.QueryRow(ctx, `select coalesce(epoch, 0)::bigint + 1 from graph_traversal_synopsis_generation where graph_id = $1`, graphID).Scan(&nextEpoch); err == pgx.ErrNoRows { + nextEpoch = 1 + } else if err != nil { + return TraversalTopologySynopsis{}, fmt.Errorf("read topology synopsis generation: %w", err) + } + for _, statement := range []string{ + `delete from graph_traversal_synopsis_degree where graph_id = $1`, + `delete from graph_traversal_synopsis_edge_count where graph_id = $1`, + `delete from graph_traversal_synopsis_node_count where graph_id = $1`, + `insert into graph_traversal_synopsis_node_count (graph_id, epoch, kind_id, node_count) +select $1::int4, $2::bigint, kind_id, count(*)::bigint +from node cross join lateral unnest(kind_ids) as kinds(kind_id) +where graph_id = $1::int4 group by kind_id`, + `insert into graph_traversal_synopsis_edge_count (graph_id, epoch, direction, kind_id, edge_count, distinct_start_count, distinct_end_count) +select $1::int4, $2::bigint, direction, kind_id, edge_count, distinct_start_count, distinct_end_count +from ( + select 'outbound'::text as direction, kind_id, count(*)::bigint as edge_count, count(distinct start_id)::bigint as distinct_start_count, count(distinct end_id)::bigint as distinct_end_count from edge where graph_id = $1::int4 group by kind_id + union all + select 'inbound'::text, kind_id, count(*)::bigint, count(distinct end_id)::bigint, count(distinct start_id)::bigint from edge where graph_id = $1::int4 group by kind_id +) counts`, + `insert into graph_traversal_synopsis_degree (graph_id, epoch, direction, kind_id, bucket, node_count) +with degree as ( + select 'outbound'::text as direction, kind_id, start_id as node_id, count(*)::bigint as degree from edge where graph_id = $1::int4 group by kind_id, start_id + union all + select 'inbound'::text, kind_id, end_id, count(*)::bigint from edge where graph_id = $1::int4 group by kind_id, end_id +) +select $1::int4, $2::bigint, direction, kind_id, + case when degree = 1 then 'one' when degree <= 4 then 'two_to_four' when degree <= 16 then 'five_to_sixteen' else 'seventeen_plus' end, + count(*)::bigint +from degree group by direction, kind_id, case when degree = 1 then 'one' when degree <= 4 then 'two_to_four' when degree <= 16 then 'five_to_sixteen' else 'seventeen_plus' end`, + } { + args := []any{graphID} + if statement[0:6] == "insert" { + args = append(args, nextEpoch) + } + if _, err := tx.Exec(ctx, statement, args...); err != nil { + return TraversalTopologySynopsis{}, fmt.Errorf("publish topology synopsis detail: %w", err) + } + } + const publish = ` +insert into graph_traversal_synopsis_generation + (graph_id, epoch, source_mutation_epoch, estimator_version, schema_version, status, node_count, edge_count, refresh_started_at, refresh_completed_at, refresh_mode) +select $1, $2, $3, 'topology-fixed-suffix-counts-v1', 'topology-synopsis-schema-v2', 'ready', $4, $5, clock_timestamp(), clock_timestamp(), 'full' +where (select epoch from graph_traversal_epoch where graph_id = $1) = $3 +on conflict (graph_id) do update +set epoch = excluded.epoch, source_mutation_epoch = excluded.source_mutation_epoch, + estimator_version = excluded.estimator_version, schema_version = excluded.schema_version, + status = excluded.status, node_count = excluded.node_count, edge_count = excluded.edge_count, + built_at = clock_timestamp(), refresh_started_at = excluded.refresh_started_at, + refresh_completed_at = excluded.refresh_completed_at, refresh_mode = excluded.refresh_mode +where (select epoch from graph_traversal_epoch where graph_id = $1) = $3 +returning graph_id::int4, epoch, source_mutation_epoch, + (select epoch from graph_traversal_epoch where graph_id = $1), + estimator_version, schema_version, status, node_count, edge_count` + var synopsis TraversalTopologySynopsis + if err := tx.QueryRow(ctx, publish, graphID, nextEpoch, sourceEpoch, nodeCount, edgeCount).Scan( + &synopsis.GraphID, &synopsis.Epoch, &synopsis.SourceMutationEpoch, &synopsis.CurrentMutationEpoch, + &synopsis.EstimatorVersion, &synopsis.SchemaVersion, &synopsis.Status, &synopsis.NodeCount, &synopsis.EdgeCount, + ); err != nil { + if err == pgx.ErrNoRows { + return TraversalTopologySynopsis{}, fmt.Errorf("topology synopsis became stale during refresh") + } + return TraversalTopologySynopsis{}, fmt.Errorf("publish topology synopsis generation: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return TraversalTopologySynopsis{}, fmt.Errorf("commit topology synopsis refresh: %w", err) + } + return synopsis, nil +} + +// traversalTopologySynopsis reads the current published synopsis visible to this transaction. +func (s *transaction) traversalTopologySynopsis(graphID int32) (TraversalTopologySynopsis, error) { + var synopsis TraversalTopologySynopsis + const statement = ` +select epoch.graph_id::int4, coalesce(synopsis.epoch, 0), coalesce(synopsis.source_mutation_epoch, 0), + epoch.epoch, coalesce(synopsis.estimator_version, ''), coalesce(synopsis.schema_version, ''), coalesce(synopsis.status, ''), + coalesce(synopsis.node_count, 0), coalesce(synopsis.edge_count, 0) +from graph_traversal_epoch epoch +left join graph_traversal_synopsis_generation synopsis on synopsis.graph_id = epoch.graph_id +where epoch.graph_id = $1` + err := s.driver().QueryRow(s.ctx, statement, graphID).Scan( + &synopsis.GraphID, &synopsis.Epoch, &synopsis.SourceMutationEpoch, + &synopsis.CurrentMutationEpoch, &synopsis.EstimatorVersion, &synopsis.SchemaVersion, &synopsis.Status, + &synopsis.NodeCount, &synopsis.EdgeCount, + ) + if err != nil { + return TraversalTopologySynopsis{}, err + } + return synopsis, nil +} diff --git a/drivers/pg/topology_synopsis_test.go b/drivers/pg/topology_synopsis_test.go new file mode 100644 index 00000000..eb4605d8 --- /dev/null +++ b/drivers/pg/topology_synopsis_test.go @@ -0,0 +1,56 @@ +package pg + +import "testing" + +func TestTraversalTopologySynopsisAvailable(t *testing.T) { + testCases := []struct { + name string + synopsis TraversalTopologySynopsis + want bool + }{ + { + name: "current ready generation", + synopsis: TraversalTopologySynopsis{ + Epoch: 1, + SourceMutationEpoch: 4, + CurrentMutationEpoch: 4, + Status: "ready", + }, + want: true, + }, + { + name: "missing generation", + synopsis: TraversalTopologySynopsis{ + SourceMutationEpoch: 4, + CurrentMutationEpoch: 4, + Status: "ready", + }, + }, + { + name: "stale generation", + synopsis: TraversalTopologySynopsis{ + Epoch: 1, + SourceMutationEpoch: 3, + CurrentMutationEpoch: 4, + Status: "ready", + }, + }, + { + name: "failed generation", + synopsis: TraversalTopologySynopsis{ + Epoch: 1, + SourceMutationEpoch: 4, + CurrentMutationEpoch: 4, + Status: "failed", + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + if got := testCase.synopsis.Available(); got != testCase.want { + t.Fatalf("Available() = %t, want %t", got, testCase.want) + } + }) + } +} diff --git a/drivers/pg/transaction.go b/drivers/pg/transaction.go index ccfb10e2..c0811eee 100644 --- a/drivers/pg/transaction.go +++ b/drivers/pg/transaction.go @@ -3,8 +3,11 @@ package pg import ( "context" "fmt" + "strings" + "time" "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" @@ -15,49 +18,95 @@ import ( "github.com/specterops/dawgs/util/size" ) +// driver is the common execution surface implemented by pooled connections and explicit pgx transactions. type driver interface { + // Exec executes a statement and returns its PostgreSQL command tag. Exec(ctx context.Context, sql string, arguments ...any) (commandTag pgconn.CommandTag, err error) + + // Query executes a statement and returns its streaming row set. Query(ctx context.Context, sql string, arguments ...any) (pgx.Rows, error) + + // QueryRow executes a statement whose first row is consumed through pgx.Row. QueryRow(ctx context.Context, sql string, arguments ...any) pgx.Row } +// inspectingDriver records SQL and arguments before delegating execution to a connection or transaction. type inspectingDriver struct { + // upstreamDriver receives each operation after its SQL and arguments have been inspected. upstreamDriver driver } +// Exec inspects and forwards a non-row SQL statement to the wrapped executor. func (s inspectingDriver) Exec(ctx context.Context, sql string, arguments ...any) (commandTag pgconn.CommandTag, err error) { inspector().Inspect(sql, arguments) return s.upstreamDriver.Exec(ctx, sql, arguments...) } +// Query inspects and forwards a row-producing SQL statement to the wrapped executor. func (s inspectingDriver) Query(ctx context.Context, sql string, arguments ...any) (pgx.Rows, error) { inspector().Inspect(sql, arguments) return s.upstreamDriver.Query(ctx, sql, arguments...) } +// QueryRow inspects and forwards a single-row SQL statement to the wrapped executor. func (s inspectingDriver) QueryRow(ctx context.Context, sql string, arguments ...any) pgx.Row { inspector().Inspect(sql, arguments) return s.upstreamDriver.QueryRow(ctx, sql, arguments...) } +// transaction binds query execution, schema resolution, and an optional pgx transaction to one graph operation context. type transaction struct { - schemaManager *SchemaManager - queryExecMode pgx.QueryExecMode + // schemaManager resolves target graphs, kind identifiers, and cached Cypher translations. + schemaManager *SchemaManager + + // translationCache is selected for the physical connection leased by this + // transaction. Nil deliberately bypasses translation retention. + translationCache CypherTranslationCache + + // queryExecMode selects the pgx execution protocol supplied with each query. + queryExecMode pgx.QueryExecMode + + // queryResultsFormat selects the pgx wire format requested for returned columns. queryResultsFormat pgx.QueryResultFormats - ctx context.Context - conn *pgxpool.Conn - tx pgx.Tx - targetSchema graph.Graph - targetSchemaSet bool + + // ctx scopes all work performed by the graph transaction. + ctx context.Context + + // conn is the acquired pooled connection underlying this transaction wrapper. + conn *pgxpool.Conn + + // tx is the optional explicit PostgreSQL transaction used for transactional operations. + tx pgx.Tx + + // isolation is the PostgreSQL snapshot level selected for this transaction. + isolation pgx.TxIsoLevel + + // targetSchema identifies the graph selected explicitly for subsequent operations. + targetSchema graph.Graph + + // targetSchemaSet distinguishes an explicit target from the zero-value graph schema. + targetSchemaSet bool + + // topologyRouteDecisions is transaction-owned shadow routing state. It is + // allocated only after an explicit stable-snapshot transaction begins. + topologyRouteDecisions *topologyRouteDecisionCache } +// newTransactionWrapper configures a graph transaction and optionally begins an explicit PostgreSQL transaction. func newTransactionWrapper(ctx context.Context, conn *pgxpool.Conn, schemaManager *SchemaManager, cfg *Config, allocateTransaction bool) (*transaction, error) { + var physicalConnection *pgx.Conn + if conn != nil { + physicalConnection = conn.Conn() + } + wrapper := &transaction{ schemaManager: schemaManager, + translationCache: schemaManager.cypherTranslationCacheForConnection(physicalConnection), queryExecMode: cfg.QueryExecMode, queryResultsFormat: cfg.QueryResultFormats, ctx: ctx, conn: conn, + isolation: cfg.Options.IsoLevel, targetSchemaSet: false, } @@ -68,10 +117,14 @@ func newTransactionWrapper(ctx context.Context, conn *pgxpool.Conn, schemaManage wrapper.tx = pgxTx } } + if wrapper.tx != nil && stableSnapshotIsolation(wrapper.isolation) { + wrapper.topologyRouteDecisions = newTopologyRouteDecisionCache() + } return wrapper, nil } +// driver returns an inspected executor backed by the active transaction or, when absent, the pooled connection. func (s *transaction) driver() driver { if s.tx != nil { return inspectingDriver{ @@ -84,10 +137,12 @@ func (s *transaction) driver() driver { } } +// GraphQueryMemoryLimit returns the memory limit applied to graph query processing. func (s *transaction) GraphQueryMemoryLimit() size.Size { return s.schemaManager.graphQueryMemoryLimit } +// WithGraph selects schema as the target graph for subsequent operations. func (s *transaction) WithGraph(schema graph.Graph) graph.Transaction { s.targetSchema = schema s.targetSchemaSet = true @@ -95,13 +150,17 @@ func (s *transaction) WithGraph(schema graph.Graph) graph.Transaction { return s } +// Close rolls back an active PostgreSQL transaction and invalidates its route observations. func (s *transaction) Close() { + s.invalidateTopologyRouteDecisions() if s.tx != nil { s.tx.Rollback(s.ctx) s.tx = nil } } +// getTargetGraph resolves the explicitly selected graph or falls back to the +// driver's default graph. func (s *transaction) getTargetGraph() (model.Graph, error) { if !s.targetSchemaSet { // Look for a default graph target @@ -115,6 +174,7 @@ func (s *transaction) getTargetGraph() (model.Graph, error) { return s.schemaManager.AssertGraph(s, s.targetSchema) } +// targetGraphID resolves the database ID of the transaction's explicit or default graph target. func (s *transaction) targetGraphID() (int32, error) { if graphTarget, err := s.getTargetGraph(); err != nil { return 0, err @@ -123,6 +183,7 @@ func (s *transaction) targetGraphID() (int32, error) { } } +// CreateNode inserts a node with properties and kinds into the target graph. func (s *transaction) CreateNode(properties *graph.Properties, kinds ...graph.Kind) (*graph.Node, error) { if graphTarget, err := s.getTargetGraph(); err != nil { return nil, err @@ -150,7 +211,9 @@ func (s *transaction) CreateNode(properties *graph.Properties, kinds ...graph.Ki } } +// UpdateNode applies a node's pending kind and property changes to the target graph. func (s *transaction) UpdateNode(node *graph.Node) error { + s.invalidateTopologyRouteDecisions() var ( properties = node.Properties updateStatements []graph.Criteria @@ -178,12 +241,14 @@ func (s *transaction) UpdateNode(node *graph.Node) error { }, updateStatements...) } +// Nodes creates a node query scoped to the transaction's current graph target. func (s *transaction) Nodes() graph.NodeQuery { return &nodeQuery{ liveQuery: newLiveQuery(s.ctx, s, s.targetGraphID), } } +// CreateRelationshipByIDs inserts a relationship with the supplied endpoints, kind, and properties. func (s *transaction) CreateRelationshipByIDs(startNodeID, endNodeID graph.ID, kind graph.Kind, properties *graph.Properties) (*graph.Relationship, error) { if graphTarget, err := s.getTargetGraph(); err != nil { return nil, err @@ -213,7 +278,9 @@ func (s *transaction) CreateRelationshipByIDs(startNodeID, endNodeID graph.ID, k } } +// UpdateRelationship applies a relationship's pending property changes to the target graph. func (s *transaction) UpdateRelationship(relationship *graph.Relationship) error { + s.invalidateTopologyRouteDecisions() var ( modifiedProperties = relationship.Properties.ModifiedProperties() deletedProperties = relationship.Properties.DeletedProperties() @@ -256,12 +323,15 @@ func (s *transaction) UpdateRelationship(relationship *graph.Relationship) error return err } +// Relationships creates a relationship query scoped to the transaction's current graph target. func (s *transaction) Relationships() graph.RelationshipQuery { return &relationshipQuery{ liveQuery: newLiveQuery(s.ctx, s, s.targetGraphID), } } +// query executes SQL with the transaction's configured execution mode and +// result format, adding named parameters when present. func (s *transaction) query(query string, parameters map[string]any) (pgx.Rows, error) { queryArgs := []any{s.queryExecMode, s.queryResultsFormat} @@ -272,20 +342,181 @@ func (s *transaction) query(query string, parameters map[string]any) (pgx.Rows, return s.driver().Query(s.ctx, query, queryArgs...) } +// Query parses and translates Cypher through the schema caches, returning translation failures as graph results. func (s *transaction) Query(query string, parameters map[string]any) graph.Result { - if graphTarget, err := s.getTargetGraph(); err != nil { + if cypherMayMutate(query) { + s.invalidateTopologyRouteDecisions() + } + profile := SQLGenerationProfile{QueryClass: sqlGenerationQueryClass(query)} + if profile.QueryClass == "shortest_path" { + if provider, ok := s.schemaManager.translationCacheProvider.(StableSnapshotTraversalWorkspaceProvider); ok { + if err := provider.EnsureStableSnapshotTraversalWorkspaces(s.ctx, s.conn); err != nil { + s.recordSQLGenerationProfile(profile) + return graph.NewErrorResult(err) + } + } + } + parseStarted := time.Now() + parsedQuery, _, err := s.schemaManager.parseCache.Parse(query) + profile.Parse = time.Since(parseStarted) + if err != nil { + s.recordSQLGenerationProfile(profile) + return graph.NewErrorResult(err) + } + graphStarted := time.Now() + graphTarget, err := s.getTargetGraph() + profile.Graph = time.Since(graphStarted) + if err != nil { + s.recordSQLGenerationProfile(profile) return graph.NewErrorResult(err) + } + policyStarted := time.Now() + shape := TraversalShape{} + if s.schemaManager.shouldClassifyTraversal() { + shape, _ = s.schemaManager.classifyTraversalShape(query, parsedQuery) + } + policy, policyIdentity := s.schemaManager.effectiveTraversalPolicyForShape(query, shape, s.isolation) + topologyPolicy, topologyPolicyIdentity := s.schemaManager.topologyFixedSuffixPolicyForShape(shape, s.isolation) + topologyEstimatorVersion := "" + maximumEdgeToNodeRatioPerMille := int64(0) + if topologyPolicy.enabled() { + topologyEstimatorVersion = topologyPolicy.compiledManifest.TopologyEstimatorVersion + maximumEdgeToNodeRatioPerMille = topologyPolicy.compiledManifest.TopologyThresholds["maximum_edge_to_node_ratio_per_mille"] + } + topologyCandidate := s.topologyRouteDecision(graphTarget.ID, shape, parameters, topologyPolicyIdentity, topologyEstimatorVersion, maximumEdgeToNodeRatioPerMille, topologyPolicy.enabled(), topologyPolicy.EnableTopologyFixedSuffixFirstUse) + profile.Policy = time.Since(policyStarted) + if topologyCandidate { + policy = topologyPolicy + policyIdentity = topologyPolicyIdentity + } + s.schemaManager.observeTraversalStrategySelection(query, shape, policy) + buildTranslation := func(activePolicy TraversalPolicy) func() (translate.Result, string, error) { + return func() (translate.Result, string, error) { + var translated translate.Result + var translateErr error + translateStarted := time.Now() + if activePolicy.enabled() { + if options, optionsErr := activePolicy.productionOptionsForShape(query, shape); optionsErr != nil { + return translate.Result{}, "", optionsErr + } else { + translated, translateErr = translate.TranslateWithProductionOptions(s.ctx, parsedQuery, s.schemaManager, parameters, graphTarget.ID, options) + } + } else { + translated, translateErr = translate.Translate(s.ctx, parsedQuery, s.schemaManager, parameters, graphTarget.ID) + } + profile.Translate += time.Since(translateStarted) + if translateErr != nil { + return translate.Result{}, "", translateErr + } + formatStarted := time.Now() + formatted, formatErr := translate.Translated(translated) + profile.Format += time.Since(formatStarted) + if formatErr == nil && activePolicy.enabled() && activePolicy.compiledManifest.Version == 2 { + if anchorErr := validateTraversalPromotionSQLAnchor(activePolicy.compiledManifest, formatted); anchorErr != nil { + return translate.Result{}, "", anchorErr + } + } + return translated, formatted, formatErr + } + } + translateCached := func(activePolicy TraversalPolicy, identity string) (string, map[string]any, error) { + builder := buildTranslation(activePolicy) + if s.translationCache == nil { + translated, formatted, err := builder() + if err != nil { + return "", nil, err + } + return formatted, translated.Parameters, nil + } + return s.translationCache.TranslateWithPolicy(query, graphTarget.ID, parameters, identity, builder) + } + if topologyCandidate { + cacheStarted := time.Now() + candidateSQL, candidateParameters, candidateErr := translateCached(policy, policyIdentity) + if candidateErr != nil { + profile.Cache = time.Since(cacheStarted) + s.recordSQLGenerationProfile(profile) + return graph.NewErrorResult(candidateErr) + } + fallbackSQL, fallbackParameters, fallbackErr := translateCached(TraversalPolicy{}, policyIdentity+"-fallback") + profile.Cache = time.Since(cacheStarted) + if fallbackErr != nil { + s.recordSQLGenerationProfile(profile) + return graph.NewErrorResult(fallbackErr) + } + dispatchStarted := time.Now() + result := s.RawSuffixReverseRetry(candidateSQL, fallbackSQL, candidateParameters, fallbackParameters, SuffixReverseRetryLimits{ + OutputRows: policy.compiledManifest.Caps["output_row_limit"], + OutputBytes: policy.compiledManifest.Caps["output_bytes_limit"], + }) + profile.Dispatch = time.Since(dispatchStarted) + s.recordSQLGenerationProfile(profile) + return result + } + buildCurrentTranslation := buildTranslation(policy) + + var sqlQuery string + var translatedParameters map[string]any + cacheStarted := time.Now() + if s.translationCache == nil { + translated, translatedSQL, translateErr := buildCurrentTranslation() + if translateErr != nil { + profile.Cache = time.Since(cacheStarted) + s.recordSQLGenerationProfile(profile) + return graph.NewErrorResult(translateErr) + } + sqlQuery, translatedParameters = translatedSQL, translated.Parameters } else { - sqlQuery, bindings, err := s.schemaManager.compileText(s.ctx, query, parameters, graphTarget.ID) - if err != nil { - return graph.NewErrorResult(err) + var translateErr error + sqlQuery, translatedParameters, translateErr = s.translationCache.TranslateWithPolicy(query, graphTarget.ID, parameters, policyIdentity, buildCurrentTranslation) + if translateErr != nil { + profile.Cache = time.Since(cacheStarted) + s.recordSQLGenerationProfile(profile) + return graph.NewErrorResult(translateErr) } + } + profile.Cache = time.Since(cacheStarted) + dispatchStarted := time.Now() + result := s.raw(sqlQuery, translatedParameters) + profile.Dispatch = time.Since(dispatchStarted) + s.recordSQLGenerationProfile(profile) + return result +} + +// recordSQLGenerationProfile sends generation timing to an optional connection collector. +func (s *transaction) recordSQLGenerationProfile(profile SQLGenerationProfile) { + if collector, ok := s.schemaManager.translationCacheProvider.(SQLGenerationProfileCollector); ok { + collector.RecordSQLGenerationProfile(profile) + } +} - return s.Raw(sqlQuery, bindings) +// sqlGenerationQueryClass groups a query for low-cardinality generation timing telemetry. +func sqlGenerationQueryClass(query string) string { + if strings.Contains(strings.ToLower(query), "shortestpath") { + return "shortest_path" } + return "other" } +// cypherMayMutate conservatively detects mutation clauses that invalidate route observations. +func cypherMayMutate(value string) bool { + lower := strings.ToLower(value) + for _, keyword := range []string{" create ", " merge ", " delete ", " detach delete ", " set ", " remove "} { + if strings.Contains(" "+lower+" ", keyword) { + return true + } + } + return false +} + +// Raw executes PostgreSQL SQL directly after invalidating topology route observations. func (s *transaction) Raw(query string, parameters map[string]any) graph.Result { + s.invalidateTopologyRouteDecisions() + return s.raw(query, parameters) +} + +// raw executes PostgreSQL SQL without changing transaction routing state. +func (s *transaction) raw(query string, parameters map[string]any) graph.Result { if rows, err := s.query(query, parameters); err != nil { return graph.NewErrorResult(err) } else { @@ -297,7 +528,9 @@ func (s *transaction) Raw(query string, parameters map[string]any) graph.Result } } +// Commit invalidates route observations and commits the active PostgreSQL transaction. func (s *transaction) Commit() error { + s.invalidateTopologyRouteDecisions() if s.tx != nil { return s.tx.Commit(s.ctx) } diff --git a/drivers/pg/translation_cache.go b/drivers/pg/translation_cache.go index ed8a8e9a..3e0b2ed2 100644 --- a/drivers/pg/translation_cache.go +++ b/drivers/pg/translation_cache.go @@ -54,9 +54,9 @@ func (s *translationCacheCall) finish(reusable bool) { }) } -// TranslationCacheStats contains aggregate counters only. It intentionally +// CompilationCacheStats contains aggregate counters only. It intentionally // exposes no source text, SQL, parameter names, values, or connection data. -type TranslationCacheStats struct { +type CompilationCacheStats struct { Hits int64 Misses int64 Coalesced int64 @@ -235,7 +235,7 @@ func (s *translationCache) GetOrBuildContext(ctx context.Context, key translatio s.lock.Unlock() sql, result, err, panicked := callTranslationBuild(build) - entry, cacheable := cacheableTranslation(sql, result, parameters, err) + entry, cacheable := cacheableCompilation(sql, result, parameters, err) s.lock.Lock() currentGeneration := s.schemaGeneration.Load() @@ -309,7 +309,7 @@ func cloneTranslationCacheKey(key translationCacheKey) translationCacheKey { return key } -func cacheableTranslation(sql string, result translationCacheBuildResult, parameters map[string]any, err error) (translationCacheEntry, bool) { +func cacheableCompilation(sql string, result translationCacheBuildResult, parameters map[string]any, err error) (translationCacheEntry, bool) { if err != nil { return translationCacheEntry{}, false } @@ -381,7 +381,7 @@ func (s *translationCache) Close() { } } -func (s *translationCache) Stats() TranslationCacheStats { +func (s *translationCache) Stats() CompilationCacheStats { s.lock.Lock() defer s.lock.Unlock() @@ -390,7 +390,7 @@ func (s *translationCache) Stats() TranslationCacheStats { size = s.entries.Stats().Size() } - return TranslationCacheStats{ + return CompilationCacheStats{ Hits: s.hits.Load(), Misses: s.misses.Load(), Coalesced: s.coalesced.Load(), diff --git a/drivers/pg/translation_cache_contract.go b/drivers/pg/translation_cache_contract.go new file mode 100644 index 00000000..c66d6bdc --- /dev/null +++ b/drivers/pg/translation_cache_contract.go @@ -0,0 +1,79 @@ +package pg + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + model "github.com/specterops/dawgs/cypher/models/pgsql" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// CypherTranslationCache implements reusable Cypher-to-SQL translation. +// Implementations must not retain caller parameter values or mutable maps. +type CypherTranslationCache interface { + TranslateWithPolicy( + query string, + graphID int32, + parameters map[string]any, + policyIdentity string, + build func() (translate.Result, string, error), + ) (string, map[string]any, error) +} + +// CypherTranslationCacheProvider selects cache ownership for a physical +// connection. Returning nil deliberately bypasses translation retention. +type CypherTranslationCacheProvider interface { + CacheForConnection(conn *pgx.Conn) CypherTranslationCache +} + +// TranslationParameterTypeKey encodes sorted parameter names and negotiated +// PostgreSQL data types into an unambiguous cache-key component. +func TranslationParameterTypeKey(parameters map[string]any) string { + keys := make([]string, 0, len(parameters)) + for key := range parameters { + keys = append(keys, key) + } + sort.Strings(keys) + var key strings.Builder + for _, name := range keys { + value := parameters[name] + var typeName string + if value == nil { + typeName = "null" + } else if dataType, err := model.ValueToDataType(value); err == nil { + typeName = dataType.String() + } else { + typeName = fmt.Sprintf("invalid:%T", value) + } + key.WriteString(strconv.Itoa(len(name))) + key.WriteByte(':') + key.WriteString(name) + key.WriteString(strconv.Itoa(len(typeName))) + key.WriteByte(':') + key.WriteString(typeName) + } + return key.String() +} + +func translationParameterTypeKey(parameters map[string]any) string { + return TranslationParameterTypeKey(parameters) +} + +func cacheableTranslation(result translate.Result, parameters map[string]any) bool { + if len(result.Parameters) != len(result.ParameterSources) { + return false + } + for identifier := range result.Parameters { + source, found := result.ParameterSources[identifier] + if !found || source == "" { + return false + } + if _, found := parameters[source]; !found { + return false + } + } + return true +} diff --git a/drivers/pg/translation_cache_test.go b/drivers/pg/translation_cache_test.go index 356c15e7..5bacf6e9 100644 --- a/drivers/pg/translation_cache_test.go +++ b/drivers/pg/translation_cache_test.go @@ -13,7 +13,7 @@ import ( "github.com/stretchr/testify/require" ) -func cacheableBuild(sql string, parameters map[string]any, sources map[string]string) func() (string, translationCacheBuildResult, error) { +func cacheableCompilationBuild(sql string, parameters map[string]any, sources map[string]string) func() (string, translationCacheBuildResult, error) { return func() (string, translationCacheBuildResult, error) { return sql, translationCacheBuildResult{ parameters: parameters, @@ -27,7 +27,7 @@ func TestTranslationCacheRebindsCurrentValues(t *testing.T) { firstParameters := map[string]any{"needle": graph.ID(1)} key := translationCache.Key("RETURN $needle", 7, firstParameters) - _, bindings, err := translationCache.GetOrBuild(key, firstParameters, cacheableBuild("select @p0", map[string]any{"p0": uint64(1)}, map[string]string{"p0": "needle"})) + _, bindings, err := translationCache.GetOrBuild(key, firstParameters, cacheableCompilationBuild("select @p0", map[string]any{"p0": uint64(1)}, map[string]string{"p0": "needle"})) require.NoError(t, err) require.Equal(t, map[string]any{"p0": uint64(1)}, bindings) @@ -100,7 +100,7 @@ func TestTranslationCacheBypassesOversizedSource(t *testing.T) { } func TestCacheableTranslationRequiresExactParameterProvenance(t *testing.T) { - _, cacheable := cacheableTranslation("select @p0", translationCacheBuildResult{ + _, cacheable := cacheableCompilation("select @p0", translationCacheBuildResult{ parameters: map[string]any{"p0": int64(1)}, parameterSources: map[string]string{"p1": "value"}, }, map[string]any{"value": int64(1)}, nil) @@ -112,9 +112,9 @@ func TestTranslationCacheReportsEviction(t *testing.T) { translationCache := newTranslationCache(1) parameters := map[string]any{"value": int64(1)} - _, _, err := translationCache.GetOrBuild(translationCache.Key("RETURN $value", 1, parameters), parameters, cacheableBuild("select @p0", map[string]any{"p0": int64(1)}, map[string]string{"p0": "value"})) + _, _, err := translationCache.GetOrBuild(translationCache.Key("RETURN $value", 1, parameters), parameters, cacheableCompilationBuild("select @p0", map[string]any{"p0": int64(1)}, map[string]string{"p0": "value"})) require.NoError(t, err) - _, _, err = translationCache.GetOrBuild(translationCache.Key("RETURN $value + 1", 1, parameters), parameters, cacheableBuild("select @p0", map[string]any{"p0": int64(1)}, map[string]string{"p0": "value"})) + _, _, err = translationCache.GetOrBuild(translationCache.Key("RETURN $value + 1", 1, parameters), parameters, cacheableCompilationBuild("select @p0", map[string]any{"p0": int64(1)}, map[string]string{"p0": "value"})) require.NoError(t, err) stats := translationCache.Stats() @@ -262,7 +262,7 @@ func TestTranslationCacheWaiterCancellationDoesNotInterruptLeader(t *testing.T) func TestTranslationCacheParameterlessHitAndClose(t *testing.T) { translationCache := newTranslationCache(1) key := translationCache.Key("RETURN 1", 1, nil) - build := cacheableBuild("select 1", map[string]any{}, map[string]string{}) + build := cacheableCompilationBuild("select 1", map[string]any{}, map[string]string{}) _, bindings, err := translationCache.GetOrBuild(key, nil, build) require.NoError(t, err) @@ -357,7 +357,7 @@ func TestTranslationCachePanicReleasesWaiters(t *testing.T) { <-leaderStarted go func() { - _, _, err := translationCache.GetOrBuild(key, parameters, cacheableBuild( + _, _, err := translationCache.GetOrBuild(key, parameters, cacheableCompilationBuild( "select @p0", map[string]any{"p0": int64(1)}, map[string]string{"p0": "value"}, @@ -397,7 +397,7 @@ func TestTranslationCacheCloseReleasesWaiters(t *testing.T) { <-leaderStarted go func() { - _, _, err := translationCache.GetOrBuildContext(context.Background(), key, parameters, cacheableBuild( + _, _, err := translationCache.GetOrBuildContext(context.Background(), key, parameters, cacheableCompilationBuild( "select @p0", map[string]any{"p0": int64(1)}, map[string]string{"p0": "value"}, diff --git a/drivers/pg/traversal_policy.go b/drivers/pg/traversal_policy.go new file mode 100644 index 00000000..23e797f0 --- /dev/null +++ b/drivers/pg/traversal_policy.go @@ -0,0 +1,1047 @@ +package pg + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "path/filepath" + "slices" + "sort" + "strings" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" +) + +// TraversalPolicy is a default-off, query-allowlisted production canary. A +// generation is mandatory whenever a candidate is enabled and is included in +// the translation cache identity. +type TraversalPolicy struct { + // Generation distinguishes policy revisions in cache and rollout identities. + Generation uint64 `json:"generation"` + + // PromotionManifestSHA256 verifies the immutable promotion manifest supplied below. + PromotionManifestSHA256 string `json:"promotion_manifest_sha256"` + + // PromotionManifestJSON is the exact verified authorization document. It + // is intentionally excluded from policy serialization; its digest and + // content-derived fields form the cache identity. + PromotionManifestJSON json.RawMessage `json:"-"` + // QuerySHA256Allowlist is the exact set of query digests authorized by the manifest. + QuerySHA256Allowlist []string `json:"query_sha256_allowlist"` + + // ShortestPathExecutor selects the manifest-backed shortest-path candidate. + ShortestPathExecutor optimize.ShortestPathExecutor `json:"shortest_path_executor,omitempty"` + + // EnableExpansionOrientation enables the manifest-backed expansion orientation candidate. + EnableExpansionOrientation bool `json:"enable_expansion_orientation,omitempty"` + // EnableTopologyFixedSuffix permits the manifest-v4, snapshot-owned + // fixed-suffix candidate. It never changes ordinary routing: the + // transaction route-decision cache must independently select the arm. + EnableTopologyFixedSuffix bool `json:"enable_topology_fixed_suffix,omitempty"` + // EnableTopologyFixedSuffixFirstUse permits the manifest-v5 first-use + // selector. It is deliberately independent of the v4 cache-hit protocol. + EnableTopologyFixedSuffixFirstUse bool `json:"enable_topology_fixed_suffix_first_use,omitempty"` + // DisableExpansionOrientation is an evidence-free emergency rollback switch + // for any manifest-authorized orientation selector. + DisableExpansionOrientation bool `json:"disable_expansion_orientation,omitempty"` + // DisableTopologyFixedSuffix is the evidence-free emergency rollback + // switch for the manifest-v4 fixed-suffix candidate. + DisableTopologyFixedSuffix bool `json:"disable_topology_fixed_suffix,omitempty"` + // DisableTopologyFixedSuffixFirstUse is the emergency rollback for v5. + DisableTopologyFixedSuffixFirstUse bool `json:"disable_topology_fixed_suffix_first_use,omitempty"` + // DisableEndpointSeededReverse immediately disables the endpoint-seeded reverse candidate. + DisableEndpointSeededReverse bool `json:"disable_endpoint_seeded_reverse,omitempty"` + + // DisableInlineASPDAG immediately disables the inline ASP DAG candidate. + DisableInlineASPDAG bool `json:"disable_inline_asp_dag,omitempty"` + + // DisableInlineSPWitness immediately disables the inline shortest-path witness candidate. + DisableInlineSPWitness bool `json:"disable_inline_sp_witness,omitempty"` + // DisableInlineSPDistance is the emergency rollback switch for SP-I2-C-D. + DisableInlineSPDistance bool `json:"disable_inline_sp_distance,omitempty"` + // compiledManifest is the decoded manifest used during translation and selection. + compiledManifest traversalPromotionManifest + + // compiledBuckets maps every authorized query digest to its manifest bucket. + compiledBuckets map[string]traversalPromotionBucket + + // compiledIdentity keys translations generated under this immutable policy snapshot. + compiledIdentity string +} + +// enabled reports whether the policy changes any production translation behavior. +func (s TraversalPolicy) enabled() bool { + return s.ShortestPathExecutor != "" || s.EnableExpansionOrientation || s.EnableTopologyFixedSuffix || s.EnableTopologyFixedSuffixFirstUse || s.DisableExpansionOrientation || s.DisableTopologyFixedSuffix || s.DisableTopologyFixedSuffixFirstUse || s.DisableEndpointSeededReverse || s.DisableInlineASPDAG || s.DisableInlineSPWitness || s.DisableInlineSPDistance +} + +// rollbackActive reports whether an emergency rollback can change the SQL +// authorized by a promotion manifest. Rollback generations retain their own +// cache identity, but must not compare incumbent SQL with the candidate anchor. +func (s TraversalPolicy) rollbackActive() bool { + return s.DisableExpansionOrientation || s.DisableTopologyFixedSuffix || s.DisableTopologyFixedSuffixFirstUse || s.DisableEndpointSeededReverse || s.DisableInlineASPDAG || s.DisableInlineSPWitness || s.DisableInlineSPDistance +} + +// manifestCandidateEnabled reports whether this policy carries a candidate +// whose authorization depends on a promotion manifest. +func (s TraversalPolicy) manifestCandidateEnabled() bool { + return s.ShortestPathExecutor != "" || s.EnableExpansionOrientation || s.EnableTopologyFixedSuffix || s.EnableTopologyFixedSuffixFirstUse +} + +// rollbackSwitchCount returns the number of emergency controls enabled in the +// policy. Manifest-backed candidates may compose with at most one switch, and +// that switch must be the control dedicated to the candidate family. +func (s TraversalPolicy) rollbackSwitchCount() int { + count := 0 + for _, enabled := range []bool{ + s.DisableExpansionOrientation, + s.DisableTopologyFixedSuffix, + s.DisableTopologyFixedSuffixFirstUse, + s.DisableEndpointSeededReverse, + s.DisableInlineASPDAG, + s.DisableInlineSPWitness, + s.DisableInlineSPDistance, + } { + if enabled { + count++ + } + } + return count +} + +// matchingCandidateRollbackActive reports whether exactly one emergency +// switch is enabled and it belongs to the manifest-backed candidate. +func (s TraversalPolicy) matchingCandidateRollbackActive() bool { + if s.rollbackSwitchCount() != 1 { + return false + } + if s.EnableExpansionOrientation { + return s.DisableExpansionOrientation + } + if s.EnableTopologyFixedSuffix { + return s.DisableTopologyFixedSuffix + } + if s.EnableTopologyFixedSuffixFirstUse { + return s.DisableTopologyFixedSuffixFirstUse + } + switch s.ShortestPathExecutor { + case optimize.ShortestPathExecutorASPI1DAG: + return s.DisableInlineASPDAG + case optimize.ShortestPathExecutorI1CanonicalPredecessorWitness: + return s.DisableInlineSPWitness + case optimize.ShortestPathExecutorI2GuardedDistance: + return s.DisableInlineSPDistance + default: + return false + } +} + +// withoutManifestCandidate derives the incumbent-only form used by a matching +// emergency rollback. The installed manifest and policy remain immutable. +func (s TraversalPolicy) withoutManifestCandidate() TraversalPolicy { + s.ShortestPathExecutor = "" + s.EnableExpansionOrientation = false + s.EnableTopologyFixedSuffix = false + s.EnableTopologyFixedSuffixFirstUse = false + return s +} + +// productionOptions derives validated translation options from the active traversal policy. +func (s TraversalPolicy) productionOptions(query string) (translate.ProductionOptions, error) { + return s.productionOptionsForShape(query, TraversalShape{}) +} + +// productionOptionsForShape derives options and an authorization bucket for query and shape. +func (s TraversalPolicy) productionOptionsForShape(query string, shape TraversalShape) (translate.ProductionOptions, error) { + manifest := s.compiledManifest + if manifest.SelectorVersion == "" && len(s.PromotionManifestJSON) > 0 { + var err error + if manifest, err = decodeTraversalPromotionManifest(s.PromotionManifestJSON); err != nil { + return translate.ProductionOptions{}, fmt.Errorf("decode traversal promotion manifest: %w", err) + } + } + selectorVersion := manifest.SelectorVersion + if selectorVersion == "" { + selectorVersion = fmt.Sprintf("traversal-kill-switch-g%d", s.Generation) + if s.DisableExpansionOrientation && !s.DisableEndpointSeededReverse && !s.DisableInlineASPDAG && !s.DisableInlineSPWitness && !s.DisableInlineSPDistance { + selectorVersion = fmt.Sprintf("expansion-orientation-kill-switch-g%d", s.Generation) + } else if s.DisableEndpointSeededReverse && !s.DisableInlineASPDAG { + selectorVersion = fmt.Sprintf("endpoint-seeded-kill-switch-g%d", s.Generation) + } else if s.DisableInlineASPDAG && !s.DisableEndpointSeededReverse { + selectorVersion = fmt.Sprintf("inline-asp-kill-switch-g%d", s.Generation) + } + } + options := translate.ProductionOptions{ + ShortestPathExecutor: s.ShortestPathExecutor, + EnableExpansionOrientation: s.EnableExpansionOrientation && !s.DisableExpansionOrientation, + DisableEndpointSeededReverse: s.DisableEndpointSeededReverse, + DisableInlineASPDAG: s.DisableInlineASPDAG, + DisableInlineSPWitness: s.DisableInlineSPWitness, + DisableInlineSPDistance: s.DisableInlineSPDistance, + SelectorVersion: selectorVersion, + } + if (s.EnableTopologyFixedSuffix && !s.DisableTopologyFixedSuffix) || (s.EnableTopologyFixedSuffixFirstUse && !s.DisableTopologyFixedSuffixFirstUse) { + options.EnableTopologyFixedSuffix = true + options.TopologyFixedSuffixCaps = &translate.ProductionFixedSuffixCaps{ + SuffixRowLimit: manifest.Caps["suffix_row_limit"], + StateLimit: manifest.Caps["state_limit"], + OutputRowLimit: manifest.Caps["output_row_limit"], + OutputBytesLimit: manifest.Caps["output_bytes_limit"], + } + } + if options.EnableExpansionOrientation { + options.ExpansionOrientationPolicy = optimize.ExpansionSearchPolicy(manifest.SelectorVersion) + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG || s.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || s.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistance { + options.ShortestPathCaps = &translate.ProductionShortestPathCaps{ + StateLimit: manifest.Caps["state_limit"], + FrontierLimit: manifest.Caps["frontier_limit"], + PredecessorLimit: manifest.Caps["predecessor_limit"], + EnumerationLimit: manifest.Caps["enumeration_limit"], + OutputBytesLimit: manifest.Caps["output_bytes_limit"], + } + queryDigest := TraversalPolicyQuerySHA256(query) + if bucket, found := s.compiledBuckets[queryDigest]; found { + options.AuthorizedBucket = &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + } + } else if bucket, found := s.authorizedStructuralBucketForShape(shape); found { + options.AuthorizedBucket = &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + } + } else { + for _, bucket := range manifest.Buckets { + if !slices.Contains(bucket.QuerySHA256, queryDigest) { + continue + } + options.AuthorizedBucket = &translate.ProductionTraversalBucket{ + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + } + break + } + } + } + return options, nil +} + +// structuralBucketForShape reports an observation-only structural match. It +// does not authorize a candidate: manifest v2 remains exact-query gated until +// the structural evidence schema is independently verified. +func (s TraversalPolicy) structuralBucketForShape(shape TraversalShape) (traversalPromotionBucket, bool) { + if !shape.Available() || len(s.compiledManifest.Buckets) == 0 { + return traversalPromotionBucket{}, false + } + var matched *traversalPromotionBucket + for index := range s.compiledManifest.Buckets { + bucket := &s.compiledManifest.Buckets[index] + if bucket.Direction != shape.Direction || bucket.ObservationMode != shape.ObservationMode || + bucket.MinimumDepth != shape.MinimumDepth || bucket.MaximumDepth != shape.MaximumDepth || + bucket.RelationshipKindCount != shape.RelationshipKindCount || bucket.UntypedRelationship != shape.UntypedRelationship || + bucket.SuffixLength != shape.SuffixLength || bucket.CandidateStrategy != shape.CandidateStrategy { + continue + } + if matched != nil { + return traversalPromotionBucket{}, false + } + matched = bucket + } + if matched == nil { + return traversalPromotionBucket{}, false + } + return *matched, true +} + +// authorizedStructuralBucketForShape reports a v3 or v4 manifest-backed +// structural authorization. Version 2 buckets intentionally remain +// observation-only: their SQL anchor binds one exact query, not a reusable SQL +// template. A v4 match authorizes only the route-decision candidate path; it +// does not make ordinary translation select that candidate. +func (s TraversalPolicy) authorizedStructuralBucketForShape(shape TraversalShape) (traversalPromotionBucket, bool) { + if !shape.Available() || (s.compiledManifest.Version != 3 && s.compiledManifest.Version != 4 && s.compiledManifest.Version != 5) { + return traversalPromotionBucket{}, false + } + var matched *traversalPromotionBucket + for index := range s.compiledManifest.Buckets { + bucket := &s.compiledManifest.Buckets[index] + if bucket.StructuralShapeVersion != shape.Version || bucket.StructuralShapeSHA256 != shape.Fingerprint { + continue + } + if matched != nil { + return traversalPromotionBucket{}, false + } + matched = bucket + } + if matched == nil { + return traversalPromotionBucket{}, false + } + return *matched, true +} + +// structuralSQLTemplateSHA256 returns bucket's required template digest under manifest. +func structuralSQLTemplateSHA256(manifest traversalPromotionManifest, bucket traversalPromotionBucket) string { + return TraversalSQLTemplateSHA256(manifest.Candidate, manifest.SelectorVersion, manifest.ExecutionBoundary, TraversalShape{ + Version: bucket.StructuralShapeVersion, + Family: bucket.StructuralFamily, + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + SuffixLength: bucket.SuffixLength, + CandidateStrategy: bucket.CandidateStrategy, + Fingerprint: bucket.StructuralShapeSHA256, + }) +} + +// TraversalSQLTemplateSHA256 returns the public template-contract digest for +// a v3 or v4 structural bucket. It binds the candidate and every SQL-shaping +// static fact, but intentionally excludes Cypher identifiers and caller +// values. +func TraversalSQLTemplateSHA256(candidate, selectorVersion, executionBoundary string, shape TraversalShape) string { + if shape.Version == TraversalFixedSuffixShapeVersion { + canonical := fmt.Sprintf( + "topology-sql-template-v1|%s|%s|%s|%s|%s|%s|%s|%s|%d|%d|%d|%s", + candidate, selectorVersion, executionBoundary, + shape.Version, shape.Family, shape.Fingerprint, + shape.Direction, shape.ObservationMode, shape.MinimumDepth, shape.MaximumDepth, + shape.SuffixLength, shape.CandidateStrategy, + ) + digest := sha256.Sum256([]byte(canonical)) + return hex.EncodeToString(digest[:]) + } + canonical := fmt.Sprintf( + "structural-sql-template-v1|%s|%s|%s|%s|%s|%s|%s|%s|%d|%d|%d|%t", + candidate, selectorVersion, executionBoundary, + shape.Version, shape.Family, shape.Fingerprint, + shape.Direction, shape.ObservationMode, shape.MinimumDepth, + shape.MaximumDepth, shape.RelationshipKindCount, shape.UntypedRelationship, + ) + digest := sha256.Sum256([]byte(canonical)) + return hex.EncodeToString(digest[:]) +} + +// traversalPromotionBucket binds the workload, shape, and SQL contract that authorizes one candidate. +type traversalPromotionBucket struct { + // Name identifies the qualified workload bucket. + Name string `json:"name,omitempty"` + // QuerySHA256 contains the exact query digests admitted to this bucket. + QuerySHA256 []string `json:"query_sha256"` + // QualificationSplit assigns the workload to training, holdout, or diagnostic evidence. + QualificationSplit []string `json:"qualification_split"` + // Direction selects the traversal orientation covered by the contract. + Direction string `json:"direction,omitempty"` + // ObservationMode identifies whether the traversal observes a distance, one path, or all paths. + ObservationMode string `json:"observation_mode,omitempty"` + // MinimumDepth sets the inclusive lower traversal-depth bound. + MinimumDepth int64 `json:"minimum_depth,omitempty"` + // MaximumDepth sets the inclusive upper traversal-depth bound. + MaximumDepth int64 `json:"maximum_depth,omitempty"` + // RelationshipKindCount is the number of relationship kinds constrained by the bucket. + RelationshipKindCount int `json:"relationship_kind_count,omitempty"` + // UntypedRelationship reports whether the bucket permits any relationship kind. + UntypedRelationship bool `json:"untyped_relationship,omitempty"` + // SuffixLength binds the fixed terminal suffix width for a v4 bucket. + SuffixLength int `json:"suffix_length,omitempty"` + // CandidateStrategy binds the optimizer-provided fixed-suffix candidate. + CandidateStrategy string `json:"candidate_strategy,omitempty"` + // StructuralShapeVersion identifies the canonical structural classifier + // used when this bucket authorizes production-wide selection. + StructuralShapeVersion string `json:"structural_shape_version,omitempty"` + // StructuralFamily identifies the shortest-path family bound by the + // structural classifier. + StructuralFamily string `json:"structural_family,omitempty"` + // StructuralShapeSHA256 binds the classifier output without retaining the + // source query text. + StructuralShapeSHA256 string `json:"structural_shape_sha256,omitempty"` + // SQLTemplateSHA256 binds the candidate SQL template contract for a + // structural bucket. Unlike the v2 SQL anchor, it is independent of Cypher + // identifiers and parameters. + SQLTemplateSHA256 string `json:"sql_template_sha256,omitempty"` +} + +// traversalPromotionEvidence identifies an independently verifiable promotion-evidence artifact. +type traversalPromotionEvidence struct { + // Path identifies the evidence document relative to its manifest. + Path string `json:"path,omitempty"` + // SHA256 verifies the evidence artifact's exact content. + SHA256 string `json:"sha256"` +} + +// traversalPromotionManifest binds the immutable inputs authorized for traversal promotion. +type traversalPromotionManifest struct { + // Version identifies the immutable manifest schema and routing protocol. + Version int `json:"version"` + // Candidate identifies the execution strategy being evaluated or authorized. + Candidate string `json:"candidate"` + // SelectorVersion identifies the candidate-selection protocol. + SelectorVersion string `json:"selector_version"` + // ExecutionBoundary identifies where the candidate and fallback are coordinated. + ExecutionBoundary string `json:"execution_boundary"` + // FallbackExecutor identifies the incumbent used if the candidate cannot complete. + FallbackExecutor string `json:"fallback_executor,omitempty"` + // SourceCommit identifies the source revision qualified by the promotion evidence. + SourceCommit string `json:"source_commit"` + // SourceSHA256 binds the referenced source content by SHA-256 digest. + SourceSHA256 string `json:"source_sha256"` + // BinarySHA256 binds the referenced binary content by SHA-256 digest. + BinarySHA256 string `json:"binary_sha256"` + // CorpusSHA256 binds the referenced corpus content by SHA-256 digest. + CorpusSHA256 string `json:"corpus_sha256"` + // OperationalCandidateSQLSHA256 independently freezes the exact SQL used + // by the operational candidate matrix. + OperationalCandidateSQLSHA256 string `json:"operational_candidate_sql_sha256"` + // TopologyEstimatorVersion binds the frozen synopsis estimator for v4. + TopologyEstimatorVersion string `json:"topology_estimator_version,omitempty"` + // SynopsisSchemaVersion binds the published synopsis schema required by v4. + SynopsisSchemaVersion string `json:"synopsis_schema_version,omitempty"` + // RouteCacheProtocol binds the transaction-local v4 route-cache contract. + RouteCacheProtocol string `json:"route_cache_protocol,omitempty"` + // TopologyThresholds bind the immutable topology estimator thresholds for + // a manifest-v4 candidate. + TopologyThresholds map[string]int64 `json:"topology_thresholds,omitempty"` + // Caps maps each guarded resource dimension to its immutable limit. + Caps map[string]int64 `json:"caps"` + // Buckets contains the workload and structural scopes authorized for the candidate. + Buckets []traversalPromotionBucket `json:"buckets"` + // Evidence maps each required qualification role to its verified artifact. + Evidence map[string]traversalPromotionEvidence `json:"evidence"` +} + +// decodeTraversalPromotionManifest decodes one complete, duplicate-key-free promotion manifest. +func decodeTraversalPromotionManifest(raw []byte) (traversalPromotionManifest, error) { + var manifest traversalPromotionManifest + if len(raw) == 0 { + return manifest, fmt.Errorf("enabled traversal policy requires the verified promotion manifest JSON") + } + if err := rejectDuplicateTraversalManifestKeys(raw); err != nil { + return manifest, fmt.Errorf("decode promotion manifest: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&manifest); err != nil { + return manifest, fmt.Errorf("decode promotion manifest: %w", err) + } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + if err == nil { + return manifest, fmt.Errorf("decode promotion manifest: trailing JSON data") + } + return manifest, fmt.Errorf("decode promotion manifest trailing data: %w", err) + } + return manifest, nil +} + +// rejectDuplicateTraversalManifestKeys rejects duplicate object keys at every +// depth. The standard decoder otherwise accepts the final duplicate value, +// which makes authorization documents ambiguous across implementations. +func rejectDuplicateTraversalManifestKeys(raw []byte) error { + decoder := json.NewDecoder(bytes.NewReader(raw)) + decoder.UseNumber() + if err := rejectDuplicateTraversalJSONValue(decoder); err != nil { + return err + } + if token, err := decoder.Token(); err != io.EOF { + if err == nil { + return fmt.Errorf("trailing JSON data after %v", token) + } + return err + } + return nil +} + +// rejectDuplicateTraversalJSONValue recursively rejects duplicate keys in one JSON value. +func rejectDuplicateTraversalJSONValue(decoder *json.Decoder) error { + token, err := decoder.Token() + if err != nil { + return err + } + delimiter, composite := token.(json.Delim) + if !composite { + return nil + } + switch delimiter { + case '{': + seen := map[string]struct{}{} + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return err + } + key, ok := keyToken.(string) + if !ok { + return fmt.Errorf("object key is not a string") + } + if _, duplicate := seen[key]; duplicate { + return fmt.Errorf("duplicate JSON object key %q", key) + } + seen[key] = struct{}{} + if err := rejectDuplicateTraversalJSONValue(decoder); err != nil { + return err + } + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim('}') { + return fmt.Errorf("object has invalid closing delimiter") + } + case '[': + for decoder.More() { + if err := rejectDuplicateTraversalJSONValue(decoder); err != nil { + return err + } + } + closing, err := decoder.Token() + if err != nil { + return err + } + if closing != json.Delim(']') { + return fmt.Errorf("array has invalid closing delimiter") + } + default: + return fmt.Errorf("unexpected JSON delimiter %q", delimiter) + } + return nil +} + +// validate verifies that a policy's candidate, manifest, and rollback controls agree. +func (s TraversalPolicy) validate() error { + if !s.enabled() { + return nil + } + if s.Generation == 0 { + return fmt.Errorf("enabled traversal policy requires a nonzero generation") + } + candidateFamilies := 0 + for _, enabled := range []bool{s.ShortestPathExecutor != "", s.EnableExpansionOrientation, s.EnableTopologyFixedSuffix, s.EnableTopologyFixedSuffixFirstUse} { + if enabled { + candidateFamilies++ + } + } + if candidateFamilies > 1 { + return fmt.Errorf("one traversal policy generation may enable only one candidate family") + } + if s.manifestCandidateEnabled() && s.rollbackActive() && !s.matchingCandidateRollbackActive() { + return fmt.Errorf("a manifest-backed traversal candidate may be combined only with its single matching emergency rollback switch") + } + if !s.manifestCandidateEnabled() && s.rollbackActive() { + if s.PromotionManifestSHA256 != "" || len(s.PromotionManifestJSON) != 0 || len(s.QuerySHA256Allowlist) != 0 { + return fmt.Errorf("a standalone traversal rollback policy must not carry promotion manifest or query authorization fields") + } + return nil + } + if !lowerHexSHA256(s.PromotionManifestSHA256) { + return fmt.Errorf("enabled traversal policy requires a lowercase promotion manifest SHA-256 digest") + } + manifest, err := decodeTraversalPromotionManifest(s.PromotionManifestJSON) + if err != nil { + return err + } + digest := sha256.Sum256(s.PromotionManifestJSON) + if hex.EncodeToString(digest[:]) != s.PromotionManifestSHA256 { + return fmt.Errorf("promotion manifest content does not match its SHA-256 digest") + } + if (manifest.Version != 2 && manifest.Version != 3 && manifest.Version != 4 && manifest.Version != 5) || strings.TrimSpace(manifest.SelectorVersion) == "" { + return fmt.Errorf("promotion manifest requires version 2, 3, 4, or 5 and a selector version") + } + if strings.TrimSpace(manifest.SourceCommit) == "" || !lowerHexSHA256(manifest.SourceSHA256) || !lowerHexSHA256(manifest.BinarySHA256) || !lowerHexSHA256(manifest.CorpusSHA256) { + return fmt.Errorf("promotion manifest requires source commit and lowercase source, binary, and corpus SHA-256 digests") + } + if !lowerHexSHA256(manifest.OperationalCandidateSQLSHA256) { + return fmt.Errorf("promotion manifest requires a lowercase operational candidate SQL SHA-256 digest") + } + expectedCandidate := string(s.ShortestPathExecutor) + if s.EnableExpansionOrientation { + policy := optimize.ExpansionSearchPolicy(manifest.SelectorVersion) + if policy != optimize.ExpansionSearchPolicyOrientationProbeV1 && policy != optimize.ExpansionSearchPolicyOrientationProbeV2 { + return fmt.Errorf("unsupported production orientation selector %q", manifest.SelectorVersion) + } + expectedCandidate = string(policy) + } + if s.EnableTopologyFixedSuffix { + expectedCandidate = string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1) + } + if s.EnableTopologyFixedSuffixFirstUse { + expectedCandidate = string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) + } + if manifest.Candidate != expectedCandidate { + return fmt.Errorf("promotion manifest candidate %q does not authorize %q", manifest.Candidate, expectedCandidate) + } + expectedBoundary := "inline_statement" + if s.EnableExpansionOrientation { + expectedBoundary = "guarded_dual_arm" + } else if s.EnableTopologyFixedSuffix { + expectedBoundary = "transaction_retry" + } else if s.EnableTopologyFixedSuffixFirstUse { + expectedBoundary = "first_use_transaction_retry" + } else if s.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG || s.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness || s.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistance { + expectedBoundary = "guarded_dual_arm" + } + if manifest.ExecutionBoundary != expectedBoundary { + return fmt.Errorf("promotion manifest execution boundary %q does not authorize %q", manifest.ExecutionBoundary, expectedBoundary) + } + if len(manifest.Caps) == 0 || len(manifest.Buckets) == 0 { + return fmt.Errorf("promotion manifest requires immutable caps and authorized buckets") + } + if s.EnableExpansionOrientation { + expectedCaps := map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("%s promotion manifest requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", manifest.SelectorVersion) + } + for name, expected := range expectedCaps { + if actual, found := manifest.Caps[name]; !found || actual != expected { + return fmt.Errorf("%s promotion manifest requires %s=%d", manifest.SelectorVersion, name, expected) + } + } + if manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) { + return fmt.Errorf("%s promotion manifest requires fallback %q", manifest.SelectorVersion, optimize.ExpansionSearchStepwiseForward) + } + } + if s.EnableTopologyFixedSuffix || s.EnableTopologyFixedSuffixFirstUse { + expectedCaps := map[string]int64{ + "suffix_row_limit": optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, + "state_limit": optimize.ExpansionSearchSuffixReverseGuardStateLimit, + "output_row_limit": optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, + "output_bytes_limit": optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("topology fixed-suffix promotion manifest requires exactly suffix, state, output-row, and output-byte caps") + } + for name, expected := range expectedCaps { + if actual, found := manifest.Caps[name]; !found || actual != expected { + return fmt.Errorf("topology fixed-suffix promotion manifest requires %s=%d", name, expected) + } + } + expectedVersion, expectedSelector, expectedProtocol := 4, string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1), "topology-selected-routing-v1" + if s.EnableTopologyFixedSuffixFirstUse { + expectedVersion, expectedSelector, expectedProtocol = 5, string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1), "topology-selected-first-use-routing-v1" + } + if manifest.Version != expectedVersion || manifest.SelectorVersion != expectedSelector || manifest.FallbackExecutor != string(optimize.ExpansionSearchStepwiseForward) || manifest.TopologyEstimatorVersion != "topology-fixed-suffix-counts-v1" || manifest.SynopsisSchemaVersion != "topology-synopsis-schema-v2" || manifest.RouteCacheProtocol != expectedProtocol { + return fmt.Errorf("topology fixed-suffix promotion manifest requires versioned selector, fallback, estimator, synopsis schema, and route-cache protocol bindings") + } + if !slices.EqualFunc(sortedTopologyThresholds(manifest.TopologyThresholds), []topologyThreshold{{Name: "maximum_edge_to_node_ratio_per_mille", Value: 1000}}, func(left, right topologyThreshold) bool { + return left == right + }) { + return fmt.Errorf("topology fixed-suffix promotion manifest requires maximum_edge_to_node_ratio_per_mille=1000") + } + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorASPI1DAG { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("ASP-I1 promotion manifest requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + return fmt.Errorf("ASP-I1 promotion manifest requires positive %s", name) + } + } + if manifest.FallbackExecutor != string(optimize.ShortestPathExecutorASPA1DAG) { + return fmt.Errorf("ASP-I1 promotion manifest requires fallback %q", optimize.ShortestPathExecutorASPA1DAG) + } + for _, bucket := range manifest.Buckets { + if (bucket.Direction != "outbound" && bucket.Direction != "inbound") || bucket.ObservationMode != "all_paths" || bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 || bucket.RelationshipKindCount < 0 { + return fmt.Errorf("ASP-I1 promotion bucket does not match the supported directed all-paths depth envelope") + } + if bucket.UntypedRelationship != (bucket.RelationshipKindCount == 0) { + return fmt.Errorf("ASP-I1 promotion bucket relationship kind metadata is inconsistent") + } + } + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + expectedCaps := map[string]struct{}{ + "state_limit": {}, "predecessor_limit": {}, "enumeration_limit": {}, "output_bytes_limit": {}, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("SP-I1 canonical promotion manifest requires exactly state, predecessor, enumeration, and output-byte caps") + } + for name := range expectedCaps { + if manifest.Caps[name] <= 0 { + return fmt.Errorf("SP-I1 canonical promotion manifest requires positive %s", name) + } + } + if manifest.FallbackExecutor != string(optimize.ShortestPathExecutorS4CanonicalWitness) { + return fmt.Errorf("SP-I1 canonical promotion manifest requires fallback %q", optimize.ShortestPathExecutorS4CanonicalWitness) + } + if manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV6 { + return fmt.Errorf("SP-I1 canonical promotion manifest requires selector %q", optimize.ShortestPathSelectorStaticV6) + } + for _, bucket := range manifest.Buckets { + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationOnePath) || + bucket.MinimumDepth != 1 || bucket.MaximumDepth != 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("SP-I1 canonical promotion bucket must match the qualified inbound typed single-kind one-path depth 1..64 envelope") + } + } + } + if s.ShortestPathExecutor == optimize.ShortestPathExecutorI2GuardedDistance { + expectedCaps := map[string]int64{ + "state_limit": optimize.ShortestPathI2QualifiedStateLimit, + "frontier_limit": optimize.ShortestPathI2QualifiedFrontierLimit, + } + if len(manifest.Caps) != len(expectedCaps) { + return fmt.Errorf("SP-I2 distance promotion manifest requires exactly state and frontier caps") + } + for name, expected := range expectedCaps { + if actual, found := manifest.Caps[name]; !found || actual != expected { + return fmt.Errorf("SP-I2 distance promotion manifest requires %s=%d", name, expected) + } + } + if manifest.FallbackExecutor != string(optimize.ShortestPathExecutorS4CanonicalDistance) { + return fmt.Errorf("SP-I2 distance promotion manifest requires fallback %q", optimize.ShortestPathExecutorS4CanonicalDistance) + } + if manifest.SelectorVersion != optimize.ShortestPathSelectorStaticV8HiddenFanIn { + return fmt.Errorf("SP-I2 distance promotion manifest requires selector %q", optimize.ShortestPathSelectorStaticV8HiddenFanIn) + } + for _, bucket := range manifest.Buckets { + if bucket.Direction != "inbound" || bucket.ObservationMode != string(optimize.ShortestPathObservationDistance) || bucket.MinimumDepth != 1 || bucket.MaximumDepth < 1 || bucket.MaximumDepth > 64 || bucket.RelationshipKindCount != 1 || bucket.UntypedRelationship { + return fmt.Errorf("SP-I2 distance promotion bucket must be inbound, typed single-kind, distance-only, and depth-bounded") + } + } + } + manifestQueries := make([]string, 0) + seenBucketNames := map[string]struct{}{} + seenManifestQueries := map[string]string{} + for _, bucket := range manifest.Buckets { + if strings.TrimSpace(bucket.Name) == "" { + return fmt.Errorf("each promotion bucket requires a nonempty unique name") + } + if _, duplicate := seenBucketNames[bucket.Name]; duplicate { + return fmt.Errorf("promotion bucket %q is duplicated", bucket.Name) + } + seenBucketNames[bucket.Name] = struct{}{} + if !slices.Equal(bucket.QualificationSplit, []string{"training", "holdout"}) { + return fmt.Errorf("each promotion bucket requires exactly one training and one holdout qualification split in canonical order") + } + if manifest.Version == 3 || manifest.Version == 4 { + shape := TraversalShape{ + Version: bucket.StructuralShapeVersion, + Family: bucket.StructuralFamily, + Direction: bucket.Direction, + ObservationMode: bucket.ObservationMode, + MinimumDepth: bucket.MinimumDepth, + MaximumDepth: bucket.MaximumDepth, + RelationshipKindCount: bucket.RelationshipKindCount, + UntypedRelationship: bucket.UntypedRelationship, + SuffixLength: bucket.SuffixLength, + CandidateStrategy: bucket.CandidateStrategy, + } + expectedShapeVersion := TraversalShapeVersion + if manifest.Version == 4 { + expectedShapeVersion = TraversalFixedSuffixShapeVersion + } + if shape.Version != expectedShapeVersion || !lowerHexSHA256(bucket.StructuralShapeSHA256) || bucket.StructuralShapeSHA256 != TraversalShapeFingerprint(shape) { + return fmt.Errorf("promotion bucket %q has an invalid structural shape binding", bucket.Name) + } + if !lowerHexSHA256(bucket.SQLTemplateSHA256) || bucket.SQLTemplateSHA256 != structuralSQLTemplateSHA256(manifest, bucket) { + return fmt.Errorf("promotion bucket %q has an invalid structural SQL template binding", bucket.Name) + } + if manifest.Version == 4 && (bucket.Direction != "outbound" || bucket.ObservationMode != string(optimize.ExpansionSearchObservationFullPath) || bucket.MinimumDepth != 0 || bucket.MaximumDepth != 16 || bucket.SuffixLength != 3 || bucket.CandidateStrategy != string(optimize.ExpansionSearchSuffixSeededReverse)) { + return fmt.Errorf("topology fixed-suffix promotion bucket %q does not match the qualified outbound full-path fixed-suffix envelope", bucket.Name) + } + } + if len(bucket.QuerySHA256) == 0 { + return fmt.Errorf("promotion bucket %q requires a nonempty query allowlist", bucket.Name) + } + seenWithinBucket := map[string]struct{}{} + for _, query := range bucket.QuerySHA256 { + if !lowerHexSHA256(query) { + return fmt.Errorf("promotion bucket %q contains invalid query digest %q", bucket.Name, query) + } + if _, duplicate := seenWithinBucket[query]; duplicate { + return fmt.Errorf("promotion bucket %q duplicates query digest %q", bucket.Name, query) + } + seenWithinBucket[query] = struct{}{} + if owner, duplicate := seenManifestQueries[query]; duplicate { + return fmt.Errorf("promotion manifest query %q is authorized by both bucket %q and bucket %q", query, owner, bucket.Name) + } + seenManifestQueries[query] = bucket.Name + } + manifestQueries = append(manifestQueries, bucket.QuerySHA256...) + } + sort.Strings(manifestQueries) + manifestQueries = slices.Compact(manifestQueries) + if manifest.Version == 2 && len(manifestQueries) != 1 { + return fmt.Errorf("promotion manifest operational SQL anchor requires exactly one authorized query digest") + } + policyQueries := append([]string(nil), s.QuerySHA256Allowlist...) + sort.Strings(policyQueries) + compactedPolicyQueries := slices.Compact(policyQueries) + if len(compactedPolicyQueries) != len(s.QuerySHA256Allowlist) { + return fmt.Errorf("query allowlist must not contain duplicate digests") + } + policyQueries = compactedPolicyQueries + if !slices.Equal(manifestQueries, policyQueries) { + return fmt.Errorf("query allowlist must exactly match the promotion manifest buckets") + } + requiredEvidenceRoles := []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} + if len(manifest.Evidence) != len(requiredEvidenceRoles) { + return fmt.Errorf("promotion manifest requires exactly the six supported evidence roles") + } + for _, role := range requiredEvidenceRoles { + evidence, found := manifest.Evidence[role] + if !found || !lowerHexSHA256(evidence.SHA256) { + return fmt.Errorf("promotion manifest requires digest-bound %s evidence", role) + } + clean := filepath.Clean(evidence.Path) + if evidence.Path == "" || filepath.IsAbs(evidence.Path) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return fmt.Errorf("promotion manifest %s evidence requires a contained relative path", role) + } + } + if len(s.QuerySHA256Allowlist) == 0 { + return fmt.Errorf("enabled traversal policy requires a nonempty query SHA-256 allowlist") + } + if s.ShortestPathExecutor != "" && !productionCanaryExecutor(s.ShortestPathExecutor) { + return fmt.Errorf("shortest-path executor %q is not production-canary eligible", s.ShortestPathExecutor) + } + for _, value := range s.QuerySHA256Allowlist { + if !lowerHexSHA256(value) { + return fmt.Errorf("query allowlist entry %q is not a SHA-256 digest", value) + } + } + return nil +} + +type topologyThreshold struct { + // Name is the manifest key for this topology estimator threshold. + Name string + + // Value is the immutable threshold value required by the estimator protocol. + Value int64 +} + +// sortedTopologyThresholds turns an unordered threshold map into a deterministic sequence. +func sortedTopologyThresholds(input map[string]int64) []topologyThreshold { + thresholds := make([]topologyThreshold, 0, len(input)) + for name, value := range input { + thresholds = append(thresholds, topologyThreshold{Name: name, Value: value}) + } + slices.SortFunc(thresholds, func(left, right topologyThreshold) int { + return strings.Compare(left.Name, right.Name) + }) + return thresholds +} + +// lowerHexSHA256 reports whether value is a lowercase hexadecimal SHA-256 digest. +func lowerHexSHA256(value string) bool { + if value != strings.ToLower(value) { + return false + } + decoded, err := hex.DecodeString(value) + return err == nil && len(decoded) == sha256.Size +} + +// validateTraversalPromotionSQLAnchor binds the SQL rendered on a production +// cache miss to the independent manifest identity before it can be executed or +// cached. The exact formatter output is hashed without normalization, matching +// GraphBench's sql_fingerprint contract. +func validateTraversalPromotionSQLAnchor(manifest traversalPromotionManifest, sqlQuery string) error { + // Emergency rollback policies intentionally carry no promotion manifest. + // Active candidates cannot reach this helper with an empty anchor because + // TraversalPolicy.validate requires one before installation. + if manifest.OperationalCandidateSQLSHA256 == "" { + return nil + } + digest := sha256.Sum256([]byte(sqlQuery)) + actual := hex.EncodeToString(digest[:]) + if actual != manifest.OperationalCandidateSQLSHA256 { + return fmt.Errorf( + "production traversal SQL SHA-256 %s does not match promotion manifest anchor %s", + actual, + manifest.OperationalCandidateSQLSHA256, + ) + } + return nil +} + +// productionCanaryExecutor reports whether executor is eligible for a production promotion manifest. +func productionCanaryExecutor(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorI2GuardedDistance: + return true + default: + return false + } +} + +// TraversalPolicyQuerySHA256 returns the stable digest used by policy +// allowlists. Only surrounding whitespace is normalized. Collapsing interior +// whitespace is unsafe because whitespace inside string literals and escaped +// identifiers is semantically significant. +func TraversalPolicyQuerySHA256(query string) string { + normalized := strings.TrimSpace(query) + digest := sha256.Sum256([]byte(normalized)) + return hex.EncodeToString(digest[:]) +} + +// SetTraversalPolicy atomically replaces production canary selection. The +// zero value disables all candidates; old cached SQL becomes unreachable +// because the effective policy identity changes immediately. +func (s *Driver) SetTraversalPolicy(policy TraversalPolicy) error { + if s == nil || s.SchemaManager == nil { + return fmt.Errorf("PostgreSQL driver is not initialized") + } + if err := policy.validate(); err != nil { + return err + } + policy.QuerySHA256Allowlist = append([]string(nil), policy.QuerySHA256Allowlist...) + policy.PromotionManifestJSON = append(json.RawMessage(nil), policy.PromotionManifestJSON...) + sort.Strings(policy.QuerySHA256Allowlist) + policy.QuerySHA256Allowlist = slices.Compact(policy.QuerySHA256Allowlist) + policy.compiledBuckets = map[string]traversalPromotionBucket{} + if len(policy.PromotionManifestJSON) > 0 { + manifest, err := decodeTraversalPromotionManifest(policy.PromotionManifestJSON) + if err != nil { + return err + } + policy.compiledManifest = manifest + for _, bucket := range manifest.Buckets { + for _, queryDigest := range bucket.QuerySHA256 { + if _, duplicate := policy.compiledBuckets[queryDigest]; duplicate { + return fmt.Errorf("promotion manifest query %q is authorized by more than one bucket", queryDigest) + } + policy.compiledBuckets[queryDigest] = bucket + } + } + } + raw, err := json.Marshal(policy) + if err != nil { + return fmt.Errorf("serialize traversal policy identity: %w", err) + } + digest := sha256.Sum256(raw) + policy.compiledIdentity = "production-policy-" + hex.EncodeToString(digest[:]) + s.traversalPolicyLock.Lock() + s.traversalPolicy = policy + s.traversalPolicyLock.Unlock() + return nil +} + +// TraversalPolicy returns an immutable snapshot of the active policy. +func (s *Driver) TraversalPolicy() TraversalPolicy { + if s == nil || s.SchemaManager == nil { + return TraversalPolicy{} + } + s.traversalPolicyLock.RLock() + defer s.traversalPolicyLock.RUnlock() + policy := s.traversalPolicy + policy.QuerySHA256Allowlist = append([]string(nil), policy.QuerySHA256Allowlist...) + policy.PromotionManifestJSON = append(json.RawMessage(nil), policy.PromotionManifestJSON...) + return policy +} + +// effectiveTraversalPolicy selects the exact-query policy effective at the requested isolation level. +func (s *SchemaManager) effectiveTraversalPolicy(query string, isolation pgx.TxIsoLevel) (TraversalPolicy, string) { + return s.effectiveTraversalPolicyForShape(query, TraversalShape{}, isolation) +} + +// effectiveTraversalPolicyForShape returns the active policy for an exact +// canary query or a v3 structurally authorized query. A zero shape preserves +// the historic exact-query behavior. +func (s *SchemaManager) effectiveTraversalPolicyForShape(query string, shape TraversalShape, isolation pgx.TxIsoLevel) (TraversalPolicy, string) { + s.traversalPolicyLock.RLock() + policy := s.traversalPolicy + s.traversalPolicyLock.RUnlock() + candidateRollback := policy.matchingCandidateRollbackActive() + standaloneRollback := !policy.manifestCandidateEnabled() && policy.rollbackActive() + if candidateRollback { + policy = policy.withoutManifestCandidate() + } + if candidateRollback || standaloneRollback { + policy.compiledManifest.OperationalCandidateSQLSHA256 = "" + } + // Manifest v4 has a separate, snapshot-owned selection path. An exact + // evidence query must not make it eligible through the ordinary translation + // cache path, otherwise a route-cache miss could silently execute it. + if policy.EnableTopologyFixedSuffix || policy.EnableTopologyFixedSuffixFirstUse || (candidateRollback && (policy.compiledManifest.Version == 4 || policy.compiledManifest.Version == 5)) { + if candidateRollback { + return policy, policy.compiledIdentity + } + return TraversalPolicy{}, "production-incumbent-v1" + } + + _, queryAuthorized := policy.compiledBuckets[TraversalPolicyQuerySHA256(query)] + _, structuralAuthorized := policy.authorizedStructuralBucketForShape(shape) + effective := policy.enabled() && (candidateRollback || standaloneRollback || queryAuthorized || structuralAuthorized) + if shortestPathExecutorRequiresStableSnapshot(policy.ShortestPathExecutor) && isolation != pgx.RepeatableRead && isolation != pgx.Serializable { + effective = false + } + if !effective { + return TraversalPolicy{}, "production-incumbent-v1" + } + return policy, policy.compiledIdentity +} + +// hasStructuralTraversalPolicy reports whether a non-rollback policy needs structural classification. +func (s *SchemaManager) hasStructuralTraversalPolicy() bool { + s.traversalPolicyLock.RLock() + defer s.traversalPolicyLock.RUnlock() + return s.traversalPolicy.manifestCandidateEnabled() && (s.traversalPolicy.compiledManifest.Version == 3 || s.traversalPolicy.compiledManifest.Version == 4 || s.traversalPolicy.compiledManifest.Version == 5) && !s.traversalPolicy.rollbackActive() +} + +// topologyFixedSuffixPolicyForShape returns a validated v4 or v5 policy only for +// the narrow structural bucket it authorizes. The caller must still own a +// stable transaction snapshot and receive a route-cache hit before it can +// execute the returned candidate. +func (s *SchemaManager) topologyFixedSuffixPolicyForShape(shape TraversalShape, isolation pgx.TxIsoLevel) (TraversalPolicy, string) { + if shape.Version != TraversalFixedSuffixShapeVersion || !stableSnapshotIsolation(isolation) { + return TraversalPolicy{}, "" + } + s.traversalPolicyLock.RLock() + policy := s.traversalPolicy + s.traversalPolicyLock.RUnlock() + if (!policy.EnableTopologyFixedSuffix && !policy.EnableTopologyFixedSuffixFirstUse) || policy.rollbackActive() || (policy.compiledManifest.Version != 4 && policy.compiledManifest.Version != 5) { + return TraversalPolicy{}, "" + } + if _, authorized := policy.authorizedStructuralBucketForShape(shape); !authorized { + return TraversalPolicy{}, "" + } + if policy.EnableTopologyFixedSuffixFirstUse { + return policy, policy.compiledIdentity + "-topology-fixed-suffix-first-use-candidate" + } + return policy, policy.compiledIdentity + "-topology-fixed-suffix-candidate" +} + +// shortestPathExecutorRequiresStableSnapshot reports whether executor reads state that requires snapshot stability. +func shortestPathExecutorRequiresStableSnapshot(executor optimize.ShortestPathExecutor) bool { + switch executor { + case optimize.ShortestPathExecutorB1AlternatingNodeDistance, + optimize.ShortestPathExecutorB1AlternatingNodeWitness, + optimize.ShortestPathExecutorB2SmallerCurrentLevelDistance, + optimize.ShortestPathExecutorB2SmallerCurrentLevelWitness, + optimize.ShortestPathExecutorASPB1AlternatingNodeDAG, + optimize.ShortestPathExecutorASPB2SmallerCurrentLevelDAG, + optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + optimize.ShortestPathExecutorASPI1DAG, + optimize.ShortestPathExecutorI2GuardedDistance: + return true + default: + return false + } +} diff --git a/drivers/pg/traversal_policy_test.go b/drivers/pg/traversal_policy_test.go new file mode 100644 index 00000000..5b1c20ad --- /dev/null +++ b/drivers/pg/traversal_policy_test.go @@ -0,0 +1,1005 @@ +package pg + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/drivers/pg/model" + "github.com/specterops/dawgs/graph" + "github.com/stretchr/testify/require" +) + +// testTraversalPolicy coordinates PostgreSQL driver behavior for test traversal policy. +func testTraversalPolicy(query string, executor optimize.ShortestPathExecutor, orientation bool) TraversalPolicy { + candidate := string(executor) + if orientation { + candidate = "orientation-probe-v1" + } + queryDigest := TraversalPolicyQuerySHA256(query) + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"path": role + ".json", "sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"} + } + boundary := map[bool]string{ + true: "guarded_dual_arm", + false: "inline_statement", + }[orientation] + selectorVersion := "test-selector-v1" + caps := map[string]int64{"state_limit": 1000} + bucket := map[string]any{"name": "qualified-query", "query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}} + fallback := "" + if orientation { + selectorVersion = string(optimize.ExpansionSearchPolicyOrientationProbeV1) + caps = map[string]int64{ + "root_row_limit": optimize.ExpansionSearchOrientationRootRowLimit, + "reverse_seed_row_limit": optimize.ExpansionSearchOrientationReverseSeedRowLimit, + "directional_degree_row_limit": optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit, + "state_limit": optimize.ExpansionSearchOrientationStateLimit, + } + fallback = string(optimize.ExpansionSearchStepwiseForward) + } + if executor == optimize.ShortestPathExecutorASPI1DAG { + boundary = "guarded_dual_arm" + caps = map[string]int64{ + "state_limit": 1000, "predecessor_limit": 900, "enumeration_limit": 800, "output_bytes_limit": 70000, + } + fallback = string(optimize.ShortestPathExecutorASPA1DAG) + bucket["direction"] = "outbound" + bucket["observation_mode"] = "all_paths" + bucket["minimum_depth"] = 1 + bucket["maximum_depth"] = 4 + bucket["relationship_kind_count"] = 1 + bucket["untyped_relationship"] = false + } + if executor == optimize.ShortestPathExecutorI1CanonicalPredecessorWitness { + selectorVersion = optimize.ShortestPathSelectorStaticV6 + boundary = "guarded_dual_arm" + caps = map[string]int64{ + "state_limit": 1000, "predecessor_limit": 900, "enumeration_limit": 800, "output_bytes_limit": 70000, + } + fallback = string(optimize.ShortestPathExecutorS4CanonicalWitness) + bucket["direction"] = "inbound" + bucket["observation_mode"] = "one_path" + bucket["minimum_depth"] = 1 + bucket["maximum_depth"] = 64 + bucket["relationship_kind_count"] = 1 + bucket["untyped_relationship"] = false + } + if executor == optimize.ShortestPathExecutorI2GuardedDistance { + selectorVersion = optimize.ShortestPathSelectorStaticV8HiddenFanIn + boundary = "guarded_dual_arm" + caps = map[string]int64{ + "state_limit": optimize.ShortestPathI2QualifiedStateLimit, + "frontier_limit": optimize.ShortestPathI2QualifiedFrontierLimit, + } + fallback = string(optimize.ShortestPathExecutorS4CanonicalDistance) + bucket["direction"] = "inbound" + bucket["observation_mode"] = "distance" + bucket["minimum_depth"] = 1 + bucket["maximum_depth"] = 32 + bucket["relationship_kind_count"] = 1 + bucket["untyped_relationship"] = false + } + raw, err := json.Marshal(map[string]any{ + "version": 2, "candidate": candidate, "selector_version": selectorVersion, + "source_commit": "deadbeef", "source_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "binary_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "corpus_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + "operational_candidate_sql_sha256": "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + "execution_boundary": boundary, + "fallback_executor": fallback, + "caps": caps, + "buckets": []map[string]any{bucket}, + "evidence": evidence, + }) + if err != nil { + panic(err) + } + digest := sha256.Sum256(raw) + return TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: hex.EncodeToString(digest[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{queryDigest}, + ShortestPathExecutor: executor, + EnableExpansionOrientation: orientation, + } +} + +// rewriteTestTraversalPolicyManifest coordinates PostgreSQL driver behavior for rewrite test traversal policy manifest. +func rewriteTestTraversalPolicyManifest(t *testing.T, policy TraversalPolicy, mutate func(*traversalPromotionManifest)) TraversalPolicy { + t.Helper() + + var manifest traversalPromotionManifest + require.NoError(t, json.Unmarshal(policy.PromotionManifestJSON, &manifest)) + mutate(&manifest) + + raw, err := json.Marshal(manifest) + require.NoError(t, err) + digest := sha256.Sum256(raw) + policy.PromotionManifestJSON = raw + policy.PromotionManifestSHA256 = hex.EncodeToString(digest[:]) + return policy +} + +func testTopologyFixedSuffixPolicy(t *testing.T, evidenceQuery string, shape TraversalShape) TraversalPolicy { + t.Helper() + manifest := traversalPromotionManifest{ + Version: 4, + Candidate: string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1), + SelectorVersion: string(optimize.ExpansionSearchPolicyTopologyFixedSuffixV1), + ExecutionBoundary: "transaction_retry", + FallbackExecutor: string(optimize.ExpansionSearchStepwiseForward), + SourceCommit: "test-topology-fixed-suffix", + SourceSHA256: strings.Repeat("0", sha256.Size*2), + BinarySHA256: strings.Repeat("1", sha256.Size*2), + CorpusSHA256: strings.Repeat("2", sha256.Size*2), + OperationalCandidateSQLSHA256: strings.Repeat("3", sha256.Size*2), + TopologyEstimatorVersion: "topology-fixed-suffix-counts-v1", + SynopsisSchemaVersion: "topology-synopsis-schema-v2", + RouteCacheProtocol: "topology-selected-routing-v1", + TopologyThresholds: map[string]int64{"maximum_edge_to_node_ratio_per_mille": 1000}, + Caps: map[string]int64{ + "suffix_row_limit": optimize.ExpansionSearchSuffixReverseGuardSuffixRowLimit, + "state_limit": optimize.ExpansionSearchSuffixReverseGuardStateLimit, + "output_row_limit": optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, + "output_bytes_limit": optimize.ExpansionSearchSuffixReverseRetryOutputBytesLimit, + }, + } + bucket := traversalPromotionBucket{ + Name: "topology-fixed-suffix", + QuerySHA256: []string{TraversalPolicyQuerySHA256(evidenceQuery)}, + QualificationSplit: []string{"training", "holdout"}, + Direction: shape.Direction, + ObservationMode: shape.ObservationMode, + MinimumDepth: shape.MinimumDepth, + MaximumDepth: shape.MaximumDepth, + SuffixLength: shape.SuffixLength, + CandidateStrategy: shape.CandidateStrategy, + StructuralShapeVersion: shape.Version, + StructuralFamily: shape.Family, + StructuralShapeSHA256: shape.Fingerprint, + } + manifest.Buckets = []traversalPromotionBucket{bucket} + manifest.Buckets[0].SQLTemplateSHA256 = structuralSQLTemplateSHA256(manifest, manifest.Buckets[0]) + manifest.Evidence = map[string]traversalPromotionEvidence{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + manifest.Evidence[role] = traversalPromotionEvidence{Path: role + ".json", SHA256: strings.Repeat("4", sha256.Size*2)} + } + raw, err := json.Marshal(manifest) + require.NoError(t, err) + digest := sha256.Sum256(raw) + return TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: hex.EncodeToString(digest[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{bucket.QuerySHA256[0]}, + EnableTopologyFixedSuffix: true, + } +} + +func TestTraversalPolicyV4RequiresRouteOwnedFixedSuffixSelection(t *testing.T) { + query := `MATCH (root:Root) WHERE root.key = $key MATCH route = (root)-[:Expand*0..16]->()-[:Enter]->(:Middle)-[:Continue]->(:NearTerminal)-[:Complete]->(:Terminal) RETURN route` + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + shape, err := traversalShapeForQuery(parsed) + require.NoError(t, err) + require.Equal(t, TraversalFixedSuffixShapeVersion, shape.Version) + + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + policy := testTopologyFixedSuffixPolicy(t, query, shape) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + ordinary, identity := driver.SchemaManager.effectiveTraversalPolicyForShape(query, shape, pgx.RepeatableRead) + require.False(t, ordinary.enabled()) + require.Equal(t, "production-incumbent-v1", identity) + + topology, topologyIdentity := driver.SchemaManager.topologyFixedSuffixPolicyForShape(shape, pgx.RepeatableRead) + require.True(t, topology.enabled()) + require.Contains(t, topologyIdentity, "topology-fixed-suffix-candidate") + options, err := topology.productionOptionsForShape(query, shape) + require.NoError(t, err) + require.True(t, options.EnableTopologyFixedSuffix) + require.Equal(t, optimize.ExpansionSearchSuffixReverseRetryOutputRowLimit, options.TopologyFixedSuffixCaps.OutputRowLimit) + + invalid := rewriteTestTraversalPolicyManifest(t, policy, func(manifest *traversalPromotionManifest) { + manifest.RouteCacheProtocol = "unknown" + }) + require.ErrorContains(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(invalid), "route-cache protocol") + invalid = rewriteTestTraversalPolicyManifest(t, policy, func(manifest *traversalPromotionManifest) { + manifest.TopologyThresholds["maximum_edge_to_node_ratio_per_mille"] = 999 + }) + require.ErrorContains(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(invalid), "maximum_edge_to_node_ratio_per_mille=1000") +} + +func TestTraversalPolicyV5AuthorizesSeparateFirstUseFixedSuffixSelection(t *testing.T) { + query := `MATCH (root:Root) WHERE root.key = $key MATCH route = (root)-[:Expand*0..16]->()-[:Enter]->(:Middle)-[:Continue]->(:NearTerminal)-[:Complete]->(:Terminal) RETURN route` + parsed, err := frontend.ParseCypher(frontend.NewContext(), query) + require.NoError(t, err) + shape, err := traversalShapeForQuery(parsed) + require.NoError(t, err) + + policy := testTopologyFixedSuffixPolicy(t, query, shape) + policy.EnableTopologyFixedSuffix = false + policy.EnableTopologyFixedSuffixFirstUse = true + policy = rewriteTestTraversalPolicyManifest(t, policy, func(manifest *traversalPromotionManifest) { + manifest.Version = 5 + manifest.Candidate = string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) + manifest.SelectorVersion = string(optimize.ExpansionSearchPolicyTopologyFixedSuffixFirstUseV1) + manifest.ExecutionBoundary = "first_use_transaction_retry" + manifest.RouteCacheProtocol = "topology-selected-first-use-routing-v1" + manifest.Buckets[0].SQLTemplateSHA256 = structuralSQLTemplateSHA256(*manifest, manifest.Buckets[0]) + }) + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(policy)) + + ordinary, _ := driver.SchemaManager.effectiveTraversalPolicyForShape(query, shape, pgx.RepeatableRead) + require.False(t, ordinary.enabled()) + topology, identity := driver.SchemaManager.topologyFixedSuffixPolicyForShape(shape, pgx.RepeatableRead) + require.True(t, topology.EnableTopologyFixedSuffixFirstUse) + require.Contains(t, identity, "first-use-candidate") +} + +// TestTraversalPolicyAuthorizesGuardedInlineASPOnlyWithStableSnapshotAndExactCaps verifies traversal policy authorizes guarded inline asp only with stable snapshot and exact caps behavior. +func TestTraversalPolicyAuthorizesGuardedInlineASPOnlyWithStableSnapshotAndExactCaps(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH p = allShortestPaths((s)-[:MemberOf*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p" + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorASPI1DAG, false) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + effective, _ := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + require.False(t, effective.enabled()) + effective, _ = driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.Equal(t, optimize.ShortestPathExecutorASPI1DAG, effective.ShortestPathExecutor) + options, err := effective.productionOptions(query) + require.NoError(t, err) + require.Equal(t, int64(1000), options.ShortestPathCaps.StateLimit) + require.Equal(t, int64(900), options.ShortestPathCaps.PredecessorLimit) + require.Equal(t, int64(800), options.ShortestPathCaps.EnumerationLimit) + require.Equal(t, int64(70000), options.ShortestPathCaps.OutputBytesLimit) + require.Equal(t, "outbound", options.AuthorizedBucket.Direction) +} + +// TestTraversalPolicyInlineASPKillSwitchRequiresNoEvidence verifies traversal policy inline asp kill switch requires no evidence behavior. +func TestTraversalPolicyInlineASPKillSwitchRequiresNoEvidence(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 9, + DisableInlineASPDAG: true, + })) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy("MATCH (n) RETURN n", pgx.ReadCommitted) + require.True(t, effective.DisableInlineASPDAG) + require.Empty(t, effective.ShortestPathExecutor) + require.Contains(t, identity, "production-policy-") + options, err := effective.productionOptions("MATCH (n) RETURN n") + require.NoError(t, err) + require.Equal(t, "inline-asp-kill-switch-g9", options.SelectorVersion) +} + +// TestTraversalPolicyIsAllowlistedSnapshotSafeAndImmediatelyReversible verifies traversal policy is allowlisted snapshot safe and immediately reversible behavior. +func TestTraversalPolicyIsAllowlistedSnapshotSafeAndImmediatelyReversible(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + effective, _ := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + require.False(t, effective.enabled()) + effective, candidateKey := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.True(t, effective.enabled()) + require.Contains(t, candidateKey, "production-policy-") + + effective, _ = driver.SchemaManager.effectiveTraversalPolicy("RETURN 1", pgx.RepeatableRead) + require.False(t, effective.enabled(), "queries outside the allowlist remain on incumbents") + + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{})) + effective, rollbackKey := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.False(t, effective.enabled()) + require.Equal(t, "production-incumbent-v1", rollbackKey) + require.NotEqual(t, candidateKey, rollbackKey) +} + +func TestTraversalPolicyAuthorizesAndRollsBackGuardedDistance(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) RETURN length(p)" + policy := testTraversalPolicy(query, optimize.ShortestPathExecutorI2GuardedDistance, false) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + effective, _ := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.Equal(t, optimize.ShortestPathExecutorI2GuardedDistance, effective.ShortestPathExecutor) + options, err := effective.productionOptions(query) + require.NoError(t, err) + require.Equal(t, optimize.ShortestPathSelectorStaticV8HiddenFanIn, options.SelectorVersion) + require.Equal(t, optimize.ShortestPathI2QualifiedStateLimit, options.ShortestPathCaps.StateLimit) + require.Equal(t, optimize.ShortestPathI2QualifiedFrontierLimit, options.ShortestPathCaps.FrontierLimit) + + for name, test := range map[string]struct { + capName string + value int64 + expected int64 + }{ + "non-qualified state cap": { + capName: "state_limit", value: 1000, expected: optimize.ShortestPathI2QualifiedStateLimit, + }, + "non-qualified frontier cap": { + capName: "frontier_limit", value: 100, expected: optimize.ShortestPathI2QualifiedFrontierLimit, + }, + } { + t.Run(name, func(t *testing.T) { + invalid := rewriteTestTraversalPolicyManifest(t, policy, func(manifest *traversalPromotionManifest) { + manifest.Caps[test.capName] = test.value + }) + err := (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(invalid) + require.ErrorContains(t, err, fmt.Sprintf( + "SP-I2 distance promotion manifest requires %s=%d", + test.capName, + test.expected, + )) + }) + } + + policy.DisableInlineSPDistance = true + require.NoError(t, driver.SetTraversalPolicy(policy)) + effective, _ = driver.SchemaManager.effectiveTraversalPolicy(query, pgx.RepeatableRead) + require.Empty(t, effective.ShortestPathExecutor) + require.True(t, effective.DisableInlineSPDistance) +} + +// TestTraversalPolicyFailsClosed verifies traversal policy fails closed behavior. +func TestTraversalPolicyFailsClosed(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, + EnableExpansionOrientation: true, + })) + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + QuerySHA256Allowlist: []string{"not-a-digest"}, + EnableExpansionOrientation: true, + })) + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", + QuerySHA256Allowlist: []string{TraversalPolicyQuerySHA256("RETURN 1")}, + ShortestPathExecutor: optimize.ShortestPathExecutorS3Unidirectional, + })) + require.Error(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 1, + QuerySHA256Allowlist: []string{TraversalPolicyQuerySHA256("RETURN 1")}, + EnableExpansionOrientation: true, + }), "an enabled production policy must be traceable to verified evidence") + require.ErrorContains(t, driver.SetTraversalPolicy(testTraversalPolicy( + "MATCH p = shortestPath((s)-[*1..4]->(e)) RETURN length(p)", + optimize.ShortestPathExecutorI1CanonicalDistance, + false, + )), "not production-canary eligible") +} + +// TestTraversalPolicyRequiresOperationalSQLAnchor verifies production admission +// cannot rely only on the operational report's self-declared fingerprint. +func TestTraversalPolicyRequiresOperationalSQLAnchor(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + valid := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + + for name, mutate := range map[string]func(*traversalPromotionManifest){ + "missing": func(manifest *traversalPromotionManifest) { manifest.OperationalCandidateSQLSHA256 = "" }, + "invalid": func(manifest *traversalPromotionManifest) { manifest.OperationalCandidateSQLSHA256 = "NOT-A-DIGEST" }, + } { + t.Run(name, func(t *testing.T) { + policy := rewriteTestTraversalPolicyManifest(t, valid, mutate) + err := (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(policy) + require.ErrorContains(t, err, "operational candidate SQL SHA-256 digest") + }) + } +} + +// TestDecodeTraversalPromotionManifestIsStrict verifies unknown or trailing +// content cannot change a final authorization document without rejection. +func TestDecodeTraversalPromotionManifestIsStrict(t *testing.T) { + valid := testTraversalPolicy("MATCH (n) RETURN n", optimize.ShortestPathExecutorASPI1DAG, false) + var document map[string]any + require.NoError(t, json.Unmarshal(valid.PromotionManifestJSON, &document)) + document["operational_candidate_sql_sha_256"] = document["operational_candidate_sql_sha256"] + raw, err := json.Marshal(document) + require.NoError(t, err) + _, err = decodeTraversalPromotionManifest(raw) + require.ErrorContains(t, err, "unknown field") + + _, err = decodeTraversalPromotionManifest(append(valid.PromotionManifestJSON, []byte("\n{}")...)) + require.ErrorContains(t, err, "trailing JSON data") + + duplicateTopLevel := strings.Replace(string(valid.PromotionManifestJSON), `"version":2`, `"version":2,"version":2`, 1) + _, err = decodeTraversalPromotionManifest([]byte(duplicateTopLevel)) + require.ErrorContains(t, err, `duplicate JSON object key "version"`) + + duplicateNested := strings.Replace(string(valid.PromotionManifestJSON), `"qualification_split":`, `"qualification_split":["training","holdout"],"qualification_split":`, 1) + _, err = decodeTraversalPromotionManifest([]byte(duplicateNested)) + require.ErrorContains(t, err, `duplicate JSON object key "qualification_split"`) +} + +func TestTraversalPolicyRequiresExactManifestSets(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + valid := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + + tests := map[string]struct { + mutate func(*traversalPromotionManifest) + reason string + }{ + "extra evidence role": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Evidence["invented"] = traversalPromotionEvidence{SHA256: strings.Repeat("a", 64)} + }, + reason: "exactly the six supported evidence roles", + }, + "escaping evidence path": { + mutate: func(manifest *traversalPromotionManifest) { + reference := manifest.Evidence["aa"] + reference.Path = "../aa.json" + manifest.Evidence["aa"] = reference + }, + reason: "requires a contained relative path", + }, + "duplicate split": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Buckets[0].QualificationSplit = []string{"training", "training", "holdout"} + }, + reason: "exactly one training and one holdout qualification split", + }, + "extra split": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Buckets[0].QualificationSplit = []string{"training", "holdout", "diagnostic"} + }, + reason: "exactly one training and one holdout qualification split", + }, + "duplicate query within bucket": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Buckets[0].QuerySHA256 = append(manifest.Buckets[0].QuerySHA256, manifest.Buckets[0].QuerySHA256[0]) + }, + reason: "duplicates query digest", + }, + "duplicate bucket name": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Buckets = append(manifest.Buckets, manifest.Buckets[0]) + }, + reason: "promotion bucket", + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + policy := rewriteTestTraversalPolicyManifest(t, valid, test.mutate) + err := (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(policy) + require.ErrorContains(t, err, test.reason) + }) + } + + duplicateAllowlist := valid + duplicateAllowlist.QuerySHA256Allowlist = append(duplicateAllowlist.QuerySHA256Allowlist, duplicateAllowlist.QuerySHA256Allowlist[0]) + require.ErrorContains(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(duplicateAllowlist), "must not contain duplicate digests") +} + +// TestValidateTraversalPromotionSQLAnchor verifies the production execution +// path compares exact rendered SQL with the independent manifest digest. +func TestValidateTraversalPromotionSQLAnchor(t *testing.T) { + sqlQuery := "select 1::int8 as distance" + digest := sha256.Sum256([]byte(sqlQuery)) + manifest := traversalPromotionManifest{OperationalCandidateSQLSHA256: hex.EncodeToString(digest[:])} + require.NoError(t, validateTraversalPromotionSQLAnchor(manifest, sqlQuery)) + require.ErrorContains(t, validateTraversalPromotionSQLAnchor(manifest, sqlQuery+" "), "does not match promotion manifest anchor") + require.NoError(t, validateTraversalPromotionSQLAnchor(traversalPromotionManifest{}, sqlQuery), + "evidence-free emergency rollback policies have no operational SQL anchor") +} + +// TestTraversalPolicyRollbackCompositionIsCandidateSpecific exhaustively +// verifies that each manifest-backed candidate composes only with its dedicated +// emergency control. A matching rollback derives an incumbent-only policy with +// a distinct cache identity and no candidate SQL anchor; unrelated switches +// fail installation and cannot authorize the candidate for another query. +func TestTraversalPolicyRollbackCompositionIsCandidateSpecific(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) RETURN length(p)" + unauthorizedQuery := "RETURN 1" + candidates := []struct { + name string + executor optimize.ShortestPathExecutor + orientation bool + isolation pgx.TxIsoLevel + matching string + }{ + { + name: "expansion orientation", + orientation: true, + isolation: pgx.ReadCommitted, + matching: "expansion orientation", + }, + { + name: "inline all shortest paths", + executor: optimize.ShortestPathExecutorASPI1DAG, + isolation: pgx.RepeatableRead, + matching: "inline all shortest paths", + }, + { + name: "inline shortest path witness", + executor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + isolation: pgx.RepeatableRead, + matching: "inline shortest path witness", + }, + { + name: "inline shortest path distance", + executor: optimize.ShortestPathExecutorI2GuardedDistance, + isolation: pgx.RepeatableRead, + matching: "inline shortest path distance", + }, + } + switches := []struct { + name string + disable func(*TraversalPolicy) + active func(TraversalPolicy) bool + }{ + { + name: "expansion orientation", + disable: func(policy *TraversalPolicy) { + policy.DisableExpansionOrientation = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableExpansionOrientation }, + }, + { + name: "endpoint seeded reverse", + disable: func(policy *TraversalPolicy) { + policy.DisableEndpointSeededReverse = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableEndpointSeededReverse }, + }, + { + name: "inline all shortest paths", + disable: func(policy *TraversalPolicy) { + policy.DisableInlineASPDAG = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableInlineASPDAG }, + }, + { + name: "inline shortest path witness", + disable: func(policy *TraversalPolicy) { + policy.DisableInlineSPWitness = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableInlineSPWitness }, + }, + { + name: "inline shortest path distance", + disable: func(policy *TraversalPolicy) { + policy.DisableInlineSPDistance = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableInlineSPDistance }, + }, + } + + for _, candidate := range candidates { + for _, rollback := range switches { + t.Run(candidate.name+"/"+rollback.name, func(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + policy := testTraversalPolicy(query, candidate.executor, candidate.orientation) + require.NoError(t, driver.SetTraversalPolicy(policy)) + activeCandidate, candidateIdentity := driver.SchemaManager.effectiveTraversalPolicy(query, candidate.isolation) + require.True(t, activeCandidate.manifestCandidateEnabled()) + + rollback.disable(&policy) + err := driver.SetTraversalPolicy(policy) + if rollback.name != candidate.matching { + require.ErrorContains(t, err, "single matching emergency rollback switch") + installed := driver.TraversalPolicy() + require.True(t, installed.manifestCandidateEnabled(), "rejected policy must not replace the installed candidate") + effective, identity := driver.SchemaManager.effectiveTraversalPolicy(unauthorizedQuery, candidate.isolation) + require.False(t, effective.manifestCandidateEnabled(), "an unrelated switch must not bypass query authorization") + require.Equal(t, "production-incumbent-v1", identity) + return + } + + require.NoError(t, err) + effective, rollbackIdentity := driver.SchemaManager.effectiveTraversalPolicy(query, candidate.isolation) + require.True(t, rollback.active(effective)) + require.False(t, effective.manifestCandidateEnabled()) + require.NotEqual(t, candidateIdentity, rollbackIdentity) + require.Empty(t, effective.compiledManifest.OperationalCandidateSQLSHA256) + require.NoError(t, validateTraversalPromotionSQLAnchor(effective.compiledManifest, "incumbent SQL")) + + unauthorized, unauthorizedIdentity := driver.SchemaManager.effectiveTraversalPolicy(unauthorizedQuery, candidate.isolation) + require.True(t, rollback.active(unauthorized), "matching emergency rollback remains global") + require.False(t, unauthorized.manifestCandidateEnabled()) + require.Empty(t, unauthorized.compiledManifest.OperationalCandidateSQLSHA256) + require.Equal(t, rollbackIdentity, unauthorizedIdentity) + + installed := driver.TraversalPolicy() + require.True(t, installed.manifestCandidateEnabled()) + require.NotEmpty(t, installed.compiledManifest.OperationalCandidateSQLSHA256, + "deriving the rollback policy must not mutate the installed manifest") + }) + } + } +} + +// TestTraversalPolicyRejectsMatchingRollbackWithAdditionalSwitch proves the +// matching exception cannot be broadened by adding any second emergency flag. +func TestTraversalPolicyRejectsMatchingRollbackWithAdditionalSwitch(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) RETURN length(p)" + tests := []struct { + name string + executor optimize.ShortestPathExecutor + orientation bool + disable func(*TraversalPolicy) + }{ + { + name: "expansion orientation", orientation: true, + disable: func(policy *TraversalPolicy) { + policy.DisableExpansionOrientation = true + policy.DisableEndpointSeededReverse = true + }, + }, + { + name: "inline all shortest paths", executor: optimize.ShortestPathExecutorASPI1DAG, + disable: func(policy *TraversalPolicy) { + policy.DisableInlineASPDAG = true + policy.DisableEndpointSeededReverse = true + }, + }, + { + name: "inline shortest path witness", executor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + disable: func(policy *TraversalPolicy) { + policy.DisableInlineSPWitness = true + policy.DisableEndpointSeededReverse = true + }, + }, + { + name: "inline shortest path distance", executor: optimize.ShortestPathExecutorI2GuardedDistance, + disable: func(policy *TraversalPolicy) { + policy.DisableInlineSPDistance = true + policy.DisableEndpointSeededReverse = true + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + policy := testTraversalPolicy(query, test.executor, test.orientation) + test.disable(&policy) + err := (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(policy) + require.ErrorContains(t, err, "single matching emergency rollback switch") + }) + } +} + +// TestTraversalPolicyStandaloneRollbacksRemainGlobalAndUnanchored verifies all +// evidence-free switch-only policies apply independently of query allowlists +// without carrying a manifest candidate or candidate SQL anchor. +func TestTraversalPolicyStandaloneRollbacksRemainGlobalAndUnanchored(t *testing.T) { + tests := []struct { + name string + disable func(*TraversalPolicy) + active func(TraversalPolicy) bool + }{ + { + name: "expansion orientation", + disable: func(policy *TraversalPolicy) { + policy.DisableExpansionOrientation = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableExpansionOrientation }, + }, + { + name: "endpoint seeded reverse", + disable: func(policy *TraversalPolicy) { + policy.DisableEndpointSeededReverse = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableEndpointSeededReverse }, + }, + { + name: "inline all shortest paths", + disable: func(policy *TraversalPolicy) { + policy.DisableInlineASPDAG = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableInlineASPDAG }, + }, + { + name: "inline shortest path witness", + disable: func(policy *TraversalPolicy) { + policy.DisableInlineSPWitness = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableInlineSPWitness }, + }, + { + name: "inline shortest path distance", + disable: func(policy *TraversalPolicy) { + policy.DisableInlineSPDistance = true + }, + active: func(policy TraversalPolicy) bool { return policy.DisableInlineSPDistance }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + policy := TraversalPolicy{Generation: 42} + test.disable(&policy) + require.NoError(t, driver.SetTraversalPolicy(policy)) + + effective, identity := driver.SchemaManager.effectiveTraversalPolicy("RETURN 1", pgx.ReadCommitted) + require.True(t, test.active(effective)) + require.False(t, effective.manifestCandidateEnabled()) + require.Empty(t, effective.compiledManifest.OperationalCandidateSQLSHA256) + require.Contains(t, identity, "production-policy-") + }) + } +} + +// TestTraversalPolicyStandaloneRollbackRejectsPromotionFields verifies an +// evidence-free switch cannot consume an unverified manifest selector or +// retain irrelevant authorization data in its cache identity. +func TestTraversalPolicyStandaloneRollbackRejectsPromotionFields(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..32]-(e)) RETURN length(p)" + candidate := testTraversalPolicy(query, optimize.ShortestPathExecutorI2GuardedDistance, false) + tests := map[string]func(*TraversalPolicy){ + "manifest digest": func(policy *TraversalPolicy) { + policy.PromotionManifestSHA256 = strings.Repeat("a", 64) + }, + "manifest JSON": func(policy *TraversalPolicy) { + policy.PromotionManifestJSON = json.RawMessage(`{"selector_version":"unverified-selector"}`) + }, + "query allowlist": func(policy *TraversalPolicy) { + policy.QuerySHA256Allowlist = []string{TraversalPolicyQuerySHA256(query)} + }, + "fields copied from candidate": func(policy *TraversalPolicy) { + policy.PromotionManifestSHA256 = candidate.PromotionManifestSHA256 + policy.PromotionManifestJSON = append(json.RawMessage(nil), candidate.PromotionManifestJSON...) + policy.QuerySHA256Allowlist = append([]string(nil), candidate.QuerySHA256Allowlist...) + }, + } + + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + policy := TraversalPolicy{Generation: 43, DisableEndpointSeededReverse: true} + mutate(&policy) + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.ErrorContains(t, driver.SetTraversalPolicy(policy), "must not carry promotion manifest or query authorization fields") + require.False(t, driver.TraversalPolicy().enabled(), "rejected standalone evidence must not replace the installed policy") + }) + } +} + +// TestTransactionQueryRejectsPromotionSQLAnchorDrift exercises the production +// parse/translate/format callback and proves drift is returned before Raw can +// reach the database driver. +func TestTransactionQueryRejectsPromotionSQLAnchorDrift(t *testing.T) { + query := ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) + RETURN path + ` + manager := NewSchemaManager(nil, 0) + manager.setDefaultGraph(model.Graph{ID: 1, Name: "test"}, graph.Graph{Name: "test"}) + for index, name := range []string{"ExpansionRoot", "Expand", "EnterSuffix", "SuffixHead", "ContinueSuffix", "SuffixMiddle", "CompleteSuffix", "SuffixTerminal"} { + manager.kindsByID[graph.StringKind(name)] = int16(index + 1) + } + policy := testTraversalPolicy(query, "", true) + require.NoError(t, (&Driver{SchemaManager: manager}).SetTraversalPolicy(policy)) + + tx := &transaction{schemaManager: manager, ctx: context.Background(), isolation: pgx.ReadCommitted} + result := tx.Query(query, map[string]any{"root_key": "root"}) + require.ErrorContains(t, result.Error(), "does not match promotion manifest anchor") +} + +// TestTraversalPolicySQLAnchorRequiresOneAuthorizedQuery verifies one scalar +// SQL anchor cannot ambiguously authorize several distinct query statements. +func TestTraversalPolicySQLAnchorRequiresOneAuthorizedQuery(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + valid := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + policy := rewriteTestTraversalPolicyManifest(t, valid, func(manifest *traversalPromotionManifest) { + manifest.Buckets[0].QuerySHA256 = append(manifest.Buckets[0].QuerySHA256, strings.Repeat("f", 64)) + }) + policy.QuerySHA256Allowlist = append(policy.QuerySHA256Allowlist, strings.Repeat("f", 64)) + err := (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(policy) + require.ErrorContains(t, err, "operational SQL anchor requires exactly one authorized query digest") +} + +// TestTraversalPolicyCanonicalSPRequiresExactStaticV6Envelope verifies traversal policy canonical sp requires exact static v6 envelope behavior. +func TestTraversalPolicyCanonicalSPRequiresExactStaticV6Envelope(t *testing.T) { + query := "MATCH p = shortestPath((s)<-[:MemberOf*1..64]-(e)) RETURN p" + valid := testTraversalPolicy(query, optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, false) + require.NoError(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(valid)) + + tests := map[string]struct { + // mutate retains the mutate while anonymous record is assembled or evaluated. + mutate func(*traversalPromotionManifest) + // errorContains retains the error contains while anonymous record is assembled or evaluated. + errorContains string + }{ + "selector": { + mutate: func(manifest *traversalPromotionManifest) { manifest.SelectorVersion = "sp-static-v5-contained" }, + errorContains: `requires selector "sp-static-v6"`, + }, + "outbound": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Buckets[0].Direction = "outbound" }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + "shallower maximum": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Buckets[0].MaximumDepth = 63 }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + "multiple kinds": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Buckets[0].RelationshipKindCount = 2 }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + "untyped": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Buckets[0].RelationshipKindCount = 0 + manifest.Buckets[0].UntypedRelationship = true + }, + errorContains: "qualified inbound typed single-kind one-path depth 1..64 envelope", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + policy := rewriteTestTraversalPolicyManifest(t, valid, test.mutate) + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.ErrorContains(t, driver.SetTraversalPolicy(policy), test.errorContains) + }) + } +} + +// TestTraversalPolicyQuerySHA256PreservesSemanticWhitespace verifies traversal policy query sha256 preserves semantic whitespace behavior. +func TestTraversalPolicyQuerySHA256PreservesSemanticWhitespace(t *testing.T) { + require.Equal(t, + TraversalPolicyQuerySHA256(" MATCH (n) RETURN n "), + TraversalPolicyQuerySHA256("MATCH (n) RETURN n"), + ) + require.NotEqual(t, + TraversalPolicyQuerySHA256(`RETURN "a b"`), + TraversalPolicyQuerySHA256(`RETURN "a b"`), + ) + require.NotEqual(t, + TraversalPolicyQuerySHA256("MATCH (`a b`) RETURN `a b`"), + TraversalPolicyQuerySHA256("MATCH (`a b`) RETURN `a b`"), + ) +} + +// TestTraversalPolicyAllowsGuardedOrientationWithoutSnapshotUpgrade verifies traversal policy allows guarded orientation without snapshot upgrade behavior. +func TestTraversalPolicyAllowsGuardedOrientationWithoutSnapshotUpgrade(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) RETURN id(e)" + policy := testTraversalPolicy(query, "", true) + policy.Generation = 2 + require.NoError(t, driver.SetTraversalPolicy(policy)) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + require.True(t, effective.EnableExpansionOrientation) + require.Contains(t, identity, "production-policy-") +} + +// TestTraversalPolicyAuthorizesOrientationProbeV2WithoutExecutingV1 verifies +// the manifest selector is carried into production translation options. +func TestTraversalPolicyAuthorizesOrientationProbeV2WithoutExecutingV1(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) RETURN id(e)" + policy := testTraversalPolicy(query, "", true) + policy = rewriteTestTraversalPolicyManifest(t, policy, func(manifest *traversalPromotionManifest) { + manifest.Candidate = string(optimize.ExpansionSearchPolicyOrientationProbeV2) + manifest.SelectorVersion = string(optimize.ExpansionSearchPolicyOrientationProbeV2) + }) + require.NoError(t, driver.SetTraversalPolicy(policy)) + effective, _ := driver.SchemaManager.effectiveTraversalPolicy(query, pgx.ReadCommitted) + options, err := effective.productionOptions(query) + require.NoError(t, err) + require.True(t, options.EnableExpansionOrientation) + require.Equal(t, optimize.ExpansionSearchPolicyOrientationProbeV2, options.ExpansionOrientationPolicy) + require.Equal(t, string(optimize.ExpansionSearchPolicyOrientationProbeV2), options.SelectorVersion) +} + +// TestTraversalPolicyExpansionOrientationKillSwitchRequiresNoEvidence verifies +// rollback changes the cache identity without requiring promotion evidence. +func TestTraversalPolicyExpansionOrientationKillSwitchRequiresNoEvidence(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 10, + DisableExpansionOrientation: true, + })) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy("MATCH (n) RETURN n", pgx.ReadCommitted) + require.True(t, effective.DisableExpansionOrientation) + require.Contains(t, identity, "production-policy-") + options, err := effective.productionOptions("MATCH (n) RETURN n") + require.NoError(t, err) + require.False(t, options.EnableExpansionOrientation) + require.Equal(t, "expansion-orientation-kill-switch-g10", options.SelectorVersion) +} + +// TestTraversalPolicyGuardedOrientationRequiresExactManifestContract verifies traversal policy guarded orientation requires exact manifest contract behavior. +func TestTraversalPolicyGuardedOrientationRequiresExactManifestContract(t *testing.T) { + query := "MATCH (r)-[:Expand*0..16]->()-[:Suffix]->(e) RETURN id(e)" + valid := testTraversalPolicy(query, "", true) + require.NoError(t, (&Driver{SchemaManager: NewSchemaManager(nil, 0)}).SetTraversalPolicy(valid)) + + tests := map[string]struct { + // mutate retains the mutate while anonymous record is assembled or evaluated. + mutate func(*traversalPromotionManifest) + // errorContains retains the error contains while anonymous record is assembled or evaluated. + errorContains string + }{ + "candidate": { + mutate: func(manifest *traversalPromotionManifest) { manifest.Candidate = "orientation-probe-v2" }, + errorContains: `candidate "orientation-probe-v2" does not authorize "orientation-probe-v1"`, + }, + "execution boundary": { + mutate: func(manifest *traversalPromotionManifest) { manifest.ExecutionBoundary = "inline_statement" }, + errorContains: `execution boundary "inline_statement" does not authorize "guarded_dual_arm"`, + }, + "missing cap": { + mutate: func(manifest *traversalPromotionManifest) { + delete(manifest.Caps, "root_row_limit") + }, + errorContains: "requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + "extra cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["survival_row_limit"] = 1 + }, + errorContains: "requires exactly root-row, reverse-seed-row, directional-degree-row, and state caps", + }, + "root cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["root_row_limit"] = optimize.ExpansionSearchOrientationRootRowLimit + 1 + }, + errorContains: "requires root_row_limit=512", + }, + "reverse seed cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["reverse_seed_row_limit"] = optimize.ExpansionSearchOrientationReverseSeedRowLimit + 1 + }, + errorContains: "requires reverse_seed_row_limit=512", + }, + "directional degree cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["directional_degree_row_limit"] = optimize.ExpansionSearchOrientationDirectionalDegreeRowLimit + 1 + }, + errorContains: "requires directional_degree_row_limit=16384", + }, + "state cap": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.Caps["state_limit"] = optimize.ExpansionSearchOrientationStateLimit + 1 + }, + errorContains: "requires state_limit=4096", + }, + "fallback": { + mutate: func(manifest *traversalPromotionManifest) { + manifest.FallbackExecutor = string(optimize.ExpansionSearchSuffixSeededReverse) + }, + errorContains: `requires fallback "EXPANSION-STEPWISE-FORWARD"`, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + policy := rewriteTestTraversalPolicyManifest(t, valid, test.mutate) + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.ErrorContains(t, driver.SetTraversalPolicy(policy), test.errorContains) + }) + } +} + +// TestTraversalPolicyEndpointSeededKillSwitchRequiresNoPromotionEvidence verifies traversal policy endpoint seeded kill switch requires no promotion evidence behavior. +func TestTraversalPolicyEndpointSeededKillSwitchRequiresNoPromotionEvidence(t *testing.T) { + driver := &Driver{SchemaManager: NewSchemaManager(nil, 0)} + require.NoError(t, driver.SetTraversalPolicy(TraversalPolicy{ + Generation: 7, + DisableEndpointSeededReverse: true, + })) + effective, identity := driver.SchemaManager.effectiveTraversalPolicy("MATCH (n) RETURN n", pgx.ReadCommitted) + require.True(t, effective.DisableEndpointSeededReverse) + require.Contains(t, identity, "production-policy-") + options, err := effective.productionOptions("MATCH (n) RETURN n") + require.NoError(t, err) + require.Equal(t, "endpoint-seeded-kill-switch-g7", options.SelectorVersion) +} diff --git a/drivers/pg/types.go b/drivers/pg/types.go index 211049a9..d1f3ceb2 100644 --- a/drivers/pg/types.go +++ b/drivers/pg/types.go @@ -7,14 +7,48 @@ import ( "github.com/specterops/dawgs/graph" ) +// edgeComposite is the ordered Go representation of PostgreSQL's edge composite type. type edgeComposite struct { - ID int64 - StartID int64 - EndID int64 - KindID int16 + // ID is the database identifier of the decoded relationship. + ID int64 + + // StartID is the database identifier of the relationship's start node. + StartID int64 + + // EndID is the database identifier of the relationship's end node. + EndID int64 + + // KindID is the PostgreSQL int2 identifier of the relationship kind. + KindID int16 + + // Properties contains the relationship's decoded JSON property values. Properties map[string]any } +// ScanNull rejects a null edge because the owned scalar representation has no null state. +func (s *edgeComposite) ScanNull() error { + return fmt.Errorf("cannot scan NULL into %T", s) +} + +// ScanIndex returns the destination for a PostgreSQL edge field in schema order. +func (s *edgeComposite) ScanIndex(index int) any { + switch index { + case 0: + return &s.ID + case 1: + return &s.StartID + case 2: + return &s.EndID + case 3: + return &s.KindID + case 4: + return &s.Properties + default: + return fmt.Errorf("%T only has 5 fields: index %d is out of bounds", s, index) + } +} + +// castSlice copies either a typed slice or a pgx []any representation into []T. func castSlice[T any](raw any) ([]T, error) { switch rawSlice := raw.(type) { case []T: @@ -38,6 +72,7 @@ func castSlice[T any](raw any) ([]T, error) { } } +// castMapValueAsSliceOf retrieves key from a fallback composite map and converts its value to []T. func castMapValueAsSliceOf[T any](compositeMap map[string]any, key string) ([]T, error) { if src, hasKey := compositeMap[key]; !hasKey { return nil, fmt.Errorf("composite map does not contain expected key %s", key) @@ -46,6 +81,7 @@ func castMapValueAsSliceOf[T any](compositeMap map[string]any, key string) ([]T, } } +// castAndAssignMapValue assigns a fallback composite-map field to dst, allowing lossless widening of integer values. func castAndAssignMapValue[T any](compositeMap map[string]any, key string, dst *T) error { if src, hasKey := compositeMap[key]; !hasKey { return fmt.Errorf("composite map does not contain expected key %s", key) @@ -124,52 +160,65 @@ func castAndAssignMapValue[T any](compositeMap map[string]any, key string, dst * return nil } +// nodeCompositesFromRaw converts typed or pgx fallback arrays into owned node composites. func nodeCompositesFromRaw(raw any) ([]nodeComposite, error) { - rawNodes, typeOK := raw.([]any) - if !typeOK { - return nil, fmt.Errorf("expected raw node composite array type []any but received %T", raw) - } - - nodes := make([]nodeComposite, 0, len(rawNodes)) - for _, rawNode := range rawNodes { - compositeMap, typeOK := rawNode.(map[string]any) - if !typeOK { - return nil, fmt.Errorf("unexpected type for raw node: %T", rawNode) - } - - var node nodeComposite - if err := node.FromMap(compositeMap); err != nil { - return nil, err + switch rawNodes := raw.(type) { + case []nodeComposite: + return rawNodes, nil + case []any: + nodes := make([]nodeComposite, len(rawNodes)) + for idx, rawNode := range rawNodes { + if node, typeOK := nodeCompositeFromRaw(rawNode); !typeOK { + return nil, fmt.Errorf("unexpected type for raw node at index %d: %T", idx, rawNode) + } else { + nodes[idx] = node + } } - nodes = append(nodes, node) + return nodes, nil + default: + return nil, fmt.Errorf("expected raw node composite array type []nodeComposite or []any but received %T", raw) } - - return nodes, nil } +// edgeCompositesFromRaw converts typed or pgx fallback arrays into owned edge composites. func edgeCompositesFromRaw(raw any) ([]edgeComposite, error) { - rawEdges, typeOK := raw.([]any) - if !typeOK { - return nil, fmt.Errorf("expected raw edge composite array type []any but received %T", raw) + switch rawEdges := raw.(type) { + case []edgeComposite: + return rawEdges, nil + case []any: + edges := make([]edgeComposite, len(rawEdges)) + for idx, rawEdge := range rawEdges { + if edge, typeOK := edgeCompositeFromRaw(rawEdge); !typeOK { + return nil, fmt.Errorf("unexpected type for raw edge at index %d: %T", idx, rawEdge) + } else { + edges[idx] = edge + } + } + + return edges, nil + default: + return nil, fmt.Errorf("expected raw edge composite array type []edgeComposite or []any but received %T", raw) } +} - edges := make([]edgeComposite, 0, len(rawEdges)) - for _, rawEdge := range rawEdges { - compositeMap, typeOK := rawEdge.(map[string]any) - if !typeOK { - return nil, fmt.Errorf("unexpected type for raw edge: %T", rawEdge) +// edgeCompositeFromRaw accepts an owned edge value, pointer, or pgx fallback map. +func edgeCompositeFromRaw(raw any) (edgeComposite, bool) { + switch typedRaw := raw.(type) { + case edgeComposite: + return typedRaw, true + case *edgeComposite: + if typedRaw != nil { + return *typedRaw, true } - + case map[string]any: var edge edgeComposite - if err := edge.FromMap(compositeMap); err != nil { - return nil, err + if edge.TryMap(typedRaw) { + return edge, true } - - edges = append(edges, edge) } - return edges, nil + return edgeComposite{}, false } func (s *edgeComposite) TryMap(compositeMap map[string]any) bool { @@ -215,12 +264,56 @@ func (s *edgeComposite) ToRelationship(ctx context.Context, kindMapper KindMappe return nil } +// nodeComposite is the ordered Go representation of PostgreSQL's node composite type. type nodeComposite struct { - ID int64 - KindIDs []int16 + // ID is the database identifier of the decoded node. + ID int64 + + // KindIDs contains the PostgreSQL int2 identifiers of the node's kinds. + KindIDs []int16 + + // Properties contains the node's decoded JSON property values. Properties map[string]any } +// ScanNull rejects a null node because the owned scalar representation has no null state. +func (s *nodeComposite) ScanNull() error { + return fmt.Errorf("cannot scan NULL into %T", s) +} + +// ScanIndex returns the destination for a PostgreSQL node field in schema order. +func (s *nodeComposite) ScanIndex(index int) any { + switch index { + case 0: + return &s.ID + case 1: + return &s.KindIDs + case 2: + return &s.Properties + default: + return fmt.Errorf("%T only has 3 fields: index %d is out of bounds", s, index) + } +} + +// nodeCompositeFromRaw accepts an owned node value, pointer, or pgx fallback map. +func nodeCompositeFromRaw(raw any) (nodeComposite, bool) { + switch typedRaw := raw.(type) { + case nodeComposite: + return typedRaw, true + case *nodeComposite: + if typedRaw != nil { + return *typedRaw, true + } + case map[string]any: + var node nodeComposite + if node.TryMap(typedRaw) { + return node, true + } + } + + return nodeComposite{}, false +} + func (s *nodeComposite) TryMap(compositeMap map[string]any) bool { return s.FromMap(compositeMap) == nil } @@ -256,21 +349,62 @@ func (s *nodeComposite) ToNode(ctx context.Context, kindMapper KindMapper, node return nil } +// pathComposite is the ordered Go representation of PostgreSQL's path composite type. type pathComposite struct { + // Nodes contains the path's decoded nodes in traversal order. Nodes []nodeComposite + + // Edges contains the path's decoded relationships in traversal order. Edges []edgeComposite } +// ScanNull rejects a null path because the owned scalar representation has no null state. +func (s *pathComposite) ScanNull() error { + return fmt.Errorf("cannot scan NULL into %T", s) +} + +// ScanIndex returns the destination for a PostgreSQL path field in schema order. +func (s *pathComposite) ScanIndex(index int) any { + switch index { + case 0: + return &s.Nodes + case 1: + return &s.Edges + default: + return fmt.Errorf("%T only has 2 fields: index %d is out of bounds", s, index) + } +} + +// pathCompositeFromRaw accepts an owned path value, pointer, or pgx fallback map. +func pathCompositeFromRaw(raw any) (pathComposite, bool) { + switch typedRaw := raw.(type) { + case pathComposite: + return typedRaw, true + case *pathComposite: + if typedRaw != nil { + return *typedRaw, true + } + case map[string]any: + var path pathComposite + if path.TryMap(typedRaw) { + return path, true + } + } + + return pathComposite{}, false +} + func (s *pathComposite) TryMap(compositeMap map[string]any) bool { return s.FromMap(compositeMap) == nil } +// FromMap populates a path composite from pgx's fallback map representation of its node and edge arrays. func (s *pathComposite) FromMap(compositeMap map[string]any) error { if rawNodes, hasNodes := compositeMap["nodes"]; hasNodes { if nodes, err := nodeCompositesFromRaw(rawNodes); err != nil { return err } else { - s.Nodes = append(s.Nodes, nodes...) + s.Nodes = nodes } } @@ -278,7 +412,7 @@ func (s *pathComposite) FromMap(compositeMap map[string]any) error { if edges, err := edgeCompositesFromRaw(rawEdges); err != nil { return err } else { - s.Edges = append(s.Edges, edges...) + s.Edges = edges } } diff --git a/go.mod b/go.mod index 6d2eac40..34c886c2 100644 --- a/go.mod +++ b/go.mod @@ -3,24 +3,24 @@ module github.com/specterops/dawgs go 1.26.4 require ( - cuelang.org/go v0.17.0 - github.com/RoaringBitmap/roaring/v2 v2.19.0 + cuelang.org/go v0.17.1 + github.com/RoaringBitmap/roaring/v2 v2.25.0 github.com/antlr4-go/antlr/v4 v4.13.1 github.com/axiomhq/hyperloglog v0.2.6 - github.com/bits-and-blooms/bitset v1.24.5 + github.com/bits-and-blooms/bitset v1.25.0 github.com/cespare/xxhash/v2 v2.3.0 github.com/cucumber/godog v0.16.0 github.com/fzipp/gocyclo v0.6.0 github.com/gammazero/deque v1.2.1 github.com/jackc/pgtype v1.14.4 github.com/jackc/pgx/v5 v5.10.0 - github.com/klauspost/compress v1.19.0 + github.com/klauspost/compress v1.19.2 github.com/neo4j/neo4j-go-driver/v5 v5.28.4 github.com/parquet-go/parquet-go v0.32.0 github.com/pashagolub/pgxmock/v5 v5.1.0 github.com/pelletier/go-toml/v2 v2.4.3 - github.com/stretchr/testify v1.11.1 - golang.org/x/tools v0.48.0 + github.com/stretchr/testify v1.12.1 + golang.org/x/tools v0.49.0 ) // Dawgrun requirements @@ -29,7 +29,7 @@ require ( github.com/charmbracelet/lipgloss v1.1.0 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 - github.com/jedib0t/go-pretty/v6 v6.8.2 + github.com/jedib0t/go-pretty/v6 v6.8.3 github.com/kanmu/go-sqlfmt v0.0.2-0.20200215095417-d1e63e2ee5eb github.com/mitchellh/go-wordwrap v1.0.1 github.com/specterops/go-repl v1.0.1 @@ -78,7 +78,7 @@ require ( github.com/ccojocar/zxcvbn-go v1.0.4 // indirect github.com/charithe/durationcheck v0.0.11 // indirect github.com/charmbracelet/colorprofile v0.4.3 // indirect - github.com/charmbracelet/x/ansi v0.11.7 // indirect + github.com/charmbracelet/x/ansi v0.11.8 // indirect github.com/charmbracelet/x/cellbuf v0.0.15 // indirect github.com/charmbracelet/x/term v0.2.2 // indirect github.com/ckaznocha/intrange v0.3.1 // indirect @@ -93,7 +93,7 @@ require ( github.com/denis-tingaikin/go-header v0.5.0 // indirect github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 // indirect github.com/dlclark/regexp2 v1.12.0 // indirect - github.com/dlclark/regexp2/v2 v2.2.2 // indirect + github.com/dlclark/regexp2/v2 v2.7.1 // indirect github.com/ettle/strcase v0.2.0 // indirect github.com/fatih/color v1.19.0 // indirect github.com/fatih/structtag v1.2.0 // indirect @@ -161,7 +161,7 @@ require ( github.com/ldez/tagliatelle v0.7.2 // indirect github.com/ldez/usetesting v0.5.0 // indirect github.com/leonklingele/grouper v1.1.2 // indirect - github.com/lucasb-eyer/go-colorful v1.4.0 // indirect + github.com/lucasb-eyer/go-colorful v1.4.1 // indirect github.com/macabu/inamedparam v0.2.0 // indirect github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect github.com/manuelarte/funcorder v0.5.0 // indirect @@ -169,8 +169,8 @@ require ( github.com/maratori/testpackage v1.1.2 // indirect github.com/matoous/godox v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.22 // indirect - github.com/mattn/go-runewidth v0.0.24 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mattn/go-runewidth v0.0.28 // indirect github.com/mgechev/revive v1.15.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moricho/tparallel v0.3.2 // indirect @@ -185,7 +185,6 @@ require ( github.com/parquet-go/jsonlite v1.0.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.22.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.63.0 // indirect @@ -230,7 +229,7 @@ require ( github.com/uudashr/gocognit v1.2.1 // indirect github.com/uudashr/iface v1.4.1 // indirect github.com/xen0n/gosmopolitan v1.3.0 // indirect - github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect + github.com/xo/terminfo v1.0.0 // indirect github.com/yagipy/maintidx v1.0.0 // indirect github.com/yeya24/promlinter v0.3.0 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect @@ -241,14 +240,14 @@ require ( go.augendre.info/fatcontext v0.9.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.1 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.54.0 // indirect - golang.org/x/exp v0.0.0-20260611194520-c48552f49976 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa // indirect golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 // indirect - golang.org/x/mod v0.38.0 // indirect + golang.org/x/mod v0.40.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.40.0 // indirect + golang.org/x/text v0.41.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.7.0 // indirect diff --git a/go.sum b/go.sum index 9ab22710..12ef5f1a 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ codeberg.org/polyfloyd/go-errorlint v1.9.0 h1:VkdEEmA1VBpH6ecQoMR4LdphVI3fA4RrCh codeberg.org/polyfloyd/go-errorlint v1.9.0/go.mod h1:GPRRu2LzVijNn4YkrZYJfatQIdS+TrcK8rL5Xs24qw8= cuelabs.dev/go/oci/ociregistry v0.0.0-20260601085548-328ff8e2c943 h1:XUtzi/yWlmuy8V6kkmVbbmirmUqcFe9Ce3gmEaHXf1Q= cuelabs.dev/go/oci/ociregistry v0.0.0-20260601085548-328ff8e2c943/go.mod h1:WjmQxb+W6nVNCgj8nXrF24lIz95AHwnSl36tpjDZSU8= -cuelang.org/go v0.17.0 h1:PrijS5ofUD01yiG11w74I04laXKLaBiMhEYvdt8Gb/A= -cuelang.org/go v0.17.0/go.mod h1:xlly/o1wSLvxOsi5vkQGieU0rLOt7TvUIizOFtnxHRU= +cuelang.org/go v0.17.1 h1:liOkxZDqTHrzq0USJX+6bMYOZ5PSf+wzvQr15AHpDCQ= +cuelang.org/go v0.17.1/go.mod h1:xlly/o1wSLvxOsi5vkQGieU0rLOt7TvUIizOFtnxHRU= dev.gaijin.team/go/exhaustruct/v4 v4.0.0 h1:873r7aNneqoBB3IaFIzhvt2RFYTuHgmMjoKfwODoI1Y= dev.gaijin.team/go/exhaustruct/v4 v4.0.0/go.mod h1:aZ/k2o4Y05aMJtiux15x8iXaumE88YdiB0Ai4fXOzPI= dev.gaijin.team/go/golib v0.6.0 h1:v6nnznFTs4bppib/NyU1PQxobwDHwCXXl15P7DV5Zgo= @@ -40,8 +40,8 @@ github.com/MirrexOne/unqueryvet v1.5.4 h1:38QOxShO7JmMWT+eCdDMbcUgGCOeJphVkzzRgy github.com/MirrexOne/unqueryvet v1.5.4/go.mod h1:fs9Zq6eh1LRIhsDIsxf9PONVUjYdFHdtkHIgZdJnyPU= github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= -github.com/RoaringBitmap/roaring/v2 v2.19.0 h1:zsWtVE+biht4eVl0YDLvykUqGkftR3Qc5UCbeKB4UQ4= -github.com/RoaringBitmap/roaring/v2 v2.19.0/go.mod h1:SfT3of9nYh3vis1dIbCj4Yw6KQGujTN+f345nrN/0JA= +github.com/RoaringBitmap/roaring/v2 v2.25.0 h1:HjcMG0PfmgO1rJcp2VHMarvQiulkB51qA31UH4I+j/U= +github.com/RoaringBitmap/roaring/v2 v2.25.0/go.mod h1:SfT3of9nYh3vis1dIbCj4Yw6KQGujTN+f345nrN/0JA= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/chroma/v2 v2.27.0 h1:FodwmyOBgJULFYmDqibcp9pvfDLWdtPRh9v/r5BXYZs= @@ -74,8 +74,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.24.5 h1:654xBVHc23gJMAgOTkPNoCVfiRxuIOAUnAZFtopqJ4w= -github.com/bits-and-blooms/bitset v1.24.5/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.25.0 h1:0Ro0qF4abCkM6SqWPVj29sFhAbMPAZpaDD7xhJ10beM= +github.com/bits-and-blooms/bitset v1.25.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bkielbasa/cyclop v1.2.3 h1:faIVMIGDIANuGPWH031CZJTi2ymOQBULs9H21HSMa5w= github.com/bkielbasa/cyclop v1.2.3/go.mod h1:kHTwA9Q0uZqOADdupvcFJQtp/ksSnytRMe8ztxG8Fuo= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= @@ -104,8 +104,8 @@ github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= -github.com/charmbracelet/x/ansi v0.11.7 h1:kzv1kJvjg2S3r9KHo8hDdHFQLEqn4RBCb39dAYC84jI= -github.com/charmbracelet/x/ansi v0.11.7/go.mod h1:9qGpnAVYz+8ACONkZBUWPtL7lulP9No6p1epAihUZwQ= +github.com/charmbracelet/x/ansi v0.11.8 h1:JMFwp0CgDC2+jcOB162HH5k7I3FVbgFSMMYg7dSPBQQ= +github.com/charmbracelet/x/ansi v0.11.8/go.mod h1:ZNN+3mXny/516oTQPLMPIBeSINvNJJQ8uQXDgbeJxY0= github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI= github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q= github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk= @@ -147,8 +147,8 @@ github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33 h1:ucRHb6/lvW/+mT github.com/dgryski/go-metro v0.0.0-20250106013310-edb8663e5e33/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dlclark/regexp2 v1.12.0 h1:0j4c5qQmnC6XOWNjP3PIXURXN2gWx76rd3KvgdPkCz8= github.com/dlclark/regexp2 v1.12.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2/v2 v2.2.2 h1:MYWvNYw8okuqNhwTYO587EZMiDruVa2vhV6fsGpfya0= -github.com/dlclark/regexp2/v2 v2.2.2/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= +github.com/dlclark/regexp2/v2 v2.7.1 h1:yqDtwI1ptXXvEUNpYTk2lad4jLtAcKqkzepn4savSk4= +github.com/dlclark/regexp2/v2 v2.7.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU= github.com/emicklei/proto v1.14.3 h1:zEhlzNkpP8kN6utonKMzlPfIvy82t5Kb9mufaJxSe1Q= github.com/emicklei/proto v1.14.3/go.mod h1:rn1FgRS/FANiZdD2djyH7TMA9jdRDcYQ9IEN9yvjX0A= github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= @@ -334,8 +334,8 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jedib0t/go-pretty/v6 v6.8.2 h1:FmKNr1GOyot/zqNQplE8HLhFguJaeHJTCArntnI4uxE= -github.com/jedib0t/go-pretty/v6 v6.8.2/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= +github.com/jedib0t/go-pretty/v6 v6.8.3 h1:yVSk5aemoYHCvcrtqyXklwqcgHQIQzmy/oUzFlmffSQ= +github.com/jedib0t/go-pretty/v6 v6.8.3/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= github.com/jgautheron/goconst v1.8.2 h1:y0XF7X8CikZ93fSNT6WBTb/NElBu9IjaY7CCYQrCMX4= github.com/jgautheron/goconst v1.8.2/go.mod h1:A0oxgBCHy55NQn6sYpO7UdnA9p+h7cPtoOZUmvNIako= github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjzq7gFzUs= @@ -355,8 +355,8 @@ github.com/kisielk/errcheck v1.10.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkHAIKE/contextcheck v1.1.6 h1:7HIyRcnyzxL9Lz06NGhiKvenXq7Zw6Q0UQu/ttjfJCE= github.com/kkHAIKE/contextcheck v1.1.6/go.mod h1:3dDbMRNBFaq8HFXWC1JyvDSPm43CmE6IuHam8Wr0rkg= -github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= -github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -393,8 +393,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= -github.com/lucasb-eyer/go-colorful v1.4.0 h1:UtrWVfLdarDgc44HcS7pYloGHJUjHV/4FwW4TvVgFr4= -github.com/lucasb-eyer/go-colorful v1.4.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= +github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/macabu/inamedparam v0.2.0 h1:VyPYpOc10nkhI2qeNUdh3Zket4fcZjEWe35poddBCpE= github.com/macabu/inamedparam v0.2.0/go.mod h1:+Pee9/YfGe5LJ62pYXqB89lJ+0k5bsR8Wgz/C0Zlq3U= github.com/manuelarte/embeddedstructfieldcheck v0.4.0 h1:3mAIyaGRtjK6EO9E73JlXLtiy7ha80b2ZVGyacxgfww= @@ -416,10 +416,10 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stg github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= -github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= -github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= -github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.28 h1:rPyg2ybwEKPebvpzVWe1gKBkH8EQFkxO4Y0hjBeLaBU= +github.com/mattn/go-runewidth v0.0.28/go.mod h1:3qAiGCV4Koz/yuveO58qUefmUTRm8r0IGEXZ9jeHp/8= github.com/mgechev/revive v1.15.0 h1:vJ0HzSBzfNyPbHKolgiFjHxLek9KUijhqh42yGoqZ8Q= github.com/mgechev/revive v1.15.0/go.mod h1:LlAKO3QQe9OJ0pVZzI2GPa8CbXGZ/9lNpCGvK4T/a8A= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -475,8 +475,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= @@ -574,8 +572,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tenntenn/modver v1.0.1 h1:2klLppGhDgzJrScMpkj9Ujy3rXPUspSjAcev9tSEBgA= @@ -604,8 +602,8 @@ github.com/uudashr/iface v1.4.1 h1:J16Xl1wyNX9ofhpHmQ9h9gk5rnv2A6lX/2+APLTo0zU= github.com/uudashr/iface v1.4.1/go.mod h1:pbeBPlbuU2qkNDn0mmfrxP2X+wjPMIQAy+r1MBXSXtg= github.com/xen0n/gosmopolitan v1.3.0 h1:zAZI1zefvo7gcpbCOrPSHJZJYA9ZgLfJqtKzZ5pHqQM= github.com/xen0n/gosmopolitan v1.3.0/go.mod h1:rckfr5T6o4lBtM1ga7mLGKZmLxswUoH1zxHgNXOsEt4= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xo/terminfo v1.0.0 h1:2ZpYzqWzyyytjk3TP6aJVDhkMAkc99/1xKQdA3TDTBY= +github.com/xo/terminfo v1.0.0/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM= github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk= github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs= @@ -648,8 +646,9 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -664,10 +663,10 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M= -golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa h1:QSyA8ishJCyT21kER9KwNt0b7BM3iRK4x9QXhjN5Fdk= +golang.org/x/exp v0.0.0-20260824195058-e88cd73687aa/go.mod h1:zeBbvyFKDaLwa7CH/zI8KXt7gTl14SF7sO08Pl5jBCM= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20260209203927-2842357ff358 h1:qWFG1Dj7TBjOjOvhEOkmyGPVoquqUKnIU0lEVLp8xyk= @@ -684,8 +683,8 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -702,8 +701,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -766,8 +765,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -787,8 +786,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= -golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= -golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= diff --git a/integration/BENCHMARKS.md b/integration/BENCHMARKS.md index 09c4677f..6c12994a 100644 --- a/integration/BENCHMARKS.md +++ b/integration/BENCHMARKS.md @@ -33,6 +33,92 @@ ## Shortest Paths +### PostgreSQL V2 inline predecessor-DAG qualification + +The default-off `ASP-I1-U-DAG+MAT-M0` executor resolves singleton endpoints +before recursive work, discovers distance and predecessors using identifier-only +state, and hydrates emitted paths through an inline M0 lateral operator. The +incumbent PostgreSQL SQL remains unchanged; production use still requires an +exact traversal-policy manifest and stable-snapshot transaction. + +Matched live `traversal_shapes` runs use 20 timed iterations, two warm-up +iterations, one worker, PostgreSQL `plan_cache_mode=auto`, and JIT enabled: + +| Scenario | PG V2 incumbent p50/p95 | PG V2 candidate p50/p95 | Neo4j p50/p95 | +|---|---:|---:|---:| +| Diamond, three shortest paths | 31.6ms / 216ms | 1.8ms / 4.5ms | 1.9ms / 2.7ms | +| Disconnected endpoints | 2.5ms / 42.3ms | 1.5ms / 2.2ms | 1.5ms / 2.8ms | + +The stored-workspace `ASP-B2-DAG-MIN-LEVEL` candidate did not qualify on this +fixture because workspace execution dominated the small search. Forced custom +planning also regressed both shortest-path shapes; `auto` remains the selected +plan policy. The B2 executor remains available only for diagnostic/tool runs. + +### Production-policy path + +The forced-executor measurements above establish a candidate SQL comparison, +but do not exercise the driver's manifest selection or connection-local translation +cache. `cmd/benchmark` now has a separate `production_policy` mode that loads a +GraphBench-verified manifest into `Driver.SetTraversalPolicy`, requires +Repeatable Read, and runs exactly its single allowlisted parameterized Cypher +scenario. A live PostgreSQL manual integration test renders the candidate SQL, +binds its SHA-256 into a schema-v2 manifest, installs that policy on `pg`, +and executes the route successfully. + +No forced-mode latency is relabeled as a production-policy result here: a +comparable publication requires a clean-source, GraphBench-verified manifest +whose SQL anchor and exact query digest match the current benchmark schema. +The policy route was nevertheless executed live against PostgreSQL on +2026-08-20: `TestPostgresV2BenchmarkPolicyPath` rendered and anchor-validated +the candidate statement, installed it through `SetTraversalPolicy`, and +returned the expected path. The full PostgreSQL `make test_all` suite also +passed. These are execution-validation results, not promotion-performance +evidence. + +For a new qualified manifest, first derive its SQL anchor from the actual +benchmark graph and parameterized scenario. This preflight record is not +evidence and cannot activate a production policy: + +```bash +go run ./cmd/benchmark \ + -driver pg \ + -connection "postgresql://user:password@localhost/database" \ + -dataset traversal_shapes \ + -pg-traversal-policy-preflight-manifest .coverage/provisional.json \ + -pg-traversal-policy-preflight-output .coverage/policy-preflight.json +``` + +Copy the emitted `operational_candidate_sql_sha256` into the provisional +manifest, generate and verify the complete GraphBench evidence closure, and +only then run the production-policy command below. Use a new preflight-output +path for every capture; the benchmark refuses to overwrite a manifest or prior +record. + +When that evidence is available, run: + +```bash +go run ./cmd/graphbench -promotion-manifest .coverage/promotion.json +go run ./cmd/benchmark \ + -driver pg \ + -connection "postgresql://user:password@localhost/database" \ + -dataset traversal_shapes \ + -pg-traversal-policy-manifest .coverage/promotion.json \ + -pg-traversal-policy-generation 7 \ + -pg-plan-cache-mode auto \ + -iterations 20 -warmup 2 -workers 1 +``` + +```bash +go run ./cmd/benchmark \ + -driver pg \ + -connection "postgresql://user:password@localhost/database" \ + -dataset traversal_shapes \ + -iterations 20 -warmup 2 -workers 1 \ + -pg-min-conns 0 -pg-max-conns 1 \ + -pg-shortest-path-executor 'ASP-I1-U-DAG+MAT-M0' \ + -pg-plan-cache-mode auto +``` + | Dataset | Start | End | Paths | Median | P95 | Max | | --------------- | ----- | --- | ----: | -----: | -----: | -----: | | diamond | a | d | 2 | 0.42ms | 0.68ms | 0.91ms | diff --git a/integration/cypher_template_test.go b/integration/cypher_template_test.go index 6fd5b5f6..4fd534ff 100644 --- a/integration/cypher_template_test.go +++ b/integration/cypher_template_test.go @@ -31,42 +31,97 @@ import ( "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) +// cypherTemplateFile describes the ordinary and metamorphic query families loaded from one template JSON file. type cypherTemplateFile struct { - Families []cypherTemplateFamily `json:"families,omitempty"` + // Families contains independently asserted query-template families. + Families []cypherTemplateFamily `json:"families,omitempty"` + + // Metamorphic contains families whose query variants must produce equivalent results. Metamorphic []cypherMetamorphicFamily `json:"metamorphic,omitempty"` - path string + + // path records the source file for subtest naming and diagnostics. + path string } +// cypherTemplateFamily combines a fixture and query template with the variants asserted against it. type cypherTemplateFamily struct { - Name string `json:"name"` - Fixture *opengraph.Graph `json:"fixture"` - Template string `json:"template"` - Params map[string]any `json:"params,omitempty"` + // Name identifies the family in test output. + Name string `json:"name"` + + // Fixture is loaded transactionally for every variant. + Fixture *opengraph.Graph `json:"fixture"` + + // Template is the Cypher source rendered with each variant's Vars. + Template string `json:"template"` + + // Params supplies parameters shared by every variant. + Params testutil.Params `json:"params,omitempty"` + + // NodeParams maps shared parameter names to fixture node identifiers. + NodeParams map[string]string `json:"node_params,omitempty"` + + // NodeListParams maps shared parameter names to lists of fixture node identifiers. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + + // Variants enumerates template substitutions and expected results. Variants []cypherTemplateVariant `json:"variants"` } +// cypherTemplateVariant supplies one rendering and assertion for a query-template family. type cypherTemplateVariant struct { - Name string `json:"name"` - Vars map[string]string `json:"vars,omitempty"` - Params map[string]any `json:"params,omitempty"` - Assert json.RawMessage `json:"assert"` + // Name identifies the variant in test output. + Name string `json:"name"` + + // Vars contains text substitutions applied to the Cypher template. + Vars map[string]string `json:"vars,omitempty"` + + // Params augments or overrides family-level query parameters. + Params testutil.Params `json:"params,omitempty"` + + // NodeParams augments or overrides family-level fixture-node parameters. + NodeParams map[string]string `json:"node_params,omitempty"` + + // NodeListParams augments or overrides family-level fixture-node-list parameters. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + + // Assert encodes the expected primary query result. + Assert json.RawMessage `json:"assert"` + + // PostAssertions contains state checks run after the primary query drains. + PostAssertions []stateAssertion `json:"post_assertions,omitempty"` } +// cypherMetamorphicFamily describes queries that must agree under the selected comparison modes. type cypherMetamorphicFamily struct { - Name string `json:"name"` - Fixture *opengraph.Graph `json:"fixture"` - Compare comparisonModes `json:"compare"` + // Name identifies the family in test output. + Name string `json:"name"` + + // Fixture is loaded once for the family's equivalence comparison. + Fixture *opengraph.Graph `json:"fixture"` + + // Compare selects the result dimensions used to establish equivalence. + Compare comparisonModes `json:"compare"` + + // Queries contains the query variants compared with the baseline. Queries []cypherMetamorphicQuery `json:"queries"` } +// cypherMetamorphicQuery is one named Cypher statement and parameter set in an equivalence family. type cypherMetamorphicQuery struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` + // Name identifies the query in test output. + Name string `json:"name"` + + // Cypher is the statement executed for this variant. + Cypher string `json:"cypher"` + + // Params contains the statement's query parameters. + Params testutil.Params `json:"params,omitempty"` } +// TestCypherTemplates renders every template variant and verifies its query and post-state assertions against the shared fixture. func TestCypherTemplates(t *testing.T) { templateFiles := loadCypherTemplateFiles(t) nodeKinds, edgeKinds := cypherTemplateKinds(templateFiles) @@ -85,10 +140,13 @@ func TestCypherTemplates(t *testing.T) { cypher = renderCypherTemplate(t, family.Template, variant.Vars) check = parseAssertion(t, variant.Assert) tc = testCase{ - Name: variant.Name, - Cypher: cypher, - Params: mergeParams(family.Params, variant.Params), - Fixture: family.Fixture, + Name: variant.Name, + Cypher: cypher, + Params: mergeParams(family.Params, variant.Params), + NodeParams: mergeStringMap(family.NodeParams, variant.NodeParams), + NodeListParams: mergeStringListMap(family.NodeListParams, variant.NodeListParams), + Fixture: family.Fixture, + PostAssertions: variant.PostAssertions, } ) @@ -107,6 +165,7 @@ func TestCypherTemplates(t *testing.T) { } } +// loadCypherTemplateFiles reads and decodes every JSON template file, preserving each source path for diagnostics. func loadCypherTemplateFiles(t *testing.T) []cypherTemplateFile { t.Helper() @@ -137,6 +196,8 @@ func loadCypherTemplateFiles(t *testing.T) []cypherTemplateFile { return templateFiles } +// cypherTemplateKinds collects the node and relationship kinds used by every +// inline template and metamorphic fixture. func cypherTemplateKinds(templateFiles []cypherTemplateFile) (graph.Kinds, graph.Kinds) { var nodeKinds, edgeKinds graph.Kinds @@ -161,6 +222,8 @@ func cypherTemplateKinds(templateFiles []cypherTemplateFile) (graph.Kinds, graph return nodeKinds, edgeKinds } +// renderCypherTemplate replaces named placeholders and fails if any placeholder +// remains unresolved. func renderCypherTemplate(t *testing.T, template string, vars map[string]string) string { t.Helper() @@ -176,6 +239,7 @@ func renderCypherTemplate(t *testing.T, template string, vars map[string]string) return rendered } +// mergeParams returns a copy of base with overrides taking precedence. func mergeParams(base, overrides map[string]any) map[string]any { if len(base) == 0 && len(overrides) == 0 { return nil @@ -192,6 +256,42 @@ func mergeParams(base, overrides map[string]any) map[string]any { return merged } +// mergeStringMap returns a copy of base with string overrides taking +// precedence. +func mergeStringMap(base, overrides map[string]string) map[string]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = value + } + for key, value := range overrides { + merged[key] = value + } + return merged +} + +// mergeStringListMap returns a deep-enough copy of base with list overrides +// taking precedence. +func mergeStringListMap(base, overrides map[string][]string) map[string][]string { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + + merged := make(map[string][]string, len(base)+len(overrides)) + for key, value := range base { + merged[key] = append([]string(nil), value...) + } + for key, value := range overrides { + merged[key] = append([]string(nil), value...) + } + return merged +} + +// runWithTemplateFixture executes a rendered case against its inline fixture, +// checks the query result and postconditions, and rolls the transaction back. func runWithTemplateFixture(t *testing.T, ctx context.Context, db graph.Database, tc testCase, assertion caseAssertion) { t.Helper() @@ -200,16 +300,21 @@ func runWithTemplateFixture(t *testing.T, ctx context.Context, db graph.Database } queryErrorObserved := false - session := &Session{DB: db, Ctx: ctx} + session := &Session{ + DB: db, + Ctx: ctx, + } err := session.WithRollbackFixture(t, tc.Fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { - result := tx.Query(tc.Cypher, tc.Params) - defer result.Close() + params := resolveFixtureParams(t, tc.Params, tc.NodeParams, tc.NodeListParams, idMap) + result := tx.Query(tc.Cypher, params) assertion.checkResult(t, result, newAssertionContext(idMap)) + result.Close() if assertion.expectQueryError { queryErrorObserved = true + return nil } - return nil + return runStateAssertions(t, tx, idMap, tc.PostAssertions) }) if assertion.expectQueryError && queryErrorObserved && err != nil { @@ -221,6 +326,8 @@ func runWithTemplateFixture(t *testing.T, ctx context.Context, db graph.Database } } +// runMetamorphicFamily executes every query over one fixture and requires their +// selected comparison signatures to match the first query. func runMetamorphicFamily(t *testing.T, ctx context.Context, db graph.Database, family cypherMetamorphicFamily) { t.Helper() @@ -232,7 +339,10 @@ func runMetamorphicFamily(t *testing.T, ctx context.Context, db graph.Database, t.Fatal("metamorphic cases must define at least two queries") } - session := &Session{DB: db, Ctx: ctx} + session := &Session{ + DB: db, + Ctx: ctx, + } err := session.WithRollbackFixture(t, family.Fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { assertCtx := newAssertionContext(idMap) var baselineName string @@ -277,8 +387,10 @@ func runMetamorphicFamily(t *testing.T, ctx context.Context, db graph.Database, } } +// comparisonModes accepts either one comparison-mode string or a list in template JSON. type comparisonModes []string +// UnmarshalJSON accepts either a single comparison mode or a list of modes. func (s *comparisonModes) UnmarshalJSON(raw []byte) error { var mode string if err := json.Unmarshal(raw, &mode); err == nil { @@ -295,10 +407,13 @@ func (s *comparisonModes) UnmarshalJSON(raw []byte) error { return nil } +// String joins comparison modes for use in generated subtest names. func (s comparisonModes) String() string { return strings.Join(s, ",") } +// comparisonSignature computes each requested comparison mode for a collected +// result in declaration order. func comparisonSignature(t *testing.T, result queryResult, ctx assertionContext, modes comparisonModes) []string { t.Helper() @@ -314,6 +429,8 @@ func comparisonSignature(t *testing.T, result queryResult, ctx assertionContext, return signature } +// comparisonModeSignature canonicalizes a collected result according to one +// supported metamorphic comparison mode. func comparisonModeSignature(t *testing.T, result queryResult, ctx assertionContext, mode string) string { t.Helper() @@ -346,6 +463,12 @@ func comparisonModeSignature(t *testing.T, result queryResult, ctx assertionCont signatures = append(signatures, pathEdgeKindSignature(t, path)) } signature = sortedSignatures(signatures) + case "path_relationship_records": + signatures := make([]string, 0, len(result.rows)) + for _, path := range collectPaths(t, result) { + signatures = append(signatures, pathRelationshipRecordSignature(t, path, ctx)) + } + signature = sortedSignatures(signatures) default: t.Fatalf("unknown metamorphic comparison mode %q", mode) } @@ -358,6 +481,8 @@ func comparisonModeSignature(t *testing.T, result queryResult, ctx assertionCont return mode + ":" + string(encoded) } +// firstScalarSignatures returns the canonical signature of each row's first +// projected value. func firstScalarSignatures(t *testing.T, result queryResult) []string { t.Helper() @@ -373,6 +498,7 @@ func firstScalarSignatures(t *testing.T, result queryResult) []string { return signatures } +// rowScalarSignatures renders every result row into a deterministic scalar signature. func rowScalarSignatures(result queryResult) []string { signatures := make([]string, 0, len(result.rows)) for _, row := range result.rows { @@ -382,6 +508,7 @@ func rowScalarSignatures(result queryResult) []string { return signatures } +// sortedSignatures returns a sorted copy without modifying its input. func sortedSignatures(signatures []string) []string { sorted := append([]string(nil), signatures...) sort.Strings(sorted) diff --git a/integration/cypher_test.go b/integration/cypher_test.go index b9adfc96..4ae03bae 100644 --- a/integration/cypher_test.go +++ b/integration/cypher_test.go @@ -32,25 +32,64 @@ import ( "github.com/specterops/dawgs/graph" "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/testutil" ) // caseFile represents one JSON test case file. type caseFile struct { - Dataset string `json:"dataset"` - Cases []testCase `json:"cases"` + // Dataset selects the fixture dataset loaded before executing Cases. + Dataset string `json:"dataset"` + + // Cases contains the queries and assertions decoded from this file. + Cases []testCase `json:"cases"` } // testCase is a single test: a Cypher query and an assertion on its result. // Cases with a "fixture" field run in a write transaction that rolls back, // so the inline data doesn't persist. type testCase struct { - Name string `json:"name"` - Cypher string `json:"cypher"` - Params map[string]any `json:"params,omitempty"` - Assert json.RawMessage `json:"assert"` + // Name identifies the case in test output. + Name string `json:"name"` + + // Cypher is the statement executed by the case. + Cypher string `json:"cypher"` + + // Params contains literal and generated query parameters. + Params testutil.Params `json:"params,omitempty"` + + // NodeParams maps parameter names to fixture node identifiers. + NodeParams map[string]string `json:"node_params,omitempty"` + + // NodeListParams maps parameter names to lists of fixture node identifiers. + NodeListParams map[string][]string `json:"node_list_params,omitempty"` + + // Assert encodes the expected primary result assertion. + Assert json.RawMessage `json:"assert"` + + // PostAssertions contains state checks executed after the primary result drains. + PostAssertions []stateAssertion `json:"post_assertions,omitempty"` + + // Fixture optionally supplies inline graph data loaded in a rollback transaction. Fixture *opengraph.Graph `json:"fixture,omitempty"` } +// stateAssertion runs after the primary query has been fully drained. It is +// executed in the same transaction and against the same fixture ID map. +type stateAssertion struct { + // Name optionally identifies the assertion in diagnostics. + Name string `json:"name,omitempty"` + + // Cypher is the state-inspection query executed after the primary query. + Cypher string `json:"cypher"` + + // Params contains parameters for the state-inspection query. + Params testutil.Params `json:"params,omitempty"` + + // Assert encodes the expected state-inspection result. + Assert json.RawMessage `json:"assert"` +} + +// TestCypher executes every fixture-backed case, grouping cases by dataset so each group shares one loaded graph. func TestCypher(t *testing.T) { files, err := filepath.Glob("testdata/cases/*.json") if err != nil { @@ -60,10 +99,13 @@ func TestCypher(t *testing.T) { t.Fatal("no case files found in testdata/cases/") } - // Parse all case files and group by dataset. + // group collects case files that share one fixture dataset. type group struct { + // dataset names the fixture dataset shared by files. dataset string - files []caseFile + + // files contains the parsed cases in the dataset group. + files []caseFile } var ( groups = map[string]*group{} @@ -141,11 +183,15 @@ func TestCypher(t *testing.T) { // {"contains_edge": {start,end,kind,props}} — some row/path has a relationship matching all listed fields // {"node_ids": ["a", "b"]} — exact multiset of returned fixture node IDs, order-independent // {"node_id_set": ["a", "b"]} — exact set of returned fixture node IDs, order-independent +// {"node_records": [{id,kinds,props}]} — exact returned nodes, including kinds and properties +// {"relationship_triples": [{start,end,kind}]} — exact returned relationship triples +// {"relationship_records": [{start,end,kind,props}]} — exact returned relationships and properties // {"ordered_node_ids": ["a", "b"]} — first returned node ID per row, preserving row order // {"node_list_ids": [["a", "b"]]} — exact multiset of returned node-list ID sequences // {"path_node_ids": [["a", "b"]]} — exact multiset of returned path node ID sequences // {"path_lengths": [N...]} — exact multiset of returned path edge counts // {"path_edge_kinds": [["K"...]]} — exact multiset of returned path edge kind sequences +// {"path_relationship_records": [[{start,end,kind,props}...]]} — exact ordered relationships for every returned path // {"relationship_list_kinds": [["K"...]]} — exact multiset of returned relationship-list kind sequences // // Object assertions may combine multiple keys; every assertion must pass. @@ -218,6 +264,15 @@ func parseAssertion(t *testing.T, raw json.RawMessage) caseAssertion { case "node_id_set": assertions = append(assertions, assertNodeIDs(decodeAssertionValue[[]string](t, key, val), true)) + case "node_records": + assertions = append(assertions, assertNodeRecords(decodeAssertionValue[[]nodeExpectation](t, key, val))) + + case "relationship_triples": + assertions = append(assertions, assertRelationshipRecords(decodeAssertionValue[[]edgeExpectation](t, key, val), false)) + + case "relationship_records": + assertions = append(assertions, assertRelationshipRecords(decodeAssertionValue[[]edgeExpectation](t, key, val), true)) + case "ordered_node_ids": assertions = append(assertions, assertOrderedNodeIDs(decodeAssertionValue[[]string](t, key, val))) @@ -233,6 +288,9 @@ func parseAssertion(t *testing.T, raw json.RawMessage) caseAssertion { case "path_edge_kinds": assertions = append(assertions, assertPathEdgeKinds(decodeAssertionValue[[][]string](t, key, val))) + case "path_relationship_records": + assertions = append(assertions, assertPathRelationshipRecords(decodeAssertionValue[[][]edgeExpectation](t, key, val))) + case "relationship_list_kinds": assertions = append(assertions, assertRelationshipListKinds(decodeAssertionValue[[][]string](t, key, val))) @@ -262,8 +320,9 @@ func runReadOnly(t *testing.T, ctx context.Context, db graph.Database, idMap ope var ( queryErrorObserved = false + params = resolveFixtureParams(t, tc.Params, tc.NodeParams, tc.NodeListParams, idMap) err = db.ReadTransaction(ctx, func(tx graph.Transaction) error { - result := tx.Query(tc.Cypher, tc.Params) + result := tx.Query(tc.Cypher, params) defer result.Close() assertion.checkResult(t, result, newAssertionContext(idMap)) if assertion.expectQueryError { @@ -288,16 +347,21 @@ func runWithFixture(t *testing.T, ctx context.Context, db graph.Database, tc tes t.Helper() queryErrorObserved := false - session := &Session{DB: db, Ctx: ctx} + session := &Session{ + DB: db, + Ctx: ctx, + } err := session.WithRollbackFixture(t, tc.Fixture, true, func(tx graph.Transaction, idMap opengraph.IDMap) error { - result := tx.Query(tc.Cypher, tc.Params) - defer result.Close() + params := resolveFixtureParams(t, tc.Params, tc.NodeParams, tc.NodeListParams, idMap) + result := tx.Query(tc.Cypher, params) assertion.checkResult(t, result, newAssertionContext(idMap)) + result.Close() if assertion.expectQueryError { queryErrorObserved = true + return nil } - return nil + return runStateAssertions(t, tx, idMap, tc.PostAssertions) }) if assertion.expectQueryError && queryErrorObserved && err != nil { @@ -309,15 +373,90 @@ func runWithFixture(t *testing.T, ctx context.Context, db graph.Database, tc tes } } +// resolveFixtureParams copies literal parameters and replaces fixture node +// references with their backend database IDs. +func resolveFixtureParams( + t *testing.T, + params map[string]any, + nodeParams map[string]string, + nodeListParams map[string][]string, + idMap opengraph.IDMap, +) map[string]any { + t.Helper() + + resolved := make(map[string]any, len(params)+len(nodeParams)+len(nodeListParams)) + for name, value := range params { + resolved[name] = value + } + + for paramName, fixtureID := range nodeParams { + id, found := idMap[fixtureID] + if !found { + t.Fatalf("node parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + resolved[paramName] = id.Int64() + } + + for paramName, fixtureIDs := range nodeListParams { + ids := make([]int64, len(fixtureIDs)) + for idx, fixtureID := range fixtureIDs { + id, found := idMap[fixtureID] + if !found { + t.Fatalf("node list parameter %q references unknown fixture ID %q", paramName, fixtureID) + } + ids[idx] = id.Int64() + } + resolved[paramName] = ids + } + + if len(resolved) == 0 { + return nil + } + return resolved +} + +// runStateAssertions executes and checks each postcondition query in the +// fixture's transaction. +func runStateAssertions(t *testing.T, tx graph.Transaction, idMap opengraph.IDMap, assertions []stateAssertion) error { + t.Helper() + + for idx, spec := range assertions { + name := spec.Name + if name == "" { + name = fmt.Sprintf("post assertion %d", idx+1) + } + if spec.Cypher == "" { + t.Fatalf("%s has no Cypher query", name) + } + + check := parseAssertion(t, spec.Assert) + if check.expectQueryError { + t.Fatalf("%s may not expect a query error", name) + } + + result := tx.Query(spec.Cypher, spec.Params) + check.checkResult(t, result, newAssertionContext(idMap)) + result.Close() + } + + return nil +} + // --- Assertion implementations --- +// caseAssertion selects either a normalized result check or an expected query-error check. type caseAssertion struct { - check resultAssertion + // check validates a successfully drained result. + check resultAssertion + + // expectQueryError selects the error path instead of invoking check. expectQueryError bool } +// resultAssertion validates a normalized query result using fixture-aware identity mapping. type resultAssertion func(*testing.T, queryResult, assertionContext) +// checkResult dispatches to the expected error path or the configured successful-result assertion. func (s caseAssertion) checkResult(t *testing.T, result graph.Result, ctx assertionContext) { t.Helper() @@ -333,10 +472,13 @@ func (s caseAssertion) checkResult(t *testing.T, result graph.Result, ctx assert s.check(t, collectResult(t, result), ctx) } +// assertionContext translates backend database IDs back to stable fixture identifiers. type assertionContext struct { + // fixtureIDByID maps database node IDs to their source fixture IDs. fixtureIDByID map[graph.ID]string } +// newAssertionContext reverses a fixture ID map for result assertions. func newAssertionContext(idMap opengraph.IDMap) assertionContext { ctx := assertionContext{ fixtureIDByID: make(map[graph.ID]string, len(idMap)), @@ -349,6 +491,7 @@ func newAssertionContext(idMap opengraph.IDMap) assertionContext { return ctx } +// fixtureID returns the stable fixture identifier for dbID and fails the current test if it is unknown. func (s assertionContext) fixtureID(t *testing.T, dbID graph.ID) string { t.Helper() @@ -360,16 +503,26 @@ func (s assertionContext) fixtureID(t *testing.T, dbID graph.ID) string { return "" } +// resultRow is an owned snapshot of one backend result row and its column names. type resultRow struct { - keys []string + // keys contains the row's projected column names. + keys []string + + // values contains an owned copy of the row's projected values. values []any } +// queryResult contains drained rows and the backend mapper needed to decode graph values. type queryResult struct { - rows []resultRow + // rows contains every drained row in result order. + rows []resultRow + + // mapper converts backend-specific values to graph-native representations. mapper graph.ValueMapper } +// collectResult drains a backend result into owned rows and retains its value +// mapper for graph-value assertions. func collectResult(t *testing.T, result graph.Result) queryResult { t.Helper() @@ -391,6 +544,7 @@ func collectResult(t *testing.T, result graph.Result) queryResult { return collected } +// assertQueryError drains result and requires the backend to report an execution error. func assertQueryError(t *testing.T, result graph.Result) { t.Helper() @@ -402,6 +556,7 @@ func assertQueryError(t *testing.T, result graph.Result) { } } +// decodeAssertionValue decodes assertion JSON into T and fails the current test with the assertion key on error. func decodeAssertionValue[T any](t *testing.T, key string, raw json.RawMessage) T { t.Helper() @@ -413,6 +568,7 @@ func decodeAssertionValue[T any](t *testing.T, key string, raw json.RawMessage) return value } +// assertNonEmpty requires at least one result row. func assertNonEmpty(t *testing.T, result queryResult, _ assertionContext) { t.Helper() if len(result.rows) == 0 { @@ -420,6 +576,7 @@ func assertNonEmpty(t *testing.T, result queryResult, _ assertionContext) { } } +// assertEmpty requires a result set with no rows. func assertEmpty(t *testing.T, result queryResult, _ assertionContext) { t.Helper() if len(result.rows) > 0 { @@ -427,10 +584,12 @@ func assertEmpty(t *testing.T, result queryResult, _ assertionContext) { } } +// assertNoError accepts any successfully collected result without imposing a row-shape assertion. func assertNoError(t *testing.T, _ queryResult, _ assertionContext) { t.Helper() } +// assertKeys requires every result row to expose exactly the expected projection keys in order. func assertKeys(expected []string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -448,6 +607,7 @@ func assertKeys(expected []string) resultAssertion { } } +// assertRowCount requires exactly n result rows. func assertRowCount(n int) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -457,6 +617,7 @@ func assertRowCount(n int) resultAssertion { } } +// assertAtLeastInt64 requires the first scalar result to be an integer no smaller than min. func assertAtLeastInt64(min int64) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -474,6 +635,7 @@ func assertAtLeastInt64(min int64) resultAssertion { } } +// assertExactInt64 requires one row whose first scalar is exactly expected. func assertExactInt64(expected int64) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -491,6 +653,7 @@ func assertExactInt64(expected int64) resultAssertion { } } +// assertScalarValues compares each row's first scalar with expected, optionally preserving row order. func assertScalarValues(expected []any, ordered bool) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -519,6 +682,7 @@ func assertScalarValues(expected []any, ordered bool) resultAssertion { } } +// assertRowValues compares complete scalar rows with expected, optionally preserving row order. func assertRowValues(expected [][]any, ordered bool) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -543,6 +707,8 @@ func assertRowValues(expected [][]any, ordered bool) resultAssertion { } } +// firstScalarValue returns the first projected value and fails for an empty +// result or row. func firstScalarValue(t *testing.T, result queryResult) any { t.Helper() @@ -557,6 +723,7 @@ func firstScalarValue(t *testing.T, result queryResult) any { return result.rows[0].values[0] } +// asInt64 converts supported integer representations to int64 without accepting non-integral values. func asInt64(value any) (int64, bool) { switch typedValue := value.(type) { case int: @@ -596,6 +763,7 @@ func asInt64(value any) (int64, bool) { return 0, false } +// rowScalarSignature joins deterministic scalar signatures for one projected row. func rowScalarSignature(values []any) string { parts := make([]string, len(values)) for idx, value := range values { @@ -610,6 +778,8 @@ func rowScalarSignature(values []any) string { return string(encoded) } +// scalarSignature canonicalizes nil, numeric, string, boolean, and JSON-backed +// values for backend-independent comparisons. func scalarSignature(value any) string { if value == nil { return "null:" @@ -637,6 +807,8 @@ func scalarSignature(value any) string { } } +// jsonNumberSignature recognizes a JSON number and returns its canonical +// numeric signature. func jsonNumberSignature(encoded []byte) (string, bool) { decoder := json.NewDecoder(strings.NewReader(string(encoded))) decoder.UseNumber() @@ -659,6 +831,7 @@ func jsonNumberSignature(encoded []byte) (string, bool) { return fmt.Sprintf("number:%g", value), true } +// assertContainsNodeWithProp requires any returned node to contain key with the expected string value. func assertContainsNodeWithProp(key, expected string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -676,6 +849,7 @@ func assertContainsNodeWithProp(key, expected string) resultAssertion { } } +// assertContainsNodeWithProps requires any returned node to contain the expected property subset. func assertContainsNodeWithProps(expected map[string]any) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -702,13 +876,34 @@ func assertContainsNodeWithProps(expected map[string]any) resultAssertion { } } +// edgeExpectation describes the stable identity, kind, and optional properties required of a relationship result. type edgeExpectation struct { - Start string `json:"start,omitempty"` - End string `json:"end,omitempty"` - Kind string `json:"kind,omitempty"` + // Start is the expected fixture ID of the relationship's start node. + Start string `json:"start,omitempty"` + + // End is the expected fixture ID of the relationship's end node. + End string `json:"end,omitempty"` + + // Kind is the expected relationship kind. + Kind string `json:"kind,omitempty"` + + // Props contains the expected relationship property subset. + Props map[string]any `json:"props,omitempty"` +} + +// nodeExpectation describes the stable identity, kinds, and optional properties required of a node result. +type nodeExpectation struct { + // ID is the expected fixture node identifier. + ID string `json:"id"` + + // Kinds contains the expected node kinds independent of order. + Kinds []string `json:"kinds,omitempty"` + + // Props contains the expected node property subset. Props map[string]any `json:"props,omitempty"` } +// assertContainsEdge requires any returned relationship to match expected endpoints, kind, and properties. func assertContainsEdge(expected edgeExpectation) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -723,6 +918,7 @@ func assertContainsEdge(expected edgeExpectation) resultAssertion { } } +// assertNodeIDs compares collected fixture node IDs as a multiset, optionally deduplicating them first. func assertNodeIDs(expected []string, unique bool) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -732,6 +928,74 @@ func assertNodeIDs(expected []string, unique bool) resultAssertion { } } +// assertNodeRecords compares returned nodes with expected fixture IDs, kinds, and property subsets independent of order. +func assertNodeRecords(expected []nodeExpectation) resultAssertion { + return func(t *testing.T, result queryResult, ctx assertionContext) { + t.Helper() + + got := make([]string, 0, len(expected)) + for _, row := range result.rows { + for _, rawValue := range row.values { + var node graph.Node + if result.mapper.Map(rawValue, &node) { + got = append(got, nodeRecordSignature(t, node, ctx)) + } + } + } + + want := make([]string, len(expected)) + for idx, node := range expected { + want[idx] = expectedNodeRecordSignature(node) + } + + assertStringMultiset(t, got, want, "node records") + } +} + +// assertRelationshipRecords compares returned relationships with expected records, optionally including properties. +func assertRelationshipRecords(expected []edgeExpectation, includeProperties bool) resultAssertion { + return func(t *testing.T, result queryResult, ctx assertionContext) { + t.Helper() + + relationships := collectRelationships(t, result) + got := make([]string, len(relationships)) + for idx, relationship := range relationships { + got[idx] = relationshipRecordSignature(t, relationship, ctx, includeProperties) + } + + want := make([]string, len(expected)) + for idx, relationship := range expected { + want[idx] = expectedRelationshipRecordSignature(relationship, includeProperties) + } + + assertStringMultiset(t, got, want, "relationship records") + } +} + +// assertPathRelationshipRecords compares the ordered relationship record sequence in each returned path. +func assertPathRelationshipRecords(expected [][]edgeExpectation) resultAssertion { + return func(t *testing.T, result queryResult, ctx assertionContext) { + t.Helper() + + paths := collectPaths(t, result) + got := make([]string, len(paths)) + for idx, path := range paths { + got[idx] = pathRelationshipRecordSignature(t, path, ctx) + } + want := make([]string, len(expected)) + for pathIdx, relationships := range expected { + parts := make([]string, len(relationships)) + for relationshipIdx, relationship := range relationships { + parts[relationshipIdx] = expectedRelationshipRecordSignature(relationship, true) + } + want[pathIdx] = strings.Join(parts, "\x02") + } + + assertStringMultiset(t, got, want, "ordered path relationship records") + } +} + +// assertOrderedNodeIDs compares fixture node IDs in result-row order. func assertOrderedNodeIDs(expected []string) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -759,6 +1023,7 @@ func assertOrderedNodeIDs(expected []string) resultAssertion { } } +// assertNodeListIDs compares each returned node-list projection by its ordered fixture IDs. func assertNodeListIDs(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -782,6 +1047,8 @@ func assertNodeListIDs(expected [][]string) resultAssertion { } } +// collectNodeIDs maps every returned node to a fixture ID, optionally removing +// duplicates while preserving first occurrence order. func collectNodeIDs(t *testing.T, result queryResult, ctx assertionContext, unique bool) []string { t.Helper() @@ -810,6 +1077,7 @@ func collectNodeIDs(t *testing.T, result queryResult, ctx assertionContext, uniq return ids } +// nodeListIDSignature renders a node slice as an ordered fixture-ID sequence. func nodeListIDSignature(t *testing.T, nodes []*graph.Node, ctx assertionContext) string { t.Helper() @@ -825,6 +1093,7 @@ func nodeListIDSignature(t *testing.T, nodes []*graph.Node, ctx assertionContext return strings.Join(nodeIDs, "->") } +// assertPathNodeIDs compares each returned path by its ordered fixture node IDs. func assertPathNodeIDs(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, ctx assertionContext) { t.Helper() @@ -848,6 +1117,7 @@ func assertPathNodeIDs(expected [][]string) resultAssertion { } } +// assertPathLengths compares the relationship count of every returned path. func assertPathLengths(expected []int) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -866,6 +1136,7 @@ func assertPathLengths(expected []int) resultAssertion { } } +// assertPathEdgeKinds compares each returned path by its ordered relationship kinds. func assertPathEdgeKinds(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -884,6 +1155,7 @@ func assertPathEdgeKinds(expected [][]string) resultAssertion { } } +// assertRelationshipListKinds compares each relationship-list projection by ordered kind names. func assertRelationshipListKinds(expected [][]string) resultAssertion { return func(t *testing.T, result queryResult, _ assertionContext) { t.Helper() @@ -913,6 +1185,7 @@ func assertRelationshipListKinds(expected [][]string) resultAssertion { } } +// pathNodeIDSignature renders a path as an ordered fixture-node-ID sequence. func pathNodeIDSignature(t *testing.T, path graph.Path, ctx assertionContext) string { t.Helper() @@ -928,6 +1201,7 @@ func pathNodeIDSignature(t *testing.T, path graph.Path, ctx assertionContext) st return strings.Join(nodeIDs, "->") } +// pathEdgeKindSignature renders a path as an ordered relationship-kind sequence. func pathEdgeKindSignature(t *testing.T, path graph.Path) string { t.Helper() @@ -947,6 +1221,7 @@ func pathEdgeKindSignature(t *testing.T, path graph.Path) string { return strings.Join(edgeKinds, "->") } +// relationshipListKindSignature renders a relationship-pointer slice as an ordered kind sequence. func relationshipListKindSignature(t *testing.T, relationships []*graph.Relationship) string { t.Helper() @@ -966,6 +1241,7 @@ func relationshipListKindSignature(t *testing.T, relationships []*graph.Relation return strings.Join(edgeKinds, "->") } +// relationshipValueListKindSignature renders a relationship-value slice as an ordered kind sequence. func relationshipValueListKindSignature(t *testing.T, relationships []graph.Relationship) string { t.Helper() @@ -981,6 +1257,7 @@ func relationshipValueListKindSignature(t *testing.T, relationships []graph.Rela return strings.Join(edgeKinds, "->") } +// collectPaths maps every path-valued result cell into a graph path. func collectPaths(t *testing.T, result queryResult) []graph.Path { t.Helper() @@ -997,6 +1274,8 @@ func collectPaths(t *testing.T, result queryResult) []graph.Path { return paths } +// collectRelationships maps standalone relationships and relationships nested +// in returned paths into one slice. func collectRelationships(t *testing.T, result queryResult) []graph.Relationship { t.Helper() @@ -1022,6 +1301,96 @@ func collectRelationships(t *testing.T, result queryResult) []graph.Relationship return relationships } +// nodeRecordSignature renders a returned node into a stable fixture ID, sorted kinds, and property signature. +func nodeRecordSignature(t *testing.T, node graph.Node, ctx assertionContext) string { + t.Helper() + + kinds := node.Kinds.Strings() + sort.Strings(kinds) + + return strings.Join([]string{ + ctx.fixtureID(t, node.ID), + strings.Join(kinds, ","), + propertyMapSignature(node.Properties.MapOrEmpty()), + }, "\x00") +} + +// expectedNodeRecordSignature renders a node expectation in the same canonical form as a returned node. +func expectedNodeRecordSignature(node nodeExpectation) string { + kinds := append([]string(nil), node.Kinds...) + sort.Strings(kinds) + + return strings.Join([]string{ + node.ID, + strings.Join(kinds, ","), + propertyMapSignature(node.Props), + }, "\x00") +} + +// relationshipRecordSignature renders a returned relationship into canonical fixture endpoints, kind, and optional properties. +func relationshipRecordSignature(t *testing.T, relationship graph.Relationship, ctx assertionContext, includeProperties bool) string { + t.Helper() + + kind := "" + if relationship.Kind != nil { + kind = relationship.Kind.String() + } + + parts := []string{ + ctx.fixtureID(t, relationship.StartID), + ctx.fixtureID(t, relationship.EndID), + kind, + } + if includeProperties { + parts = append(parts, propertyMapSignature(relationship.Properties.MapOrEmpty())) + } + + return strings.Join(parts, "\x00") +} + +// pathRelationshipRecordSignature renders a path's ordered relationships into one canonical comparison value. +func pathRelationshipRecordSignature(t *testing.T, path graph.Path, ctx assertionContext) string { + t.Helper() + + parts := make([]string, 0, len(path.Edges)) + for _, relationship := range path.Edges { + if relationship == nil { + t.Fatal("path contains a nil relationship") + } + parts = append(parts, relationshipRecordSignature(t, *relationship, ctx, true)) + } + return strings.Join(parts, "\x02") +} + +// expectedRelationshipRecordSignature renders a relationship expectation in the same canonical form as a returned relationship. +func expectedRelationshipRecordSignature(relationship edgeExpectation, includeProperties bool) string { + parts := []string{relationship.Start, relationship.End, relationship.Kind} + if includeProperties { + parts = append(parts, propertyMapSignature(relationship.Props)) + } + + return strings.Join(parts, "\x00") +} + +// propertyMapSignature renders properties in key order using canonical scalar +// signatures. +func propertyMapSignature(properties map[string]any) string { + keys := make([]string, 0, len(properties)) + for key := range properties { + keys = append(keys, key) + } + sort.Strings(keys) + + parts := make([]string, len(keys)) + for idx, key := range keys { + parts[idx] = key + "=" + scalarSignature(properties[key]) + } + + return strings.Join(parts, "\x01") +} + +// relationshipMatches reports whether a relationship satisfies the expected +// fixture endpoints, kind, and property subset. func relationshipMatches(t *testing.T, relationship graph.Relationship, expected edgeExpectation, ctx assertionContext) bool { t.Helper() @@ -1042,6 +1411,7 @@ func relationshipMatches(t *testing.T, relationship graph.Relationship, expected return propertiesMatch(relationship.Properties, expected.Props) } +// propertiesMatch reports whether properties contains every expected key with an equivalent value. func propertiesMatch(properties *graph.Properties, expected map[string]any) bool { if len(expected) == 0 { return true @@ -1061,6 +1431,7 @@ func propertiesMatch(properties *graph.Properties, expected map[string]any) bool return true } +// valuesEqual compares numeric values across concrete widths and delegates all other values to deep equality. func valuesEqual(actual, expected any) bool { if actualNumber, actualIsNumber := asFloat64(actual); actualIsNumber { if expectedNumber, expectedIsNumber := asFloat64(expected); expectedIsNumber { @@ -1071,6 +1442,7 @@ func valuesEqual(actual, expected any) bool { return reflect.DeepEqual(actual, expected) } +// asFloat64 converts supported numeric representations to a common comparison value. func asFloat64(value any) (float64, bool) { switch typedValue := value.(type) { case int: @@ -1102,6 +1474,7 @@ func asFloat64(value any) (float64, bool) { } } +// assertStringMultiset compares string collections after sorting copies and reports label on mismatch. func assertStringMultiset(t *testing.T, got, expected []string, label string) { t.Helper() diff --git a/integration/delegated_enrollment_legacy_builder_test.go b/integration/delegated_enrollment_legacy_builder_test.go new file mode 100644 index 00000000..99bfb7e3 --- /dev/null +++ b/integration/delegated_enrollment_legacy_builder_test.go @@ -0,0 +1,125 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderDelegatedEnrollmentDiscovery verifies legacy criteria preserve delegated-enrollment discovery results across backends. +func TestLegacyBuilderDelegatedEnrollmentDiscovery(t *testing.T) { + fixture := delegatedEnrollmentFixture() + nodeKinds, edgeKinds := fixture.Kinds() + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + + WithLegacyRelationshipQuery(t, &Session{ + DB: db, + Ctx: ctx, + }, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.In(query.EndProperty("objectid"), []string{"ca-a", "ca-b"}), + query.Kind(query.Relationship(), graph.StringKind("PublishedTo")), + query.Kind(query.Start(), graph.StringKind("CertTemplate")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 3, "raw relationship results must retain duplicate paths to one template") + + nodes, err := ops.FetchStartNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 2, nodes.Len(), "FetchStartNodes must de-duplicate repeated start nodes") + require.True(t, nodes.ContainsID(idMap["template-a"])) + require.True(t, nodes.ContainsID(idMap["template-b"])) + return nil + }) +} + +// delegatedEnrollmentFixture builds templates, enrollment endpoints, and +// duplicate paths used by the delegated-enrollment regression cases. +func delegatedEnrollmentFixture() *opengraph.Graph { + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "template-a", + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": "template-a"}, + }, + { + ID: "template-b", + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": "template-b"}, + }, + { + ID: "wrong-start", + Kinds: []string{"OtherTemplate"}, + Properties: map[string]any{"objectid": "wrong-start"}, + }, + { + ID: "ca-a", + Kinds: []string{"EnterpriseCA"}, + Properties: map[string]any{"objectid": "ca-a"}, + }, + { + ID: "ca-b", + Kinds: []string{"EnterpriseCA"}, + Properties: map[string]any{"objectid": "ca-b"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "template-a", + EndID: "ca-a", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "published-a"}, + }, + { + StartID: "template-a", + EndID: "ca-b", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "published-b"}, + }, + { + StartID: "template-b", + EndID: "ca-a", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "published-c"}, + }, + { + StartID: "wrong-start", + EndID: "ca-a", + Kind: "PublishedTo", + Properties: map[string]any{"marker": "wrong-start"}, + }, + { + StartID: "template-a", + EndID: "ca-a", + Kind: "OtherPublication", + Properties: map[string]any{"marker": "wrong-edge"}, + }, + }, + } +} diff --git a/integration/direct_write_mutations_test.go b/integration/direct_write_mutations_test.go new file mode 100644 index 00000000..6dba96f2 --- /dev/null +++ b/integration/direct_write_mutations_test.go @@ -0,0 +1,1139 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "context" + "fmt" + "math" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +const ( + // directWriteObjectID is the identity property used by node selectors and upserts. + directWriteObjectID = "objectid" + + // directWriteLastSeen is the mutable timestamp property used to verify update semantics. + directWriteLastSeen = "lastseen" +) + +var ( + // directWriteDeleteRelationshipKind identifies relationships targeted by direct delete tests. + directWriteDeleteRelationshipKind = graph.StringKind("WriteDeleteRelationship") + + // directWriteCreateRelationshipKind identifies relationships created and conflict-merged by batch tests. + directWriteCreateRelationshipKind = graph.StringKind("WriteCreateRelationship") + + // directWriteCreateRelationshipOther identifies non-target relationships that must survive create tests. + directWriteCreateRelationshipOther = graph.StringKind("WriteCreateRelationshipOther") + + // directWriteUpsertNodeKind identifies nodes targeted by identity-based upserts. + directWriteUpsertNodeKind = graph.StringKind("WriteUpsertNode") + + // directWriteUpsertNodeKindA is the first kind used to verify multi-kind node updates. + directWriteUpsertNodeKindA = graph.StringKind("WriteUpsertNodeA") + + // directWriteUpsertNodeKindB is the second kind used to verify multi-kind node updates. + directWriteUpsertNodeKindB = graph.StringKind("WriteUpsertNodeB") + + // directWriteUpsertNodeKindC is the replacement kind used to verify kind-set mutation. + directWriteUpsertNodeKindC = graph.StringKind("WriteUpsertNodeC") + + // directWriteUpsertRelationshipKind identifies relationships targeted by identity-based upserts. + directWriteUpsertRelationshipKind = graph.StringKind("WriteUpsertRelationship") + + // directWriteUpsertRelationshipOther identifies non-target relationships that must survive upserts. + directWriteUpsertRelationshipOther = graph.StringKind("WriteUpsertRelationshipOther") + + // directWriteEnsureRelationshipKind identifies relationships created or updated by read-then-write tests. + directWriteEnsureRelationshipKind = graph.StringKind("WriteEnsureRelationship") + + // directWriteEntityKind is the common base kind assigned to direct-write fixture nodes. + directWriteEntityKind = graph.StringKind("Entity") + + // directWriteGroupKind identifies group nodes used by get-or-create tests. + directWriteGroupKind = graph.StringKind("Group") + + // directWriteUnrelatedKind marks nodes that selectors must not mutate. + directWriteUnrelatedKind = graph.StringKind("WriteUnrelated") + + // directWriteSuffixKind marks nodes used to exercise suffix-selector updates. + directWriteSuffixKind = graph.StringKind("WriteSuffix") + + // directWriteMissingKind is intentionally absent from the fixture for miss-path assertions. + directWriteMissingKind = graph.StringKind("WriteMissing") + + // directWriteScanKind identifies nodes used by kind-scan selectors. + directWriteScanKind = graph.StringKind("WriteKindScan") + + // directWriteEndpointKind identifies relationship endpoint nodes in mutation fixtures. + directWriteEndpointKind = graph.StringKind("WriteEndpoint") + + // directWriteBoundarySizes exercises empty, exact, adjacent, and repeated batch-flush thresholds. + directWriteBoundarySizes = []int{0, 1, 1_000, 1_999, 2_000, 2_001, 4_001, 8_001} +) + +// TestDirectWriteDeleteRelationshipBoundariesAndSurvivors verifies batched relationship deletion at flush boundaries preserves non-target edges. +func TestDirectWriteDeleteRelationshipBoundariesAndSurvivors(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range directWriteBoundarySizes { + t.Run(fmt.Sprintf("WRITE-01 size %d", size), func(t *testing.T) { + _, _ = directWriteLoadDirectWriteFixture(t, ctx, db, size) + ids := directWriteFetchRelationshipIDs(t, ctx, db, func() graph.Criteria { + return query.And( + query.Kind(query.Relationship(), directWriteDeleteRelationshipKind), + query.Equals(query.RelationshipProperty("deletebatch"), true), + ) + }) + require.Len(t, ids, size) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() WHERE r.marker = 'same-kind-survivor' RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + require.Equal(t, int64(size), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpdateRelationship]->() RETURN count(r)")) + require.Equal(t, directWriteIncidentCount(size), countByCypher(t, ctx, db, "MATCH ()-[r:WriteIncident]->() RETURN count(r)")) + }) + } + + t.Run("WRITE-01 duplicate and missing IDs are harmless", func(t *testing.T) { + directWriteLoadDirectWriteFixture(t, ctx, db, 3) + ids := directWriteFetchRelationshipIDs(t, ctx, db, func() graph.Criteria { + return query.And( + query.Kind(query.Relationship(), directWriteDeleteRelationshipKind), + query.Equals(query.RelationshipProperty("deletebatch"), true), + ) + }) + require.Len(t, ids, 3) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range []graph.ID{ids[0], ids[0], graph.ID(math.MaxInt64 - 7)} { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() WHERE r.marker = 'same-kind-survivor' RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + }) +} + +// TestDirectWriteDeleteNodeBoundariesAndCascades verifies batched node deletion removes incident edges and preserves unrelated nodes. +func TestDirectWriteDeleteNodeBoundariesAndCascades(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range directWriteBoundarySizes { + t.Run(fmt.Sprintf("WRITE-02 size %d", size), func(t *testing.T) { + _, idMap := directWriteLoadDirectWriteFixture(t, ctx, db, size) + ids := make([]graph.ID, 0, size) + for _, targetName := range testutil.FixtureNames("write-target", size) { + ids = append(ids, idMap[targetName]) + } + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteNode(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(0), countByCypher(t, ctx, db, "MATCH (n:WriteDeleteNode) RETURN count(n)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + }) + } + + t.Run("WRITE-02 duplicate missing isolated self low high and mixed directions", func(t *testing.T) { + _, idMap := directWriteLoadDirectWriteFixture(t, ctx, db, 8) + targetIDs := testutil.FixtureNames("write-target", 8) + isolated := directWriteCreateNode(t, ctx, db, directWriteProperties(directWriteObjectID, "write-isolated"), graph.StringKind("WriteDeleteNode")) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, targetName := range targetIDs { + if err := batch.DeleteNode(idMap[targetName]); err != nil { + return err + } + } + if err := batch.DeleteNode(isolated.ID); err != nil { + return err + } + if err := batch.DeleteNode(idMap[targetIDs[0]]); err != nil { + return err + } + return batch.DeleteNode(graph.ID(math.MaxInt64 - 11)) + }, graph.WithBatchSize(3))) + + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)")) + require.Equal(t, int64(0), countByCypher(t, ctx, db, "MATCH ()-[r:WriteIncident]->() RETURN count(r)")) + }) +} + +// TestDirectWriteCreateRelationshipConflictMerge verifies duplicate relationship keys merge properties without colliding with distinct endpoint tuples. +func TestDirectWriteCreateRelationshipConflictMerge(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + a, b, c := directWriteCreateEndpoints(t, ctx, db, "create-a", "create-b", "create-c") + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + updates := []struct { + // start is the relationship start node ID. + start graph.ID + + // end is the relationship end node ID. + end graph.ID + + // kind is the relationship kind to create or merge. + kind graph.Kind + + // properties supplies the values merged into the relationship. + properties *graph.Properties + }{ + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("firstseen", "2026-01-01T00:00:00Z", "custom", "first", "preserved", "yes"), + }, + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("lastseen", "2026-01-02T00:00:00Z", "custom", "within"), + }, + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("custom", "last", "nullable", nil), + }, + { + start: b.ID, + end: a.ID, + kind: directWriteCreateRelationshipKind, + properties: directWriteProperties("marker", "reverse"), + }, + { + start: a.ID, + end: b.ID, + kind: directWriteCreateRelationshipOther, + properties: directWriteProperties("marker", "other-kind"), + }, + { + start: a.ID, + end: c.ID, + kind: directWriteCreateRelationshipKind, + properties: graph.NewProperties(), + }, + } + for _, update := range updates { + if err := batch.CreateRelationshipByIDs(update.start, update.end, update.kind, update.properties); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + primary := directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteCreateRelationshipKind) + require.Equal(t, "2026-01-01T00:00:00Z", directWriteStringProperty(t, primary.Properties, "firstseen")) + require.Equal(t, "2026-01-02T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "preserved")) + // Neo4j removes a property set to null while PostgreSQL retains a JSONB null + // key. The shared graph API exposes nil in both cases. + require.Nil(t, primary.Properties.Get("nullable").Any()) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, b.ID, a.ID, directWriteCreateRelationshipKind)) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteCreateRelationshipOther)) + require.Empty(t, directWriteFetchRelationship(t, ctx, db, a.ID, c.ID, directWriteCreateRelationshipKind).Properties.MapOrEmpty()) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationshipOther]->() RETURN count(r)")) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.CreateRelationshipByIDs(a.ID, b.ID, directWriteCreateRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-03T00:00:00Z", + "retry", "yes", + )) + })) + primary = directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteCreateRelationshipKind) + require.Equal(t, "2026-01-03T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "retry")) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteCreateRelationshipOther]->() RETURN count(r)")) +} + +// TestDirectWriteUpdateNodeBySemanticsAndBoundaries verifies identity-based node updates, replacements, and misses across flush boundaries. +func TestDirectWriteUpdateNodeBySemanticsAndBoundaries(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range []int{1_000, 1_999, 2_000, 2_001} { + t.Run(fmt.Sprintf("WRITE-04 size %d", size), func(t *testing.T) { + ClearGraph(t, db, ctx) + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateNodeBy(directWriteNodeUpdate( + fmt.Sprintf("node-boundary-%04d", idx), + directWriteUpsertNodeKind, + directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "ordinal", idx), + )); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + require.Equal(t, int64(size), countByCypher(t, ctx, db, "MATCH (n:WriteUpsertNode) RETURN count(n)")) + first := directWriteFetchNodeByObjectID(t, ctx, db, "node-boundary-0000") + require.Equal(t, "2026-01-02T00:00:00Z", directWriteStringProperty(t, first.Properties, directWriteLastSeen)) + }) + } + + t.Run("WRITE-04 insert update duplicates retry lastseen and kind merge", func(t *testing.T) { + ClearGraph(t, db, ctx) + existing := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "node-existing", + directWriteLastSeen, "2026-01-01T00:00:00Z", + "preserved", "yes", + ), directWriteUpsertNodeKindA) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + updates := []graph.NodeUpdate{ + directWriteNodeUpdate("node-new", directWriteUpsertNodeKindA, directWriteProperties(directWriteLastSeen, "2026-01-01T00:00:00Z", "custom", "first")), + directWriteNodeUpdate("node-new", directWriteUpsertNodeKindB, directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "custom", "within")), + directWriteNodeUpdate("node-new", directWriteUpsertNodeKindC, directWriteProperties(directWriteLastSeen, "2026-01-03T00:00:00Z", "custom", "last")), + directWriteNodeUpdate("node-existing", directWriteUpsertNodeKindB, directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "changed", true)), + } + for _, update := range updates { + if err := batch.UpdateNodeBy(update); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + inserted := directWriteFetchNodeByObjectID(t, ctx, db, "node-new") + require.Equal(t, "2026-01-03T00:00:00Z", directWriteStringProperty(t, inserted.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, inserted.Properties, "custom")) + require.True(t, inserted.Kinds.ContainsOneOf(directWriteUpsertNodeKindA)) + require.True(t, inserted.Kinds.ContainsOneOf(directWriteUpsertNodeKindB)) + require.True(t, inserted.Kinds.ContainsOneOf(directWriteUpsertNodeKindC)) + + updated := directWriteFetchNodeByObjectID(t, ctx, db, "node-existing") + require.Equal(t, existing.ID, updated.ID) + require.Equal(t, "yes", directWriteStringProperty(t, updated.Properties, "preserved")) + require.True(t, updated.Properties.Get("changed").Any().(bool)) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.UpdateNodeBy(directWriteNodeUpdate("node-new", directWriteUpsertNodeKind, directWriteProperties( + directWriteLastSeen, "2026-01-04T00:00:00Z", + "retry", "yes", + ))) + })) + inserted = directWriteFetchNodeByObjectID(t, ctx, db, "node-new") + require.Equal(t, "2026-01-04T00:00:00Z", directWriteStringProperty(t, inserted.Properties, directWriteLastSeen)) + require.Equal(t, "yes", directWriteStringProperty(t, inserted.Properties, "retry")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'node-new' RETURN count(n)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'node-existing' RETURN count(n)")) + }) +} + +// TestDirectWriteUpdateRelationshipBySemanticsAndBoundaries verifies relationship upsert semantics and survivor isolation across flush boundaries. +func TestDirectWriteUpdateRelationshipBySemanticsAndBoundaries(t *testing.T) { + db, ctx := directWriteSetup(t) + + for _, size := range []int{1_000, 1_999, 2_000, 2_001} { + t.Run(fmt.Sprintf("WRITE-05 size %d", size), func(t *testing.T) { + ClearGraph(t, db, ctx) + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateRelationshipBy(directWriteRelationshipUpdate( + fmt.Sprintf("rel-source-%04d", idx), + fmt.Sprintf("rel-target-%04d", idx), + directWriteUpsertRelationshipKind, + directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "ordinal", idx), + )); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000))) + require.Equal(t, int64(size*2), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(size), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)")) + }) + } + + t.Run("WRITE-05 endpoint upsert duplicate retry reverse kind and property merge", func(t *testing.T) { + ClearGraph(t, db, ctx) + a := directWriteCreateNode(t, ctx, db, directWriteProperties(directWriteObjectID, "rel-a", "preserved", "start"), directWriteEndpointKind) + b := directWriteCreateNode(t, ctx, db, directWriteProperties(directWriteObjectID, "rel-b", "preserved", "end"), directWriteEndpointKind) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + updates := []graph.RelationshipUpdate{ + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties(directWriteLastSeen, "2026-01-01T00:00:00Z", "custom", "first", "preserved", "yes")), + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties(directWriteLastSeen, "2026-01-02T00:00:00Z", "custom", "within")), + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties(directWriteLastSeen, "2026-01-03T00:00:00Z", "custom", "last")), + directWriteRelationshipUpdate("rel-b", "rel-a", directWriteUpsertRelationshipKind, directWriteProperties("marker", "reverse")), + directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipOther, directWriteProperties("marker", "other-kind")), + directWriteRelationshipUpdate("rel-missing-a", "rel-missing-b", directWriteUpsertRelationshipKind, directWriteProperties("marker", "missing-endpoints")), + } + for _, update := range updates { + if err := batch.UpdateRelationshipBy(update); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2))) + + primary := directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteUpsertRelationshipKind) + require.Equal(t, "2026-01-03T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "preserved")) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, b.ID, a.ID, directWriteUpsertRelationshipKind)) + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteUpsertRelationshipOther)) + missingStart := directWriteFetchNodeByObjectID(t, ctx, db, "rel-missing-a") + missingEnd := directWriteFetchNodeByObjectID(t, ctx, db, "rel-missing-b") + require.NotNil(t, directWriteFetchRelationship(t, ctx, db, missingStart.ID, missingEnd.ID, directWriteUpsertRelationshipKind)) + require.Equal(t, int64(4), countByCypher(t, ctx, db, "MATCH (n:WriteEndpoint) RETURN count(n)")) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationshipOther]->() RETURN count(r)")) + require.Equal(t, "start", directWriteStringProperty(t, directWriteFetchNodeByObjectID(t, ctx, db, "rel-a").Properties, "preserved")) + require.Equal(t, "end", directWriteStringProperty(t, directWriteFetchNodeByObjectID(t, ctx, db, "rel-b").Properties, "preserved")) + + require.NoError(t, db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.UpdateRelationshipBy(directWriteRelationshipUpdate("rel-a", "rel-b", directWriteUpsertRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-04T00:00:00Z", + "retry", "yes", + ))) + })) + primary = directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteUpsertRelationshipKind) + require.Equal(t, "2026-01-04T00:00:00Z", directWriteStringProperty(t, primary.Properties, directWriteLastSeen)) + require.Equal(t, "last", directWriteStringProperty(t, primary.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, primary.Properties, "retry")) + require.Equal(t, int64(3), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:WriteUpsertRelationshipOther]->() RETURN count(r)")) + }) +} + +// TestDirectWriteReadThenCreateOrUpdateRelationship verifies the read-then-write path updates an existing edge or creates the missing edge exactly once. +func TestDirectWriteReadThenCreateOrUpdateRelationship(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + a, b, _ := directWriteCreateEndpoints(t, ctx, db, "ensure-a", "ensure-b", "ensure-unused") + + // A reverse-direction relationship is a decoy, not an existing exact key. + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + _, err := tx.CreateRelationshipByIDs(b.ID, a.ID, directWriteEnsureRelationshipKind, directWriteProperties("marker", "reverse")) + return err + })) + + createdID, created, err := directWriteEnsureRelationship(ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-01T00:00:00Z", + "custom", "created", + )) + require.NoError(t, err) + require.True(t, created) + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH ()-[r:WriteEnsureRelationship]->() RETURN count(r)")) + + updatedID, created, err := directWriteEnsureRelationship(ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-02T00:00:00Z", + "custom", "updated", + "newproperty", "yes", + )) + require.NoError(t, err) + require.False(t, created) + require.Equal(t, createdID, updatedID) + + repeatedID, created, err := directWriteEnsureRelationship(ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind, directWriteProperties( + directWriteLastSeen, "2026-01-02T00:00:00Z", + "custom", "updated", + "newproperty", "yes", + )) + require.NoError(t, err) + require.False(t, created) + require.Equal(t, createdID, repeatedID) + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH ()-[r:WriteEnsureRelationship]->() RETURN count(r)")) + + relationship := directWriteFetchRelationship(t, ctx, db, a.ID, b.ID, directWriteEnsureRelationshipKind) + require.Equal(t, "2026-01-02T00:00:00Z", directWriteStringProperty(t, relationship.Properties, directWriteLastSeen)) + require.Equal(t, "updated", directWriteStringProperty(t, relationship.Properties, "custom")) + require.Equal(t, "yes", directWriteStringProperty(t, relationship.Properties, "newproperty")) + reverse := directWriteFetchRelationship(t, ctx, db, b.ID, a.ID, directWriteEnsureRelationshipKind) + require.Equal(t, "reverse", directWriteStringProperty(t, reverse.Properties, "marker")) +} + +// TestDirectWriteFullNodeUpdateAfterSelectors verifies selector results can be fully replaced without mutating unmatched nodes. +func TestDirectWriteFullNodeUpdateAfterSelectors(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + + suffix := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "S-1-5-21-512", + "name", "old suffix name", + "preserved", "suffix", + ), directWriteEntityKind, directWriteSuffixKind, directWriteUnrelatedKind) + missing := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "missing-name", + "preserved", "missing", + ), directWriteEntityKind, directWriteMissingKind, directWriteUnrelatedKind) + scan := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "kind-scan", + "name", "old scan name", + "preserved", "scan", + ), directWriteEntityKind, directWriteScanKind, directWriteUnrelatedKind) + directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "S-1-5-21-513", + "name", "decoy", + ), directWriteEntityKind, directWriteUnrelatedKind) + + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + selectedSuffix, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.And( + query.Kind(query.Node(), directWriteSuffixKind), + query.StringEndsWith(query.NodeProperty(directWriteObjectID), "-512"), + ) + }).First() + if err != nil { + return err + } + selectedSuffix.Properties.Set("name", "new suffix name") + if err := tx.UpdateNode(selectedSuffix); err != nil { + return err + } + + selectedMissing, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.And( + query.Kind(query.Node(), directWriteMissingKind), + query.Not(query.Exists(query.NodeProperty("name"))), + ) + }).First() + if err != nil { + return err + } + selectedMissing.AddKinds(directWriteGroupKind) + if err := tx.UpdateNode(selectedMissing); err != nil { + return err + } + + selectedScan, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.Kind(query.Node(), directWriteScanKind) + }).First() + if err != nil { + return err + } + selectedScan.Properties.Set("name", "new scan name") + selectedScan.AddKinds(directWriteGroupKind) + return tx.UpdateNode(selectedScan) + })) + + updatedSuffix := directWriteFetchNodeByID(t, ctx, db, suffix.ID) + require.Equal(t, "new suffix name", directWriteStringProperty(t, updatedSuffix.Properties, "name")) + require.Equal(t, "suffix", directWriteStringProperty(t, updatedSuffix.Properties, "preserved")) + require.True(t, updatedSuffix.Kinds.ContainsOneOf(directWriteUnrelatedKind)) + require.False(t, updatedSuffix.Kinds.ContainsOneOf(directWriteGroupKind)) + + updatedMissing := directWriteFetchNodeByID(t, ctx, db, missing.ID) + require.False(t, updatedMissing.Properties.Exists("name")) + require.Equal(t, "missing", directWriteStringProperty(t, updatedMissing.Properties, "preserved")) + require.True(t, updatedMissing.Kinds.ContainsOneOf(directWriteGroupKind)) + require.True(t, updatedMissing.Kinds.ContainsOneOf(directWriteUnrelatedKind)) + + updatedScan := directWriteFetchNodeByID(t, ctx, db, scan.ID) + require.Equal(t, "new scan name", directWriteStringProperty(t, updatedScan.Properties, "name")) + require.Equal(t, "scan", directWriteStringProperty(t, updatedScan.Properties, "preserved")) + require.True(t, updatedScan.Kinds.ContainsOneOf(directWriteGroupKind)) + require.True(t, updatedScan.Kinds.ContainsOneOf(directWriteUnrelatedKind)) +} + +// TestDirectWriteExactKeyMissThenCreateNode verifies an exact-key miss followed by creation yields one correctly keyed node. +func TestDirectWriteExactKeyMissThenCreateNode(t *testing.T) { + db, ctx := directWriteSetup(t) + ClearGraph(t, db, ctx) + + _, err := directWriteFindNodeByObjectID(ctx, db, "well-known-new") + require.Error(t, err) + require.True(t, graph.IsErrNotFound(err), "selector must report an exact-key miss before the driver create") + + completeProperties := directWriteProperties( + directWriteObjectID, "well-known-new", + "name", "Well Known Group", + "domainsid", "S-1-5-21", + "domainfqdn", "example.test", + directWriteLastSeen, "2026-01-01T00:00:00Z", + ) + created, wasCreated, err := directWriteGetOrCreateGroup(ctx, db, completeProperties) + require.NoError(t, err) + require.True(t, wasCreated) + require.True(t, created.Kinds.ContainsOneOf(directWriteEntityKind)) + require.True(t, created.Kinds.ContainsOneOf(directWriteGroupKind)) + require.Equal(t, "Well Known Group", directWriteStringProperty(t, created.Properties, "name")) + require.Equal(t, "S-1-5-21", directWriteStringProperty(t, created.Properties, "domainsid")) + require.Equal(t, "example.test", directWriteStringProperty(t, created.Properties, "domainfqdn")) + + selectorHit, err := directWriteFindNodeByObjectID(ctx, db, "well-known-new") + require.NoError(t, err) + require.Equal(t, created.ID, selectorHit.ID) + + repeated, wasCreated, err := directWriteGetOrCreateGroup(ctx, db, completeProperties) + require.NoError(t, err) + require.False(t, wasCreated) + require.Equal(t, created.ID, repeated.ID) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'well-known-new' RETURN count(n)")) + + existing := directWriteCreateNode(t, ctx, db, directWriteProperties( + directWriteObjectID, "well-known-existing", + "name", "Existing", + "preserved", "yes", + ), directWriteEntityKind, directWriteUnrelatedKind) + existingResult, wasCreated, err := directWriteGetOrCreateGroup(ctx, db, directWriteProperties( + directWriteObjectID, "well-known-existing", + "name", "replacement ignored", + )) + require.NoError(t, err) + require.False(t, wasCreated) + require.Equal(t, existing.ID, existingResult.ID) + require.True(t, existingResult.Kinds.ContainsOneOf(directWriteGroupKind)) + require.True(t, existingResult.Kinds.ContainsOneOf(directWriteUnrelatedKind)) + require.Equal(t, "yes", directWriteStringProperty(t, existingResult.Properties, "preserved")) + require.Equal(t, "Existing", directWriteStringProperty(t, existingResult.Properties, "name")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH (n) WHERE n.objectid = 'well-known-existing' RETURN count(n)")) +} + +// BenchmarkMutationSafeDirectWrites measures guarded direct-write workloads across representative batch sizes. +func BenchmarkMutationSafeDirectWrites(b *testing.B) { + session := Open(b, Options{ + Schema: directWriteSchema(), + CleanupMode: CleanupGraph, + }) + + for _, size := range []int{1_000, 2_000, 2_001} { + b.Run(fmt.Sprintf("size-%d", size), func(b *testing.B) { + b.Run("WRITE-01 DeleteRelationship", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + if _, err := opengraph.WriteGraph(session.Ctx, session.DB, testutil.NewDirectWriteScaleFixture(size)); err != nil { + b.Fatalf("load fixture: %v", err) + } + ids, err := directWriteRelationshipIDs(session.Ctx, session.DB, func() graph.Criteria { + return query.And( + query.Kind(query.Relationship(), directWriteDeleteRelationshipKind), + query.Equals(query.RelationshipProperty("deletebatch"), true), + ) + }) + if err != nil { + b.Fatalf("select relationship IDs: %v", err) + } + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("delete relationships: %v", err) + } + b.StopTimer() + if remaining, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteDeleteRelationship]->() RETURN count(r)"); err != nil || remaining != 1 { + b.Fatalf("remaining relationships: got %d, err %v", remaining, err) + } + } + }) + + b.Run("WRITE-02 DeleteNode cascade", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + idMap, err := opengraph.WriteGraph(session.Ctx, session.DB, testutil.NewDirectWriteScaleFixture(size)) + if err != nil { + b.Fatalf("load fixture: %v", err) + } + ids := make([]graph.ID, 0, size) + for _, name := range testutil.FixtureNames("write-target", size) { + ids = append(ids, idMap[name]) + } + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteNode(id); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("delete nodes: %v", err) + } + b.StopTimer() + if remaining, err := directWriteCount(session.Ctx, session.DB, "MATCH (n:WriteDeleteNode) RETURN count(n)"); err != nil || remaining != 0 { + b.Fatalf("remaining nodes: got %d, err %v", remaining, err) + } + if survivors, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteSurvivor]->() RETURN count(r)"); err != nil || survivors != 1 { + b.Fatalf("survivor relationships: got %d, err %v", survivors, err) + } + } + }) + + b.Run("WRITE-03 CreateRelationship conflict merge", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + idMap, err := opengraph.WriteGraph(session.Ctx, session.DB, testutil.NewDirectWriteScaleFixture(size)) + if err != nil { + b.Fatalf("load fixture: %v", err) + } + rootID := idMap["write-root"] + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for idx, name := range testutil.FixtureNames("write-target", size) { + if err := batch.CreateRelationshipByIDs(rootID, idMap[name], directWriteCreateRelationshipKind, directWriteProperties("ordinal", idx, "custom", "first")); err != nil { + return err + } + if err := batch.CreateRelationshipByIDs(rootID, idMap[name], directWriteCreateRelationshipKind, directWriteProperties("custom", "last")); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("create relationships: %v", err) + } + b.StopTimer() + if created, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteCreateRelationship]->() RETURN count(r)"); err != nil || created != int64(size) { + b.Fatalf("created relationships: got %d, want %d, err %v", created, size, err) + } + if merged, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteCreateRelationship]->() WHERE r.custom = 'last' RETURN count(r)"); err != nil || merged != int64(size) { + b.Fatalf("merged relationships: got %d, want %d, err %v", merged, size, err) + } + } + }) + + b.Run("WRITE-04 UpdateNodeBy", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateNodeBy(directWriteNodeUpdate(fmt.Sprintf("bench-node-%04d", idx), directWriteUpsertNodeKind, directWriteProperties("ordinal", idx))); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("update nodes: %v", err) + } + b.StopTimer() + if updated, err := directWriteCount(session.Ctx, session.DB, "MATCH (n:WriteUpsertNode) RETURN count(n)"); err != nil || updated != int64(size) { + b.Fatalf("updated nodes: got %d, want %d, err %v", updated, size, err) + } + } + }) + + b.Run("WRITE-05 UpdateRelationshipBy", func(b *testing.B) { + b.ReportAllocs() + for range b.N { + b.StopTimer() + directWriteClearBenchmarkGraph(b, session) + b.StartTimer() + if err := session.DB.BatchOperation(session.Ctx, func(batch graph.Batch) error { + for idx := range size { + if err := batch.UpdateRelationshipBy(directWriteRelationshipUpdate( + fmt.Sprintf("bench-source-%04d", idx), + fmt.Sprintf("bench-target-%04d", idx), + directWriteUpsertRelationshipKind, + directWriteProperties("ordinal", idx), + )); err != nil { + return err + } + } + return nil + }, graph.WithBatchSize(2_000)); err != nil { + b.Fatalf("update relationships: %v", err) + } + b.StopTimer() + if updated, err := directWriteCount(session.Ctx, session.DB, "MATCH ()-[r:WriteUpsertRelationship]->() RETURN count(r)"); err != nil || updated != int64(size) { + b.Fatalf("updated relationships: got %d, want %d, err %v", updated, size, err) + } + } + }) + }) + } +} + +// directWriteSetup opens a guarded integration session with the mutation fixture schema and returns its database context. +func directWriteSetup(t *testing.T) (graph.Database, context.Context) { + t.Helper() + session := Open(t, Options{ + Schema: directWriteSchema(), + CleanupMode: CleanupGraph, + }) + return session.DB, session.Ctx +} + +// directWriteSchema returns the graph schema containing every kind used by direct-write fixtures and assertions. +func directWriteSchema() *graph.Schema { + nodeKinds, edgeKinds := directWriteKinds() + graphSchema := graph.Graph{ + Name: "integration_test", + Nodes: nodeKinds, + Edges: edgeKinds, + NodeConstraints: []graph.Constraint{{ + Field: directWriteObjectID, + Type: graph.BTreeIndex, + }}, + } + return &graph.Schema{ + Graphs: []graph.Graph{graphSchema}, + DefaultGraph: graphSchema, + } +} + +// directWriteKinds returns every node and relationship kind required by the +// direct-write fixture and mutation cases. +func directWriteKinds() (graph.Kinds, graph.Kinds) { + fixtureNodeKinds, fixtureEdgeKinds := testutil.NewDirectWriteScaleFixture(2).Kinds() + nodeKinds := fixtureNodeKinds.Add( + directWriteUpsertNodeKind, + directWriteUpsertNodeKindA, + directWriteUpsertNodeKindB, + directWriteUpsertNodeKindC, + directWriteEntityKind, + directWriteGroupKind, + directWriteUnrelatedKind, + directWriteSuffixKind, + directWriteMissingKind, + directWriteScanKind, + ) + edgeKinds := fixtureEdgeKinds.Add( + directWriteCreateRelationshipKind, + directWriteCreateRelationshipOther, + directWriteUpsertRelationshipKind, + directWriteUpsertRelationshipOther, + directWriteEnsureRelationshipKind, + ) + return nodeKinds, edgeKinds +} + +// directWriteLoadDirectWriteFixture clears the database, loads a generated +// direct-write graph, and returns both the fixture and its database ID map. +func directWriteLoadDirectWriteFixture(t *testing.T, ctx context.Context, db graph.Database, size int) (*opengraph.Graph, opengraph.IDMap) { + t.Helper() + ClearGraph(t, db, ctx) + fixture := testutil.NewDirectWriteScaleFixture(size) + idMap, err := opengraph.WriteGraph(ctx, db, fixture) + require.NoError(t, err) + return fixture, idMap +} + +// directWriteCreateEndpoints creates the three endpoint nodes required by relationship mutation cases. +func directWriteCreateEndpoints(t *testing.T, ctx context.Context, db graph.Database, objectIDs ...string) (*graph.Node, *graph.Node, *graph.Node) { + t.Helper() + require.Len(t, objectIDs, 3) + created := make([]*graph.Node, 0, len(objectIDs)) + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + for _, objectID := range objectIDs { + node, err := tx.CreateNode(directWriteProperties(directWriteObjectID, objectID), directWriteEndpointKind) + if err != nil { + return err + } + created = append(created, node) + } + return nil + })) + return created[0], created[1], created[2] +} + +// directWriteCreateNode creates one node in a committed transaction and returns its database-assigned identity. +func directWriteCreateNode(t *testing.T, ctx context.Context, db graph.Database, properties *graph.Properties, kinds ...graph.Kind) *graph.Node { + t.Helper() + var created *graph.Node + require.NoError(t, db.WriteTransaction(ctx, func(tx graph.Transaction) error { + var err error + created, err = tx.CreateNode(properties, kinds...) + return err + })) + return created +} + +// directWriteProperties constructs a property bag from alternating string keys and values. +func directWriteProperties(keyValues ...any) *graph.Properties { + properties := graph.NewProperties() + for idx := 0; idx < len(keyValues); idx += 2 { + properties.Set(keyValues[idx].(string), keyValues[idx+1]) + } + return properties +} + +// directWriteIncidentCount returns the expected number of fixture relationships incident to targets nodes. +func directWriteIncidentCount(targets int) int64 { + switch targets { + case 0: + return 0 + case 1: + return 1 + default: + return int64(targets + 1) + } +} + +// directWriteNodeUpdate builds an identity-property node upsert while preserving objectID in the replacement properties. +func directWriteNodeUpdate(objectID string, kind graph.Kind, properties *graph.Properties) graph.NodeUpdate { + properties = properties.Clone().Set(directWriteObjectID, objectID) + return graph.NodeUpdate{ + Node: graph.PrepareNode(properties, kind), + IdentityProperties: []string{directWriteObjectID}, + } +} + +// directWriteRelationshipUpdate builds a relationship upsert whose endpoints are selected by objectID. +func directWriteRelationshipUpdate(startObjectID, endObjectID string, kind graph.Kind, properties *graph.Properties) graph.RelationshipUpdate { + return graph.RelationshipUpdate{ + Start: graph.PrepareNode( + directWriteProperties(directWriteObjectID, startObjectID), + directWriteEndpointKind, + ), + StartIdentityProperties: []string{directWriteObjectID}, + End: graph.PrepareNode( + directWriteProperties(directWriteObjectID, endObjectID), + directWriteEndpointKind, + ), + EndIdentityProperties: []string{directWriteObjectID}, + Relationship: graph.PrepareRelationship(properties, kind), + } +} + +// directWriteFetchRelationshipIDs returns matching relationship IDs and fails the current test on query error. +func directWriteFetchRelationshipIDs(t *testing.T, ctx context.Context, db graph.Database, criteria graph.CriteriaProvider) []graph.ID { + t.Helper() + ids, err := directWriteRelationshipIDs(ctx, db, criteria) + require.NoError(t, err) + return ids +} + +// directWriteRelationshipIDs queries the IDs of relationships matching criteria in a read transaction. +func directWriteRelationshipIDs(ctx context.Context, db graph.Database, criteria graph.CriteriaProvider) ([]graph.ID, error) { + var ids []graph.ID + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + ids, err = ops.FetchRelationshipIDs(tx.Relationships().Filterf(criteria)) + return err + }); err != nil { + return nil, err + } + + return ids, nil +} + +// directWriteFetchRelationship returns the relationship with the exact endpoints and kind, failing the current test when absent. +func directWriteFetchRelationship(t *testing.T, ctx context.Context, db graph.Database, startID, endID graph.ID, kind graph.Kind) *graph.Relationship { + t.Helper() + var relationship *graph.Relationship + require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + relationship, err = tx.Relationships().Filterf(func() graph.Criteria { + return query.And( + query.Equals(query.StartID(), startID), + query.Equals(query.EndID(), endID), + query.Kind(query.Relationship(), kind), + ) + }).First() + return err + })) + return relationship +} + +// directWriteFetchNodeByObjectID returns the node selected by objectID and fails the current test on lookup error. +func directWriteFetchNodeByObjectID(t *testing.T, ctx context.Context, db graph.Database, objectID string) *graph.Node { + t.Helper() + node, err := directWriteFindNodeByObjectID(ctx, db, objectID) + require.NoError(t, err) + return node +} + +// directWriteFindNodeByObjectID queries the single node selected by objectID. +func directWriteFindNodeByObjectID(ctx context.Context, db graph.Database, objectID string) (*graph.Node, error) { + var node *graph.Node + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + node, err = tx.Nodes().Filterf(func() graph.Criteria { + return query.Equals(query.NodeProperty(directWriteObjectID), objectID) + }).First() + return err + }); err != nil { + return nil, err + } + + return node, nil +} + +// directWriteFetchNodeByID returns the node selected by database ID and fails the current test on lookup error. +func directWriteFetchNodeByID(t *testing.T, ctx context.Context, db graph.Database, id graph.ID) *graph.Node { + t.Helper() + var node *graph.Node + require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + node, err = tx.Nodes().Filter(query.Equals(query.NodeID(), id)).First() + return err + })) + return node +} + +// directWriteStringProperty reads key as a string and fails the current test when the value is absent or incompatible. +func directWriteStringProperty(t *testing.T, properties *graph.Properties, key string) string { + t.Helper() + value, err := properties.Get(key).String() + require.NoError(t, err) + return value +} + +// directWriteEnsureRelationship updates the exact relationship when present or creates it when absent, reporting which path ran. +func directWriteEnsureRelationship(ctx context.Context, db graph.Database, startID, endID graph.ID, kind graph.Kind, properties *graph.Properties) (graph.ID, bool, error) { + var ( + id graph.ID + created bool + ) + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if relationship, err := tx.Relationships().Filterf(func() graph.Criteria { + return query.And( + query.Equals(query.StartID(), startID), + query.Equals(query.EndID(), endID), + query.Kind(query.Relationship(), kind), + ) + }).First(); err != nil { + if !graph.IsErrNotFound(err) { + return err + } + + if createdRelationship, err := tx.CreateRelationshipByIDs(startID, endID, kind, properties); err != nil { + return err + } else { + id = createdRelationship.ID + created = true + return nil + } + } else { + relationship.Properties.Merge(properties) + id = relationship.ID + return tx.UpdateRelationship(relationship) + } + }); err != nil { + return 0, false, err + } + + return id, created, nil +} + +// directWriteGetOrCreateGroup returns the group selected by objectID or creates it atomically when missing. +func directWriteGetOrCreateGroup(ctx context.Context, db graph.Database, properties *graph.Properties) (*graph.Node, bool, error) { + objectID, err := properties.Get(directWriteObjectID).String() + if err != nil { + return nil, false, err + } + + var ( + result *graph.Node + created bool + ) + if err := db.WriteTransaction(ctx, func(tx graph.Transaction) error { + if existing, err := tx.Nodes().Filterf(func() graph.Criteria { + return query.Equals(query.NodeProperty(directWriteObjectID), objectID) + }).First(); err != nil { + if !graph.IsErrNotFound(err) { + return err + } + + if createdNode, err := tx.CreateNode(properties.Clone(), directWriteEntityKind, directWriteGroupKind); err != nil { + return err + } else { + result = createdNode + created = true + return nil + } + } else { + result = existing + if !result.Kinds.ContainsOneOf(directWriteGroupKind) { + result.AddKinds(directWriteGroupKind) + return tx.UpdateNode(result) + } + + return nil + } + }); err != nil { + return nil, false, err + } + + return result, created, nil +} + +// directWriteClearBenchmarkGraph removes every benchmark node and its incident relationships before the next iteration. +func directWriteClearBenchmarkGraph(b *testing.B, session *Session) { + b.Helper() + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + return tx.Nodes().Delete() + }); err != nil { + b.Fatalf("clear benchmark graph: %v", err) + } +} + +// directWriteCount executes a scalar Cypher count query and returns its first value. +func directWriteCount(ctx context.Context, db graph.Database, cypher string) (int64, error) { + var count int64 + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + result := tx.Query(cypher, nil) + defer result.Close() + + if !result.Next() { + return result.Error() + } + if err := result.Scan(&count); err != nil { + return err + } + return result.Error() + }); err != nil { + return 0, err + } + + return count, nil +} diff --git a/integration/harness.go b/integration/harness.go index fa568613..d1a96dac 100644 --- a/integration/harness.go +++ b/integration/harness.go @@ -36,17 +36,24 @@ import ( "github.com/specterops/dawgs/util/size" ) +// ConnectionStringEnv names the default environment variable read by integration sessions. const ConnectionStringEnv = "CONNECTION_STRING" var ( - localDatasetFlag = flag.String("local-dataset", "", "name of a local dataset to test (e.g. local/phantom)") + // localDatasetFlag optionally restricts the integration harness to one local dataset. + localDatasetFlag = flag.String("local-dataset", "", "name of a local dataset to test (e.g. local/phantom)") + + // errFixtureRollback is the sentinel returned to force successful fixture transactions to roll back. errFixtureRollback = errors.New("fixture rollback") ) type CleanupMode int const ( + // CleanupGraph removes graph data when an integration session closes. CleanupGraph CleanupMode = iota + + // CloseOnly closes an integration session without deleting graph data. CloseOnly ) @@ -90,7 +97,8 @@ func DriverFromConnectionString(connStr string) (string, error) { } } -func Open(t *testing.T, opts Options) *Session { +// Open validates the configured disposable target, initializes its schema, and returns an integration session registered for cleanup. +func Open(t testing.TB, opts Options) *Session { t.Helper() ctx := context.Background() @@ -106,7 +114,6 @@ func Open(t *testing.T, opts Options) *Session { } t.Fatalf("%s env var is not set", connEnv) } - driver, err := DriverFromConnectionString(connStr) if err != nil { t.Fatalf("failed to detect driver: %v", err) @@ -226,6 +233,7 @@ func (s *Session) WithRollback(t *testing.T, delegate func(tx graph.Transaction) return s.withRollback(t, delegate) } +// withRollback runs delegate in a write transaction and converts the fixture rollback sentinel into success. func (s *Session) withRollback(t *testing.T, delegate func(tx graph.Transaction) error) error { t.Helper() @@ -243,7 +251,8 @@ func (s *Session) withRollback(t *testing.T, delegate func(tx graph.Transaction) return err } -func buildSchema(t *testing.T, opts Options) *graph.Schema { +// buildSchema combines kinds discovered from selected datasets with explicitly requested kinds. +func buildSchema(t testing.TB, opts Options) *graph.Schema { t.Helper() nodeKinds, edgeKinds := collectKinds(t, opts.Datasets, opts.datasetPath()) @@ -270,7 +279,7 @@ func buildSchema(t *testing.T, opts Options) *graph.Schema { } // collectKinds parses the given datasets and returns the union of all node and edge kinds. -func collectKinds(t *testing.T, datasets []string, datasetPath func(name string) string) (graph.Kinds, graph.Kinds) { +func collectKinds(t testing.TB, datasets []string, datasetPath func(name string) string) (graph.Kinds, graph.Kinds) { t.Helper() var nodeKinds, edgeKinds graph.Kinds @@ -295,6 +304,7 @@ func collectKinds(t *testing.T, datasets []string, datasetPath func(name string) return nodeKinds, edgeKinds } +// datasetPath returns the configured dataset resolver or the repository testdata resolver. func (s *Options) datasetPath() func(name string) string { if s.DatasetPath != nil { return s.DatasetPath @@ -305,6 +315,8 @@ func (s *Options) datasetPath() func(name string) string { } } +// graphQueryMemoryLimit returns the backend's configured query memory limit, +// defaulting to unlimited when the driver does not expose one. func (s Options) graphQueryMemoryLimit() size.Size { if s.GraphQueryMemoryLimit == 0 { return size.Gibibyte diff --git a/integration/legacy_query_harness.go b/integration/legacy_query_harness.go new file mode 100644 index 00000000..f53244dc --- /dev/null +++ b/integration/legacy_query_harness.go @@ -0,0 +1,75 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" +) + +// WithLegacyNodeQuery executes legacy query-builder criteria directly through +// the selected backend and keeps fixture setup, execution, and assertions in a +// single rollback transaction. +func WithLegacyNodeQuery( + t *testing.T, + session *Session, + fixture *opengraph.Graph, + criteriaProvider func(idMap opengraph.IDMap) graph.Criteria, + delegate func(query graph.NodeQuery, idMap opengraph.IDMap) error, +) { + t.Helper() + + err := session.WithRollbackFixture(t, fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { + query := tx.Nodes() + if criteriaProvider != nil { + query = query.Filter(criteriaProvider(idMap)) + } + + return delegate(query, idMap) + }) + if err != nil { + t.Fatalf("legacy node query failed: %v", err) + } +} + +// WithLegacyRelationshipQuery is the relationship-query counterpart to +// WithLegacyNodeQuery. +func WithLegacyRelationshipQuery( + t *testing.T, + session *Session, + fixture *opengraph.Graph, + criteriaProvider func(idMap opengraph.IDMap) graph.Criteria, + delegate func(query graph.RelationshipQuery, idMap opengraph.IDMap) error, +) { + t.Helper() + + err := session.WithRollbackFixture(t, fixture, false, func(tx graph.Transaction, idMap opengraph.IDMap) error { + query := tx.Relationships() + if criteriaProvider != nil { + query = query.Filter(criteriaProvider(idMap)) + } + + return delegate(query, idMap) + }) + if err != nil { + t.Fatalf("legacy relationship query failed: %v", err) + } +} diff --git a/integration/logical_forms_legacy_builder_test.go b/integration/logical_forms_legacy_builder_test.go new file mode 100644 index 00000000..d1517606 --- /dev/null +++ b/integration/logical_forms_legacy_builder_test.go @@ -0,0 +1,441 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "sort" + "testing" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderLogicalForms verifies legacy logical predicates preserve grouping, precedence, and result identity. +func TestLegacyBuilderLogicalForms(t *testing.T) { + logicFixture := logicalFormsFixture() + projectionFixture := logicalProjectionFixture() + logicNodeKinds, logicEdgeKinds := logicFixture.Kinds() + projectionNodeKinds, projectionEdgeKinds := projectionFixture.Kinds() + + db, ctx := SetupDBWithKindsNoGraphCleanup( + t, + logicNodeKinds.Add(projectionNodeKinds...), + logicEdgeKinds.Add(projectionEdgeKinds...), + ) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + + t.Run("LOGIC-01 branch-local relationship kinds", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, logicFixture, func(idMap opengraph.IDMap) graph.Criteria { + forwardID := idMap["direction-forward"] + reverseID := idMap["direction-reverse"] + return query.And( + query.Kind(query.Start(), graph.StringKind("LogicDomain")), + query.Kind(query.End(), graph.StringKind("LogicDomain")), + query.Or( + query.And( + query.Equals(query.StartID(), forwardID), + query.Equals(query.EndID(), reverseID), + query.KindIn(query.Relationship(), graph.StringKind("LogicKindA")), + ), + query.And( + query.Equals(query.StartID(), reverseID), + query.Equals(query.EndID(), forwardID), + query.KindIn(query.Relationship(), graph.StringKind("LogicKindB")), + ), + ), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + var ids []graph.ID + err := relationshipQuery.FetchIDs(func(cursor graph.Cursor[graph.ID]) error { + for id := range cursor.Chan() { + ids = append(ids, id) + } + return cursor.Error() + }) + require.NoError(t, err) + require.Len(t, ids, 2, "both invalid kind/direction combinations must remain excluded") + require.NotEqual(t, ids[0], ids[1]) + return nil + }) + }) + + t.Run("LOGIC-02 cross-binding temporal disjunction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, logicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("LogicStaleTrust")), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + var markers []string + err := relationshipQuery.Fetch(func(cursor graph.Cursor[*graph.Relationship]) error { + for relationship := range cursor.Chan() { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + return cursor.Error() + }) + require.NoError(t, err) + sort.Strings(markers) + require.Equal(t, []string{"older-both", "older-end-only", "older-start-only"}, markers) + return nil + }) + }) + + t.Run("LOGIC-03 scoped negation and null-aware age predicate", func(t *testing.T) { + threshold := time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC) + WithLegacyNodeQuery(t, session, logicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("LogicProtected"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + ) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + var fixtureIDs []string + err := nodeQuery.FetchIDs(func(cursor graph.Cursor[graph.ID]) error { + for id := range cursor.Chan() { + fixtureIDs = append(fixtureIDs, regressionFixtureID(t, idMap, id)) + } + return cursor.Error() + }) + require.NoError(t, err) + sort.Strings(fixtureIDs) + require.Equal(t, []string{"candidate-missing", "candidate-null", "candidate-older", "direction-forward", "direction-reverse", "early-a", "early-b", "equal-a", "equal-b", "late-a", "late-b"}, fixtureIDs) + return nil + }) + }) + + t.Run("LOGIC-05 projection order and Go result types", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("LogicProjectionEdge")), + query.Equals(query.StartID(), idMap["projection-start"]), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + err := relationshipQuery.FetchDirection(graph.DirectionInbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := make([]graph.DirectionalResult, 0, 1) + for result := range cursor.Chan() { + results = append(results, result) + } + require.NoError(t, cursor.Error()) + require.Len(t, results, 1) + require.IsType(t, &graph.Relationship{}, results[0].Relationship) + require.IsType(t, &graph.Node{}, results[0].Node) + require.Equal(t, idMap["projection-end"], results[0].Node.ID) + return nil + }) + require.NoError(t, err) + + err = relationshipQuery.Query(func(result graph.Result) error { + require.True(t, result.Next()) + var ( + nodeID, relationshipID graph.ID + nodeKinds graph.Kinds + relationshipKind graph.Kind + ) + + require.NoError(t, result.Scan(&nodeID, &nodeKinds, &relationshipID, &relationshipKind)) + require.Equal(t, idMap["projection-end"], nodeID) + require.Equal(t, graph.StringKind("LogicProjectionEdge"), relationshipKind) + require.Contains(t, nodeKinds, graph.StringKind("LogicProjectionEnd")) + require.NotZero(t, relationshipID) + require.False(t, result.Next()) + return result.Error() + }, query.Returning( + query.EndID(), + query.KindsOf(query.End()), + query.RelationshipID(), + query.KindsOf(query.Relationship()), + )) + require.NoError(t, err) + + err = relationshipQuery.FetchTriples(func(cursor graph.Cursor[graph.RelationshipTripleResult]) error { + triples := make([]graph.RelationshipTripleResult, 0, 1) + for triple := range cursor.Chan() { + triples = append(triples, triple) + } + require.NoError(t, cursor.Error()) + require.Len(t, triples, 1) + require.Equal(t, []graph.RelationshipTripleResult{{ + ID: triples[0].ID, + StartID: idMap["projection-start"], + EndID: idMap["projection-end"], + }}, triples) + return nil + }) + require.NoError(t, err) + + err = relationshipQuery.FetchIDs(func(cursor graph.Cursor[graph.ID]) error { + ids := make([]graph.ID, 0, 1) + for id := range cursor.Chan() { + ids = append(ids, id) + } + require.NoError(t, cursor.Error()) + require.Len(t, ids, 1) + return nil + }) + require.NoError(t, err) + + err = relationshipQuery.Fetch(func(cursor graph.Cursor[*graph.Relationship]) error { + relationships := make([]*graph.Relationship, 0, 1) + for relationship := range cursor.Chan() { + relationships = append(relationships, relationship) + } + require.NoError(t, cursor.Error()) + require.Len(t, relationships, 1) + require.IsType(t, &graph.Relationship{}, relationships[0]) + return nil + }) + require.NoError(t, err) + return nil + }) + }) +} + +// logicalFormsFixture builds direction, null, time, and boolean property cases +// for logical criteria regressions. +func logicalFormsFixture() *opengraph.Graph { + day := func(day int) time.Time { + return time.Date(2026, time.January, day, 0, 0, 0, 0, time.UTC) + } + + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "direction-forward", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"name": "forward"}, + }, + { + ID: "direction-reverse", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"name": "reverse"}, + }, + { + ID: "early-a", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(2)}, + }, + { + ID: "early-b", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(2)}, + }, + { + ID: "equal-a", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(3)}, + }, + { + ID: "equal-b", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(3)}, + }, + { + ID: "late-a", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4)}, + }, + { + ID: "late-b", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4)}, + }, + { + ID: "late-b-newer", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4), "lastseen": day(4)}, + }, + { + ID: "late-b-missing", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4), "lastseen": day(4)}, + }, + { + ID: "late-b-null", + Kinds: []string{"LogicDomain"}, + Properties: map[string]any{"lastcollected": day(4), "lastseen": day(4)}, + }, + { + ID: "candidate-missing", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{}, + }, + { + ID: "candidate-null", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": nil}, + }, + { + ID: "candidate-older", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": day(2)}, + }, + { + ID: "candidate-equal", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": day(3)}, + }, + { + ID: "candidate-newer", + Kinds: []string{"LogicCandidate"}, + Properties: map[string]any{"lastseen": day(4)}, + }, + { + ID: "protected-missing", + Kinds: []string{"LogicProtected"}, + Properties: map[string]any{}, + }, + { + ID: "protected-null", + Kinds: []string{"LogicProtected"}, + Properties: map[string]any{"lastseen": nil}, + }, + { + ID: "protected-older", + Kinds: []string{"LogicProtected"}, + Properties: map[string]any{"lastseen": day(2)}, + }, + { + ID: "multi-kind-protected", + Kinds: []string{"LogicCandidate", "LogicProtected"}, + Properties: map[string]any{"lastseen": day(2)}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "direction-forward", + EndID: "direction-reverse", + Kind: "LogicKindA", + Properties: map[string]any{"marker": "valid-forward"}, + }, + { + StartID: "direction-reverse", + EndID: "direction-forward", + Kind: "LogicKindB", + Properties: map[string]any{"marker": "valid-reverse"}, + }, + { + StartID: "direction-forward", + EndID: "direction-reverse", + Kind: "LogicKindB", + Properties: map[string]any{"marker": "invalid-forward-kind"}, + }, + { + StartID: "direction-reverse", + EndID: "direction-forward", + Kind: "LogicKindA", + Properties: map[string]any{"marker": "invalid-reverse-kind"}, + }, + { + StartID: "late-a", + EndID: "early-a", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "older-start-only"}, + }, + { + StartID: "early-a", + EndID: "late-a", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "older-end-only"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "older-both"}, + }, + { + StartID: "equal-a", + EndID: "equal-b", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(3), "marker": "equal"}, + }, + { + StartID: "late-a", + EndID: "late-b-newer", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": day(5), "marker": "newer"}, + }, + { + StartID: "late-a", + EndID: "late-b-missing", + Kind: "LogicStaleTrust", + Properties: map[string]any{"marker": "missing"}, + }, + { + StartID: "late-a", + EndID: "late-b-null", + Kind: "LogicStaleTrust", + Properties: map[string]any{"lastseen": nil, "marker": "null"}, + }, + }, + } +} + +// logicalProjectionFixture builds the single relationship used to verify +// projection and fetch behavior for logical criteria. +func logicalProjectionFixture() *opengraph.Graph { + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "projection-start", + Kinds: []string{"LogicProjectionStart"}, + Properties: map[string]any{"name": "start"}, + }, + { + ID: "projection-end", + Kinds: []string{"LogicProjectionEnd", "LogicProjectionEntity"}, + Properties: map[string]any{"name": "end"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "projection-start", + EndID: "projection-end", + Kind: "LogicProjectionEdge", + Properties: map[string]any{"marker": "projection"}, + }, + }, + } +} + +// regressionFixtureID resolves a database node ID back to its stable fixture identifier and fails when unmapped. +func regressionFixtureID(t *testing.T, idMap opengraph.IDMap, id graph.ID) string { + t.Helper() + for fixtureID, databaseID := range idMap { + if databaseID == id { + return fixtureID + } + } + t.Fatalf("database ID %d is absent from fixture ID map", id) + return "" +} diff --git a/integration/pgsql_delete_by_kind_test.go b/integration/pgsql_delete_by_kind_test.go index 917db51b..08d910d6 100644 --- a/integration/pgsql_delete_by_kind_test.go +++ b/integration/pgsql_delete_by_kind_test.go @@ -29,6 +29,7 @@ import ( // nodesByKindDeleter mirrors the capability the BloodHound delete path detects on the PostgreSQL driver. type nodesByKindDeleter interface { + // DeleteNodesByKinds deletes nodes matching any included kind unless they match an excluded kind. DeleteNodesByKinds(ctx context.Context, includeAny graph.Kinds, excludeAny graph.Kinds) error } diff --git a/integration/pgsql_delete_relationships_by_kind_test.go b/integration/pgsql_delete_relationships_by_kind_test.go index 5600bb70..81601b24 100644 --- a/integration/pgsql_delete_relationships_by_kind_test.go +++ b/integration/pgsql_delete_relationships_by_kind_test.go @@ -29,6 +29,7 @@ import ( // relationshipsByKindDeleter mirrors the capability the BloodHound delete path detects on the PostgreSQL driver. type relationshipsByKindDeleter interface { + // DeleteRelationshipsByKinds deletes relationships matching any supplied kind. DeleteRelationshipsByKinds(ctx context.Context, kinds graph.Kinds) error } diff --git a/integration/pgsql_inline_asp_test.go b/integration/pgsql_inline_asp_test.go new file mode 100644 index 00000000..d7e58c91 --- /dev/null +++ b/integration/pgsql_inline_asp_test.go @@ -0,0 +1,685 @@ +// Copyright 2026 Specter Ops, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +var ( + // inlineASPNodeKind contains the frozen inline asp node kind declaration consulted by package validation. + inlineASPNodeKind = graph.StringKind("InlineASPNode") + + // inlineASPEdgeOne contains the frozen inline asp edge one declaration consulted by package validation. + inlineASPEdgeOne = graph.StringKind("InlineASPEdgeOne") + + // inlineASPEdgeTwo contains the frozen inline asp edge two declaration consulted by package validation. + inlineASPEdgeTwo = graph.StringKind("InlineASPEdgeTwo") +) + +// inlineASPCypher reserves the stable protocol value used to recognize inline asp cypher across artifacts and executions. +const inlineASPCypher = ` + MATCH p = allShortestPaths((s)-[:InlineASPEdgeOne|InlineASPEdgeTwo*1..4]->(e)) + WHERE id(s) = $start_id AND id(e) = $end_id + RETURN p +` + +// TestPostgreSQLInlineASPMatchesA1AndFallsBackWithoutPartialRows exercises the +// typed guarded statement at the real PostgreSQL boundary. A tiny state cap +// must select exact A1 and return the same complete relationship-distinct bag. +func TestPostgreSQLInlineASPMatchesA1AndFallsBackWithoutPartialRows(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{inlineASPNodeKind}, + ExtraEdgeKinds: graph.Kinds{inlineASPEdgeOne, inlineASPEdgeTwo}, + }) + + var startID, endID, disconnectedID, deepStartID, deepEndID graph.ID + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + start, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + left, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + right, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + end, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + startID, endID = start.ID, end.ID + disconnected, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + disconnectedID = disconnected.ID + deepStart, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepMiddleOne, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepMiddleTwo, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepEnd, err := tx.CreateNode(graph.NewProperties(), inlineASPNodeKind) + if err != nil { + return err + } + deepStartID, deepEndID = deepStart.ID, deepEnd.ID + for _, edge := range []struct { + // start retains the start while anonymous record is assembled or evaluated. + start graph.ID + // end retains the end while anonymous record is assembled or evaluated. + end graph.ID + // kind retains the kind while anonymous record is assembled or evaluated. + kind graph.Kind + }{ + {start.ID, left.ID, inlineASPEdgeOne}, + {left.ID, end.ID, inlineASPEdgeOne}, + {start.ID, right.ID, inlineASPEdgeTwo}, + {right.ID, end.ID, inlineASPEdgeTwo}, + {left.ID, left.ID, inlineASPEdgeOne}, + {left.ID, start.ID, inlineASPEdgeTwo}, + {deepStart.ID, deepMiddleOne.ID, inlineASPEdgeOne}, + {deepMiddleOne.ID, deepMiddleTwo.ID, inlineASPEdgeOne}, + {deepMiddleTwo.ID, deepEnd.ID, inlineASPEdgeOne}, + } { + if _, err := tx.CreateRelationshipByIDs(edge.start, edge.end, edge.kind, graph.NewProperties()); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load inline ASP fixture: %v", err) + } + + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), inlineASPCypher) + if err != nil { + t.Fatalf("parse inline ASP query: %v", err) + } + parameters := map[string]any{"start_id": int64(startID), "end_id": int64(endID)} + + a1, err := translate.Translate(session.Ctx, regularQuery, pgDriver.KindMapper(), parameters, defaultGraph.ID) + if err != nil { + t.Fatalf("translate A1: %v", err) + } + i1, err := translate.TranslateForTool(session.Ctx, regularQuery, pgDriver.KindMapper(), parameters, defaultGraph.ID, + translate.ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG}) + if err != nil { + t.Fatalf("translate I1: %v", err) + } + fallback, err := translate.TranslateWithProductionOptions(session.Ctx, regularQuery, pgDriver.KindMapper(), parameters, defaultGraph.ID, + translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 100, + PredecessorLimit: 100, + EnumerationLimit: 1, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "outbound", + ObservationMode: "all_paths", + MinimumDepth: 1, + MaximumDepth: 4, + RelationshipKindCount: 2, + UntypedRelationship: false, + }, + SelectorVersion: "asp-i1-integration-fallback-v1", + }) + if err != nil { + t.Fatalf("translate I1 fallback: %v", err) + } + + a1Rows := executeInlineASPTranslation(t, session, a1) + i1Rows, candidateReceipt := executeInlineASPTranslationWithReceipt(t, session, i1, "inline-asp-candidate") + fallbackRows, fallbackReceipt := executeInlineASPTranslationWithReceipt(t, session, fallback, "inline-asp-fallback") + if len(a1Rows) != 2 { + t.Fatalf("expected two relationship-distinct shortest paths, got %d: %v", len(a1Rows), a1Rows) + } + if fmt.Sprint(a1Rows) != fmt.Sprint(i1Rows) { + t.Fatalf("inline I1 differs from A1: A1=%v I1=%v", a1Rows, i1Rows) + } + if !containsAll(candidateReceipt, "ASP-I1-U-DAG+MAT-M0", "inline_predecessor_dag", "false", "1") { + t.Fatalf("candidate runtime receipt is incomplete: %s", candidateReceipt) + } + if fmt.Sprint(a1Rows) != fmt.Sprint(fallbackRows) { + t.Fatalf("guarded fallback differs from A1: A1=%v fallback=%v", a1Rows, fallbackRows) + } + if !containsAll(fallbackReceipt, "ASP-A1-DAG", "exact_a1_fallback", "true", "1") { + t.Fatalf("fallback runtime receipt is incomplete: %s", fallbackReceipt) + } + + // ASP-A1 reaches its spd_* predecessor workspace only beyond the two-hop + // preflight. Prove that a fresh stable-snapshot session can execute it. + session.PGPool.Reset() + freshA1Rows, freshA1Receipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, + map[string]any{"start_id": int64(deepStartID), "end_id": int64(deepEndID)}, + "inline-asp-fresh-repeatable-a1", optimize.ShortestPathExecutorASPA1DAG, + pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if len(freshA1Rows) != 1 || !containsAll(freshA1Receipt, "ASP-A1-DAG") { + t.Fatalf("fresh repeatable-read session did not execute recursive A1: rows=%v receipt=%s", freshA1Rows, freshA1Receipt) + } + candidatePlan := explainInlineASPTranslation(t, session, i1) + requireOrientationSubplanMetric(t, candidatePlan, "asp_i1_fallback_rows", "Actual Rows", 0) + fallbackPlan := explainInlineASPTranslation(t, session, fallback) + requireOrientationSubplanMetric(t, fallbackPlan, "asp_i1_candidate_rows", "Actual Rows", 0) + + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // query retains the query while anonymous record is assembled or evaluated. + query string + // parameters retains the parameters while anonymous record is assembled or evaluated. + parameters map[string]any + }{ + { + name: "inbound", + query: `MATCH p = allShortestPaths((e)<-[:InlineASPEdgeOne|InlineASPEdgeTwo*1..4]-(s)) + WHERE id(s) = $start_id AND id(e) = $end_id RETURN p`, + parameters: parameters, + }, + { + name: "no path", + query: inlineASPCypher, + parameters: map[string]any{"start_id": int64(startID), "end_id": int64(disconnectedID)}, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + query, err := frontend.ParseCypher(frontend.NewContext(), testCase.query) + if err != nil { + t.Fatalf("parse query: %v", err) + } + a1Translation, err := translate.Translate(session.Ctx, query, pgDriver.KindMapper(), testCase.parameters, defaultGraph.ID) + if err != nil { + t.Fatalf("translate A1: %v", err) + } + i1Translation, err := translate.TranslateForTool(session.Ctx, query, pgDriver.KindMapper(), testCase.parameters, defaultGraph.ID, + translate.ToolOptions{ForceShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG}) + if err != nil { + t.Fatalf("translate I1: %v", err) + } + expected := executeInlineASPTranslation(t, session, a1Translation) + actual := executeInlineASPTranslation(t, session, i1Translation) + if testCase.name == "no path" { + var receipt string + actual, receipt = executeInlineASPTranslationWithReceipt(t, session, i1Translation, "inline-asp-no-path") + if !containsAll(receipt, "ASP-I1-U-DAG+MAT-M0", "inline_no_path", "false", "1") { + t.Fatalf("no-path runtime receipt is incomplete: %s", receipt) + } + } + if fmt.Sprint(expected) != fmt.Sprint(actual) { + t.Fatalf("I1 differs from A1: A1=%v I1=%v", expected, actual) + } + }) + } + + t.Run("driver policy requires stable snapshot and rolls back immediately", func(t *testing.T) { + policy := inlineASPTraversalPolicy(t, pgDriver, defaultGraph.ID, inlineASPCypher, parameters) + if err := pgDriver.SetTraversalPolicy(policy); err != nil { + t.Fatalf("set inline ASP policy: %v", err) + } + t.Cleanup(func() { _ = pgDriver.SetTraversalPolicy(pg.TraversalPolicy{}) }) + + readCommittedRows, readCommittedReceipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, parameters, + "inline-asp-policy-read-committed", optimize.ShortestPathExecutorASPA1DAG) + if fmt.Sprint(a1Rows) != fmt.Sprint(readCommittedRows) || !containsAll(readCommittedReceipt, "ASP-A1-DAG") { + t.Fatalf("read-committed policy did not preserve A1: rows=%v receipt=%s", readCommittedRows, readCommittedReceipt) + } + + // Force the stable-snapshot execution onto a fresh PostgreSQL session. + // Its incumbent fallback workspace must be created before BEGIN READ ONLY. + session.PGPool.Reset() + repeatableRows, repeatableReceipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, parameters, + "inline-asp-policy-repeatable", optimize.ShortestPathExecutorASPI1DAG, pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(a1Rows) != fmt.Sprint(repeatableRows) || !containsAll(repeatableReceipt, "ASP-I1-U-DAG+MAT-M0", "inline_predecessor_dag") { + t.Fatalf("repeatable-read policy did not execute I1: rows=%v receipt=%s", repeatableRows, repeatableReceipt) + } + + if err := pgDriver.SetTraversalPolicy(pg.TraversalPolicy{ + Generation: policy.Generation + 1, + DisableInlineASPDAG: true, + }); err != nil { + t.Fatalf("activate inline ASP rollback: %v", err) + } + rollbackRows, rollbackReceipt := executeDriverCypherWithReceipt(t, session, inlineASPCypher, parameters, + "inline-asp-policy-rollback", optimize.ShortestPathExecutorASPA1DAG, pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(a1Rows) != fmt.Sprint(rollbackRows) || !containsAll(rollbackReceipt, "ASP-A1-DAG") || strings.Contains(rollbackReceipt, "ASP-I1-U-DAG+MAT-M0") { + t.Fatalf("rollback did not immediately restore A1: rows=%v receipt=%s", rollbackRows, rollbackReceipt) + } + }) + + t.Run("canonical inline witness falls back to S4 before exposing rows", func(t *testing.T) { + const shortestCypher = `MATCH p = shortestPath((s)<-[:InlineASPEdgeOne*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id RETURN p` + query, err := frontend.ParseCypher(frontend.NewContext(), shortestCypher) + if err != nil { + t.Fatalf("parse canonical shortest query: %v", err) + } + deepParameters := map[string]any{"start_id": int64(deepEndID), "end_id": int64(deepStartID)} + incumbent, err := translate.Translate(session.Ctx, query, pgDriver.KindMapper(), deepParameters, defaultGraph.ID) + if err != nil { + t.Fatalf("translate shortest incumbent: %v", err) + } + candidate, err := translate.TranslateWithProductionOptions(session.Ctx, query, pgDriver.KindMapper(), deepParameters, defaultGraph.ID, + translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 1, + PredecessorLimit: 100, + EnumerationLimit: 100, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + }, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + }) + if err != nil { + t.Fatalf("translate canonical shortest candidate: %v", err) + } + expected := executeInlineASPTranslation(t, session, incumbent) + actual, receipt := executeInlineASPTranslationWithReceipt(t, session, candidate, "sp-i1-s4-fallback", optimize.ShortestPathExecutorI1CanonicalPredecessorWitness) + if fmt.Sprint(expected) != fmt.Sprint(actual) { + t.Fatalf("canonical fallback differs from incumbent: incumbent=%v candidate=%v", expected, actual) + } + if !containsAll(receipt, "exact_s4_fallback", "SP-S4-C-WE+MAT-M0", "exact_relationship_trail_fallback", "SP-S3-U-E+MAT-M0", "2") { + t.Fatalf("canonical fallback receipt does not contain the complete event chain: %s", receipt) + } + }) + + t.Run("canonical driver policy requires stable snapshot and rolls back immediately", func(t *testing.T) { + const shortestCypher = `MATCH p = shortestPath((s)<-[:InlineASPEdgeOne*1..64]-(e)) + WHERE id(s) = $start_id AND id(e) = $end_id RETURN p` + parameters := map[string]any{"start_id": int64(deepEndID), "end_id": int64(deepStartID)} + policy := inlineCanonicalSPTraversalPolicy(t, pgDriver, defaultGraph.ID, shortestCypher, parameters) + if err := pgDriver.SetTraversalPolicy(policy); err != nil { + t.Fatalf("set canonical SP policy: %v", err) + } + t.Cleanup(func() { _ = pgDriver.SetTraversalPolicy(pg.TraversalPolicy{}) }) + + incumbentRows, incumbentReceipt := executeDriverCypherWithReceipt(t, session, shortestCypher, parameters, + "sp-i1-policy-read-committed", optimize.ShortestPathExecutorS4CanonicalWitness) + if len(incumbentRows) != 1 || !containsAll(incumbentReceipt, "SP-S4-C-WE+MAT-M0", "compact_workspace_witness") || + strings.Contains(incumbentReceipt, "SP-I1-C-WE+MAT-M0") { + t.Fatalf("read-committed policy did not preserve the S4 incumbent: rows=%v receipt=%s", incumbentRows, incumbentReceipt) + } + + // Exercise admission on a fresh connection so all session-local fallback + // workspace is initialized before the stable-snapshot transaction begins. + session.PGPool.Reset() + candidateRows, candidateReceipt := executeDriverCypherWithReceipt(t, session, shortestCypher, parameters, + "sp-i1-policy-repeatable", optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(incumbentRows) != fmt.Sprint(candidateRows) || + !containsAll(candidateReceipt, "SP-I1-C-WE+MAT-M0", "inline_canonical_witness") || + strings.Contains(candidateReceipt, "SP-S4-C-WE+MAT-M0") { + t.Fatalf("repeatable-read policy did not execute canonical I1: rows=%v receipt=%s", candidateRows, candidateReceipt) + } + + if err := pgDriver.SetTraversalPolicy(pg.TraversalPolicy{ + Generation: policy.Generation + 1, + DisableInlineSPWitness: true, + }); err != nil { + t.Fatalf("activate canonical SP rollback: %v", err) + } + rollbackRows, rollbackReceipt := executeDriverCypherWithReceipt(t, session, shortestCypher, parameters, + "sp-i1-policy-rollback", optimize.ShortestPathExecutorS4CanonicalWitness, + pg.OptionSetTransactionIsolation(pgx.RepeatableRead)) + if fmt.Sprint(incumbentRows) != fmt.Sprint(rollbackRows) || + !containsAll(rollbackReceipt, "SP-S4-C-WE+MAT-M0", "compact_workspace_witness") || + strings.Contains(rollbackReceipt, "SP-I1-C-WE+MAT-M0") { + t.Fatalf("canonical SP rollback did not immediately restore S4: rows=%v receipt=%s", rollbackRows, rollbackReceipt) + } + }) +} + +// inlineASPTraversalPolicy prepares or inspects test evidence for inline asp traversal policy. +func inlineASPTraversalPolicy(t *testing.T, pgDriver *pg.Driver, graphID int32, query string, parameters map[string]any) pg.TraversalPolicy { + t.Helper() + queryDigest := pg.TraversalPolicyQuerySHA256(query) + productionOptions := translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 1000, + PredecessorLimit: 1000, + EnumerationLimit: 1000, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "outbound", + ObservationMode: "all_paths", + MinimumDepth: 1, + MaximumDepth: 4, + RelationshipKindCount: 2, + UntypedRelationship: false, + }, + SelectorVersion: "asp-i1-driver-integration-v1", + } + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"path": role + ".json", "sha256": strings.Repeat("01", sha256.Size)} + } + raw, err := json.Marshal(map[string]any{ + "version": 2, "candidate": string(optimize.ShortestPathExecutorASPI1DAG), "selector_version": "asp-i1-driver-integration-v1", + "source_commit": "integration", "source_sha256": strings.Repeat("0", 64), + "binary_sha256": strings.Repeat("0", 64), "corpus_sha256": strings.Repeat("0", 64), + "operational_candidate_sql_sha256": inlineProductionSQLSHA256(t, pgDriver, graphID, query, parameters, productionOptions), + "execution_boundary": "guarded_dual_arm", "fallback_executor": string(optimize.ShortestPathExecutorASPA1DAG), + "caps": map[string]int64{"state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20}, + "buckets": []map[string]any{{ + "name": "inline-asp-integration", + "query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}, + "direction": "outbound", "observation_mode": "all_paths", "minimum_depth": 1, "maximum_depth": 4, + "relationship_kind_count": 2, "untyped_relationship": false, + }}, + "evidence": evidence, + }) + if err != nil { + t.Fatalf("encode inline ASP policy: %v", err) + } + digest := sha256.Sum256(raw) + return pg.TraversalPolicy{ + Generation: 1, + PromotionManifestSHA256: hex.EncodeToString(digest[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{queryDigest}, + ShortestPathExecutor: optimize.ShortestPathExecutorASPI1DAG, + } +} + +// inlineCanonicalSPTraversalPolicy prepares or inspects test evidence for inline canonical sp traversal policy. +func inlineCanonicalSPTraversalPolicy(t *testing.T, pgDriver *pg.Driver, graphID int32, query string, parameters map[string]any) pg.TraversalPolicy { + t.Helper() + queryDigest := pg.TraversalPolicyQuerySHA256(query) + productionOptions := translate.ProductionOptions{ + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + ShortestPathCaps: &translate.ProductionShortestPathCaps{ + StateLimit: 1000, + PredecessorLimit: 1000, + EnumerationLimit: 1000, + OutputBytesLimit: 1 << 20, + }, + AuthorizedBucket: &translate.ProductionTraversalBucket{ + Direction: "inbound", + ObservationMode: "one_path", + MinimumDepth: 1, + MaximumDepth: 64, + RelationshipKindCount: 1, + UntypedRelationship: false, + }, + SelectorVersion: optimize.ShortestPathSelectorStaticV6, + } + evidence := map[string]map[string]string{} + for _, role := range []string{"aa", "confirmation", "performance", "resource", "reference_closure", "operational"} { + evidence[role] = map[string]string{"path": role + ".json", "sha256": strings.Repeat("01", sha256.Size)} + } + raw, err := json.Marshal(map[string]any{ + "version": 2, "candidate": string(optimize.ShortestPathExecutorI1CanonicalPredecessorWitness), "selector_version": optimize.ShortestPathSelectorStaticV6, + "source_commit": "integration", "source_sha256": strings.Repeat("0", 64), + "binary_sha256": strings.Repeat("0", 64), "corpus_sha256": strings.Repeat("0", 64), + "operational_candidate_sql_sha256": inlineProductionSQLSHA256(t, pgDriver, graphID, query, parameters, productionOptions), + "execution_boundary": "guarded_dual_arm", "fallback_executor": string(optimize.ShortestPathExecutorS4CanonicalWitness), + "caps": map[string]int64{"state_limit": 1000, "predecessor_limit": 1000, "enumeration_limit": 1000, "output_bytes_limit": 1 << 20}, + "buckets": []map[string]any{{ + "name": "inline-canonical-sp-integration", + "query_sha256": []string{queryDigest}, "qualification_split": []string{"training", "holdout"}, + "direction": "inbound", "observation_mode": "one_path", "minimum_depth": 1, "maximum_depth": 64, + "relationship_kind_count": 1, "untyped_relationship": false, + }}, + "evidence": evidence, + }) + if err != nil { + t.Fatalf("encode canonical SP policy: %v", err) + } + digest := sha256.Sum256(raw) + return pg.TraversalPolicy{ + Generation: 2, + PromotionManifestSHA256: hex.EncodeToString(digest[:]), + PromotionManifestJSON: raw, + QuerySHA256Allowlist: []string{queryDigest}, + ShortestPathExecutor: optimize.ShortestPathExecutorI1CanonicalPredecessorWitness, + } +} + +// inlineProductionSQLSHA256 renders the exact production candidate statement +// bound by an integration-test promotion manifest. +func inlineProductionSQLSHA256(t *testing.T, pgDriver *pg.Driver, graphID int32, query string, parameters map[string]any, options translate.ProductionOptions) string { + t.Helper() + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), query) + if err != nil { + t.Fatalf("parse production candidate query: %v", err) + } + translation, err := translate.TranslateWithProductionOptions(t.Context(), regularQuery, pgDriver.KindMapper(), parameters, graphID, options) + if err != nil { + t.Fatalf("translate production candidate query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render production candidate query: %v", err) + } + digest := sha256.Sum256([]byte(sqlQuery)) + return hex.EncodeToString(digest[:]) +} + +// explainInlineASPTranslation prepares or inspects test evidence for explain inline asp translation. +func explainInlineASPTranslation(t *testing.T, session *Session, translation translate.Result) any { + t.Helper() + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render translated query: %v", err) + } + var plan any + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + result := tx.Raw("explain (analyze, timing off, summary off, format json) "+sqlQuery, translation.Parameters) + defer result.Close() + if !result.Next() { + if err := result.Error(); err != nil { + return err + } + return errors.New("PostgreSQL EXPLAIN returned no rows") + } + values := result.Values() + if len(values) == 0 { + return errors.New("PostgreSQL EXPLAIN returned an empty row") + } + parsed, err := normalizeExplainPlan(values[0]) + if err != nil { + return err + } + plan = parsed + return result.Error() + }); err != nil { + t.Fatalf("explain inline ASP query: %v", err) + } + return plan +} + +// executeInlineASPTranslationWithReceipt prepares or inspects test evidence for execute inline asp translation with receipt. +func executeInlineASPTranslationWithReceipt(t *testing.T, session *Session, translation translate.Result, invocation string, requested ...optimize.ShortestPathExecutor) ([]string, string) { + t.Helper() + requestedIdentity := optimize.ShortestPathExecutorASPI1DAG + if len(requested) > 0 { + requestedIdentity = requested[0] + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render translated query: %v", err) + } + var rows []string + var receipt string + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, "requested": string(requestedIdentity), + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Raw(sqlQuery, translation.Parameters) + for result.Next() { + rows = append(rows, fmt.Sprint(result.Values())) + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw(`select + coalesce(document ->> 'runtime_identity', ''), + coalesce(document ->> 'runtime_branch', ''), + coalesce(document ->> 'fallback_executed', ''), + coalesce(document ->> 'record_count', ''), + coalesce(document ->> 'events', '') + from (select public.read_traversal_runtime_attestation_v1(@invocation) document) receipt`, map[string]any{"invocation": invocation}) + if read.Next() { + receipt = fmt.Sprint(read.Values()) + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }); err != nil { + t.Fatalf("execute translated query with receipt: %v\nSQL: %s", err, sqlQuery) + } + sort.Strings(rows) + return rows, receipt +} + +// executeDriverCypherWithReceipt prepares or inspects test evidence for execute driver cypher with receipt. +func executeDriverCypherWithReceipt(t *testing.T, session *Session, cypher string, parameters map[string]any, invocation string, + requested optimize.ShortestPathExecutor, options ...graph.TransactionOption) ([]string, string) { + t.Helper() + var rows []string + var receipt string + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, "requested": string(requested), + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Query(cypher, parameters) + for result.Next() { + rows = append(rows, fmt.Sprint(result.Values())) + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw("select coalesce(public.read_traversal_runtime_attestation_v1(@invocation)::text, '')", map[string]any{"invocation": invocation}) + if read.Next() { + values := read.Values() + if len(values) > 0 { + receipt = fmt.Sprint(values[0]) + } + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }, append(options, pg.OptionInitializeTraversalRuntimeAttestation())...); err != nil { + t.Fatalf("execute driver Cypher with receipt: %v", err) + } + sort.Strings(rows) + return rows, receipt +} + +// containsAll reports whether every required fragment occurs in the inspected SQL text. +func containsAll(value string, fragments ...string) bool { + for _, fragment := range fragments { + if !strings.Contains(value, fragment) { + return false + } + } + return true +} + +// executeInlineASPTranslation prepares or inspects test evidence for execute inline asp translation. +func executeInlineASPTranslation(t *testing.T, session *Session, translation translate.Result) []string { + t.Helper() + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render translated query: %v", err) + } + var rows []string + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + result := tx.Raw(sqlQuery, translation.Parameters) + defer result.Close() + for result.Next() { + rows = append(rows, fmt.Sprint(result.Values())) + } + return result.Error() + }); err != nil { + t.Fatalf("execute translated query: %v", err) + } + sort.Strings(rows) + return rows +} diff --git a/integration/pgsql_orientation_execution_plan_test.go b/integration/pgsql_orientation_execution_plan_test.go new file mode 100644 index 00000000..9067f0d8 --- /dev/null +++ b/integration/pgsql_orientation_execution_plan_test.go @@ -0,0 +1,813 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/specterops/dawgs/cypher/frontend" + "github.com/specterops/dawgs/cypher/models/pgsql/optimize" + "github.com/specterops/dawgs/cypher/models/pgsql/translate" + "github.com/specterops/dawgs/drivers/pg" + "github.com/specterops/dawgs/graph" +) + +// orientationExecutionPlanCypher reserves the stable protocol value used to recognize orientation execution plan cypher across artifacts and executions. +const orientationExecutionPlanCypher = ` + MATCH (root:ExpansionRoot) + WHERE root.root_key = $root_key + MATCH path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) + RETURN path +` + +var ( + // orientationRootKind contains the frozen orientation root kind declaration consulted by package validation. + orientationRootKind = graph.StringKind("ExpansionRoot") + + // orientationExpansionKind contains the frozen orientation expansion kind declaration consulted by package validation. + orientationExpansionKind = graph.StringKind("ExpansionNode") + + // orientationSuffixHeadKind contains the frozen orientation suffix head kind declaration consulted by package validation. + orientationSuffixHeadKind = graph.StringKind("SuffixHead") + + // orientationSuffixMidKind contains the frozen orientation suffix mid kind declaration consulted by package validation. + orientationSuffixMidKind = graph.StringKind("SuffixMiddle") + + // orientationSuffixEndKind contains the frozen orientation suffix end kind declaration consulted by package validation. + orientationSuffixEndKind = graph.StringKind("SuffixTerminal") + + // orientationExpandEdge contains the frozen orientation expand edge declaration consulted by package validation. + orientationExpandEdge = graph.StringKind("Expand") + + // orientationSuffixEdgeOne contains the frozen orientation suffix edge one declaration consulted by package validation. + orientationSuffixEdgeOne = graph.StringKind("EnterSuffix") + + // orientationSuffixEdgeTwo contains the frozen orientation suffix edge two declaration consulted by package validation. + orientationSuffixEdgeTwo = graph.StringKind("ContinueSuffix") + + // orientationSuffixEdgeThree contains the frozen orientation suffix edge three declaration consulted by package validation. + orientationSuffixEdgeThree = graph.StringKind("CompleteSuffix") +) + +// TestPostgreSQLGuardedOrientationInactiveArmLoops proves the emitted +// marker-first LATERAL dependencies at the PostgreSQL execution boundary. The +// forward case must leave reverse recursion uninitialized; the reverse case +// must leave the exact materialized incumbent uninitialized. +func TestPostgreSQLGuardedOrientationInactiveArmLoops(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // reverseDominates indicates whether reverse dominates applies. + reverseDominates bool + // expectedReverseLoops retains the expected reverse loops while anonymous record is assembled or evaluated. + expectedReverseLoops int64 + // expectedIncumbentLoops retains the expected incumbent loops while anonymous record is assembled or evaluated. + expectedIncumbentLoops int64 + // expectedCandidateMarkers retains the expected candidate markers while anonymous record is assembled or evaluated. + expectedCandidateMarkers int64 + // expectedIncumbentMarkers retains the expected incumbent markers while anonymous record is assembled or evaluated. + expectedIncumbentMarkers int64 + }{ + { + name: "forward policy does not initialize reverse recursion", + reverseDominates: false, + expectedReverseLoops: 0, + expectedIncumbentLoops: 1, + expectedCandidateMarkers: 0, + expectedIncumbentMarkers: 1, + }, + { + name: "reverse policy does not initialize exact incumbent", + reverseDominates: true, + expectedReverseLoops: 1, + expectedIncumbentLoops: 0, + expectedCandidateMarkers: 1, + expectedIncumbentMarkers: 0, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + session.ClearGraph(t) + loadOrientationExecutionFixture(t, session, testCase.reverseDominates) + + plan := explainGuardedOrientation(t, session) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_candidate", "Actual Rows", testCase.expectedCandidateMarkers) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_incumbent", "Actual Rows", testCase.expectedIncumbentMarkers) + requireOrientationSubplanMetric(t, plan, "_orientation_reverse", "Actual Loops", testCase.expectedReverseLoops) + requireOrientationSubplanMetric(t, plan, "_orientation_incumbent", "Actual Loops", testCase.expectedIncumbentLoops) + }) + } +} + +// TestPostgreSQLOrientationProbeV2ChangesOnlyItsVersionedDecision proves the +// depth-weighted v2 formula at the real PostgreSQL boundary while retaining +// v1's frozen choice for the same graph and statement. +func TestPostgreSQLOrientationProbeV2ChangesOnlyItsVersionedDecision(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + loadOrientationV2CrossoverFixture(t, session) + for _, testCase := range []struct { + // policy retains the policy while anonymous record is assembled or evaluated. + policy optimize.ExpansionSearchPolicy + // expectedCandidateMarkers retains the expected candidate markers while anonymous record is assembled or evaluated. + expectedCandidateMarkers int64 + // expectedIncumbentMarkers retains the expected incumbent markers while anonymous record is assembled or evaluated. + expectedIncumbentMarkers int64 + }{ + { + policy: optimize.ExpansionSearchPolicyOrientationProbeV1, + expectedIncumbentMarkers: 1, + }, + { + policy: optimize.ExpansionSearchPolicyOrientationProbeV2, + expectedCandidateMarkers: 1, + }, + } { + t.Run(string(testCase.policy), func(t *testing.T) { + plan := explainGuardedOrientationPolicy(t, session, testCase.policy) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_candidate", "Actual Rows", testCase.expectedCandidateMarkers) + requireOrientationSubplanMetric(t, plan, "_orientation_executed_incumbent", "Actual Rows", testCase.expectedIncumbentMarkers) + }) + } +} + +// TestPostgreSQLShadowOrientationAttestsEmptyIncumbent proves the shadow +// statement records its only executable arm even when that arm returns no +// rows. The marker must be outside the incumbent LATERAL boundary or an empty +// result would leave the timed receipt unprovable. +func TestPostgreSQLShadowOrientationAttestsEmptyIncumbent(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), orientationExecutionPlanCypher) + if err != nil { + t.Fatalf("parse shadow orientation query: %v", err) + } + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + translation, err := translate.TranslateForTool( + session.Ctx, + regularQuery, + pgDriver.KindMapper(), + map[string]any{"root_key": "missing-orientation-plan-root"}, + defaultGraph.ID, + translate.ToolOptions{EnableExpansionOrientationShadow: true}, + ) + if err != nil { + t.Fatalf("translate shadow orientation query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render shadow orientation query: %v", err) + } + + const invocation = "shadow-orientation-empty-incumbent" + var ( + rowCount int + receipt string + ) + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, + "requested": "EXPANSION-SUFFIX-SEEDED-REVERSE", + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Raw(sqlQuery, translation.Parameters) + for result.Next() { + rowCount++ + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw("select coalesce(public.read_traversal_runtime_attestation_v1(@invocation)::text, '')", map[string]any{"invocation": invocation}) + if read.Next() && len(read.Values()) > 0 { + receipt = fmt.Sprint(read.Values()[0]) + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }); err != nil { + t.Fatalf("execute empty shadow orientation query: %v\nSQL: %s", err, sqlQuery) + } + if rowCount != 0 { + t.Fatalf("empty shadow incumbent returned %d rows", rowCount) + } + for _, fragment := range []string{`"runtime_identity": "EXPANSION-STEPWISE-FORWARD"`, `"runtime_branch": "shadow_incumbent"`, `"fallback_executed": false`, `"record_count": 1`} { + if !strings.Contains(receipt, fragment) { + t.Fatalf("empty shadow incumbent receipt lacks %q: %s", fragment, receipt) + } + } +} + +// TestPostgreSQLGuardedOrientationFallbackReceipts proves both cap+1 fallback +// paths produce one truthful incumbent receipt. Probe overflow skips reverse +// recursion entirely; state overflow performs only the bounded reverse +// admission probe before executing the exact forward fallback. +func TestPostgreSQLGuardedOrientationFallbackReceipts(t *testing.T) { + session := Open(t, Options{ + RequireDriver: pg.DriverName, + SkipIfNoConnection: true, + SkipIfDriverMismatch: true, + CleanupMode: CleanupGraph, + ExtraNodeKinds: graph.Kinds{ + orientationRootKind, + orientationExpansionKind, + orientationSuffixHeadKind, + orientationSuffixMidKind, + orientationSuffixEndKind, + }, + ExtraEdgeKinds: graph.Kinds{ + orientationExpandEdge, + orientationSuffixEdgeOne, + orientationSuffixEdgeTwo, + orientationSuffixEdgeThree, + }, + }) + + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // rootKey retains the root key while anonymous record is assembled or evaluated. + rootKey string + // load retains the load while anonymous record is assembled or evaluated. + load func(*testing.T, *Session) + // expectedRows records the number of expected rows. + expectedRows int + }{ + { + name: "probe overflow", + rootKey: "orientation-probe-overflow-root", + load: loadOrientationProbeOverflowFixture, + expectedRows: 0, + }, + { + name: "state overflow", + rootKey: "orientation-state-overflow-root", + load: loadOrientationStateOverflowFixture, + expectedRows: 4096, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + session.ClearGraph(t) + testCase.load(t, session) + rowCount, receipt := executeGuardedOrientationWithReceipt(t, session, testCase.rootKey) + if rowCount != testCase.expectedRows { + t.Fatalf("guarded orientation returned %d rows, want %d", rowCount, testCase.expectedRows) + } + for _, fragment := range []string{`"runtime_identity": "EXPANSION-STEPWISE-FORWARD"`, `"runtime_branch": "exact_forward_incumbent"`, `"fallback_executed": true`, `"record_count": 1`} { + if !strings.Contains(receipt, fragment) { + t.Fatalf("guarded orientation receipt lacks %q: %s", fragment, receipt) + } + } + }) + } +} + +// executeGuardedOrientationWithReceipt prepares or inspects test evidence for execute guarded orientation with receipt. +func executeGuardedOrientationWithReceipt(t *testing.T, session *Session, rootKey string) (int, string) { + t.Helper() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), orientationExecutionPlanCypher) + if err != nil { + t.Fatalf("parse guarded orientation query: %v", err) + } + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + translation, err := translate.TranslateForTool( + session.Ctx, + regularQuery, + pgDriver.KindMapper(), + map[string]any{"root_key": rootKey}, + defaultGraph.ID, + translate.ToolOptions{EnableExpansionOrientationTournament: true}, + ) + if err != nil { + t.Fatalf("translate guarded orientation query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render guarded orientation query: %v", err) + } + + invocation := "guarded-" + rootKey + var ( + rowCount int + receipt string + ) + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + arm := tx.Raw("select public.begin_traversal_runtime_attestation_v1(@invocation, @requested)", map[string]any{ + "invocation": invocation, + "requested": "EXPANSION-SUFFIX-SEEDED-REVERSE", + }) + for arm.Next() { + } + if err := arm.Error(); err != nil { + arm.Close() + return err + } + arm.Close() + + result := tx.Raw(sqlQuery, translation.Parameters) + for result.Next() { + rowCount++ + } + if err := result.Error(); err != nil { + result.Close() + return err + } + result.Close() + + read := tx.Raw("select coalesce(public.read_traversal_runtime_attestation_v1(@invocation)::text, '')", map[string]any{"invocation": invocation}) + if read.Next() && len(read.Values()) > 0 { + receipt = fmt.Sprint(read.Values()[0]) + } + if err := read.Error(); err != nil { + read.Close() + return err + } + read.Close() + + clear := tx.Raw("select public.clear_traversal_runtime_attestation_v1(@invocation)", map[string]any{"invocation": invocation}) + for clear.Next() { + } + err := clear.Error() + clear.Close() + return err + }); err != nil { + t.Fatalf("execute guarded orientation query: %v\nSQL: %s", err, sqlQuery) + } + return rowCount, receipt +} + +// loadOrientationProbeOverflowFixture loads orientation probe overflow fixture. +func loadOrientationProbeOverflowFixture(t *testing.T, session *Session) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + if _, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-probe-overflow-root"}), orientationRootKind); err != nil { + return err + } + for index := 0; index <= 512; index++ { + if _, err := createOrientationSuffix(tx); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load orientation probe-overflow fixture: %v", err) + } +} + +// loadOrientationStateOverflowFixture loads orientation state overflow fixture. +func loadOrientationStateOverflowFixture(t *testing.T, session *Session) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + root, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-state-overflow-root"}), orientationRootKind) + if err != nil { + return err + } + boundary, err := createOrientationSuffix(tx) + if err != nil { + return err + } + first, err := createOrientationNodes(tx, 16) + if err != nil { + return err + } + second, err := createOrientationNodes(tx, 32) + if err != nil { + return err + } + third, err := createOrientationNodes(tx, 8) + if err != nil { + return err + } + for _, node := range first { + if _, err := tx.CreateRelationshipByIDs(root.ID, node.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + if err := connectOrientationLayers(tx, first, second); err != nil { + return err + } + if err := connectOrientationLayers(tx, second, third); err != nil { + return err + } + for _, node := range third { + if _, err := tx.CreateRelationshipByIDs(node.ID, boundary.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load orientation state-overflow fixture: %v", err) + } +} + +// createOrientationSuffix creates orientation suffix. +func createOrientationSuffix(tx graph.Transaction) (*graph.Node, error) { + boundary, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return nil, err + } + head, err := tx.CreateNode(graph.NewProperties(), orientationSuffixHeadKind) + if err != nil { + return nil, err + } + middle, err := tx.CreateNode(graph.NewProperties(), orientationSuffixMidKind) + if err != nil { + return nil, err + } + terminal, err := tx.CreateNode(graph.NewProperties(), orientationSuffixEndKind) + if err != nil { + return nil, err + } + for _, edge := range []struct { + // start identifies the relationship's source node. + start graph.ID + + // end identifies the relationship's target node. + end graph.ID + + // kind identifies the relationship kind used by the suffix query. + kind graph.Kind + }{ + { + start: boundary.ID, + end: head.ID, + kind: orientationSuffixEdgeOne, + }, + { + start: head.ID, + end: middle.ID, + kind: orientationSuffixEdgeTwo, + }, + { + start: middle.ID, + end: terminal.ID, + kind: orientationSuffixEdgeThree, + }, + } { + if _, err := tx.CreateRelationshipByIDs(edge.start, edge.end, edge.kind, graph.NewProperties()); err != nil { + return nil, err + } + } + return boundary, nil +} + +// createOrientationNodes creates orientation nodes. +func createOrientationNodes(tx graph.Transaction, count int) ([]*graph.Node, error) { + nodes := make([]*graph.Node, 0, count) + for index := 0; index < count; index++ { + node, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return nil, err + } + nodes = append(nodes, node) + } + return nodes, nil +} + +// connectOrientationLayers prepares or inspects test evidence for connect orientation layers. +func connectOrientationLayers(tx graph.Transaction, left, right []*graph.Node) error { + for _, start := range left { + for _, end := range right { + if _, err := tx.CreateRelationshipByIDs(start.ID, end.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + } + return nil +} + +// loadOrientationExecutionFixture loads orientation execution fixture. +func loadOrientationExecutionFixture(t *testing.T, session *Session, reverseDominates bool) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + root, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-plan-root"}), orientationRootKind) + if err != nil { + return err + } + + addSuffix := func(connectRoot bool) error { + boundary, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return err + } + head, err := tx.CreateNode(graph.NewProperties(), orientationSuffixHeadKind) + if err != nil { + return err + } + middle, err := tx.CreateNode(graph.NewProperties(), orientationSuffixMidKind) + if err != nil { + return err + } + terminal, err := tx.CreateNode(graph.NewProperties(), orientationSuffixEndKind) + if err != nil { + return err + } + if connectRoot { + if _, err := tx.CreateRelationshipByIDs(root.ID, boundary.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + for _, edge := range []struct { + // start identifies the relationship's source node. + start graph.ID + + // end identifies the relationship's target node. + end graph.ID + + // kind identifies the relationship kind used by the suffix query. + kind graph.Kind + }{ + { + start: boundary.ID, + end: head.ID, + kind: orientationSuffixEdgeOne, + }, + { + start: head.ID, + end: middle.ID, + kind: orientationSuffixEdgeTwo, + }, + { + start: middle.ID, + end: terminal.ID, + kind: orientationSuffixEdgeThree, + }, + } { + if _, err := tx.CreateRelationshipByIDs(edge.start, edge.end, edge.kind, graph.NewProperties()); err != nil { + return err + } + } + return nil + } + + if err := addSuffix(true); err != nil { + return err + } + if reverseDominates { + // One reverse seed but many typed forward neighbors makes reverse + // strictly dominate orientation-probe-v1's 4:3 hysteresis rule. + for index := 0; index < 24; index++ { + decoy, err := tx.CreateNode(graph.NewProperties(), orientationExpansionKind) + if err != nil { + return err + } + if _, err := tx.CreateRelationshipByIDs(root.ID, decoy.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + } else { + // Many disconnected suffix seeds overwhelm the one useful forward + // neighbor, so the incumbent wins decisively. + for index := 0; index < 20; index++ { + if err := addSuffix(false); err != nil { + return err + } + } + } + return nil + }); err != nil { + t.Fatalf("load guarded orientation fixture: %v", err) + } +} + +// loadOrientationV2CrossoverFixture loads orientation v2 crossover fixture. +func loadOrientationV2CrossoverFixture(t *testing.T, session *Session) { + t.Helper() + + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + root, err := tx.CreateNode(graph.AsProperties(map[string]any{"root_key": "orientation-plan-root"}), orientationRootKind) + if err != nil { + return err + } + for range 4 { + boundary, err := createOrientationSuffix(tx) + if err != nil { + return err + } + if _, err := tx.CreateRelationshipByIDs(root.ID, boundary.ID, orientationExpandEdge, graph.NewProperties()); err != nil { + return err + } + } + return nil + }); err != nil { + t.Fatalf("load orientation v2 crossover fixture: %v", err) + } +} + +// explainGuardedOrientation prepares or inspects test evidence for explain guarded orientation. +func explainGuardedOrientation(t *testing.T, session *Session) any { + return explainGuardedOrientationPolicy(t, session, optimize.ExpansionSearchPolicyOrientationProbeV1) +} + +// explainGuardedOrientationPolicy prepares or inspects test evidence for explain guarded orientation policy. +func explainGuardedOrientationPolicy(t *testing.T, session *Session, policy optimize.ExpansionSearchPolicy) any { + t.Helper() + + regularQuery, err := frontend.ParseCypher(frontend.NewContext(), orientationExecutionPlanCypher) + if err != nil { + t.Fatalf("parse guarded orientation query: %v", err) + } + pgDriver, ok := session.DB.(*pg.Driver) + if !ok { + t.Fatalf("expected PostgreSQL driver, found %T", session.DB) + } + defaultGraph, ok := pgDriver.DefaultGraph() + if !ok { + t.Fatal("PostgreSQL default graph is not set") + } + translation, err := translate.TranslateForTool( + session.Ctx, + regularQuery, + pgDriver.KindMapper(), + map[string]any{"root_key": "orientation-plan-root"}, + defaultGraph.ID, + translate.ToolOptions{ + ExpansionOrientationPolicy: policy, + EnableExpansionOrientationTournament: true, + }, + ) + if err != nil { + t.Fatalf("translate guarded orientation query: %v", err) + } + sqlQuery, err := translate.Translated(translation) + if err != nil { + t.Fatalf("render guarded orientation query: %v", err) + } + + var plan any + if err := session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { + result := tx.Raw("explain (analyze, timing off, summary off, format json) "+sqlQuery, translation.Parameters) + defer result.Close() + if !result.Next() { + if err := result.Error(); err != nil { + return err + } + return errors.New("PostgreSQL EXPLAIN returned no rows") + } + values := result.Values() + if len(values) == 0 { + return errors.New("PostgreSQL EXPLAIN returned an empty row") + } + parsed, err := normalizeExplainPlan(values[0]) + if err != nil { + return err + } + plan = parsed + return result.Error() + }); err != nil { + t.Fatalf("explain guarded orientation query: %v", err) + } + return plan +} + +// requireOrientationSubplanMetric prepares or inspects test evidence for require orientation subplan metric. +func requireOrientationSubplanMetric(t *testing.T, plan any, suffix, metric string, expected int64) { + t.Helper() + + subplan, found := findOrientationSubplan(plan, suffix) + if !found { + t.Fatalf("PostgreSQL JSON plan has no subplan ending in %q", suffix) + } + actual, ok := postgresPlanInt64(subplan[metric]) + if !ok { + t.Fatalf("orientation subplan %q has no numeric %s", subplan["Subplan Name"], metric) + } + if actual != expected { + t.Fatalf("orientation subplan %q %s: got %d, want %d", subplan["Subplan Name"], metric, actual, expected) + } +} + +// findOrientationSubplan prepares or inspects test evidence for find orientation subplan. +func findOrientationSubplan(value any, suffix string) (map[string]any, bool) { + switch typed := value.(type) { + case []any: + for _, child := range typed { + if found, ok := findOrientationSubplan(child, suffix); ok { + return found, true + } + } + case map[string]any: + if name, ok := typed["Subplan Name"].(string); ok && strings.HasSuffix(name, suffix) { + return typed, true + } + for _, child := range typed { + if found, ok := findOrientationSubplan(child, suffix); ok { + return found, true + } + } + } + return nil, false +} + +// postgresPlanInt64 prepares or inspects test evidence for postgres plan int64. +func postgresPlanInt64(value any) (int64, bool) { + switch typed := value.(type) { + case float64: + return int64(typed), typed == float64(int64(typed)) + case int64: + return typed, true + case int: + return int64(typed), true + default: + return 0, false + } +} diff --git a/integration/pgsql_translation_cache_test.go b/integration/pgsql_translation_cache_test.go index 55413de6..bbdd65ac 100644 --- a/integration/pgsql_translation_cache_test.go +++ b/integration/pgsql_translation_cache_test.go @@ -56,7 +56,7 @@ func TestPostgreSQLFetchStartNodesUsesBuilderCompilationCache(t *testing.T) { return err })) - before := driver.TranslationCacheStats() + before := driver.CompilationCacheStats() require.NoError(t, session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { firstNodes, err := ops.FetchStartNodes(tx.Relationships().Filter(query.InIDs(query.Start(), first.ID))) if err != nil { @@ -75,7 +75,7 @@ func TestPostgreSQLFetchStartNodesUsesBuilderCompilationCache(t *testing.T) { return nil })) - after := driver.TranslationCacheStats() + after := driver.CompilationCacheStats() require.Equal(t, before.Misses+1, after.Misses) require.GreaterOrEqual(t, after.Hits, before.Hits+1) } @@ -123,7 +123,7 @@ func TestPostgreSQLFetchStartNodesUnoptimizedBypassesCache(t *testing.T) { return err })) - before := driver.TranslationCacheStats() + before := driver.CompilationCacheStats() require.NoError(t, session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { if _, err := ops.FetchStartNodes(tx.Relationships().Filter(query.InIDs(query.Start(), first.ID))); err != nil { return err @@ -132,7 +132,7 @@ func TestPostgreSQLFetchStartNodesUnoptimizedBypassesCache(t *testing.T) { return err })) - after := driver.TranslationCacheStats() + after := driver.CompilationCacheStats() require.Equal(t, before.Hits, after.Hits) require.Equal(t, before.Misses, after.Misses) require.Equal(t, before.Insertions, after.Insertions) @@ -170,7 +170,7 @@ func TestPostgreSQLRawCypherQueryRebindsCachedParameters(t *testing.T) { return err })) - before := driver.TranslationCacheStats() + before := driver.TranslationCacheStats().Aggregate require.NoError(t, session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { firstID, err := rawCypherNodeID(tx, "first") if err != nil { @@ -186,12 +186,12 @@ func TestPostgreSQLRawCypherQueryRebindsCachedParameters(t *testing.T) { return nil })) - after := driver.TranslationCacheStats() + after := driver.TranslationCacheStats().Aggregate require.Equal(t, before.Misses+1, after.Misses) require.GreaterOrEqual(t, after.Hits, before.Hits+1) } -func TestPostgreSQLRawCypherQueryUnoptimizedBypassesCache(t *testing.T) { +func TestPostgreSQLRawCypherQueryUsesConnectionCacheWhenCompilerOptimizationDisabled(t *testing.T) { previous := pg.SetOptimizedTranslation(false) t.Cleanup(func() { pg.SetOptimizedTranslation(previous) @@ -208,7 +208,7 @@ func TestPostgreSQLRawCypherQueryUnoptimizedBypassesCache(t *testing.T) { driver, ok := session.DB.(*pg.Driver) require.True(t, ok) - before := driver.TranslationCacheStats() + before := driver.TranslationCacheStats().Aggregate require.NoError(t, session.DB.ReadTransaction(session.Ctx, func(tx graph.Transaction) error { for _, name := range []string{"first", "second"} { result := tx.Query("RETURN $name", map[string]any{"name": name}) @@ -221,11 +221,10 @@ func TestPostgreSQLRawCypherQueryUnoptimizedBypassesCache(t *testing.T) { return nil })) - after := driver.TranslationCacheStats() - require.Equal(t, before.Hits, after.Hits) - require.Equal(t, before.Misses, after.Misses) - require.Equal(t, before.Bypasses+2, after.Bypasses) - require.Equal(t, before.UnoptimizedCompilations+2, after.UnoptimizedCompilations) + after := driver.TranslationCacheStats().Aggregate + require.Equal(t, before.Misses+1, after.Misses) + require.GreaterOrEqual(t, after.Hits, before.Hits+1) + require.Equal(t, before.Insertions+1, after.Insertions) } func rawCypherNodeID(tx graph.Transaction, name string) (graph.ID, error) { @@ -272,7 +271,7 @@ func TestPostgreSQLNodeUpdateRebindsCachedBuilderParameters(t *testing.T) { return err })) - before := driver.TranslationCacheStats() + before := driver.CompilationCacheStats() require.NoError(t, session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { firstProperties := graph.NewProperties().Set("name", "after-first") if err := tx.Nodes().Filter(query.InIDs(query.Node(), first.ID)).Update(firstProperties); err != nil { @@ -282,7 +281,7 @@ func TestPostgreSQLNodeUpdateRebindsCachedBuilderParameters(t *testing.T) { secondProperties := graph.NewProperties().Set("name", "after-second") return tx.Nodes().Filter(query.InIDs(query.Node(), second.ID)).Update(secondProperties) })) - after := driver.TranslationCacheStats() + after := driver.CompilationCacheStats() require.Equal(t, before.Misses+1, after.Misses) require.GreaterOrEqual(t, after.Hits, before.Hits+1) @@ -353,14 +352,14 @@ func TestPostgreSQLRelationshipUpdateRebindsCachedBuilderParameters(t *testing.T return err })) - before := driver.TranslationCacheStats() + before := driver.CompilationCacheStats() require.NoError(t, session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { if err := tx.Relationships().Filter(query.InIDs(query.Relationship(), first.ID)).Update(graph.NewProperties().Set("name", "after-first")); err != nil { return err } return tx.Relationships().Filter(query.InIDs(query.Relationship(), second.ID)).Update(graph.NewProperties().Set("name", "after-second")) })) - after := driver.TranslationCacheStats() + after := driver.CompilationCacheStats() require.Equal(t, before.Misses+1, after.Misses) require.GreaterOrEqual(t, after.Hits, before.Hits+1) diff --git a/integration/regression_fixture.go b/integration/regression_fixture.go new file mode 100644 index 00000000..030072ed --- /dev/null +++ b/integration/regression_fixture.go @@ -0,0 +1,146 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "fmt" + + "github.com/specterops/dawgs/opengraph" +) + +// defaultRegressionFanout is the relationship fanout used when a regression +// fixture does not request an explicit size. +const defaultRegressionFanout = 32 + +// FixtureNames returns deterministic fixture identifiers without committing +// large handwritten lists to the corpus. +func FixtureNames(prefix string, count int) []string { + if count < 0 { + count = 0 + } + + width := len(fmt.Sprintf("%d", max(count-1, 0))) + if width < 2 { + width = 2 + } + + values := make([]string, count) + for idx := range count { + values[idx] = fmt.Sprintf("%s-%0*d", prefix, width, idx) + } + + return values +} + +// FixtureKinds returns deterministic synthetic kind names for list-cardinality +// tests. +func FixtureKinds(count int) []string { + if count < 0 { + count = 0 + } + + kinds := make([]string, count) + for idx := range count { + kinds[idx] = fmt.Sprintf("RegressionKind%02d", idx+1) + } + + return kinds +} + +// NewReconciliationFixture builds the reusable reconciliation fixture. It includes +// typed and multi-kind endpoints, duplicate relationship kinds, missing and +// explicit-null properties, timestamps, both directions, and a deterministic +// high-degree anchor. A non-positive fanout selects a small production-like +// default. +func NewReconciliationFixture(fanout int) *opengraph.Graph { + if fanout <= 0 { + fanout = defaultRegressionFanout + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "anchor", + Kinds: []string{"ADEntity", "Computer", "Entity"}, + Properties: map[string]any{"objectid": "anchor-id", "lastcollected": "2026-01-02T00:00:00Z", "name": "anchor"}, + }, + { + ID: "typed-end", + Kinds: []string{"ADEntity", "Group", "Entity"}, + Properties: map[string]any{"objectid": "typed-end-id", "lastcollected": "2026-01-03T00:00:00Z", "name": "typed-end"}, + }, + { + ID: "missing-lastseen", + Kinds: []string{"ADEntity", "Entity"}, + Properties: map[string]any{"objectid": "missing-id"}, + }, + { + ID: "null-lastseen", + Kinds: []string{"ADEntity", "Entity"}, + Properties: map[string]any{"objectid": "null-id", "lastseen": nil}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "anchor", + EndID: "typed-end", + Kind: "MemberOf", + Properties: map[string]any{"lastseen": "2026-01-01T00:00:00Z", "isprimarygroup": false, "marker": "duplicate-a"}, + }, + { + StartID: "anchor", + EndID: "typed-end", + Kind: "MemberOf", + Properties: map[string]any{"lastseen": "2026-01-04T00:00:00Z", "isprimarygroup": true, "marker": "duplicate-b"}, + }, + { + StartID: "typed-end", + EndID: "anchor", + Kind: "MemberOf", + Properties: map[string]any{"marker": "reverse"}, + }, + { + StartID: "anchor", + EndID: "missing-lastseen", + Kind: "HasSession", + Properties: map[string]any{"marker": "missing-lastseen"}, + }, + { + StartID: "anchor", + EndID: "null-lastseen", + Kind: "HasSession", + Properties: map[string]any{"lastseen": nil, "marker": "null-lastseen"}, + }, + }, + } + + for _, fixtureID := range FixtureNames("fanout", fanout) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: fixtureID, + Kinds: []string{"ADEntity", "Entity", "User"}, + Properties: map[string]any{"objectid": fixtureID, "name": fixtureID}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "anchor", + EndID: fixtureID, + Kind: "FanoutEdge", + Properties: map[string]any{"lastseen": "2026-01-01T00:00:00Z"}, + }) + } + + return fixture +} diff --git a/integration/regression_fixture_test.go b/integration/regression_fixture_test.go new file mode 100644 index 00000000..bcc960d8 --- /dev/null +++ b/integration/regression_fixture_test.go @@ -0,0 +1,48 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestFixtureNamesAreDeterministic verifies fixture identifiers are stable and zero-padded for a given prefix and count. +func TestFixtureNamesAreDeterministic(t *testing.T) { + require.Equal(t, []string{"id-00", "id-01", "id-02"}, FixtureNames("id", 3)) + require.Equal(t, []string{"RegressionKind01", "RegressionKind02"}, FixtureKinds(2)) + require.Equal(t, FixtureNames("id", 1_000), FixtureNames("id", 1_000)) + require.Empty(t, FixtureNames("id", -1)) +} + +// TestNewReconciliationFixtureIncludesRequiredShapes verifies the reconciliation fixture contains every typed, null, directional, and fanout shape required by regressions. +func TestNewReconciliationFixtureIncludesRequiredShapes(t *testing.T) { + fixture := NewReconciliationFixture(4) + require.Len(t, fixture.Nodes, 8) + require.Len(t, fixture.Edges, 9) + require.Equal(t, "fanout-00", fixture.Nodes[4].ID) + require.Equal(t, "fanout-03", fixture.Nodes[7].ID) + require.Equal(t, "FanoutEdge", fixture.Edges[5].Kind) + require.Equal(t, "fanout-03", fixture.Edges[8].EndID) + + nodeKinds, edgeKinds := fixture.Kinds() + require.Contains(t, nodeKinds.Strings(), "Computer") + require.Contains(t, nodeKinds.Strings(), "Group") + require.Contains(t, edgeKinds.Strings(), "MemberOf") + require.Contains(t, edgeKinds.Strings(), "HasSession") +} diff --git a/integration/relationship_scans_node_lookups_legacy_builder_test.go b/integration/relationship_scans_node_lookups_legacy_builder_test.go new file mode 100644 index 00000000..9d4a179c --- /dev/null +++ b/integration/relationship_scans_node_lookups_legacy_builder_test.go @@ -0,0 +1,557 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "sort" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderRelationshipScansAndNodeLookups verifies legacy scan and lookup forms preserve expected records and ordering. +func TestLegacyBuilderRelationshipScansAndNodeLookups(t *testing.T) { + wideFixture := regressionTemplateFixture(t, "SCAN-01 through SCAN-04 wide relationship filters") + anchoredFixture := regressionTemplateFixture(t, "SCAN-05 through SCAN-08 anchored scans and projections") + basicFixture := regressionTemplateFixture(t, "LOOKUP-01 through LOOKUP-08 node predicates and projections") + advancedFixture := regressionTemplateFixture(t, "LOOKUP-09 through LOOKUP-14 and LOOKUP-16 advanced lookups") + countFixture := regressionTemplateFixture(t, "LOOKUP-15 dense graph counts") + + var nodeKinds, edgeKinds graph.Kinds + for _, fixture := range []*opengraph.Graph{wideFixture, anchoredFixture, basicFixture, advancedFixture, countFixture} { + nextNodeKinds, nextEdgeKinds := fixture.Kinds() + nodeKinds = nodeKinds.Add(nextNodeKinds...) + edgeKinds = edgeKinds.Add(nextEdgeKinds...) + } + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + + t.Run("SCAN-01 base endpoints and relationship IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.KindIn(query.Start(), graph.StringKind("ADBase"), graph.StringKind("AZBase")), + query.Kind(query.Relationship(), graph.StringKind("PostProcessed")), + query.KindIn(query.End(), graph.StringKind("ADBase"), graph.StringKind("AZBase")), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchRelationshipIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, 4) + return nil + }) + }) + + t.Run("SCAN-02 non-Meta relationship hydration", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Start(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + query.KindIn(query.Relationship(), graph.StringKind("TrackerA"), graph.StringKind("TrackerB")), + query.Not(query.KindIn(query.End(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"tracker-a", "tracker-b"})) + }) + + t.Run("SCAN-03 present lastseen relationship IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Start(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + query.Kind(query.Relationship(), graph.StringKind("MigratedEdge")), + query.Exists(query.RelationshipProperty("lastseen")), + query.Not(query.KindIn(query.End(), graph.StringKind("Meta"), graph.StringKind("MetaDetail"))), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchRelationshipIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, 1) + return nil + }) + }) + + t.Run("SCAN-04 raw ownership representatives", func(t *testing.T) { + for kind, expected := range map[string]string{"OwnsRaw": "owns", "WriteOwnerRaw": "write-owner"} { + t.Run(kind, func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, wideFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind(kind)), + query.Kind(query.Start(), graph.StringKind("Entity")), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{expected})) + }) + } + }) + + t.Run("SCAN-05 consolidated nine-kind inbound scan", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Start(), graph.StringKind("Entity")), + query.KindIn(query.Relationship(), scanLookupNineKinds()...), + query.Equals(query.EndID(), idMap["target"]), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + seenKinds := map[graph.Kind]int{} + err := ops.ForEachStartNode(relationshipQuery, func(relationship *graph.Relationship, node *graph.Node) error { + require.True(t, node.Kinds.ContainsOneOf(graph.StringKind("Entity"))) + seenKinds[relationship.Kind]++ + return nil + }) + require.NoError(t, err) + require.Len(t, seenKinds, 9) + for _, count := range seenKinds { + require.Equal(t, 1, count) + } + return nil + }) + }) + + t.Run("SCAN-06 FetchKinds contract", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Kind(query.End(), graph.StringKind("Computer")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchKinds(func(cursor graph.Cursor[graph.RelationshipKindsResult]) error { + var results []graph.RelationshipKindsResult + for result := range cursor.Chan() { + results = append(results, result) + } + require.NoError(t, cursor.Error()) + require.Len(t, results, 1) + require.Equal(t, idMap["source-01"], results[0].StartID) + require.Equal(t, idMap["target"], results[0].EndID) + require.Equal(t, graph.StringKind("LocalToComputer"), results[0].Kind) + require.NotZero(t, results[0].ID) + return nil + }) + }) + }) + + t.Run("SCAN-07 directed endpoint pairs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(opengraph.IDMap) graph.Criteria { + return query.KindIn(query.Relationship(), graph.StringKind("MemberOf"), graph.StringKind("MemberOfLocalGroup")) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchTriples(func(cursor graph.Cursor[graph.RelationshipTripleResult]) error { + count := 0 + duplicatePairCount := 0 + for result := range cursor.Chan() { + count++ + if result.StartID == idMap["source-01"] && result.EndID == idMap["target"] { + duplicatePairCount++ + } + } + require.NoError(t, cursor.Error()) + require.Equal(t, 3, count) + require.Equal(t, 2, duplicatePairCount) + return nil + }) + }) + }) + + t.Run("SCAN-08 both ESC scenarios", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the ESC scenario subtest. + name string + + // scenarioB selects the alternate endpoint exclusion criteria. + scenarioB bool + + // expected is the number of relationships the scenario should return. + expected int + }{ + { + name: "scenario A", + expected: 3, + }, + { + name: "scenario B", + scenarioB: true, + expected: 2, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchoredFixture, func(idMap opengraph.IDMap) graph.Criteria { + criteria := []graph.Criteria{ + query.KindIn(query.Start(), graph.StringKind("Group"), graph.StringKind("User"), graph.StringKind("Computer")), + query.InIDs(query.EndID(), idMap["victim-computer"], idMap["victim-other"], idMap["victim-unused"]), + } + if testCase.scenarioB { + criteria = append(criteria, + query.Kind(query.End(), graph.StringKind("Computer")), + query.KindIn(query.Relationship(), graph.StringKind("GenericAll"), graph.StringKind("GenericWrite"), graph.StringKind("Owns"), graph.StringKind("WriteOwner"), graph.StringKind("WriteDACL")), + ) + } else { + criteria = append(criteria, query.KindIn(query.Relationship(), graph.StringKind("GenericAll"), graph.StringKind("GenericWrite"), graph.StringKind("Owns"), graph.StringKind("WriteOwner"), graph.StringKind("WriteDACL"), graph.StringKind("WritePublicInformation"))) + } + return query.And(criteria...) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchStartNodeIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, testCase.expected) + return nil + }) + }) + } + }) + + t.Run("LOOKUP-01 kind scans and hydration", func(t *testing.T) { + WithLegacyNodeQuery(t, session, basicFixture, func(opengraph.IDMap) graph.Criteria { + return query.KindIn(query.Node(), graph.StringKind("Group"), graph.StringKind("User")) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchNodes(nodeQuery) + require.NoError(t, err) + require.Len(t, nodes, 8) + return nil + }) + }) + + t.Run("LOOKUP-02 equality First", func(t *testing.T) { + WithLegacyNodeQuery(t, session, basicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("objectid"), "S-1-5-21-100"), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + node, err := nodeQuery.Limit(1).First() + require.NoError(t, err) + require.NotNil(t, node) + return nil + }) + }) + + t.Run("LOOKUP-03 boolean projection order and type", func(t *testing.T) { + WithLegacyNodeQuery(t, session, basicFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("hasura"), true), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + return nodeQuery.Query(func(results graph.Result) error { + count := 0 + for results.Next() { + var ( + id graph.ID + hasURA bool + ) + + require.NoError(t, results.Scan(&id, &hasURA)) + require.NotZero(t, id) + require.True(t, hasURA) + count++ + } + require.NoError(t, results.Error()) + require.Equal(t, 1, count) + return nil + }, query.Returning(query.NodeID(), query.NodeProperty("hasura"))) + }) + }) + + t.Run("LOOKUP-04 case-sensitive prefix", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"adminsdholder"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Container")), + query.StringStartsWith(query.NodeProperty("distinguishedname"), "CN=ADMINSDHOLDER,CN=SYSTEM,"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + ) + }) + }) + + t.Run("LOOKUP-05 case-insensitive contains candidates", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"ci-contains-exact", "ci-contains-substring"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Entity")), + query.CaseInsensitiveStringContains(query.NodeProperty("objectid"), "Approver_GUID"), + ) + }) + }) + + t.Run("LOOKUP-06 required and excluded kinds", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"entity-only"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Entity")), + query.Not(query.KindIn(query.Node(), graph.StringKind("Group"), graph.StringKind("LocalGroup"))), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + ) + }) + }) + + t.Run("LOOKUP-07 missing and null properties", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"name-missing", "name-null"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Lookup")), + query.Not(query.Exists(query.NodeProperty("name"))), + ) + }) + }) + + t.Run("LOOKUP-08 nullable approver disjunction", func(t *testing.T) { + assertScanLookupNodeIDs(t, session, basicFixture, []string{"role-both", "role-group", "role-user"}, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("AZRole")), + query.Equals(query.NodeProperty("tenantid"), "tenant-1"), + query.Equals(query.NodeProperty("approvalrequired"), true), + query.Or( + query.IsNotNull(query.NodeProperty("userapprovers")), + query.IsNotNull(query.NodeProperty("groupapprovers")), + ), + ) + }) + }) + + t.Run("LOOKUP-09 duplicate ID list hydration", func(t *testing.T) { + WithLegacyNodeQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.InIDs(query.NodeID(), idMap["hydrate-a"], idMap["hydrate-a"], idMap["hydrate-b"]) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + nodes, err := ops.FetchNodes(nodeQuery) + require.NoError(t, err) + require.Equal(t, []string{"hydrate-a", "hydrate-b"}, scanLookupFixtureIDs(t, idMap, scanLookupNodeIDs(nodes))) + return nil + }) + }) + + t.Run("LOOKUP-10 nested negated account flags", func(t *testing.T) { + WithLegacyNodeQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + ids := make([]graph.ID, 0, 16) + for _, first := range []string{"m", "n", "f", "t"} { + for _, second := range []string{"m", "n", "f", "t"} { + ids = append(ids, idMap["flags-"+first+second]) + } + } + return query.And( + query.Kind(query.Node(), graph.StringKind("User")), + query.Not(query.And(query.Exists(query.NodeProperty("gmsa")), query.Equals(query.NodeProperty("gmsa"), true))), + query.Not(query.And(query.Exists(query.NodeProperty("msa")), query.Equals(query.NodeProperty("msa"), true))), + query.InIDs(query.NodeID(), ids...), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchNodes(nodeQuery) + require.NoError(t, err) + require.Len(t, nodes, 9) + return nil + }) + }) + + t.Run("LOOKUP-11 tenant adjacency property list", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["tenant"]), + query.Kind(query.Relationship(), graph.StringKind("Contains")), + query.KindIn(query.End(), graph.StringKind("AZRole"), graph.StringKind("AZServicePrincipal")), + query.In(query.EndProperty("roletemplateid"), []string{"role-a", "role-b", "role-multi"}), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchEndNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 3, nodes.Len()) + return nil + }) + }) + + t.Run("LOOKUP-12 exact edge key First", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["edge-start"]), + query.Equals(query.EndID(), idMap["edge-end"]), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationship, err := relationshipQuery.Limit(1).First() + require.NoError(t, err) + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + require.Equal(t, "exact-edge", marker) + return nil + }) + }) + + t.Run("LOOKUP-13 suffix and bound endpoint", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, advancedFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Equals(query.EndID(), idMap["local-target"]), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + nodes, err := ops.FetchStartNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 2, nodes.Len()) + return nil + }) + }) + + t.Run("LOOKUP-14 descending node property", func(t *testing.T) { + WithLegacyNodeQuery(t, session, advancedFixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Node(), graph.StringKind("Domain")), + query.Exists(query.NodeProperty("name")), + ) + }, func(nodeQuery graph.NodeQuery, _ opengraph.IDMap) error { + var names []string + err := nodeQuery.OrderBy(query.Order(query.NodeProperty("name"), query.Descending())).Fetch(func(cursor graph.Cursor[*graph.Node]) error { + for node := range cursor.Chan() { + name, err := node.Properties.Get("name").String() + require.NoError(t, err) + names = append(names, name) + } + return cursor.Error() + }) + require.NoError(t, err) + require.Equal(t, []string{"Gamma", "Beta", "Beta", "Alpha"}, names) + return nil + }) + }) + + t.Run("LOOKUP-15 direct sequential counts", func(t *testing.T) { + for _, testCase := range []struct { + // family names the template fixture used by the count subtest. + family string + + // expectedNodes is the fixture's expected node count. + expectedNodes int64 + + // expectedEdges is the fixture's expected relationship count. + expectedEdges int64 + }{ + {family: "LOOKUP-15 empty graph counts"}, + { + family: "LOOKUP-15 node-only graph counts", + expectedNodes: 3, + }, + { + family: "LOOKUP-15 edge-bearing graph counts", + expectedNodes: 2, + expectedEdges: 1, + }, + { + family: "LOOKUP-15 dense graph counts", + expectedNodes: 4, + expectedEdges: 6, + }, + } { + t.Run(testCase.family, func(t *testing.T) { + fixture := regressionTemplateFixture(t, testCase.family) + err := session.WithRollbackFixture(t, fixture, false, func(tx graph.Transaction, _ opengraph.IDMap) error { + nodeCount, err := tx.Nodes().Count() + require.NoError(t, err) + edgeCount, err := tx.Relationships().Count() + require.NoError(t, err) + require.Equal(t, testCase.expectedNodes, nodeCount) + require.Equal(t, testCase.expectedEdges, edgeCount) + return nil + }) + require.NoError(t, err) + }) + } + }) + + t.Run("LOOKUP-16 four-property LDAP and LDAPS forms", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the LDAP or LDAPS property combination. + name string + + // kind optionally restricts the matched endpoint kind. + kind graph.Kind + + // available names the property that records protocol availability. + available string + + // protection names the protocol protection property. + protection string + + // expected is the object ID of the endpoint that should match. + expected string + }{ + { + name: "typed LDAP", + kind: graph.StringKind("Computer"), + available: "ldapavailable", + protection: "ldapsigning", + expected: "ntlm-ldap-good", + }, + { + name: "untyped LDAPS", + available: "ldapsavailable", + protection: "epa", + expected: "ntlm-ldaps-good", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + assertScanLookupNodeIDs(t, session, advancedFixture, []string{testCase.expected}, func(opengraph.IDMap) graph.Criteria { + criteria := []graph.Criteria{ + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty(testCase.available), true), + query.Equals(query.NodeProperty(testCase.protection), false), + } + if testCase.kind != nil { + criteria = append([]graph.Criteria{query.Kind(query.Node(), testCase.kind)}, criteria...) + } + return query.And(criteria...) + }) + }) + } + }) +} + +// scanLookupNineKinds returns the nine synthetic relationship kinds used by wide-kind scan cases. +func scanLookupNineKinds() graph.Kinds { + kinds := make(graph.Kinds, 9) + for idx := range kinds { + kinds[idx] = graph.StringKind("ScanEdge0" + string(rune('1'+idx))) + } + return kinds +} + +// scanLookupNodeIDs extracts database IDs from a node result slice without reordering it. +func scanLookupNodeIDs(nodes []*graph.Node) []graph.ID { + ids := make([]graph.ID, len(nodes)) + for idx, node := range nodes { + ids[idx] = node.ID + } + return ids +} + +// scanLookupFixtureIDs maps database IDs to fixture IDs and sorts them for stable comparison. +func scanLookupFixtureIDs(t *testing.T, idMap opengraph.IDMap, ids []graph.ID) []string { + t.Helper() + fixtureIDs := make([]string, len(ids)) + for idx, id := range ids { + fixtureIDs[idx] = regressionFixtureID(t, idMap, id) + } + sort.Strings(fixtureIDs) + return fixtureIDs +} + +// assertScanLookupNodeIDs executes criteria through the legacy node query and compares the resulting fixture IDs. +func assertScanLookupNodeIDs(t *testing.T, session *Session, fixture *opengraph.Graph, expected []string, criteria func(opengraph.IDMap) graph.Criteria) { + t.Helper() + WithLegacyNodeQuery(t, session, fixture, criteria, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + ids, err := ops.FetchNodeIDs(nodeQuery) + require.NoError(t, err) + require.Equal(t, expected, scanLookupFixtureIDs(t, idMap, ids)) + return nil + }) +} diff --git a/integration/standalone_hops_legacy_builder_test.go b/integration/standalone_hops_legacy_builder_test.go new file mode 100644 index 00000000..b669ad81 --- /dev/null +++ b/integration/standalone_hops_legacy_builder_test.go @@ -0,0 +1,309 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "fmt" + "sort" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderStandaloneHops verifies legacy one-hop queries preserve direction, kinds, and endpoint projections. +func TestLegacyBuilderStandaloneHops(t *testing.T) { + anchorFixture := regressionTemplateFixture(t, "HOP-01 through HOP-03 anchored direction and relationship-kind cardinality") + idFixture := regressionTemplateFixture(t, "HOP-04 and HOP-05 endpoint kinds and ID constraints") + predicateFixture := regressionTemplateFixture(t, "HOP-06 through HOP-08 scalar nested and collection endpoint predicates") + projectionFixture := regressionTemplateFixture(t, "HOP-09 and HOP-10 two-sided sets and directional projections") + + var nodeKinds, edgeKinds graph.Kinds + for _, fixture := range []*opengraph.Graph{anchorFixture, idFixture, predicateFixture, projectionFixture} { + nextNodeKinds, nextEdgeKinds := fixture.Kinds() + nodeKinds = nodeKinds.Add(nextNodeKinds...) + edgeKinds = edgeKinds.Add(nextEdgeKinds...) + } + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + + t.Run("HOP-01 outbound full direction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchorFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["out-one"]), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionInbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["out-one-target"], results[0].Node.ID) + return nil + }) + }) + }) + + t.Run("HOP-02 inbound full direction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, anchorFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.EndID(), idMap["in-one"]), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionOutbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["in-one-source"], results[0].Node.ID) + return nil + }) + }) + }) + + t.Run("HOP-03 thirty kinds preserve anchor orientation", func(t *testing.T) { + kinds := make(graph.Kinds, 30) + for idx := range kinds { + kinds[idx] = graph.StringKind(fmt.Sprintf("HopKind%02d", idx+1)) + } + WithLegacyRelationshipQuery(t, session, anchorFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["kind-center"]), + query.KindIn(query.Relationship(), kinds...), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 30) + require.NotContains(t, standaloneHopRelationshipMarkers(t, relationships), "out-disallowed") + return nil + }) + }) + + t.Run("HOP-04 endpoint kind disjunction", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, idFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopTypedEdge")), + query.KindIn(query.End(), graph.StringKind("HopEndA"), graph.StringKind("HopEndB")), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + nodes, err := ops.FetchEndNodes(relationshipQuery) + require.NoError(t, err) + require.Equal(t, 3, nodes.Len()) + require.True(t, nodes.ContainsID(idMap["typed-a"])) + require.True(t, nodes.ContainsID(idMap["typed-b"])) + require.True(t, nodes.ContainsID(idMap["typed-multi"])) + return nil + }) + }) + + t.Run("HOP-05 endpoint IDs and traversal anchor contradiction", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies whether the root constraint agrees with the path. + name string + + // allowedRoot is the fixture ID admitted by the root constraint. + allowedRoot string + + // expected lists the endpoint object IDs returned by the query. + expected []string + }{ + { + name: "matching", + allowedRoot: "root", + expected: []string{"id-a", "id-b"}, + }, + { + name: "contradictory", + allowedRoot: "other-root", + expected: nil, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, idFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.InIDs(query.Start(), idMap[testCase.allowedRoot]), + query.InIDs(query.EndID(), idMap["id-a"], idMap["id-b"]), + query.Kind(query.Relationship(), graph.StringKind("HopIDEdge")), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.ElementsMatch(t, testCase.expected, standaloneHopRelationshipMarkers(t, relationships)) + return nil + }) + }) + } + }) + + t.Run("HOP-06 scalar property", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, predicateFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopPropertyEdge")), + query.Equals(query.EndProperty("enabled"), true), + query.Equals(query.EndProperty("score"), 7), + query.Equals(query.EndProperty("value"), "alpha"), + query.Equals(query.EndProperty("isassignabletorole"), "true"), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"scalar-match"})) + }) + + t.Run("HOP-07 nested branch-local predicate", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, predicateFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopNestedEdge")), + query.Kind(query.End(), graph.StringKind("HopTemplate")), + query.Or( + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.GreaterThan(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authorizedsignatures"), 0), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.Equals(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + ), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"nested-v1", "nested-v2"})) + }) + + t.Run("HOP-08 collection OR scalar predicate", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, predicateFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.Equals(query.StartID(), idMap["root"]), + query.Kind(query.Relationship(), graph.StringKind("HopCollectionEdge")), + query.Or( + query.Equals(query.EndProperty("schannelauthenticationenabled"), true), + query.Equals(query.Size(query.EndProperty("effectiveekus")), 0), + query.InInverted(query.EndProperty("effectiveekus"), "1.3.6.1.5.5.7.3.2"), + ), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"collection-client", "collection-empty", "collection-scalar"})) + }) + + t.Run("HOP-09 two-sided ID lists", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["s1"], idMap["s2"]), + query.InIDs(query.EndID(), idMap["e1"], idMap["e2"]), + query.Kind(query.Relationship(), graph.StringKind("HopSetEdge")), + ) + }, assertStandaloneHopRelationshipMarkers(t, []string{"s1-e1", "s1-e2", "s2-e1", "s2-e2"})) + }) + + t.Run("HOP-10 both full directional projections", func(t *testing.T) { + t.Run("outbound", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.StartID(), idMap["s1"]), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.End(), graph.StringKind("HopProjectionEnd")), + query.Equals(query.EndProperty("active"), true), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionInbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["e1"], results[0].Node.ID) + return nil + }) + }) + }) + + t.Run("inbound", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, projectionFixture, func(idMap opengraph.IDMap) graph.Criteria { + return query.And( + query.InIDs(query.EndID(), idMap["e1"]), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.Start(), graph.StringKind("HopProjectionStart")), + query.Equals(query.StartProperty("active"), true), + ) + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + return relationshipQuery.FetchDirection(graph.DirectionOutbound, func(cursor graph.Cursor[graph.DirectionalResult]) error { + results := standaloneHopDirectionalResults(t, cursor) + require.Len(t, results, 1) + require.Equal(t, idMap["s1"], results[0].Node.ID) + return nil + }) + }) + }) + }) +} + +// regressionTemplateFixture returns the inline fixture belonging to the named +// Cypher template family. +func regressionTemplateFixture(t *testing.T, familyName string) *opengraph.Graph { + t.Helper() + for _, templateFile := range loadCypherTemplateFiles(t) { + for _, family := range templateFile.Families { + if family.Name == familyName { + return family.Fixture + } + } + } + t.Fatalf("template family %q not found", familyName) + return nil +} + +// standaloneHopDirectionalResults drains a directional cursor and fails the current test on cursor error. +func standaloneHopDirectionalResults(t *testing.T, cursor graph.Cursor[graph.DirectionalResult]) []graph.DirectionalResult { + t.Helper() + var results []graph.DirectionalResult + for result := range cursor.Chan() { + results = append(results, result) + } + require.NoError(t, cursor.Error()) + return results +} + +// standaloneHopRelationshipMarkers returns sorted marker properties from a relationship slice. +func standaloneHopRelationshipMarkers(t *testing.T, relationships []*graph.Relationship) []string { + t.Helper() + markers := make([]string, 0, len(relationships)) + for _, relationship := range relationships { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + sort.Strings(markers) + return markers +} + +// assertStandaloneHopRelationshipMarkers returns a legacy-query assertion that compares sorted relationship markers. +func assertStandaloneHopRelationshipMarkers(t *testing.T, expected []string) func(graph.RelationshipQuery, opengraph.IDMap) error { + t.Helper() + return func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Equal(t, expected, standaloneHopRelationshipMarkers(t, relationships)) + return nil + } +} diff --git a/integration/testdata/README.md b/integration/testdata/README.md new file mode 100644 index 00000000..dea35272 --- /dev/null +++ b/integration/testdata/README.md @@ -0,0 +1,73 @@ +# Integration Corpus + +Files under `cases/` execute one Cypher query per case. Files under `templates/` +share a fixture and query template across variants. Fixture-backed cases run in +a write transaction that is always rolled back. + +Mutation cases use `assert: "no_error"` (or another primary result assertion) +and one or more `post_assertions`. The primary mutation result is fully drained +and checked before post-state queries run in the same transaction. Each +post-state entry contains `cypher`, optional `params`, and `assert`. + +The assertion vocabulary includes exact fixture-backed state checks: + +- `node_id_set` for exact surviving node IDs; +- `node_records` for exact node IDs, kinds, and complete property maps; +- `relationship_triples` for exact directed start/end/kind triples; +- `relationship_records` for exact triples and complete property maps; +- `exact_int` and `row_count` for counts. + +Every new reconciliation or post-processing mutation fixture must contain a +positive match and applicable decoys for direction, kind, property, fixture ID, +missing/null property state, and relationship property. Reuse +`NewReconciliationFixture`, `FixtureNames`, and `FixtureKinds` from the +`integration` package for deterministic Go integration cases and large +cardinality lists. + +Tagged datetime parameters decode to `time.Time`: + +```json +{ + "params": { + "threshold": { + "$type": "datetime", + "value": "2026-01-02T03:04:05Z" + } + } +} +``` + +Raw Cypher may instead use an explicit conversion such as +`datetime($threshold)`. Legacy query-builder cases must pass `time.Time` +directly. + +Large string-list parameters use the same tagged parameter decoder without a +large handwritten JSON array: + +```json +{ + "params": { + "object_ids": { + "$type": "string_list", + "prefix": "missing", + "count": 1000, + "include": ["target-id"] + } + } +} +``` + +Fixture-backed cases and template variants can bind database IDs without +hard-coding them. `node_params` maps a query parameter to one fixture node ID; +`node_list_params` maps a parameter to an ordered list of fixture node IDs: + +```json +{ + "node_params": {"start_id": "start"}, + "node_list_params": {"end_ids": ["end-a", "end-b"]} +} +``` + +The integration runner and `cmd/plancorpus` resolve these fields after loading +the fixture, so semantic execution and plan capture use the same ID-anchored +query shape. diff --git a/integration/testdata/adcs_fanout.json b/integration/testdata/adcs_fanout.json deleted file mode 100644 index dafbb835..00000000 --- a/integration/testdata/adcs_fanout.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "graph": { - "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-2643190041-1319121918-239771340-513"}}, - {"id": "p1-a", "kinds": ["Group"]}, - {"id": "p1-b", "kinds": ["Group"]}, - {"id": "p1-c", "kinds": ["Group"]}, - {"id": "p2-good", "kinds": ["Group"]}, - {"id": "p2-disabled", "kinds": ["Group"]}, - {"id": "p2-wrong-ca", "kinds": ["Group"]}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "other-ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "other-domain", "kinds": ["Domain"]}, - {"id": "template-good", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "template-alt", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "template-disabled", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": false, "requiresmanagerapproval": true, "enrolleesuppliessubject": false, "schemaversion": 2, "authorizedsignatures": 1}}, - {"id": "template-wrong-ca", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "other-root", "kinds": ["RootCA"]} - ], - "edges": [ - {"start_id": "n", "end_id": "p1-a", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p1-b", "kind": "MemberOf"}, - {"start_id": "p1-b", "end_id": "p1-c", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p2-good", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p2-disabled", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "p2-wrong-ca", "kind": "MemberOf"}, - {"start_id": "n", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "p1-a", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "p1-b", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "p1-c", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "p2-good", "end_id": "template-good", "kind": "GenericAll"}, - {"start_id": "p2-good", "end_id": "template-alt", "kind": "Enroll"}, - {"start_id": "p2-disabled", "end_id": "template-disabled", "kind": "AllExtendedRights"}, - {"start_id": "p2-wrong-ca", "end_id": "template-wrong-ca", "kind": "GenericAll"}, - {"start_id": "template-good", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-alt", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-disabled", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-wrong-ca", "end_id": "other-ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "other-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"}, - {"start_id": "other-root", "end_id": "other-domain", "kind": "RootCAFor"} - ] - } -} diff --git a/integration/testdata/cases/expand_into.json b/integration/testdata/cases/expand_into.json new file mode 100644 index 00000000..4f7a8638 --- /dev/null +++ b/integration/testdata/cases/expand_into.json @@ -0,0 +1,90 @@ +{ + "dataset": "expand_into", + "cases": [ + { + "name": "fixed one-hop ExpandInto preserves typed relationship identity", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves wildcard cross-kind multiplicity", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH (s)-[r]->(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves multi-kind relationship rows", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto reapplies duplicate outer pair multiplicity", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' WITH s, e, [1, 2] AS copies UNWIND copies AS copy MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]->(e) RETURN copy, type(r) ORDER BY copy, type(r)", + "assert": {"ordered_row_values": [[1, "ExpandIntoKindA"], [1, "ExpandIntoKindB"], [2, "ExpandIntoKindA"], [2, "ExpandIntoKindB"]]} + }, + { + "name": "fixed one-hop ExpandInto recognizes node endpoints introduced by UNWIND", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' WITH collect(s) AS sources, e UNWIND sources AS source MATCH (source)-[r:ExpandIntoKindA]->(e) RETURN r.slot", + "assert": {"ordered_row_values": [["a"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves self loop", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'loop' AND e.name = 'loop' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop ExpandInto missing pair is empty", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'missing' AND e.name = 'target' MATCH (s)-[r]->(e) RETURN r", + "assert": "empty" + }, + { + "name": "fixed one-hop ExpandInto preserves a pair when the source has lower degree", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'low-source' AND e.name = 'high-target' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r.slot", + "assert": {"ordered_row_values": [["low-source-match"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves a pair when the target has lower degree", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'high-source' AND e.name = 'low-target' MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN r.slot", + "assert": {"ordered_row_values": [["low-target-match"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves reversed directionless cross-kind multiplicity", + "cypher": "MATCH (s:ExpandIntoTarget), (e:ExpandIntoSource) WHERE s.name = 'target' AND e.name = 'source' MATCH (s)-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves inbound cross-kind multiplicity", + "cypher": "MATCH (s:ExpandIntoTarget), (e:ExpandIntoSource) WHERE s.name = 'target' AND e.name = 'source' MATCH (s)<-[r:ExpandIntoKindA|ExpandIntoKindB]-(e) RETURN type(r), r.slot ORDER BY type(r)", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "a"], ["ExpandIntoKindB", "b"]]} + }, + { + "name": "fixed one-hop ExpandInto emits a directionless self loop once", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'loop' AND e.name = 'loop' MATCH (s)-[r:ExpandIntoKindA]-(e) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop directionless traversal emits an unbound self loop once", + "cypher": "MATCH (s:ExpandIntoSource)-[r:ExpandIntoKindA]-(e:ExpandIntoTarget) WHERE r.slot = 'loop' RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop directionless traversal emits a single-bound self loop once", + "cypher": "MATCH (s:ExpandIntoSource) WHERE s.name = 'loop' MATCH (s)-[r:ExpandIntoKindA]-(e:ExpandIntoTarget) RETURN type(r), r.slot", + "assert": {"ordered_row_values": [["ExpandIntoKindA", "loop"]]} + }, + { + "name": "fixed one-hop ExpandInto preserves arbitrary varying bound pairs", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE (s.name = 'source' AND e.name = 'target') OR (s.name = 'low-source' AND e.name = 'high-target') MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN s.name, e.name, r.slot ORDER BY s.name", + "assert": {"ordered_row_values": [["low-source", "high-target", "low-source-match"], ["source", "target", "a"]]} + }, + { + "name": "optional fixed one-hop ExpandInto preserves null relationship rows", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'missing' AND e.name = 'target' OPTIONAL MATCH (s)-[r:ExpandIntoKindA]->(e) RETURN s.name, e.name, r.slot", + "assert": {"ordered_row_values": [["missing", "target", null]]} + }, + { + "name": "fixed one-hop ExpandInto composes with multiple path bindings", + "cypher": "MATCH (s:ExpandIntoSource), (e:ExpandIntoTarget) WHERE s.name = 'source' AND e.name = 'target' MATCH p = (s)-[r:ExpandIntoKindA]->(e) MATCH q = (s)-[r2:ExpandIntoKindB]->(e) RETURN length(p), length(q), r.slot, r2.slot", + "assert": {"ordered_row_values": [[1, 1, "a", "b"]]} + } + ] +} diff --git a/integration/testdata/cases/mutation_post_state_inline.json b/integration/testdata/cases/mutation_post_state_inline.json new file mode 100644 index 00000000..6dffb02d --- /dev/null +++ b/integration/testdata/cases/mutation_post_state_inline.json @@ -0,0 +1,67 @@ +{ + "cases": [ + { + "name": "relationship mutation assertions preserve every decoy", + "cypher": "MATCH (s:NodeKind1)-[r:EdgeKind1]->(e:NodeKind2) WHERE e.objectid = $object_id AND r.shoulddelete = $should_delete DELETE r", + "params": { + "object_id": "target-id", + "should_delete": true + }, + "fixture": { + "nodes": [ + {"id": "source", "kinds": ["NodeKind1"], "properties": {"name": "source"}}, + {"id": "source-opposite", "kinds": ["NodeKind1"], "properties": {"name": "source-opposite"}}, + {"id": "source-missing", "kinds": ["NodeKind1"], "properties": {"name": "source-missing"}}, + {"id": "target", "kinds": ["NodeKind2"], "properties": {"objectid": "target-id"}}, + {"id": "wrong-kind", "kinds": ["NodeKind1"], "properties": {"objectid": "target-id"}}, + {"id": "wrong-id", "kinds": ["NodeKind2"], "properties": {"objectid": "decoy-id"}} + ], + "edges": [ + {"start_id": "source", "end_id": "target", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "target"}}, + {"start_id": "source-opposite", "end_id": "target", "kind": "EdgeKind1", "properties": {"shoulddelete": false, "marker": "opposite-property"}}, + {"start_id": "source-missing", "end_id": "target", "kind": "EdgeKind1", "properties": {"marker": "missing-property"}}, + {"start_id": "source", "end_id": "target", "kind": "EdgeKind2", "properties": {"shoulddelete": true, "marker": "wrong-edge-kind"}}, + {"start_id": "source", "end_id": "wrong-kind", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "wrong-node-kind"}}, + {"start_id": "source", "end_id": "wrong-id", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "wrong-object-id"}}, + {"start_id": "target", "end_id": "source", "kind": "EdgeKind1", "properties": {"shoulddelete": true, "marker": "reverse-direction"}} + ] + }, + "assert": "no_error", + "post_assertions": [ + { + "name": "exact surviving nodes and properties", + "cypher": "MATCH (n) RETURN n", + "assert": { + "node_records": [ + {"id": "source", "kinds": ["NodeKind1"], "props": {"name": "source"}}, + {"id": "source-opposite", "kinds": ["NodeKind1"], "props": {"name": "source-opposite"}}, + {"id": "source-missing", "kinds": ["NodeKind1"], "props": {"name": "source-missing"}}, + {"id": "target", "kinds": ["NodeKind2"], "props": {"objectid": "target-id"}}, + {"id": "wrong-kind", "kinds": ["NodeKind1"], "props": {"objectid": "target-id"}}, + {"id": "wrong-id", "kinds": ["NodeKind2"], "props": {"objectid": "decoy-id"}} + ] + } + }, + { + "name": "exact surviving relationships and properties", + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": { + "relationship_records": [ + {"start": "source-opposite", "end": "target", "kind": "EdgeKind1", "props": {"shoulddelete": false, "marker": "opposite-property"}}, + {"start": "source-missing", "end": "target", "kind": "EdgeKind1", "props": {"marker": "missing-property"}}, + {"start": "source", "end": "target", "kind": "EdgeKind2", "props": {"shoulddelete": true, "marker": "wrong-edge-kind"}}, + {"start": "source", "end": "wrong-kind", "kind": "EdgeKind1", "props": {"shoulddelete": true, "marker": "wrong-node-kind"}}, + {"start": "source", "end": "wrong-id", "kind": "EdgeKind1", "props": {"shoulddelete": true, "marker": "wrong-object-id"}}, + {"start": "target", "end": "source", "kind": "EdgeKind1", "props": {"shoulddelete": true, "marker": "reverse-direction"}} + ] + } + }, + { + "name": "exact surviving relationship count", + "cypher": "MATCH ()-[r]->() RETURN count(r)", + "assert": {"exact_int": 6} + } + ] + } + ] +} diff --git a/integration/testdata/cases/optimizer_inline.json b/integration/testdata/cases/optimizer_inline.json index 1cd9d601..64d667ac 100644 --- a/integration/testdata/cases/optimizer_inline.json +++ b/integration/testdata/cases/optimizer_inline.json @@ -1,189 +1,194 @@ { "cases": [ { - "name": "return two ADCS-style paths with shared CA and domain endpoints", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'S-1-5-21-2643190041-1319121918-239771340-513' MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p1, p2", + "name": "return two fixed-suffix expansion paths with shared endpoints", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-shared-endpoints-root' MATCH direct_path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN direct_path, predicate_path", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-2643190041-1319121918-239771340-513"}}, - {"id": "p1-mid", "kinds": ["Group"]}, - {"id": "p2-mid", "kinds": ["Group"]}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "template", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]}, - {"id": "unused-template", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": false, "requiresmanagerapproval": true, "enrolleesuppliessubject": false, "schemaversion": 2, "authorizedsignatures": 1}} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-shared-endpoints-root"}}, + {"id": "direct-mid", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid", "kinds": ["ExpansionNode"]}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "suffix-middle", "kinds": ["SuffixMiddle"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "predicate", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-predicate", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}} ], "edges": [ - {"start_id": "n", "end_id": "p1-mid", "kind": "MemberOf"}, - {"start_id": "p1-mid", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "n", "end_id": "p2-mid", "kind": "MemberOf"}, - {"start_id": "p2-mid", "end_id": "template", "kind": "GenericAll"}, - {"start_id": "template", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "p2-mid", "end_id": "unused-template", "kind": "AllExtendedRights"} + {"start_id": "root", "end_id": "direct-mid", "kind": "Expand"}, + {"start_id": "direct-mid", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "suffix-head", "end_id": "suffix-middle", "kind": "ContinueSuffix"}, + {"start_id": "suffix-middle", "end_id": "suffix-terminal", "kind": "CompleteSuffix"}, + {"start_id": "root", "end_id": "predicate-mid", "kind": "Expand"}, + {"start_id": "predicate-mid", "end_id": "predicate", "kind": "OptionA"}, + {"start_id": "predicate", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "predicate-mid", "end_id": "unused-predicate", "kind": "OptionB"}, + {"start_id": "predicate-mid", "end_id": "unused-predicate", "kind": "OptionC"} ] }, "assert": { - "keys": ["p1", "p2"], + "keys": ["direct_path", "predicate_path"], "row_count": 1, "path_lengths": [4, 5], "path_node_ids": [ - ["n", "p1-mid", "ca", "store", "domain"], - ["n", "p2-mid", "template", "ca", "root", "domain"] + ["root", "direct-mid", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "predicate-mid", "predicate", "suffix-head", "bridge", "suffix-terminal"] ], "path_edge_kinds": [ - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"] + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"] ], - "contains_node_with_props": {"objectid": "S-1-5-21-2643190041-1319121918-239771340-513"}, - "contains_edge": {"start": "template", "end": "ca", "kind": "PublishedTo"} + "contains_node_with_props": {"root_key": "fixed-suffix-shared-endpoints-root"}, + "contains_edge": {"start": "predicate", "end": "suffix-head", "kind": "JoinSuffix"} } }, { - "name": "ADCS template predicate accepts both OR branches and rejects false alternatives", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'optimizer-or-source' MATCH p = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca:EnterpriseCA)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d:Domain) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p", + "name": "fixed-suffix predicate accepts both OR branches and rejects false alternatives", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-predicate-root' MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head:SuffixHead)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal:SuffixTerminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN predicate_path", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "optimizer-or-source"}}, - {"id": "mid-v1", "kinds": ["Group"]}, - {"id": "mid-sig", "kinds": ["Group"]}, - {"id": "mid-bad", "kinds": ["Group"]}, - {"id": "template-v1", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 2}}, - {"id": "template-sig", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "template-bad", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 1}}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]}, - {"id": "domain", "kinds": ["Domain"]} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-predicate-root"}}, + {"id": "mid-version", "kinds": ["ExpansionNode"]}, + {"id": "mid-approval", "kinds": ["ExpansionNode"]}, + {"id": "mid-rejected", "kinds": ["ExpansionNode"]}, + {"id": "predicate-version", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 2}}, + {"id": "predicate-approval", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "predicate-rejected", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 1}}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]} ], "edges": [ - {"start_id": "n", "end_id": "mid-v1", "kind": "MemberOf"}, - {"start_id": "mid-v1", "end_id": "template-v1", "kind": "GenericAll"}, - {"start_id": "n", "end_id": "mid-sig", "kind": "MemberOf"}, - {"start_id": "mid-sig", "end_id": "template-sig", "kind": "Enroll"}, - {"start_id": "n", "end_id": "mid-bad", "kind": "MemberOf"}, - {"start_id": "mid-bad", "end_id": "template-bad", "kind": "AllExtendedRights"}, - {"start_id": "template-v1", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-sig", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-bad", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"} + {"start_id": "root", "end_id": "mid-version", "kind": "Expand"}, + {"start_id": "mid-version", "end_id": "predicate-version", "kind": "OptionA"}, + {"start_id": "root", "end_id": "mid-approval", "kind": "Expand"}, + {"start_id": "mid-approval", "end_id": "predicate-approval", "kind": "OptionB"}, + {"start_id": "root", "end_id": "mid-rejected", "kind": "Expand"}, + {"start_id": "mid-rejected", "end_id": "predicate-rejected", "kind": "OptionC"}, + {"start_id": "predicate-version", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-approval", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-rejected", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"} ] }, "assert": { "row_count": 2, "path_node_ids": [ - ["n", "mid-v1", "template-v1", "ca", "root", "domain"], - ["n", "mid-sig", "template-sig", "ca", "root", "domain"] + ["root", "mid-version", "predicate-version", "suffix-head", "bridge", "suffix-terminal"], + ["root", "mid-approval", "predicate-approval", "suffix-head", "bridge", "suffix-terminal"] ], "path_edge_kinds": [ - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "Enroll", "PublishedTo", "IssuedSignedBy", "RootCAFor"] + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionB", "JoinSuffix", "HeadToBridge", "ReachTerminal"] ] } }, { - "name": "ADCS fanout returns every p1 and p2 path pair without endpoint collapse", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'optimizer-fanout-source' MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN p1, p2", + "name": "fixed-suffix fanout returns every direct and predicate path pair without endpoint collapse", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-fanout-root' MATCH direct_path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN direct_path, predicate_path", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "optimizer-fanout-source"}}, - {"id": "p1-a", "kinds": ["Group"]}, - {"id": "p1-b", "kinds": ["Group"]}, - {"id": "p2-a", "kinds": ["Group"]}, - {"id": "p2-b", "kinds": ["Group"]}, - {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-fanout-root"}}, + {"id": "direct-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "direct-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-a", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "predicate-b", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "predicate-unused", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "suffix-middle", "kinds": ["SuffixMiddle"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]} ], "edges": [ - {"start_id": "n", "end_id": "p1-a", "kind": "MemberOf"}, - {"start_id": "p1-a", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "n", "end_id": "p1-b", "kind": "MemberOf"}, - {"start_id": "p1-b", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "n", "end_id": "p2-a", "kind": "MemberOf"}, - {"start_id": "p2-a", "end_id": "template-a", "kind": "GenericAll"}, - {"start_id": "n", "end_id": "p2-b", "kind": "MemberOf"}, - {"start_id": "p2-b", "end_id": "template-b", "kind": "AllExtendedRights"}, - {"start_id": "template-a", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-b", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"} + {"start_id": "root", "end_id": "direct-mid-a", "kind": "Expand"}, + {"start_id": "direct-mid-a", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "root", "end_id": "direct-mid-b", "kind": "Expand"}, + {"start_id": "direct-mid-b", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "suffix-head", "end_id": "suffix-middle", "kind": "ContinueSuffix"}, + {"start_id": "suffix-middle", "end_id": "suffix-terminal", "kind": "CompleteSuffix"}, + {"start_id": "root", "end_id": "predicate-mid-a", "kind": "Expand"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-a", "kind": "OptionA"}, + {"start_id": "root", "end_id": "predicate-mid-b", "kind": "Expand"}, + {"start_id": "predicate-mid-b", "end_id": "predicate-b", "kind": "OptionC"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-unused", "kind": "OptionB"}, + {"start_id": "predicate-a", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-b", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"} ] }, "assert": { "row_count": 4, "path_node_ids": [ - ["n", "p1-a", "ca", "store", "domain"], - ["n", "p1-a", "ca", "store", "domain"], - ["n", "p1-b", "ca", "store", "domain"], - ["n", "p1-b", "ca", "store", "domain"], - ["n", "p2-a", "template-a", "ca", "root", "domain"], - ["n", "p2-a", "template-a", "ca", "root", "domain"], - ["n", "p2-b", "template-b", "ca", "root", "domain"], - ["n", "p2-b", "template-b", "ca", "root", "domain"] + ["root", "direct-mid-a", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "direct-mid-a", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "direct-mid-b", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "direct-mid-b", "suffix-head", "suffix-middle", "suffix-terminal"], + ["root", "predicate-mid-a", "predicate-a", "suffix-head", "bridge", "suffix-terminal"], + ["root", "predicate-mid-a", "predicate-a", "suffix-head", "bridge", "suffix-terminal"], + ["root", "predicate-mid-b", "predicate-b", "suffix-head", "bridge", "suffix-terminal"], + ["root", "predicate-mid-b", "predicate-b", "suffix-head", "bridge", "suffix-terminal"] ], "path_edge_kinds": [ - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "Enroll", "TrustedForNTAuth", "NTAuthStoreFor"], - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "GenericAll", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "AllExtendedRights", "PublishedTo", "IssuedSignedBy", "RootCAFor"], - ["MemberOf", "AllExtendedRights", "PublishedTo", "IssuedSignedBy", "RootCAFor"] + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "EnterSuffix", "ContinueSuffix", "CompleteSuffix"], + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionA", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionC", "JoinSuffix", "HeadToBridge", "ReachTerminal"], + ["Expand", "OptionC", "JoinSuffix", "HeadToBridge", "ReachTerminal"] ] } }, { - "name": "ADCS fanout endpoint projection preserves row multiplicity", - "cypher": "MATCH (n:Group) WHERE n.objectid = 'optimizer-endpoint-fanout-source' MATCH p1 = (n)-[:MemberOf*0..]->()-[:Enroll]->(ca:EnterpriseCA)-[:TrustedForNTAuth]->(:NTAuthStore)-[:NTAuthStoreFor]->(d:Domain) MATCH p2 = (n)-[:MemberOf*0..]->()-[:GenericAll|Enroll|AllExtendedRights]->(ct:CertTemplate)-[:PublishedTo]->(ca)-[:IssuedSignedBy|EnterpriseCAFor*1..]->(:RootCA)-[:RootCAFor]->(d) WHERE ct.authenticationenabled = true AND ct.requiresmanagerapproval = false AND ct.enrolleesuppliessubject = true AND (ct.schemaversion = 1 OR ct.authorizedsignatures = 0) RETURN count(*) AS rows, count(distinct id(ca)) AS ca_count, count(distinct id(d)) AS domain_count, count(distinct id(ct)) AS template_count", + "name": "fixed-suffix fanout endpoint projection preserves row multiplicity", + "cypher": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'fixed-suffix-endpoint-fanout-root' MATCH direct_path = (root)-[:Expand*0..16]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) MATCH predicate_path = (root)-[:Expand*0..16]->()-[:OptionA|OptionB|OptionC]->(predicate:PredicateNode)-[:JoinSuffix]->(head)-[:HeadToBridge|HeadToAlternateBridge*1..16]->(:BridgeNode)-[:ReachTerminal]->(terminal) WHERE predicate.eligible = true AND predicate.requires_review = false AND predicate.allows_direct = true AND (predicate.version = 1 OR predicate.required_approvals = 0) RETURN count(*) AS rows, count(distinct id(head)) AS head_count, count(distinct id(terminal)) AS terminal_count, count(distinct id(predicate)) AS predicate_count", "fixture": { "nodes": [ - {"id": "n", "kinds": ["Group"], "properties": {"objectid": "optimizer-endpoint-fanout-source"}}, - {"id": "p1-a", "kinds": ["Group"]}, - {"id": "p1-b", "kinds": ["Group"]}, - {"id": "p2-a", "kinds": ["Group"]}, - {"id": "p2-b", "kinds": ["Group"]}, - {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 1, "authorizedsignatures": 1}}, - {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"authenticationenabled": true, "requiresmanagerapproval": false, "enrolleesuppliessubject": true, "schemaversion": 2, "authorizedsignatures": 0}}, - {"id": "ca", "kinds": ["EnterpriseCA"]}, - {"id": "store", "kinds": ["NTAuthStore"]}, - {"id": "domain", "kinds": ["Domain"]}, - {"id": "root", "kinds": ["RootCA"]}, - {"id": "unused-root", "kinds": ["RootCA"]} + {"id": "root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-endpoint-fanout-root"}}, + {"id": "direct-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "direct-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-a", "kinds": ["ExpansionNode"]}, + {"id": "predicate-mid-b", "kinds": ["ExpansionNode"]}, + {"id": "predicate-a", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "predicate-b", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "predicate-unused", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}}, + {"id": "suffix-head", "kinds": ["SuffixHead"]}, + {"id": "suffix-middle", "kinds": ["SuffixMiddle"]}, + {"id": "suffix-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "bridge", "kinds": ["BridgeNode"]}, + {"id": "unused-bridge", "kinds": ["BridgeNode"]} ], "edges": [ - {"start_id": "n", "end_id": "p1-a", "kind": "MemberOf"}, - {"start_id": "p1-a", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "n", "end_id": "p1-b", "kind": "MemberOf"}, - {"start_id": "p1-b", "end_id": "ca", "kind": "Enroll"}, - {"start_id": "ca", "end_id": "store", "kind": "TrustedForNTAuth"}, - {"start_id": "store", "end_id": "domain", "kind": "NTAuthStoreFor"}, - {"start_id": "n", "end_id": "p2-a", "kind": "MemberOf"}, - {"start_id": "p2-a", "end_id": "template-a", "kind": "GenericAll"}, - {"start_id": "n", "end_id": "p2-b", "kind": "MemberOf"}, - {"start_id": "p2-b", "end_id": "template-b", "kind": "AllExtendedRights"}, - {"start_id": "template-a", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "template-b", "end_id": "ca", "kind": "PublishedTo"}, - {"start_id": "ca", "end_id": "root", "kind": "IssuedSignedBy"}, - {"start_id": "ca", "end_id": "unused-root", "kind": "EnterpriseCAFor"}, - {"start_id": "root", "end_id": "domain", "kind": "RootCAFor"} + {"start_id": "root", "end_id": "direct-mid-a", "kind": "Expand"}, + {"start_id": "direct-mid-a", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "root", "end_id": "direct-mid-b", "kind": "Expand"}, + {"start_id": "direct-mid-b", "end_id": "suffix-head", "kind": "EnterSuffix"}, + {"start_id": "suffix-head", "end_id": "suffix-middle", "kind": "ContinueSuffix"}, + {"start_id": "suffix-middle", "end_id": "suffix-terminal", "kind": "CompleteSuffix"}, + {"start_id": "root", "end_id": "predicate-mid-a", "kind": "Expand"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-a", "kind": "OptionA"}, + {"start_id": "root", "end_id": "predicate-mid-b", "kind": "Expand"}, + {"start_id": "predicate-mid-b", "end_id": "predicate-b", "kind": "OptionC"}, + {"start_id": "predicate-mid-a", "end_id": "predicate-unused", "kind": "OptionB"}, + {"start_id": "predicate-a", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "predicate-b", "end_id": "suffix-head", "kind": "JoinSuffix"}, + {"start_id": "suffix-head", "end_id": "bridge", "kind": "HeadToBridge"}, + {"start_id": "suffix-head", "end_id": "unused-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "bridge", "end_id": "suffix-terminal", "kind": "ReachTerminal"} ] }, "assert": {"row_values": [[4, 1, 1, 2]]} diff --git a/integration/testdata/cases/shortest_bound.json b/integration/testdata/cases/shortest_bound.json new file mode 100644 index 00000000..199021df --- /dev/null +++ b/integration/testdata/cases/shortest_bound.json @@ -0,0 +1,82 @@ +{ + "dataset": "shortest_bound", + "cases": [ + { + "name": "bound pair shortest prefers direct edge and hydrates in order", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "direct"}, + "assert": { + "row_count": 1, + "path_node_ids": [["start", "direct"]], + "path_edge_kinds": [["BoundEdge"]], + "contains_edge": {"start": "start", "end": "direct", "kind": "BoundEdge", "props": {"route": "direct"}} + } + }, + { + "name": "bound pair shortest disconnected endpoints return empty", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "disconnected"}, + "assert": "empty" + }, + { + "name": "bound pair shortest respects direction", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "wrong-direction"}, + "assert": "empty" + }, + { + "name": "bound pair shortest respects relationship kind", + "cypher": "MATCH p = shortestPath((s)-[:BoundEdge*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "typed-end"}, + "assert": "empty" + }, + { + "name": "bound pair shortest respects maximum depth", + "cypher": "MATCH p = shortestPath((s)-[:BoundEdge*1..2]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "cycle-end"}, + "assert": "empty" + }, + { + "name": "bound pair shortest handles cycles without relationship reuse", + "cypher": "MATCH p = shortestPath((s)-[:BoundEdge*1..4]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "cycle-end"}, + "assert": { + "row_count": 1, + "path_node_ids": [["start", "cycle-a", "cycle-b", "cycle-end"]], + "path_edge_kinds": [["BoundEdge", "BoundEdge", "BoundEdge"]] + } + }, + { + "name": "bound pair shortest missing endpoint id returns empty", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "params": {"start_id": -1}, + "node_params": {"end_id": "direct"}, + "assert": "empty" + }, + { + "name": "bound pair shortest null endpoint parameter returns empty", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "params": {"start_id": null}, + "node_params": {"end_id": "direct"}, + "assert": "empty" + }, + { + "name": "bound pair shortest same endpoint keeps error contract", + "cypher": "MATCH p = shortestPath((s)-[*1..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "start"}, + "assert": "query_error" + }, + { + "name": "bound pair zero depth returns the same endpoint", + "cypher": "MATCH p = shortestPath((s)-[*0..0]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "start"}, + "assert": {"row_count": 1, "path_node_ids": [["start"]], "path_edge_kinds": [[]]} + }, + { + "name": "bound pair unbounded zero minimum returns the same endpoint", + "cypher": "MATCH p = shortestPath((s)-[*0..]->(e)) WHERE id(s) = $start_id AND id(e) = $end_id RETURN p LIMIT 1", + "node_params": {"start_id": "start", "end_id": "start"}, + "assert": {"row_count": 1, "path_node_ids": [["start"]], "path_edge_kinds": [[]]} + } + ] +} diff --git a/integration/testdata/expand_into.json b/integration/testdata/expand_into.json new file mode 100644 index 00000000..4eec0f82 --- /dev/null +++ b/integration/testdata/expand_into.json @@ -0,0 +1,39 @@ +{ + "graph": { + "nodes": [ + {"id": "pair-source", "kinds": ["ExpandIntoSource"], "properties": {"name": "source"}}, + {"id": "pair-target", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target"}}, + {"id": "pair-missing", "kinds": ["ExpandIntoSource"], "properties": {"name": "missing"}}, + {"id": "pair-loop", "kinds": ["ExpandIntoSource", "ExpandIntoTarget"], "properties": {"name": "loop"}}, + {"id": "low-source", "kinds": ["ExpandIntoSource"], "properties": {"name": "low-source"}}, + {"id": "high-target", "kinds": ["ExpandIntoTarget"], "properties": {"name": "high-target"}}, + {"id": "high-source", "kinds": ["ExpandIntoSource"], "properties": {"name": "high-source"}}, + {"id": "low-target", "kinds": ["ExpandIntoTarget"], "properties": {"name": "low-target"}}, + {"id": "source-decoy-1", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-1"}}, + {"id": "source-decoy-2", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-2"}}, + {"id": "source-decoy-3", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-3"}}, + {"id": "source-decoy-4", "kinds": ["ExpandIntoSource"], "properties": {"name": "source-decoy-4"}}, + {"id": "target-decoy-1", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-1"}}, + {"id": "target-decoy-2", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-2"}}, + {"id": "target-decoy-3", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-3"}}, + {"id": "target-decoy-4", "kinds": ["ExpandIntoTarget"], "properties": {"name": "target-decoy-4"}} + ], + "edges": [ + {"start_id": "pair-source", "end_id": "pair-target", "kind": "ExpandIntoKindA", "properties": {"slot": "a"}}, + {"start_id": "pair-source", "end_id": "pair-target", "kind": "ExpandIntoKindB", "properties": {"slot": "b"}}, + {"start_id": "pair-loop", "end_id": "pair-loop", "kind": "ExpandIntoKindA", "properties": {"slot": "loop"}}, + {"start_id": "pair-source", "end_id": "pair-loop", "kind": "ExpandIntoDecoy", "properties": {"slot": "decoy-out"}}, + {"start_id": "pair-loop", "end_id": "pair-target", "kind": "ExpandIntoDecoy", "properties": {"slot": "decoy-in"}}, + {"start_id": "low-source", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "low-source-match"}}, + {"start_id": "source-decoy-1", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-1"}}, + {"start_id": "source-decoy-2", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-2"}}, + {"start_id": "source-decoy-3", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-3"}}, + {"start_id": "source-decoy-4", "end_id": "high-target", "kind": "ExpandIntoKindA", "properties": {"slot": "high-target-4"}}, + {"start_id": "high-source", "end_id": "low-target", "kind": "ExpandIntoKindA", "properties": {"slot": "low-target-match"}}, + {"start_id": "high-source", "end_id": "target-decoy-1", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-1"}}, + {"start_id": "high-source", "end_id": "target-decoy-2", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-2"}}, + {"start_id": "high-source", "end_id": "target-decoy-3", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-3"}}, + {"start_id": "high-source", "end_id": "target-decoy-4", "kind": "ExpandIntoKindA", "properties": {"slot": "high-source-4"}} + ] + } +} diff --git a/integration/testdata/fixed_suffix_expansion_adversarial.json b/integration/testdata/fixed_suffix_expansion_adversarial.json new file mode 100644 index 00000000..7aec947c --- /dev/null +++ b/integration/testdata/fixed_suffix_expansion_adversarial.json @@ -0,0 +1,73 @@ +{ + "graph": { + "nodes": [ + {"id":"boundary-terminal","kinds":["SuffixTerminal"]}, + {"id":"boundary-root","kinds":["ExpansionRoot"],"properties":{"root_key":"suffix-overflow-adversarial-root"}}, + {"id":"boundary-boundary-9001","kinds":["ExpansionNode"]}, + {"id":"boundary-head-a","kinds":["SuffixHead"]}, + {"id":"boundary-middle-a","kinds":["SuffixMiddle"]}, + {"id":"boundary-head-b","kinds":["SuffixHead"]}, + {"id":"boundary-middle-b","kinds":["SuffixMiddle"]}, + {"id":"boundary-lane-0001","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0002","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0003","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0004","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0005","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0006","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0007","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0008","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0009","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0010","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0011","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0012","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0013","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0014","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0015","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0016","kinds":["ExpansionNode"]}, + {"id":"boundary-lane-0017","kinds":["ExpansionNode"]} + ], + "edges": [ + {"start_id":"boundary-root","end_id":"boundary-lane-0001","kind":"Expand","properties":{"ordinal":1}}, + {"start_id":"boundary-lane-0001","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":101}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0002","kind":"Expand","properties":{"ordinal":2}}, + {"start_id":"boundary-lane-0002","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":102}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0003","kind":"Expand","properties":{"ordinal":3}}, + {"start_id":"boundary-lane-0003","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":103}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0004","kind":"Expand","properties":{"ordinal":4}}, + {"start_id":"boundary-lane-0004","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":104}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0005","kind":"Expand","properties":{"ordinal":5}}, + {"start_id":"boundary-lane-0005","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":105}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0006","kind":"Expand","properties":{"ordinal":6}}, + {"start_id":"boundary-lane-0006","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":106}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0007","kind":"Expand","properties":{"ordinal":7}}, + {"start_id":"boundary-lane-0007","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":107}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0008","kind":"Expand","properties":{"ordinal":8}}, + {"start_id":"boundary-lane-0008","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":108}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0009","kind":"Expand","properties":{"ordinal":9}}, + {"start_id":"boundary-lane-0009","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":109}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0010","kind":"Expand","properties":{"ordinal":10}}, + {"start_id":"boundary-lane-0010","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":110}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0011","kind":"Expand","properties":{"ordinal":11}}, + {"start_id":"boundary-lane-0011","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":111}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0012","kind":"Expand","properties":{"ordinal":12}}, + {"start_id":"boundary-lane-0012","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":112}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0013","kind":"Expand","properties":{"ordinal":13}}, + {"start_id":"boundary-lane-0013","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":113}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0014","kind":"Expand","properties":{"ordinal":14}}, + {"start_id":"boundary-lane-0014","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":114}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0015","kind":"Expand","properties":{"ordinal":15}}, + {"start_id":"boundary-lane-0015","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":115}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0016","kind":"Expand","properties":{"ordinal":16}}, + {"start_id":"boundary-lane-0016","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":116}}, + {"start_id":"boundary-root","end_id":"boundary-lane-0017","kind":"Expand","properties":{"ordinal":17}}, + {"start_id":"boundary-lane-0017","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":117}}, + {"start_id":"boundary-boundary-9001","end_id":"boundary-boundary-9001","kind":"Expand","properties":{"ordinal":201}}, + {"start_id":"boundary-boundary-9001","end_id":"boundary-head-a","kind":"EnterSuffix","properties":{"ordinal":301}}, + {"start_id":"boundary-head-a","end_id":"boundary-middle-a","kind":"ContinueSuffix","properties":{"ordinal":302}}, + {"start_id":"boundary-middle-a","end_id":"boundary-terminal","kind":"CompleteSuffix","properties":{"ordinal":303}}, + {"start_id":"boundary-boundary-9001","end_id":"boundary-head-b","kind":"EnterSuffix","properties":{"ordinal":401}}, + {"start_id":"boundary-head-b","end_id":"boundary-middle-b","kind":"ContinueSuffix","properties":{"ordinal":402}}, + {"start_id":"boundary-middle-b","end_id":"boundary-terminal","kind":"CompleteSuffix","properties":{"ordinal":403}} + ] + } +} diff --git a/integration/testdata/fixed_suffix_expansion_fanout.json b/integration/testdata/fixed_suffix_expansion_fanout.json new file mode 100644 index 00000000..6f93a829 --- /dev/null +++ b/integration/testdata/fixed_suffix_expansion_fanout.json @@ -0,0 +1,50 @@ +{ + "graph": { + "nodes": [ + {"id": "fse-root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "fixed-suffix-fanout-root"}}, + {"id": "fse-expansion-a", "kinds": ["ExpansionNode"]}, + {"id": "fse-expansion-b", "kinds": ["ExpansionNode"]}, + {"id": "fse-expansion-c", "kinds": ["ExpansionNode"]}, + {"id": "fse-option-good", "kinds": ["ExpansionNode"]}, + {"id": "fse-option-disabled", "kinds": ["ExpansionNode"]}, + {"id": "fse-option-wrong-head", "kinds": ["ExpansionNode"]}, + {"id": "fse-head", "kinds": ["SuffixHead"]}, + {"id": "fse-other-head", "kinds": ["SuffixHead"]}, + {"id": "fse-middle", "kinds": ["SuffixMiddle"]}, + {"id": "fse-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "fse-other-terminal", "kinds": ["SuffixTerminal"]}, + {"id": "fse-predicate-good", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "fse-predicate-alt", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 2, "required_approvals": 0}}, + {"id": "fse-predicate-disabled", "kinds": ["PredicateNode"], "properties": {"eligible": false, "requires_review": true, "allows_direct": false, "version": 2, "required_approvals": 1}}, + {"id": "fse-predicate-wrong-head", "kinds": ["PredicateNode"], "properties": {"eligible": true, "requires_review": false, "allows_direct": true, "version": 1, "required_approvals": 1}}, + {"id": "fse-bridge", "kinds": ["BridgeNode"]}, + {"id": "fse-other-bridge", "kinds": ["BridgeNode"]} + ], + "edges": [ + {"start_id": "fse-root", "end_id": "fse-expansion-a", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-expansion-b", "kind": "Expand"}, + {"start_id": "fse-expansion-b", "end_id": "fse-expansion-c", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-option-good", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-option-disabled", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-option-wrong-head", "kind": "Expand"}, + {"start_id": "fse-root", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-expansion-a", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-expansion-b", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-expansion-c", "end_id": "fse-head", "kind": "EnterSuffix"}, + {"start_id": "fse-head", "end_id": "fse-middle", "kind": "ContinueSuffix"}, + {"start_id": "fse-middle", "end_id": "fse-terminal", "kind": "CompleteSuffix"}, + {"start_id": "fse-option-good", "end_id": "fse-predicate-good", "kind": "OptionA"}, + {"start_id": "fse-option-good", "end_id": "fse-predicate-alt", "kind": "OptionB"}, + {"start_id": "fse-option-disabled", "end_id": "fse-predicate-disabled", "kind": "OptionC"}, + {"start_id": "fse-option-wrong-head", "end_id": "fse-predicate-wrong-head", "kind": "OptionA"}, + {"start_id": "fse-predicate-good", "end_id": "fse-head", "kind": "JoinSuffix"}, + {"start_id": "fse-predicate-alt", "end_id": "fse-head", "kind": "JoinSuffix"}, + {"start_id": "fse-predicate-disabled", "end_id": "fse-head", "kind": "JoinSuffix"}, + {"start_id": "fse-predicate-wrong-head", "end_id": "fse-other-head", "kind": "JoinSuffix"}, + {"start_id": "fse-head", "end_id": "fse-bridge", "kind": "HeadToBridge"}, + {"start_id": "fse-head", "end_id": "fse-other-bridge", "kind": "HeadToAlternateBridge"}, + {"start_id": "fse-bridge", "end_id": "fse-terminal", "kind": "ReachTerminal"}, + {"start_id": "fse-other-bridge", "end_id": "fse-other-terminal", "kind": "ReachTerminal"} + ] + } +} diff --git a/integration/testdata/shortest_bound.json b/integration/testdata/shortest_bound.json new file mode 100644 index 00000000..0ce18a91 --- /dev/null +++ b/integration/testdata/shortest_bound.json @@ -0,0 +1,33 @@ +{ + "graph": { + "nodes": [ + {"id": "start", "kinds": ["BoundNode"], "properties": {"name": "start"}}, + {"id": "direct", "kinds": ["BoundNode"], "properties": {"name": "direct"}}, + {"id": "long-mid", "kinds": ["BoundNode"]}, + {"id": "diamond-left", "kinds": ["BoundNode"]}, + {"id": "diamond-right", "kinds": ["BoundNode"]}, + {"id": "diamond-end", "kinds": ["BoundNode"]}, + {"id": "cycle-a", "kinds": ["BoundNode"]}, + {"id": "cycle-b", "kinds": ["BoundNode"]}, + {"id": "cycle-end", "kinds": ["BoundNode"]}, + {"id": "typed-end", "kinds": ["BoundNode"]}, + {"id": "disconnected", "kinds": ["BoundNode"]}, + {"id": "wrong-direction", "kinds": ["BoundNode"]} + ], + "edges": [ + {"start_id": "start", "end_id": "direct", "kind": "BoundEdge", "properties": {"route": "direct"}}, + {"start_id": "start", "end_id": "long-mid", "kind": "BoundEdge"}, + {"start_id": "long-mid", "end_id": "direct", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "diamond-left", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "diamond-right", "kind": "BoundEdge"}, + {"start_id": "diamond-left", "end_id": "diamond-end", "kind": "BoundEdge"}, + {"start_id": "diamond-right", "end_id": "diamond-end", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "cycle-a", "kind": "BoundEdge"}, + {"start_id": "cycle-a", "end_id": "cycle-b", "kind": "BoundEdge"}, + {"start_id": "cycle-b", "end_id": "cycle-a", "kind": "BoundEdge"}, + {"start_id": "cycle-b", "end_id": "cycle-end", "kind": "BoundEdge"}, + {"start_id": "start", "end_id": "typed-end", "kind": "OtherBoundEdge"}, + {"start_id": "wrong-direction", "end_id": "start", "kind": "BoundEdge"} + ] + } +} diff --git a/integration/testdata/templates/advanced_lookup_shapes.json b/integration/testdata/templates/advanced_lookup_shapes.json new file mode 100644 index 00000000..cdc137bc --- /dev/null +++ b/integration/testdata/templates/advanced_lookup_shapes.json @@ -0,0 +1,93 @@ +{ + "families": [ + { + "name": "LOOKUP-09 through LOOKUP-14 and LOOKUP-16 advanced lookups", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "hydrate-a", "kinds": ["Hydrate"], "properties": {"name": "hydrate-a", "value": 1}}, + {"id": "hydrate-b", "kinds": ["Hydrate"], "properties": {"name": "hydrate-b", "value": 2}}, + {"id": "hydrate-c", "kinds": ["Hydrate"], "properties": {"name": "hydrate-c", "value": 3}}, + {"id": "flags-mm", "kinds": ["User"], "properties": {"name": "flags-mm"}}, + {"id": "flags-mn", "kinds": ["User"], "properties": {"name": "flags-mn", "msa": null}}, + {"id": "flags-mf", "kinds": ["User"], "properties": {"name": "flags-mf", "msa": false}}, + {"id": "flags-mt", "kinds": ["User"], "properties": {"name": "flags-mt", "msa": true}}, + {"id": "flags-nm", "kinds": ["User"], "properties": {"name": "flags-nm", "gmsa": null}}, + {"id": "flags-nn", "kinds": ["User"], "properties": {"name": "flags-nn", "gmsa": null, "msa": null}}, + {"id": "flags-nf", "kinds": ["User"], "properties": {"name": "flags-nf", "gmsa": null, "msa": false}}, + {"id": "flags-nt", "kinds": ["User"], "properties": {"name": "flags-nt", "gmsa": null, "msa": true}}, + {"id": "flags-fm", "kinds": ["User"], "properties": {"name": "flags-fm", "gmsa": false}}, + {"id": "flags-fn", "kinds": ["User"], "properties": {"name": "flags-fn", "gmsa": false, "msa": null}}, + {"id": "flags-ff", "kinds": ["User"], "properties": {"name": "flags-ff", "gmsa": false, "msa": false}}, + {"id": "flags-ft", "kinds": ["User"], "properties": {"name": "flags-ft", "gmsa": false, "msa": true}}, + {"id": "flags-tm", "kinds": ["User"], "properties": {"name": "flags-tm", "gmsa": true}}, + {"id": "flags-tn", "kinds": ["User"], "properties": {"name": "flags-tn", "gmsa": true, "msa": null}}, + {"id": "flags-tf", "kinds": ["User"], "properties": {"name": "flags-tf", "gmsa": true, "msa": false}}, + {"id": "flags-tt", "kinds": ["User"], "properties": {"name": "flags-tt", "gmsa": true, "msa": true}}, + {"id": "tenant", "kinds": ["Tenant"], "properties": {"name": "tenant", "objectid": "tenant-1"}}, + {"id": "role-a", "kinds": ["AZRole"], "properties": {"name": "role-a", "roletemplateid": "role-a", "enabled": true, "state": "active"}}, + {"id": "role-b", "kinds": ["AZServicePrincipal"], "properties": {"name": "role-b", "roletemplateid": "role-b", "enabled": false, "state": "inactive"}}, + {"id": "role-multi", "kinds": ["AZRole", "AZServicePrincipal"], "properties": {"name": "role-multi", "roletemplateid": "role-multi", "enabled": true, "state": "active"}}, + {"id": "role-wrong-kind", "kinds": ["Other"], "properties": {"name": "role-wrong-kind", "roletemplateid": "role-a", "enabled": true, "state": "active"}}, + {"id": "edge-start", "kinds": ["Entity"], "properties": {"name": "edge-start"}}, + {"id": "edge-end", "kinds": ["Entity"], "properties": {"name": "edge-end"}}, + {"id": "local-good", "kinds": ["LocalGroup", "Entity"], "properties": {"name": "local-good", "objectid": "S-1-5-21-555"}}, + {"id": "local-good-2", "kinds": ["LocalGroup", "Entity"], "properties": {"name": "local-good-2", "objectid": "OTHER-555"}}, + {"id": "local-wrong-suffix", "kinds": ["LocalGroup", "Entity"], "properties": {"name": "local-wrong-suffix", "objectid": "S-1-5-21-556"}}, + {"id": "local-target", "kinds": ["Computer"], "properties": {"name": "local-target"}}, + {"id": "local-other-target", "kinds": ["Computer"], "properties": {"name": "local-other-target"}}, + {"id": "domain-missing", "kinds": ["Domain"], "properties": {"objectid": "domain-missing"}}, + {"id": "domain-alpha", "kinds": ["Domain"], "properties": {"name": "Alpha"}}, + {"id": "domain-beta-a", "kinds": ["Domain"], "properties": {"name": "Beta"}}, + {"id": "domain-beta-b", "kinds": ["Domain"], "properties": {"name": "Beta"}}, + {"id": "domain-multi", "kinds": ["Domain", "Other"], "properties": {"name": "Gamma"}}, + {"id": "ntlm-ldap-good", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-good", "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": true, "ldapsigning": false}}, + {"id": "ntlm-ldap-domain", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-domain", "domainsid": "S-1-5-99", "isdc": true, "ldapavailable": true, "ldapsigning": false}}, + {"id": "ntlm-ldap-isdc", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-isdc", "domainsid": "S-1-5-21", "isdc": false, "ldapavailable": true, "ldapsigning": false}}, + {"id": "ntlm-ldap-available", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-available", "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": false, "ldapsigning": false}}, + {"id": "ntlm-ldap-signing", "kinds": ["Computer"], "properties": {"name": "ntlm-ldap-signing", "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": true, "ldapsigning": true}}, + {"id": "ntlm-ldaps-good", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-good", "domainsid": "S-1-5-21", "isdc": true, "ldapsavailable": true, "epa": false}}, + {"id": "ntlm-ldaps-domain", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-domain", "domainsid": "S-1-5-99", "isdc": true, "ldapsavailable": true, "epa": false}}, + {"id": "ntlm-ldaps-isdc", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-isdc", "domainsid": "S-1-5-21", "isdc": false, "ldapsavailable": true, "epa": false}}, + {"id": "ntlm-ldaps-available", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-available", "domainsid": "S-1-5-21", "isdc": true, "ldapsavailable": false, "epa": false}}, + {"id": "ntlm-ldaps-epa", "kinds": ["Other"], "properties": {"name": "ntlm-ldaps-epa", "domainsid": "S-1-5-21", "isdc": true, "ldapsavailable": true, "epa": true}} + ], + "edges": [ + {"start_id": "tenant", "end_id": "role-a", "kind": "Contains", "properties": {"marker": "contains-role-a"}}, + {"start_id": "tenant", "end_id": "role-b", "kind": "Contains", "properties": {"marker": "contains-role-b"}}, + {"start_id": "tenant", "end_id": "role-multi", "kind": "Contains", "properties": {"marker": "contains-role-multi"}}, + {"start_id": "tenant", "end_id": "role-wrong-kind", "kind": "Contains", "properties": {"marker": "contains-wrong-kind"}}, + {"start_id": "edge-start", "end_id": "edge-end", "kind": "MemberOf", "properties": {"marker": "exact-edge"}}, + {"start_id": "edge-end", "end_id": "edge-start", "kind": "MemberOf", "properties": {"marker": "reverse-edge"}}, + {"start_id": "edge-start", "end_id": "edge-end", "kind": "WrongEdge", "properties": {"marker": "wrong-edge"}}, + {"start_id": "local-good", "end_id": "local-target", "kind": "LocalToComputer", "properties": {"marker": "local-good"}}, + {"start_id": "local-good-2", "end_id": "local-target", "kind": "LocalToComputer", "properties": {"marker": "local-good-2"}}, + {"start_id": "local-wrong-suffix", "end_id": "local-target", "kind": "LocalToComputer", "properties": {"marker": "local-wrong-suffix"}}, + {"start_id": "local-good", "end_id": "local-other-target", "kind": "LocalToComputer", "properties": {"marker": "local-wrong-end"}}, + {"start_id": "local-good", "end_id": "local-target", "kind": "WrongLocal", "properties": {"marker": "local-wrong-kind"}} + ] + }, + "variants": [ + {"name": "LOOKUP-09 empty ID list", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": []}, "assert": "empty"}, + {"name": "LOOKUP-09 single ID full hydration", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["hydrate-a"]}, "assert": {"node_records": [{"id": "hydrate-a", "kinds": ["Hydrate"], "props": {"name": "hydrate-a", "value": 1}}]}}, + {"name": "LOOKUP-09 duplicate IDs do not duplicate nodes", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["hydrate-a", "hydrate-a", "hydrate-b", "hydrate-a"]}, "assert": {"node_id_set": ["hydrate-a", "hydrate-b"], "row_count": 2}}, + {"name": "LOOKUP-09 thirty-two-entry sparse list", "vars": {"query": "MATCH (n) WHERE id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b", "hydrate-c", "hydrate-a", "hydrate-b"]}, "assert": {"node_id_set": ["hydrate-a", "hydrate-b", "hydrate-c"], "row_count": 3}}, + {"name": "LOOKUP-10 all missing null and boolean flag combinations", "vars": {"query": "MATCH (n:User) WHERE NOT (n.gmsa IS NOT NULL AND n.gmsa = true) AND NOT (n.msa IS NOT NULL AND n.msa = true) AND id(n) IN $ids RETURN n"}, "node_list_params": {"ids": ["flags-mm", "flags-mn", "flags-mf", "flags-mt", "flags-nm", "flags-nn", "flags-nf", "flags-nt", "flags-fm", "flags-fn", "flags-ff", "flags-ft", "flags-tm", "flags-tn", "flags-tf", "flags-tt"]}, "assert": {"node_id_set": ["flags-mm", "flags-mn", "flags-mf", "flags-nm", "flags-nn", "flags-nf", "flags-fm", "flags-fn", "flags-ff"]}}, + {"name": "LOOKUP-11 empty role-template list", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.roletemplateid IN $roles RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"roles": []}, "assert": "empty"}, + {"name": "LOOKUP-11 single role kind and single role-template ID", "vars": {"query": "MATCH (s)-[:Contains]->(e:AZRole) WHERE id(s) = $tenant AND e.roletemplateid IN $roles RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"roles": ["role-a"]}, "assert": {"node_id_set": ["role-a"]}}, + {"name": "LOOKUP-11 thousand-entry role-template list", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.roletemplateid IN $roles RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"roles": {"$type": "string_list", "prefix": "missing-role-", "count": 1000, "include": ["role-a", "role-b", "role-multi"]}}, "assert": {"node_id_set": ["role-a", "role-b", "role-multi"]}}, + {"name": "LOOKUP-11 endpoint boolean equality", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.enabled = true RETURN e"}, "node_params": {"tenant": "tenant"}, "assert": {"node_id_set": ["role-a", "role-multi"]}}, + {"name": "LOOKUP-11 endpoint string equality", "vars": {"query": "MATCH (s)-[:Contains]->(e) WHERE id(s) = $tenant AND (e:AZRole OR e:AZServicePrincipal) AND e.state = $state RETURN e"}, "node_params": {"tenant": "tenant"}, "params": {"state": "active"}, "assert": {"node_id_set": ["role-a", "role-multi"]}}, + {"name": "LOOKUP-12 exact edge key First hit", "vars": {"query": "MATCH (s)-[r:MemberOf]->(e) WHERE id(s) = $start_id AND id(e) = $end_id RETURN r LIMIT 1"}, "node_params": {"start_id": "edge-start", "end_id": "edge-end"}, "assert": {"relationship_records": [{"start": "edge-start", "end": "edge-end", "kind": "MemberOf", "props": {"marker": "exact-edge"}}]}}, + {"name": "LOOKUP-12 exact edge key no hit", "vars": {"query": "MATCH (s)-[r:MemberOf]->(e) WHERE id(s) = $start_id AND id(e) = $end_id RETURN r LIMIT 1"}, "node_params": {"start_id": "edge-start", "end_id": "hydrate-a"}, "assert": "empty"}, + {"name": "LOOKUP-13 full start node suffix and bound end", "vars": {"query": "MATCH (s)-[:LocalToComputer]->(e) WHERE s.objectid ENDS WITH $suffix AND id(e) = $end_id RETURN s"}, "params": {"suffix": "-555"}, "node_params": {"end_id": "local-target"}, "assert": {"node_id_set": ["local-good", "local-good-2"]}}, + {"name": "LOOKUP-13 start ID suffix and bound end", "vars": {"query": "MATCH (s)-[:LocalToComputer]->(e) WHERE s.objectid ENDS WITH $suffix AND id(e) = $end_id RETURN id(s)"}, "params": {"suffix": "-555"}, "node_params": {"end_id": "local-target"}, "assert": {"keys": ["id(s)"], "row_count": 2}}, + {"name": "LOOKUP-14 descending order includes missing equal distinct and multi-kind", "vars": {"query": "MATCH (n:Domain) RETURN n ORDER BY n.name DESC"}, "assert": {"node_id_set": ["domain-missing", "domain-alpha", "domain-beta-a", "domain-beta-b", "domain-multi"], "row_count": 5}}, + {"name": "LOOKUP-14 secondary ID key makes equal-property ties deterministic", "vars": {"query": "MATCH (n:Domain) WHERE n.name IS NOT NULL RETURN n ORDER BY n.name DESC, id(n) ASC"}, "assert": {"ordered_node_ids": ["domain-multi", "domain-beta-a", "domain-beta-b", "domain-alpha"]}}, + {"name": "LOOKUP-16 typed LDAP ID projection with one decoy per leaf", "vars": {"query": "MATCH (n:Computer) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapavailable = true AND n.ldapsigning = false RETURN id(n)"}, "params": {"domain": "S-1-5-21"}, "assert": {"keys": ["id(n)"], "row_count": 1}}, + {"name": "LOOKUP-16 typed LDAP full-node projection", "vars": {"query": "MATCH (n:Computer) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapavailable = true AND n.ldapsigning = false RETURN n"}, "params": {"domain": "S-1-5-21"}, "assert": {"node_id_set": ["ntlm-ldap-good"]}}, + {"name": "LOOKUP-16 untyped LDAPS full-node projection with one decoy per leaf", "vars": {"query": "MATCH (n) WHERE n.domainsid = $domain AND n.isdc = true AND n.ldapsavailable = true AND n.epa = false RETURN n"}, "params": {"domain": "S-1-5-21"}, "assert": {"node_id_set": ["ntlm-ldaps-good"]}} + ] + } + ] +} diff --git a/integration/testdata/templates/basic_lookup_shapes.json b/integration/testdata/templates/basic_lookup_shapes.json new file mode 100644 index 00000000..4935b1dd --- /dev/null +++ b/integration/testdata/templates/basic_lookup_shapes.json @@ -0,0 +1,72 @@ +{ + "families": [ + { + "name": "LOOKUP-01 through LOOKUP-08 node predicates and projections", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "group", "kinds": ["Group", "Entity"], "properties": {"name": "group", "objectid": "S-1-5-21-512", "domainsid": "S-1-5-21"}}, + {"id": "user", "kinds": ["User", "Entity"], "properties": {"name": "user", "objectid": "S-1-5-21-513", "domainsid": "S-1-5-21"}}, + {"id": "multi", "kinds": ["Group", "User", "Entity"], "properties": {"name": "multi", "objectid": "S-1-5-21-514", "domainsid": "S-1-5-21"}}, + {"id": "local-group", "kinds": ["Group", "LocalGroup", "Entity"], "properties": {"name": "local-group", "objectid": "S-1-5-21-512", "domainsid": "S-1-5-21"}}, + {"id": "entity-only", "kinds": ["Entity"], "properties": {"name": "entity-only", "objectid": "S-1-5-21-512", "domainsid": "S-1-5-21"}}, + {"id": "tenant", "kinds": ["Tenant"], "properties": {"name": "tenant", "objectid": "tenant-1"}}, + {"id": "computer-hit-a", "kinds": ["Computer"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-100", "enabled": true}}, + {"id": "computer-hit-b", "kinds": ["Computer", "Entity"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-100", "enabled": true}}, + {"id": "computer-disabled", "kinds": ["Computer"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-101", "enabled": false}}, + {"id": "objectid-untyped", "kinds": ["Other"], "properties": {"name": "dc.example.test", "objectid": "S-1-5-21-100", "enabled": true}}, + {"id": "ura-true", "kinds": ["Computer"], "properties": {"name": "ura-true", "hasura": true}}, + {"id": "ura-false", "kinds": ["Computer"], "properties": {"name": "ura-false", "hasura": false}}, + {"id": "ura-null", "kinds": ["Computer"], "properties": {"name": "ura-null", "hasura": null}}, + {"id": "ura-missing", "kinds": ["Computer"], "properties": {"name": "ura-missing"}}, + {"id": "adminsdholder", "kinds": ["Container"], "properties": {"name": "admin", "distinguishedname": "CN=ADMINSDHOLDER,CN=SYSTEM,DC=EXAMPLE,DC=TEST", "domainsid": "S-1-5-21"}}, + {"id": "admin-wrong-case", "kinds": ["Container"], "properties": {"name": "admin-case", "distinguishedname": "cn=adminsdholder,CN=SYSTEM,DC=EXAMPLE,DC=TEST", "domainsid": "S-1-5-21"}}, + {"id": "admin-wrong-domain", "kinds": ["Container"], "properties": {"name": "admin-domain", "distinguishedname": "CN=ADMINSDHOLDER,CN=SYSTEM,DC=OTHER", "domainsid": "S-1-5-99"}}, + {"id": "suffix-a", "kinds": ["Group"], "properties": {"name": "suffix-a", "objectid": "OBJECT-S-1"}}, + {"id": "suffix-b", "kinds": ["Group"], "properties": {"name": "suffix-b", "objectid": "OBJECT-S-2"}}, + {"id": "suffix-case", "kinds": ["Group"], "properties": {"name": "suffix-case", "objectid": "OBJECT-s-1"}}, + {"id": "suffix-wrong-kind", "kinds": ["User"], "properties": {"name": "suffix-user", "objectid": "OBJECT-S-1"}}, + {"id": "ci-prefix-exact", "kinds": ["Lookup"], "properties": {"name": "Remote Desktop Users Alpha"}}, + {"id": "ci-prefix-mixed", "kinds": ["Lookup"], "properties": {"name": "rEmOtE dEsKtOp UsErS Beta"}}, + {"id": "ci-prefix-literal", "kinds": ["Lookup"], "properties": {"name": "Remote%_Desktop Literal"}}, + {"id": "ci-prefix-wild-decoy", "kinds": ["Lookup"], "properties": {"name": "RemoteXXDesktop Decoy"}}, + {"id": "ci-contains-exact", "kinds": ["Entity"], "properties": {"name": "approver-exact", "objectid": "Approver_GUID"}}, + {"id": "ci-contains-substring", "kinds": ["Entity"], "properties": {"name": "approver-substring", "objectid": "prefix-APPROVER_guid-suffix"}}, + {"id": "ci-contains-decoy", "kinds": ["Entity"], "properties": {"name": "approver-decoy", "objectid": "different-guid"}}, + {"id": "name-missing", "kinds": ["Lookup"], "properties": {"objectid": "missing"}}, + {"id": "name-null", "kinds": ["Lookup"], "properties": {"name": null, "objectid": "null"}}, + {"id": "name-empty", "kinds": ["Lookup"], "properties": {"name": "", "objectid": "empty"}}, + {"id": "name-populated", "kinds": ["Lookup"], "properties": {"name": "populated", "objectid": "populated"}}, + {"id": "role-user", "kinds": ["AZRole"], "properties": {"name": "role-user", "tenantid": "tenant-1", "approvalrequired": true, "userapprovers": ["u1"]}}, + {"id": "role-group", "kinds": ["AZRole"], "properties": {"name": "role-group", "tenantid": "tenant-1", "approvalrequired": true, "groupapprovers": ["g1"]}}, + {"id": "role-both", "kinds": ["AZRole"], "properties": {"name": "role-both", "tenantid": "tenant-1", "approvalrequired": true, "userapprovers": ["u1"], "groupapprovers": ["g1"]}}, + {"id": "role-neither", "kinds": ["AZRole"], "properties": {"name": "role-neither", "tenantid": "tenant-1", "approvalrequired": true}}, + {"id": "role-null", "kinds": ["AZRole"], "properties": {"name": "role-null", "tenantid": "tenant-1", "approvalrequired": true, "userapprovers": null, "groupapprovers": null}}, + {"id": "role-wrong-tenant", "kinds": ["AZRole"], "properties": {"name": "role-wrong-tenant", "tenantid": "tenant-2", "approvalrequired": true, "userapprovers": ["u1"]}}, + {"id": "role-not-required", "kinds": ["AZRole"], "properties": {"name": "role-not-required", "tenantid": "tenant-1", "approvalrequired": false, "userapprovers": ["u1"]}} + ] + }, + "variants": [ + {"name": "LOOKUP-01 one kind ID projection", "vars": {"query": "MATCH (n:Group) RETURN id(n)"}, "assert": {"keys": ["id(n)"], "row_count": 6}}, + {"name": "LOOKUP-01 many kinds include multi-kind node once", "vars": {"query": "MATCH (n) WHERE n:Group OR n:User RETURN n"}, "assert": {"node_id_set": ["group", "user", "multi", "local-group", "suffix-a", "suffix-b", "suffix-case", "suffix-wrong-kind"]}}, + {"name": "LOOKUP-01 exact kind full hydration", "vars": {"query": "MATCH (n:Tenant) RETURN n"}, "assert": {"node_records": [{"id": "tenant", "kinds": ["Tenant"], "props": {"name": "tenant", "objectid": "tenant-1"}}]}}, + {"name": "LOOKUP-02 indexed kind and object ID First with multiple hits", "vars": {"query": "MATCH (n:Computer) WHERE n.objectid = $objectid RETURN n LIMIT 1"}, "params": {"objectid": "S-1-5-21-100"}, "assert": {"row_count": 1}}, + {"name": "LOOKUP-02 indexed kind no hit", "vars": {"query": "MATCH (n:Computer) WHERE n.objectid = $objectid RETURN n LIMIT 1"}, "params": {"objectid": "missing"}, "assert": "empty"}, + {"name": "LOOKUP-02 no-kind object ID includes untyped node", "vars": {"query": "MATCH (n) WHERE n.objectid = $objectid RETURN n"}, "params": {"objectid": "S-1-5-21-100"}, "assert": {"node_id_set": ["computer-hit-a", "computer-hit-b", "objectid-untyped"]}}, + {"name": "LOOKUP-02 two equalities string and boolean", "vars": {"query": "MATCH (n) WHERE n.name = $name AND n.enabled = $enabled RETURN id(n)"}, "params": {"name": "dc.example.test", "enabled": true}, "assert": {"keys": ["id(n)"], "row_count": 3}}, + {"name": "LOOKUP-03 true boolean and two-column projection", "vars": {"query": "MATCH (n:Computer) WHERE n.hasura = $value RETURN id(n), n.hasura"}, "params": {"value": true}, "assert": {"keys": ["id(n)", "n.hasura"], "row_count": 1}}, + {"name": "LOOKUP-03 false excludes null and missing", "vars": {"query": "MATCH (n:Computer) WHERE n.hasura = $value RETURN id(n), n.hasura"}, "params": {"value": false}, "assert": {"keys": ["id(n)", "n.hasura"], "row_count": 1}}, + {"name": "LOOKUP-04 case-sensitive AdminSDHolder prefix and domain", "vars": {"query": "MATCH (n:Container) WHERE n.distinguishedname STARTS WITH $prefix AND n.domainsid = $domain RETURN n"}, "params": {"prefix": "CN=ADMINSDHOLDER,CN=SYSTEM,", "domain": "S-1-5-21"}, "assert": {"node_id_set": ["adminsdholder"]}}, + {"name": "LOOKUP-04 OR of two suffixes is case-sensitive", "vars": {"query": "MATCH (n:Group) WHERE n.objectid ENDS WITH $a OR n.objectid ENDS WITH $b RETURN n"}, "params": {"a": "-S-1", "b": "-S-2"}, "assert": {"node_id_set": ["suffix-a", "suffix-b"]}}, + {"name": "LOOKUP-05 case-insensitive prefix exact and mixed case", "vars": {"query": "MATCH (n:Lookup) WHERE toLower(n.name) STARTS WITH $prefix RETURN n"}, "params": {"prefix": "remote desktop users"}, "assert": {"node_id_set": ["ci-prefix-exact", "ci-prefix-mixed"]}}, + {"name": "LOOKUP-05 percent and underscore remain literal", "vars": {"query": "MATCH (n:Lookup) WHERE toLower(n.name) STARTS WITH $prefix RETURN n"}, "params": {"prefix": "remote%_"}, "assert": {"node_id_set": ["ci-prefix-literal"]}}, + {"name": "LOOKUP-05 contains retains substring candidate", "vars": {"query": "MATCH (n:Entity) WHERE toLower(n.objectid) CONTAINS $fragment RETURN n"}, "params": {"fragment": "approver_guid"}, "assert": {"node_id_set": ["ci-contains-exact", "ci-contains-substring"]}}, + {"name": "LOOKUP-06 included kind group plus Entity suffix and domain", "vars": {"query": "MATCH (n) WHERE (n:Group OR n:User) AND n:Entity AND n.objectid ENDS WITH $suffix AND n.domainsid = $domain RETURN n"}, "params": {"suffix": "-512", "domain": "S-1-5-21"}, "assert": {"node_id_set": ["group", "local-group"]}}, + {"name": "LOOKUP-06 Entity excluding Group and LocalGroup", "vars": {"query": "MATCH (n:Entity) WHERE NOT (n:Group OR n:LocalGroup) AND n.objectid ENDS WITH $suffix RETURN n"}, "params": {"suffix": "-512"}, "assert": {"node_id_set": ["entity-only"]}}, + {"name": "LOOKUP-07 missing and explicit null names", "vars": {"query": "MATCH (n:Lookup) WHERE n.name IS NULL RETURN n"}, "assert": {"node_id_set": ["name-missing", "name-null"]}}, + {"name": "LOOKUP-07 empty and populated names are present", "vars": {"query": "MATCH (n:Lookup) WHERE n.name IS NOT NULL RETURN n"}, "assert": {"node_id_set": ["ci-prefix-exact", "ci-prefix-mixed", "ci-prefix-literal", "ci-prefix-wild-decoy", "name-empty", "name-populated"]}}, + {"name": "LOOKUP-08 either or both approver properties present", "vars": {"query": "MATCH (n:AZRole) WHERE n.tenantid = $tenant AND n.approvalrequired = true AND (n.userapprovers IS NOT NULL OR n.groupapprovers IS NOT NULL) RETURN n"}, "params": {"tenant": "tenant-1"}, "assert": {"node_id_set": ["role-user", "role-group", "role-both"]}} + ] + } + ] +} diff --git a/integration/testdata/templates/count_shapes.json b/integration/testdata/templates/count_shapes.json new file mode 100644 index 00000000..8db95c23 --- /dev/null +++ b/integration/testdata/templates/count_shapes.json @@ -0,0 +1,70 @@ +{ + "families": [ + { + "name": "LOOKUP-15 empty graph counts", + "template": "{{query}}", + "fixture": {"nodes": [], "edges": []}, + "variants": [ + {"name": "LOOKUP-15 empty node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 0}}, + {"name": "LOOKUP-15 empty relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 0}} + ] + }, + { + "name": "LOOKUP-15 node-only graph counts", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "node-a", "kinds": ["CountNode"], "properties": {"name": "a"}}, + {"id": "node-b", "kinds": ["CountNode"], "properties": {"name": "b"}}, + {"id": "node-c", "kinds": ["CountNode"], "properties": {"name": "c"}} + ], + "edges": [] + }, + "variants": [ + {"name": "LOOKUP-15 node-only node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 3}}, + {"name": "LOOKUP-15 node-only relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 0}} + ] + }, + { + "name": "LOOKUP-15 edge-bearing graph counts", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "node-a", "kinds": ["CountNode"], "properties": {"name": "a"}}, + {"id": "node-b", "kinds": ["CountNode"], "properties": {"name": "b"}} + ], + "edges": [ + {"start_id": "node-a", "end_id": "node-b", "kind": "CountEdge", "properties": {"marker": "edge"}} + ] + }, + "variants": [ + {"name": "LOOKUP-15 edge-bearing node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 2}}, + {"name": "LOOKUP-15 edge-bearing relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 1}} + ] + }, + { + "name": "LOOKUP-15 dense graph counts", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "node-a", "kinds": ["CountNode"], "properties": {"name": "a"}}, + {"id": "node-b", "kinds": ["CountNode"], "properties": {"name": "b"}}, + {"id": "node-c", "kinds": ["CountNode"], "properties": {"name": "c"}}, + {"id": "node-d", "kinds": ["CountNode"], "properties": {"name": "d"}} + ], + "edges": [ + {"start_id": "node-a", "end_id": "node-b", "kind": "CountEdge", "properties": {"marker": "a-b"}}, + {"start_id": "node-a", "end_id": "node-c", "kind": "CountEdge", "properties": {"marker": "a-c"}}, + {"start_id": "node-a", "end_id": "node-d", "kind": "CountEdge", "properties": {"marker": "a-d"}}, + {"start_id": "node-b", "end_id": "node-a", "kind": "CountEdge", "properties": {"marker": "b-a"}}, + {"start_id": "node-c", "end_id": "node-a", "kind": "CountEdge", "properties": {"marker": "c-a"}}, + {"start_id": "node-d", "end_id": "node-a", "kind": "CountEdge", "properties": {"marker": "d-a"}} + ] + }, + "variants": [ + {"name": "LOOKUP-15 dense node count", "vars": {"query": "MATCH (n) RETURN count(n)"}, "assert": {"exact_int": 4}}, + {"name": "LOOKUP-15 dense relationship count", "vars": {"query": "MATCH ()-[r]->() RETURN count(r)"}, "assert": {"exact_int": 6}} + ] + } + ] +} diff --git a/integration/testdata/templates/fixed_suffix_expansion_shapes.json b/integration/testdata/templates/fixed_suffix_expansion_shapes.json new file mode 100644 index 00000000..8bb338d5 --- /dev/null +++ b/integration/testdata/templates/fixed_suffix_expansion_shapes.json @@ -0,0 +1,48 @@ +{ + "families": [ + { + "name": "Bounded fixed-suffix expansion semantics", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "fse-root", "kinds": ["ExpansionRoot"], "properties": {"root_key": "semantic-fse-root"}}, + {"id": "fse-mid", "kinds": ["ExpansionNode"], "properties": {"name": "mid"}}, + {"id": "fse-boundary-a", "kinds": ["ExpansionNode"], "properties": {"enabled": true}}, + {"id": "fse-boundary-b", "kinds": ["ExpansionNode"], "properties": {"enabled": true}}, + {"id": "fse-head", "kinds": ["SuffixHead"], "properties": {"name": "head"}}, + {"id": "fse-middle", "kinds": ["SuffixMiddle"], "properties": {"name": "middle"}}, + {"id": "fse-terminal", "kinds": ["SuffixTerminal"], "properties": {"name": "terminal"}}, + {"id": "fse-decoy-head", "kinds": ["SuffixHead"], "properties": {"name": "decoy"}} + ], + "edges": [ + {"start_id": "fse-root", "end_id": "fse-mid", "kind": "Expand", "properties": {"ordinal": 1}}, + {"start_id": "fse-mid", "end_id": "fse-mid", "kind": "Expand", "properties": {"ordinal": 2}}, + {"start_id": "fse-mid", "end_id": "fse-boundary-a", "kind": "Expand", "properties": {"ordinal": 3}}, + {"start_id": "fse-mid", "end_id": "fse-boundary-b", "kind": "Expand", "properties": {"ordinal": 4}}, + {"start_id": "fse-boundary-a", "end_id": "fse-head", "kind": "EnterSuffix", "properties": {"ordinal": 5}}, + {"start_id": "fse-boundary-b", "end_id": "fse-head", "kind": "EnterSuffix", "properties": {"ordinal": 6}}, + {"start_id": "fse-head", "end_id": "fse-middle", "kind": "ContinueSuffix", "properties": {"ordinal": 7}}, + {"start_id": "fse-middle", "end_id": "fse-terminal", "kind": "CompleteSuffix", "properties": {"ordinal": 8}}, + {"start_id": "fse-boundary-a", "end_id": "fse-decoy-head", "kind": "WrongEnterSuffix", "properties": {"ordinal": 9}} + ] + }, + "variants": [ + { + "name": "Bounded endpoint observation preserves physical suffix multiplicity", + "vars": {"query": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'semantic-fse-root' MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) RETURN id(head), id(terminal)"}, + "assert": {"keys": ["id(head)", "id(terminal)"], "row_count": 2} + }, + { + "name": "Bounded full path observation retains relationship-distinct paths", + "vars": {"query": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'semantic-fse-root' MATCH p = (root)-[:Expand*0..2]->()-[:EnterSuffix]->(:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(:SuffixTerminal) RETURN p"}, + "assert": {"keys": ["p"], "row_count": 2} + }, + { + "name": "Bounded downstream WITH aggregation preserves bag semantics", + "vars": {"query": "MATCH (root:ExpansionRoot) WHERE root.root_key = 'semantic-fse-root' MATCH (root)-[:Expand*0..2]->()-[:EnterSuffix]->(head:SuffixHead)-[:ContinueSuffix]->(:SuffixMiddle)-[:CompleteSuffix]->(terminal:SuffixTerminal) WITH head, terminal, count(*) AS trails RETURN trails"}, + "assert": {"keys": ["trails"], "row_values": [[2]], "row_count": 1} + } + ] + } + ] +} diff --git a/integration/testdata/templates/lowering_regression_shapes.json b/integration/testdata/templates/lowering_regression_shapes.json new file mode 100644 index 00000000..89410c43 --- /dev/null +++ b/integration/testdata/templates/lowering_regression_shapes.json @@ -0,0 +1,108 @@ +{ + "families": [ + { + "name": "greedy projection and zero-depth shortest path retain full values", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "common", "kinds": ["LoweringNode"], "properties": {"root": true, "terminal": true}}, + {"id": "other-root", "kinds": ["LoweringNode"], "properties": {"root": true}}, + {"id": "target", "kinds": ["LoweringNode"], "properties": {"terminal": true}} + ], + "edges": [ + {"start_id": "common", "end_id": "target", "kind": "LoweringEdge", "properties": {"route": "common-target"}}, + {"start_id": "other-root", "end_id": "target", "kind": "LoweringEdge", "properties": {"route": "other-target"}} + ] + }, + "variants": [ + { + "name": "RETURN star materializes nodes and paths including the equal endpoint pair", + "vars": {"query": "MATCH p = shortestPath((s:LoweringNode)-[:LoweringEdge*0..4]->(e:LoweringNode)) WHERE s.root = true AND e.terminal = true RETURN *"}, + "assert": { + "row_count": 3, + "node_id_set": ["common", "other-root", "target"], + "path_node_ids": [["common"], ["common", "target"], ["other-root", "target"]] + } + }, + { + "name": "WITH star observes the full path before a scalar final projection", + "vars": {"query": "MATCH p = shortestPath((s:LoweringNode)-[:LoweringEdge*0..4]->(e:LoweringNode)) WHERE s.root = true AND e.terminal = true WITH * RETURN p"}, + "assert": { + "row_count": 3, + "path_node_ids": [["common"], ["common", "target"], ["other-root", "target"]] + } + } + ] + }, + { + "name": "guarded endpoint seeded expansion preserves complete trail semantics", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "excluded-computer", "kinds": ["Computer"], "properties": {"name": "excluded"}}, + {"id": "good-computer", "kinds": ["Computer"], "properties": {"name": "good"}}, + {"id": "excluded-user", "kinds": ["User"], "properties": {"name": "excluded-user"}}, + {"id": "good-user", "kinds": ["User"], "properties": {"name": "good-user"}}, + {"id": "middle", "kinds": ["Group"], "properties": {"objectid": "MIDDLE"}}, + {"id": "alternate-middle", "kinds": ["Group"], "properties": {"objectid": "ALTERNATE-MIDDLE"}}, + {"id": "excluded-group", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-516"}}, + {"id": "terminal", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-512"}}, + {"id": "decoy", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-513"}} + ], + "edges": [ + {"start_id": "excluded-computer", "end_id": "excluded-group", "kind": "MemberOf", "properties": {"marker": "exclude"}}, + {"start_id": "excluded-computer", "end_id": "excluded-user", "kind": "HasSession", "properties": {"marker": "session-excluded"}}, + {"start_id": "good-computer", "end_id": "good-user", "kind": "HasSession", "properties": {"marker": "session-good"}}, + {"start_id": "good-user", "end_id": "good-user", "kind": "MemberOf", "properties": {"marker": "loop"}}, + {"start_id": "good-user", "end_id": "middle", "kind": "MemberOf", "properties": {"marker": "first"}}, + {"start_id": "middle", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "second"}}, + {"start_id": "good-user", "end_id": "alternate-middle", "kind": "MemberOf", "properties": {"marker": "alternate-first"}}, + {"start_id": "alternate-middle", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "alternate-second"}}, + {"start_id": "good-user", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "direct"}}, + {"start_id": "good-user", "end_id": "decoy", "kind": "MemberOf", "properties": {"marker": "decoy"}} + ] + }, + "variants": [ + { + "name": "compound exclusion query returns ordered hydrated relationships", + "vars": {"query": "MATCH (s)-[:MemberOf*0..]->(excluded:Group) WHERE excluded.objectid ENDS WITH '-516' WITH COLLECT(s) AS exclude MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' AND NOT c IN exclude RETURN p LIMIT 1000"}, + "assert": { + "row_count": 6, + "path_relationship_records": [ + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"terminal","kind":"MemberOf","props":{"marker":"direct"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"middle","kind":"MemberOf","props":{"marker":"first"}},{"start":"middle","end":"terminal","kind":"MemberOf","props":{"marker":"second"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"alternate-middle","kind":"MemberOf","props":{"marker":"alternate-first"}},{"start":"alternate-middle","end":"terminal","kind":"MemberOf","props":{"marker":"alternate-second"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"good-user","kind":"MemberOf","props":{"marker":"loop"}},{"start":"good-user","end":"terminal","kind":"MemberOf","props":{"marker":"direct"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"good-user","kind":"MemberOf","props":{"marker":"loop"}},{"start":"good-user","end":"middle","kind":"MemberOf","props":{"marker":"first"}},{"start":"middle","end":"terminal","kind":"MemberOf","props":{"marker":"second"}}], + [{"start":"good-computer","end":"good-user","kind":"HasSession","props":{"marker":"session-good"}},{"start":"good-user","end":"good-user","kind":"MemberOf","props":{"marker":"loop"}},{"start":"good-user","end":"alternate-middle","kind":"MemberOf","props":{"marker":"alternate-first"}},{"start":"alternate-middle","end":"terminal","kind":"MemberOf","props":{"marker":"alternate-second"}}] + ] + } + } + ] + } + ], + "metamorphic": [ + { + "name": "guarded endpoint lowering matches incumbent traversal", + "fixture": { + "nodes": [ + {"id": "computer", "kinds": ["Computer"], "properties": {}}, + {"id": "user", "kinds": ["User"], "properties": {}}, + {"id": "middle", "kinds": ["Group"], "properties": {}}, + {"id": "terminal", "kinds": ["Group"], "properties": {"objectid": "S-1-5-21-512"}} + ], + "edges": [ + {"start_id": "computer", "end_id": "user", "kind": "HasSession", "properties": {"marker": "session"}}, + {"start_id": "user", "end_id": "user", "kind": "MemberOf", "properties": {"marker": "loop"}}, + {"start_id": "user", "end_id": "middle", "kind": "MemberOf", "properties": {"marker": "first"}}, + {"start_id": "middle", "end_id": "terminal", "kind": "MemberOf", "properties": {"marker": "second"}} + ] + }, + "compare": ["path_node_ids", "path_relationship_records"], + "queries": [ + {"name": "guarded endpoint seeded", "cypher": "MATCH p = (c:Computer)-[:HasSession]->(:User)-[:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p"}, + {"name": "incumbent relationship variable", "cypher": "MATCH p = (c:Computer)-[:HasSession]->(:User)-[rels:MemberOf*1..]->(g:Group) WHERE g.objectid ENDS WITH '-512' RETURN p"} + ] + } + ] +} diff --git a/integration/testdata/templates/mutation_post_state_shapes.json b/integration/testdata/templates/mutation_post_state_shapes.json new file mode 100644 index 00000000..f5a4d67b --- /dev/null +++ b/integration/testdata/templates/mutation_post_state_shapes.json @@ -0,0 +1,67 @@ +{ + "families": [ + { + "name": "mutation rollback restores the original fixture", + "template": "MATCH (n:DeleteTarget) WHERE n.objectid = $object_id DETACH DELETE n", + "params": {"object_id": "delete-me"}, + "fixture": { + "nodes": [ + {"id": "victim", "kinds": ["DeleteTarget", "Entity"], "properties": {"objectid": "delete-me", "marker": "victim"}}, + {"id": "survivor", "kinds": ["Entity"], "properties": {"objectid": "keep-me", "marker": "survivor"}}, + {"id": "kind-decoy", "kinds": ["Entity"], "properties": {"objectid": "delete-me", "marker": "wrong-kind"}}, + {"id": "property-decoy", "kinds": ["DeleteTarget"], "properties": {"objectid": "keep-me", "marker": "wrong-property"}} + ], + "edges": [ + {"start_id": "survivor", "end_id": "victim", "kind": "Incident", "properties": {"direction": "inbound"}}, + {"start_id": "victim", "end_id": "survivor", "kind": "Incident", "properties": {"direction": "outbound"}}, + {"start_id": "victim", "end_id": "victim", "kind": "Incident", "properties": {"direction": "self"}}, + {"start_id": "survivor", "end_id": "property-decoy", "kind": "Survives", "properties": {"marker": "keep-edge"}} + ] + }, + "variants": [ + { + "name": "first execution", + "assert": "no_error", + "post_assertions": [ + { + "cypher": "MATCH (n) RETURN n", + "assert": { + "node_records": [ + {"id": "survivor", "kinds": ["Entity"], "props": {"objectid": "keep-me", "marker": "survivor"}}, + {"id": "kind-decoy", "kinds": ["Entity"], "props": {"objectid": "delete-me", "marker": "wrong-kind"}}, + {"id": "property-decoy", "kinds": ["DeleteTarget"], "props": {"objectid": "keep-me", "marker": "wrong-property"}} + ] + } + }, + { + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": {"relationship_records": [{"start": "survivor", "end": "property-decoy", "kind": "Survives", "props": {"marker": "keep-edge"}}]} + }, + { + "cypher": "MATCH (n) RETURN count(n)", + "assert": {"exact_int": 3} + } + ] + }, + { + "name": "identical execution after rollback", + "assert": "no_error", + "post_assertions": [ + { + "cypher": "MATCH (n) RETURN n", + "assert": {"node_id_set": ["survivor", "kind-decoy", "property-decoy"]} + }, + { + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": {"relationship_triples": [{"start": "survivor", "end": "property-decoy", "kind": "Survives"}]} + }, + { + "cypher": "MATCH ()-[r]->() RETURN count(r)", + "assert": {"exact_int": 1} + } + ] + } + ] + } + ] +} diff --git a/integration/testdata/templates/post_processing_hop_shapes.json b/integration/testdata/templates/post_processing_hop_shapes.json new file mode 100644 index 00000000..211db629 --- /dev/null +++ b/integration/testdata/templates/post_processing_hop_shapes.json @@ -0,0 +1,280 @@ +{ + "families": [ + { + "name": "HOP-01 through HOP-03 anchored direction and relationship-kind cardinality", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "out-zero", "kinds": ["HopAnchor"], "properties": {"name": "out-zero"}}, + {"id": "out-one", "kinds": ["HopAnchor"], "properties": {"name": "out-one"}}, + {"id": "out-high", "kinds": ["HopAnchor"], "properties": {"name": "out-high"}}, + {"id": "in-zero", "kinds": ["HopAnchor"], "properties": {"name": "in-zero"}}, + {"id": "in-one", "kinds": ["HopAnchor"], "properties": {"name": "in-one"}}, + {"id": "in-high", "kinds": ["HopAnchor"], "properties": {"name": "in-high"}}, + {"id": "out-one-target", "kinds": ["HopEndpoint"], "properties": {"name": "out-one-target"}}, + {"id": "in-one-source", "kinds": ["HopEndpoint"], "properties": {"name": "in-one-source"}}, + {"id": "out-high-01", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-01"}}, + {"id": "out-high-02", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-02"}}, + {"id": "out-high-03", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-03"}}, + {"id": "out-high-04", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-04"}}, + {"id": "out-high-05", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-05"}}, + {"id": "out-high-06", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-06"}}, + {"id": "out-high-07", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-07"}}, + {"id": "out-high-08", "kinds": ["HopEndpoint"], "properties": {"name": "out-high-08"}}, + {"id": "in-high-01", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-01"}}, + {"id": "in-high-02", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-02"}}, + {"id": "in-high-03", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-03"}}, + {"id": "in-high-04", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-04"}}, + {"id": "in-high-05", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-05"}}, + {"id": "in-high-06", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-06"}}, + {"id": "in-high-07", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-07"}}, + {"id": "in-high-08", "kinds": ["HopEndpoint"], "properties": {"name": "in-high-08"}}, + {"id": "kind-center", "kinds": ["HopAnchor"], "properties": {"name": "kind-center"}}, + {"id": "kind-peer", "kinds": ["HopEndpoint"], "properties": {"name": "kind-peer"}} + ], + "edges": [ + {"start_id": "out-one", "end_id": "out-one-target", "kind": "HopKind01", "properties": {"marker": "out-one"}}, + {"start_id": "out-high", "end_id": "out-high-01", "kind": "HopKind01", "properties": {"marker": "out-high-01"}}, + {"start_id": "out-high", "end_id": "out-high-02", "kind": "HopKind01", "properties": {"marker": "out-high-02"}}, + {"start_id": "out-high", "end_id": "out-high-03", "kind": "HopKind01", "properties": {"marker": "out-high-03"}}, + {"start_id": "out-high", "end_id": "out-high-04", "kind": "HopKind01", "properties": {"marker": "out-high-04"}}, + {"start_id": "out-high", "end_id": "out-high-05", "kind": "HopKind01", "properties": {"marker": "out-high-05"}}, + {"start_id": "out-high", "end_id": "out-high-06", "kind": "HopKind01", "properties": {"marker": "out-high-06"}}, + {"start_id": "out-high", "end_id": "out-high-07", "kind": "HopKind01", "properties": {"marker": "out-high-07"}}, + {"start_id": "out-high", "end_id": "out-high-08", "kind": "HopKind01", "properties": {"marker": "out-high-08"}}, + {"start_id": "in-one-source", "end_id": "in-one", "kind": "HopKind01", "properties": {"marker": "in-one"}}, + {"start_id": "in-high-01", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-01"}}, + {"start_id": "in-high-02", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-02"}}, + {"start_id": "in-high-03", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-03"}}, + {"start_id": "in-high-04", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-04"}}, + {"start_id": "in-high-05", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-05"}}, + {"start_id": "in-high-06", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-06"}}, + {"start_id": "in-high-07", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-07"}}, + {"start_id": "in-high-08", "end_id": "in-high", "kind": "HopKind01", "properties": {"marker": "in-high-08"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind01", "properties": {"marker": "out-kind-01"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind02", "properties": {"marker": "out-kind-02"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind03", "properties": {"marker": "out-kind-03"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind04", "properties": {"marker": "out-kind-04"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind05", "properties": {"marker": "out-kind-05"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind06", "properties": {"marker": "out-kind-06"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind07", "properties": {"marker": "out-kind-07"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind08", "properties": {"marker": "out-kind-08"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind09", "properties": {"marker": "out-kind-09"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind10", "properties": {"marker": "out-kind-10"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind11", "properties": {"marker": "out-kind-11"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind12", "properties": {"marker": "out-kind-12"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind13", "properties": {"marker": "out-kind-13"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind14", "properties": {"marker": "out-kind-14"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind15", "properties": {"marker": "out-kind-15"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind16", "properties": {"marker": "out-kind-16"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind17", "properties": {"marker": "out-kind-17"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind18", "properties": {"marker": "out-kind-18"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind19", "properties": {"marker": "out-kind-19"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind20", "properties": {"marker": "out-kind-20"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind21", "properties": {"marker": "out-kind-21"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind22", "properties": {"marker": "out-kind-22"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind23", "properties": {"marker": "out-kind-23"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind24", "properties": {"marker": "out-kind-24"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind25", "properties": {"marker": "out-kind-25"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind26", "properties": {"marker": "out-kind-26"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind27", "properties": {"marker": "out-kind-27"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind28", "properties": {"marker": "out-kind-28"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind29", "properties": {"marker": "out-kind-29"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopKind30", "properties": {"marker": "out-kind-30"}}, + {"start_id": "kind-center", "end_id": "kind-peer", "kind": "HopDisallowed", "properties": {"marker": "out-disallowed"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind01", "properties": {"marker": "in-kind-01"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind02", "properties": {"marker": "in-kind-02"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind03", "properties": {"marker": "in-kind-03"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind04", "properties": {"marker": "in-kind-04"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind05", "properties": {"marker": "in-kind-05"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind06", "properties": {"marker": "in-kind-06"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind07", "properties": {"marker": "in-kind-07"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind08", "properties": {"marker": "in-kind-08"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind09", "properties": {"marker": "in-kind-09"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind10", "properties": {"marker": "in-kind-10"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind11", "properties": {"marker": "in-kind-11"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind12", "properties": {"marker": "in-kind-12"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind13", "properties": {"marker": "in-kind-13"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind14", "properties": {"marker": "in-kind-14"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind15", "properties": {"marker": "in-kind-15"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind16", "properties": {"marker": "in-kind-16"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind17", "properties": {"marker": "in-kind-17"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind18", "properties": {"marker": "in-kind-18"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind19", "properties": {"marker": "in-kind-19"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind20", "properties": {"marker": "in-kind-20"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind21", "properties": {"marker": "in-kind-21"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind22", "properties": {"marker": "in-kind-22"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind23", "properties": {"marker": "in-kind-23"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind24", "properties": {"marker": "in-kind-24"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind25", "properties": {"marker": "in-kind-25"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind26", "properties": {"marker": "in-kind-26"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind27", "properties": {"marker": "in-kind-27"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind28", "properties": {"marker": "in-kind-28"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind29", "properties": {"marker": "in-kind-29"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopKind30", "properties": {"marker": "in-kind-30"}}, + {"start_id": "kind-peer", "end_id": "kind-center", "kind": "HopDisallowed", "properties": {"marker": "in-disallowed"}} + ] + }, + "variants": [ + {"name": "HOP-01 exact anchor zero fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) = $anchor RETURN r, e"}, "node_params": {"anchor": "out-zero"}, "assert": "empty"}, + {"name": "HOP-01 exact anchor one fanout full hydration", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) = $anchor RETURN r, e"}, "node_params": {"anchor": "out-one"}, "assert": {"keys": ["r", "e"], "row_count": 1, "node_id_set": ["out-one-target"], "relationship_records": [{"start": "out-one", "end": "out-one-target", "kind": "HopKind01", "props": {"marker": "out-one"}}]}}, + {"name": "HOP-01 one-element IN high fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(s) IN $anchors RETURN r, e"}, "node_list_params": {"anchors": ["out-high"]}, "assert": {"keys": ["r", "e"], "row_count": 8, "node_id_set": ["out-high-01", "out-high-02", "out-high-03", "out-high-04", "out-high-05", "out-high-06", "out-high-07", "out-high-08"]}}, + {"name": "HOP-02 exact anchor zero inbound fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) = $anchor RETURN r, s"}, "node_params": {"anchor": "in-zero"}, "assert": "empty"}, + {"name": "HOP-02 exact anchor one inbound fanout full hydration", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) = $anchor RETURN r, s"}, "node_params": {"anchor": "in-one"}, "assert": {"keys": ["r", "s"], "row_count": 1, "node_id_set": ["in-one-source"], "relationship_records": [{"start": "in-one-source", "end": "in-one", "kind": "HopKind01", "props": {"marker": "in-one"}}]}}, + {"name": "HOP-02 one-element IN high inbound fanout", "vars": {"query": "MATCH (s)-[r:HopKind01]->(e) WHERE id(e) IN $anchors RETURN r, s"}, "node_list_params": {"anchors": ["in-high"]}, "assert": {"keys": ["r", "s"], "row_count": 8, "node_id_set": ["in-high-01", "in-high-02", "in-high-03", "in-high-04", "in-high-05", "in-high-06", "in-high-07", "in-high-08"]}}, + {"name": "HOP-03 outbound two kinds", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02]->() WHERE id(s) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["out-kind-01", "out-kind-02"]}}, + {"name": "HOP-03 outbound five kinds", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05]->() WHERE id(s) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["out-kind-01", "out-kind-02", "out-kind-03", "out-kind-04", "out-kind-05"]}}, + {"name": "HOP-03 outbound nine kinds", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09]->() WHERE id(s) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["out-kind-01", "out-kind-02", "out-kind-03", "out-kind-04", "out-kind-05", "out-kind-06", "out-kind-07", "out-kind-08", "out-kind-09"]}}, + {"name": "HOP-03 outbound thirty kinds full direction", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09|HopKind10|HopKind11|HopKind12|HopKind13|HopKind14|HopKind15|HopKind16|HopKind17|HopKind18|HopKind19|HopKind20|HopKind21|HopKind22|HopKind23|HopKind24|HopKind25|HopKind26|HopKind27|HopKind28|HopKind29|HopKind30]->(e) WHERE id(s) = $anchor RETURN r, e"}, "node_params": {"anchor": "kind-center"}, "assert": {"keys": ["r", "e"], "row_count": 30, "contains_edge": {"start": "kind-center", "end": "kind-peer", "kind": "HopKind30", "props": {"marker": "out-kind-30"}}}}, + {"name": "HOP-03 inbound two kinds", "vars": {"query": "MATCH ()-[r:HopKind01|HopKind02]->(e) WHERE id(e) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["in-kind-01", "in-kind-02"]}}, + {"name": "HOP-03 inbound five kinds", "vars": {"query": "MATCH ()-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05]->(e) WHERE id(e) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["in-kind-01", "in-kind-02", "in-kind-03", "in-kind-04", "in-kind-05"]}}, + {"name": "HOP-03 inbound nine kinds", "vars": {"query": "MATCH ()-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09]->(e) WHERE id(e) = $anchor RETURN r.marker"}, "node_params": {"anchor": "kind-center"}, "assert": {"scalar_values": ["in-kind-01", "in-kind-02", "in-kind-03", "in-kind-04", "in-kind-05", "in-kind-06", "in-kind-07", "in-kind-08", "in-kind-09"]}}, + {"name": "HOP-03 inbound thirty kinds full direction", "vars": {"query": "MATCH (s)-[r:HopKind01|HopKind02|HopKind03|HopKind04|HopKind05|HopKind06|HopKind07|HopKind08|HopKind09|HopKind10|HopKind11|HopKind12|HopKind13|HopKind14|HopKind15|HopKind16|HopKind17|HopKind18|HopKind19|HopKind20|HopKind21|HopKind22|HopKind23|HopKind24|HopKind25|HopKind26|HopKind27|HopKind28|HopKind29|HopKind30]->(e) WHERE id(e) = $anchor RETURN r, s"}, "node_params": {"anchor": "kind-center"}, "assert": {"keys": ["r", "s"], "row_count": 30, "contains_edge": {"start": "kind-peer", "end": "kind-center", "kind": "HopKind30", "props": {"marker": "in-kind-30"}}}} + ] + }, + { + "name": "HOP-04 and HOP-05 endpoint kinds and ID constraints", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "root", "kinds": ["HopAnchor"], "properties": {"name": "root"}}, + {"id": "other-root", "kinds": ["HopAnchor"], "properties": {"name": "other-root"}}, + {"id": "typed-a", "kinds": ["HopEndA"], "properties": {"name": "typed-a"}}, + {"id": "typed-b", "kinds": ["HopEndB"], "properties": {"name": "typed-b"}}, + {"id": "typed-multi", "kinds": ["HopEndA", "HopEndB"], "properties": {"name": "typed-multi"}}, + {"id": "typed-wrong", "kinds": ["HopWrongEnd"], "properties": {"name": "typed-wrong"}}, + {"id": "id-a", "kinds": ["HopEndpoint"], "properties": {"name": "id-a"}}, + {"id": "id-b", "kinds": ["HopEndpoint"], "properties": {"name": "id-b"}}, + {"id": "id-decoy", "kinds": ["HopEndpoint"], "properties": {"name": "id-decoy"}} + ], + "edges": [ + {"start_id": "root", "end_id": "typed-a", "kind": "HopTypedEdge", "properties": {"marker": "typed-a"}}, + {"start_id": "root", "end_id": "typed-b", "kind": "HopTypedEdge", "properties": {"marker": "typed-b"}}, + {"start_id": "root", "end_id": "typed-multi", "kind": "HopTypedEdge", "properties": {"marker": "typed-multi"}}, + {"start_id": "root", "end_id": "typed-wrong", "kind": "HopTypedEdge", "properties": {"marker": "typed-wrong"}}, + {"start_id": "root", "end_id": "typed-a", "kind": "HopWrongEdge", "properties": {"marker": "wrong-edge"}}, + {"start_id": "typed-a", "end_id": "root", "kind": "HopTypedEdge", "properties": {"marker": "wrong-direction"}}, + {"start_id": "root", "end_id": "id-a", "kind": "HopIDEdge", "properties": {"marker": "id-a"}}, + {"start_id": "root", "end_id": "id-b", "kind": "HopIDEdge", "properties": {"marker": "id-b"}}, + {"start_id": "root", "end_id": "id-decoy", "kind": "HopIDEdge", "properties": {"marker": "id-decoy"}}, + {"start_id": "other-root", "end_id": "id-a", "kind": "HopIDEdge", "properties": {"marker": "wrong-root"}} + ] + }, + "variants": [ + {"name": "HOP-04 single endpoint kind includes multi-kind node", "vars": {"query": "MATCH (s)-[r:HopTypedEdge]->(e:HopEndA) WHERE id(s) = $root RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["typed-a", "typed-multi"]}}, + {"name": "HOP-04 endpoint kind disjunction excludes wrong kind edge and direction", "vars": {"query": "MATCH (s)-[r:HopTypedEdge]->(e) WHERE id(s) = $root AND (e:HopEndA OR e:HopEndB) RETURN r, e"}, "node_params": {"root": "root"}, "assert": {"keys": ["r", "e"], "row_count": 3, "node_id_set": ["typed-a", "typed-b", "typed-multi"]}}, + {"name": "HOP-05 empty end ID list", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"end_ids": []}, "assert": "empty"}, + {"name": "HOP-05 single end ID equality", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) = $end_id RETURN r.marker"}, "node_params": {"root": "root", "end_id": "id-a"}, "assert": {"scalar_values": ["id-a"]}}, + {"name": "HOP-05 duplicate end IDs do not duplicate relationship rows", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"end_ids": ["id-a", "id-a"]}, "assert": {"scalar_values": ["id-a"]}}, + {"name": "HOP-05 small matching end ID list", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"end_ids": ["id-a", "id-b"]}, "assert": {"scalar_values": ["id-a", "id-b"]}}, + {"name": "HOP-05 matching traversal anchor ID constraint", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(s) IN $allowed_roots AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"allowed_roots": ["root"], "end_ids": ["id-a", "id-b"]}, "assert": {"scalar_values": ["id-a", "id-b"]}}, + {"name": "HOP-05 contradictory traversal anchor ID constraint", "vars": {"query": "MATCH (s)-[r:HopIDEdge]->(e) WHERE id(s) = $root AND id(s) IN $allowed_roots AND id(e) IN $end_ids RETURN r.marker"}, "node_params": {"root": "root"}, "node_list_params": {"allowed_roots": ["other-root"], "end_ids": ["id-a", "id-b"]}, "assert": "empty"} + ] + }, + { + "name": "HOP-06 through HOP-08 scalar nested and collection endpoint predicates", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "root", "kinds": ["HopAnchor"], "properties": {"name": "root"}}, + {"id": "scalar-match", "kinds": ["HopPropertyEnd"], "properties": {"enabled": true, "score": 7, "value": "alpha", "isassignabletorole": "true"}}, + {"id": "scalar-false", "kinds": ["HopPropertyEnd"], "properties": {"enabled": false, "score": 0, "value": "", "isassignabletorole": "false"}}, + {"id": "scalar-missing", "kinds": ["HopPropertyEnd"], "properties": {}}, + {"id": "scalar-null", "kinds": ["HopPropertyEnd"], "properties": {"enabled": null, "score": null, "value": null, "isassignabletorole": null}}, + {"id": "nested-v2", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": true}}, + {"id": "nested-v1", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 1, "authorizedsignatures": 9, "authenticationenabled": true}}, + {"id": "nested-manager", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": true, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": true}}, + {"id": "nested-signatures", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 1, "authenticationenabled": true}}, + {"id": "nested-auth", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": false}}, + {"id": "nested-cross", "kinds": ["HopTemplate"], "properties": {"requiresmanagerapproval": false, "schemaversion": 1, "authorizedsignatures": 0, "authenticationenabled": false}}, + {"id": "nested-wrong-kind", "kinds": ["HopWrongEnd"], "properties": {"requiresmanagerapproval": false, "schemaversion": 2, "authorizedsignatures": 0, "authenticationenabled": true}}, + {"id": "collection-empty", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": []}}, + {"id": "collection-client", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": ["1.3.6.1.5.5.7.3.2"]}}, + {"id": "collection-scalar", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": true, "effectiveekus": ["other"]}}, + {"id": "collection-other", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": ["other"]}}, + {"id": "collection-missing", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false}}, + {"id": "collection-null", "kinds": ["HopCollectionEnd"], "properties": {"schannelauthenticationenabled": false, "effectiveekus": null}} + ], + "edges": [ + {"start_id": "root", "end_id": "scalar-match", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-match"}}, + {"start_id": "root", "end_id": "scalar-false", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-false"}}, + {"start_id": "root", "end_id": "scalar-missing", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-missing"}}, + {"start_id": "root", "end_id": "scalar-null", "kind": "HopPropertyEdge", "properties": {"marker": "scalar-null"}}, + {"start_id": "root", "end_id": "nested-v2", "kind": "HopNestedEdge", "properties": {"marker": "nested-v2"}}, + {"start_id": "root", "end_id": "nested-v1", "kind": "HopNestedEdge", "properties": {"marker": "nested-v1"}}, + {"start_id": "root", "end_id": "nested-manager", "kind": "HopNestedEdge", "properties": {"marker": "nested-manager"}}, + {"start_id": "root", "end_id": "nested-signatures", "kind": "HopNestedEdge", "properties": {"marker": "nested-signatures"}}, + {"start_id": "root", "end_id": "nested-auth", "kind": "HopNestedEdge", "properties": {"marker": "nested-auth"}}, + {"start_id": "root", "end_id": "nested-cross", "kind": "HopNestedEdge", "properties": {"marker": "nested-cross"}}, + {"start_id": "root", "end_id": "nested-wrong-kind", "kind": "HopNestedEdge", "properties": {"marker": "nested-wrong-kind"}}, + {"start_id": "root", "end_id": "nested-v2", "kind": "HopWrongEdge", "properties": {"marker": "nested-wrong-edge"}}, + {"start_id": "root", "end_id": "collection-empty", "kind": "HopCollectionEdge", "properties": {"marker": "collection-empty"}}, + {"start_id": "root", "end_id": "collection-client", "kind": "HopCollectionEdge", "properties": {"marker": "collection-client"}}, + {"start_id": "root", "end_id": "collection-scalar", "kind": "HopCollectionEdge", "properties": {"marker": "collection-scalar"}}, + {"start_id": "root", "end_id": "collection-other", "kind": "HopCollectionEdge", "properties": {"marker": "collection-other"}}, + {"start_id": "root", "end_id": "collection-missing", "kind": "HopCollectionEdge", "properties": {"marker": "collection-missing"}}, + {"start_id": "root", "end_id": "collection-null", "kind": "HopCollectionEdge", "properties": {"marker": "collection-null"}} + ] + }, + "variants": [ + {"name": "HOP-06 boolean true excludes false missing and null", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.enabled = true RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-match"]}}, + {"name": "HOP-06 boolean false", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.enabled = false RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-false"]}}, + {"name": "HOP-06 numeric equality", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.score = 7 RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-match"]}}, + {"name": "HOP-06 string equality", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.value = 'alpha' RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["scalar-match"]}}, + {"name": "HOP-06 production string true value", "vars": {"query": "MATCH (s)-[r:HopPropertyEdge]->(e) WHERE id(s) = $root AND e.isassignabletorole = 'true' RETURN r, e"}, "node_params": {"root": "root"}, "assert": {"keys": ["r", "e"], "row_count": 1, "node_id_set": ["scalar-match"]}}, + {"name": "HOP-07 exact branch-local nested truth table", "vars": {"query": "MATCH (s)-[r:HopNestedEdge]->(e:HopTemplate) WHERE id(s) = $root AND ((e.requiresmanagerapproval = false AND e.schemaversion > 1 AND e.authorizedsignatures = 0 AND e.authenticationenabled = true) OR (e.requiresmanagerapproval = false AND e.schemaversion = 1 AND e.authenticationenabled = true)) RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["nested-v2", "nested-v1"]}}, + {"name": "HOP-07 full directional hydration", "vars": {"query": "MATCH (s)-[r:HopNestedEdge]->(e:HopTemplate) WHERE id(s) = $root AND ((e.requiresmanagerapproval = false AND e.schemaversion > 1 AND e.authorizedsignatures = 0 AND e.authenticationenabled = true) OR (e.requiresmanagerapproval = false AND e.schemaversion = 1 AND e.authenticationenabled = true)) RETURN r, e"}, "node_params": {"root": "root"}, "assert": {"keys": ["r", "e"], "row_count": 2, "node_id_set": ["nested-v2", "nested-v1"]}}, + {"name": "HOP-08 empty collection", "vars": {"query": "MATCH (s)-[r:HopCollectionEdge]->(e) WHERE id(s) = $root AND size(e.effectiveekus) = 0 RETURN r.marker"}, "node_params": {"root": "root"}, "assert": {"scalar_values": ["collection-empty"]}}, + {"name": "HOP-08 value membership", "vars": {"query": "MATCH (s)-[r:HopCollectionEdge]->(e) WHERE id(s) = $root AND $eku IN e.effectiveekus RETURN r.marker"}, "node_params": {"root": "root"}, "params": {"eku": "1.3.6.1.5.5.7.3.2"}, "assert": {"scalar_values": ["collection-client"]}}, + {"name": "HOP-08 nested collection OR scalar predicate", "vars": {"query": "MATCH (s)-[r:HopCollectionEdge]->(e) WHERE id(s) = $root AND (e.schannelauthenticationenabled = true OR size(e.effectiveekus) = 0 OR $eku IN e.effectiveekus) RETURN r, e"}, "node_params": {"root": "root"}, "params": {"eku": "1.3.6.1.5.5.7.3.2"}, "assert": {"keys": ["r", "e"], "row_count": 3, "node_id_set": ["collection-empty", "collection-client", "collection-scalar"]}} + ] + }, + { + "name": "HOP-09 and HOP-10 two-sided sets and directional projections", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "s1", "kinds": ["HopProjectionStart"], "properties": {"name": "s1", "active": true}}, + {"id": "s2", "kinds": ["HopProjectionStart"], "properties": {"name": "s2", "active": true}}, + {"id": "s3", "kinds": ["HopProjectionStart"], "properties": {"name": "s3", "active": false}}, + {"id": "e1", "kinds": ["HopProjectionEnd"], "properties": {"name": "e1", "active": true}}, + {"id": "e2", "kinds": ["HopProjectionEnd"], "properties": {"name": "e2", "active": true}}, + {"id": "e3", "kinds": ["HopProjectionEnd"], "properties": {"name": "e3", "active": false}}, + {"id": "common", "kinds": ["HopProjectionStart", "HopProjectionEnd"], "properties": {"name": "common", "active": true}}, + {"id": "wrong-kind-start", "kinds": ["HopWrongStart"], "properties": {"name": "wrong-kind-start", "active": true}}, + {"id": "wrong-kind-end", "kinds": ["HopWrongEnd"], "properties": {"name": "wrong-kind-end", "active": true}} + ], + "edges": [ + {"start_id": "s1", "end_id": "e1", "kind": "HopSetEdge", "properties": {"marker": "s1-e1"}}, + {"start_id": "s1", "end_id": "e2", "kind": "HopSetEdge", "properties": {"marker": "s1-e2"}}, + {"start_id": "s2", "end_id": "e1", "kind": "HopSetEdge", "properties": {"marker": "s2-e1"}}, + {"start_id": "s2", "end_id": "e2", "kind": "HopSetEdge", "properties": {"marker": "s2-e2"}}, + {"start_id": "s3", "end_id": "e3", "kind": "HopSetEdge", "properties": {"marker": "s3-e3"}}, + {"start_id": "common", "end_id": "common", "kind": "HopSetEdge", "properties": {"marker": "common-self"}}, + {"start_id": "s1", "end_id": "e1", "kind": "HopWrongEdge", "properties": {"marker": "wrong-edge"}}, + {"start_id": "e1", "end_id": "s1", "kind": "HopSetEdge", "properties": {"marker": "wrong-direction"}}, + {"start_id": "s1", "end_id": "e1", "kind": "HopProjectionEdge", "properties": {"marker": "projection-out"}}, + {"start_id": "s2", "end_id": "e2", "kind": "HopProjectionEdge", "properties": {"marker": "projection-second"}}, + {"start_id": "s3", "end_id": "e3", "kind": "HopProjectionEdge", "properties": {"marker": "projection-inactive"}}, + {"start_id": "wrong-kind-start", "end_id": "e1", "kind": "HopProjectionEdge", "properties": {"marker": "projection-wrong-start"}}, + {"start_id": "s1", "end_id": "wrong-kind-end", "kind": "HopProjectionEdge", "properties": {"marker": "projection-wrong-end"}} + ] + }, + "variants": [ + {"name": "HOP-09 empty start list", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": [], "end_ids": ["e1", "e2"]}, "assert": "empty"}, + {"name": "HOP-09 empty end list", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["s1", "s2"], "end_ids": []}, "assert": "empty"}, + {"name": "HOP-09 singleton sets", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["s1"], "end_ids": ["e1"]}, "assert": {"scalar_values": ["s1-e1"]}}, + {"name": "HOP-09 duplicate IDs do not duplicate rows", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["s1", "s1"], "end_ids": ["e1", "e1"]}, "assert": {"scalar_values": ["s1-e1"]}}, + {"name": "HOP-09 dense small bipartite sets", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r, e"}, "node_list_params": {"start_ids": ["s1", "s2"], "end_ids": ["e1", "e2"]}, "assert": {"keys": ["r", "e"], "row_count": 4, "node_id_set": ["e1", "e2"]}}, + {"name": "HOP-09 overlapping start and end sets retain self edge", "vars": {"query": "MATCH (s)-[r:HopSetEdge]->(e) WHERE id(s) IN $start_ids AND id(e) IN $end_ids RETURN r.marker"}, "node_list_params": {"start_ids": ["common"], "end_ids": ["common"]}, "assert": {"scalar_values": ["common-self"]}}, + {"name": "HOP-10 outbound full relationship and endpoint", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN r, e"}, "node_params": {"start_id": "s1"}, "assert": {"keys": ["r", "e"], "row_count": 1, "node_id_set": ["e1"], "relationship_records": [{"start": "s1", "end": "e1", "kind": "HopProjectionEdge", "props": {"marker": "projection-out"}}]}}, + {"name": "HOP-10 outbound endpoint node projection", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN e"}, "node_params": {"start_id": "s1"}, "assert": {"node_id_set": ["e1"]}}, + {"name": "HOP-10 outbound endpoint ID projection", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN id(e)"}, "node_params": {"start_id": "s1"}, "assert": {"keys": ["id(e)"], "row_count": 1}}, + {"name": "HOP-10 relationship-only projection", "vars": {"query": "MATCH (s)-[r:HopProjectionEdge]->(e:HopProjectionEnd) WHERE id(s) = $start_id AND e.active = true RETURN r"}, "node_params": {"start_id": "s1"}, "assert": {"relationship_records": [{"start": "s1", "end": "e1", "kind": "HopProjectionEdge", "props": {"marker": "projection-out"}}]}}, + {"name": "HOP-10 inbound full relationship and endpoint", "vars": {"query": "MATCH (s:HopProjectionStart)-[r:HopProjectionEdge]->(e) WHERE id(e) = $end_id AND s.active = true RETURN r, s"}, "node_params": {"end_id": "e1"}, "assert": {"keys": ["r", "s"], "row_count": 1, "node_id_set": ["s1"], "relationship_records": [{"start": "s1", "end": "e1", "kind": "HopProjectionEdge", "props": {"marker": "projection-out"}}]}}, + {"name": "HOP-10 inbound endpoint node projection", "vars": {"query": "MATCH (s:HopProjectionStart)-[r:HopProjectionEdge]->(e) WHERE id(e) = $end_id AND s.active = true RETURN s"}, "node_params": {"end_id": "e1"}, "assert": {"node_id_set": ["s1"]}}, + {"name": "HOP-10 inbound endpoint ID projection", "vars": {"query": "MATCH (s:HopProjectionStart)-[r:HopProjectionEdge]->(e) WHERE id(e) = $end_id AND s.active = true RETURN id(s)"}, "node_params": {"end_id": "e1"}, "assert": {"keys": ["id(s)"], "row_count": 1}} + ] + } + ] +} diff --git a/integration/testdata/templates/post_processing_shapes.json b/integration/testdata/templates/post_processing_shapes.json new file mode 100644 index 00000000..0076e1cd --- /dev/null +++ b/integration/testdata/templates/post_processing_shapes.json @@ -0,0 +1,143 @@ +{ + "families": [ + { + "name": "LOGIC-03 scoped kind negation with null-aware age predicate", + "template": "MATCH (n) WHERE NOT n:LogicProtected AND (n.lastseen IS NULL OR datetime(n.lastseen) < datetime($threshold)) RETURN n", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "missing", "kinds": ["LogicCandidate"], "properties": {"name": "missing"}}, + {"id": "null", "kinds": ["LogicCandidate"], "properties": {"name": "null", "lastseen": null}}, + {"id": "older", "kinds": ["LogicCandidate"], "properties": {"name": "older", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "equal", "kinds": ["LogicCandidate"], "properties": {"name": "equal", "lastseen": "2026-01-03T00:00:00Z"}}, + {"id": "newer", "kinds": ["LogicCandidate"], "properties": {"name": "newer", "lastseen": "2026-01-04T00:00:00Z"}}, + {"id": "protected-missing", "kinds": ["LogicProtected"], "properties": {"name": "protected-missing"}}, + {"id": "protected-null", "kinds": ["LogicProtected"], "properties": {"name": "protected-null", "lastseen": null}}, + {"id": "protected-older", "kinds": ["LogicProtected"], "properties": {"name": "protected-older", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "multi-kind-protected", "kinds": ["LogicCandidate", "LogicProtected"], "properties": {"name": "multi-kind-protected", "lastseen": "2026-01-02T00:00:00Z"}} + ] + }, + "variants": [ + { + "name": "missing null older equal newer and protected truth table", + "assert": {"node_id_set": ["missing", "null", "older"]} + } + ] + }, + { + "name": "PRUNE-01 stale relationship selection with protected kinds", + "template": "MATCH ()-[r]->() WHERE {{excluded}} AND datetime(r.lastseen) < datetime($threshold) RETURN {{projection}}", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "a", "kinds": ["PruneEndpoint"], "properties": {"name": "a"}}, + {"id": "b", "kinds": ["PruneEndpoint"], "properties": {"name": "b"}}, + {"id": "b-equal", "kinds": ["PruneEndpoint"], "properties": {"name": "b-equal"}}, + {"id": "b-new", "kinds": ["PruneEndpoint"], "properties": {"name": "b-new"}}, + {"id": "b-missing", "kinds": ["PruneEndpoint"], "properties": {"name": "b-missing"}}, + {"id": "b-null", "kinds": ["PruneEndpoint"], "properties": {"name": "b-null"}} + ], + "edges": [ + {"start_id": "a", "end_id": "b", "kind": "CandidateRel", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "candidate-old"}}, + {"start_id": "a", "end_id": "b-equal", "kind": "CandidateRel", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "candidate-equal"}}, + {"start_id": "a", "end_id": "b-new", "kind": "CandidateRel", "properties": {"lastseen": "2026-01-04T00:00:00Z", "marker": "candidate-new"}}, + {"start_id": "a", "end_id": "b-missing", "kind": "CandidateRel", "properties": {"marker": "candidate-missing"}}, + {"start_id": "a", "end_id": "b-null", "kind": "CandidateRel", "properties": {"lastseen": null, "marker": "candidate-null"}}, + {"start_id": "a", "end_id": "b", "kind": "HasSession", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "session-old"}}, + {"start_id": "a", "end_id": "b", "kind": "MetaIncludes", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "meta-includes-old"}} + ] + }, + "variants": [ + { + "name": "one excluded kind leaves other old kinds eligible", + "vars": {"excluded": "NOT r:HasSession", "projection": "r.marker"}, + "assert": {"scalar_values": ["candidate-old", "meta-includes-old"]} + }, + { + "name": "several excluded kinds select only old candidate relationship IDs", + "vars": {"excluded": "NOT (r:HasSession OR r:MetaIncludes)", "projection": "id(r)"}, + "assert": {"keys": ["id(r)"], "row_count": 1} + }, + { + "name": "several excluded kinds exact old equal new missing and null matrix", + "vars": {"excluded": "NOT (r:HasSession OR r:MetaIncludes)", "projection": "r.marker"}, + "assert": {"scalar_values": ["candidate-old"]} + } + ] + }, + { + "name": "PRUNE-02 stale or unobserved HasSession selection", + "template": "MATCH ()-[r:HasSession]->() WHERE r.lastseen IS NULL OR datetime(r.lastseen) < datetime($threshold) RETURN {{projection}}", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "a", "kinds": ["PruneEndpoint"], "properties": {"name": "a"}}, + {"id": "b", "kinds": ["PruneEndpoint"], "properties": {"name": "b"}}, + {"id": "b-null", "kinds": ["PruneEndpoint"], "properties": {"name": "b-null"}}, + {"id": "b-old", "kinds": ["PruneEndpoint"], "properties": {"name": "b-old"}}, + {"id": "b-equal", "kinds": ["PruneEndpoint"], "properties": {"name": "b-equal"}}, + {"id": "b-new", "kinds": ["PruneEndpoint"], "properties": {"name": "b-new"}} + ], + "edges": [ + {"start_id": "a", "end_id": "b", "kind": "HasSession", "properties": {"marker": "session-missing"}}, + {"start_id": "a", "end_id": "b-null", "kind": "HasSession", "properties": {"lastseen": null, "marker": "session-null"}}, + {"start_id": "a", "end_id": "b-old", "kind": "HasSession", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "session-old"}}, + {"start_id": "a", "end_id": "b-equal", "kind": "HasSession", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "session-equal"}}, + {"start_id": "a", "end_id": "b-new", "kind": "HasSession", "properties": {"lastseen": "2026-01-04T00:00:00Z", "marker": "session-new"}}, + {"start_id": "a", "end_id": "b", "kind": "OtherSession", "properties": {"marker": "wrong-kind-missing"}}, + {"start_id": "a", "end_id": "b-old", "kind": "OtherSession", "properties": {"lastseen": "2026-01-02T00:00:00Z", "marker": "wrong-kind-old"}} + ] + }, + "variants": [ + {"name": "returns missing null and old HasSession relationship IDs", "vars": {"projection": "id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 3}}, + {"name": "exact missing null old equal newer and wrong-kind matrix", "vars": {"projection": "r.marker"}, "assert": {"scalar_values": ["session-missing", "session-null", "session-old"]}} + ] + }, + { + "name": "PRUNE-03 stale or unobserved node selection with protected kinds", + "template": "MATCH (n) WHERE NOT (n:Domain OR n:Tenant OR n:Meta OR n:MetaIncludes OR n:MigrationData) AND (n.lastseen IS NULL OR datetime(n.lastseen) < datetime($threshold)) RETURN {{projection}}", + "params": {"threshold": "2026-01-03T00:00:00Z"}, + "fixture": { + "nodes": [ + {"id": "candidate-missing", "kinds": ["CandidateNode"], "properties": {"name": "candidate-missing"}}, + {"id": "candidate-null", "kinds": ["CandidateNode"], "properties": {"name": "candidate-null", "lastseen": null}}, + {"id": "candidate-old", "kinds": ["CandidateNode"], "properties": {"name": "candidate-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "candidate-equal", "kinds": ["CandidateNode"], "properties": {"name": "candidate-equal", "lastseen": "2026-01-03T00:00:00Z"}}, + {"id": "candidate-new", "kinds": ["CandidateNode"], "properties": {"name": "candidate-new", "lastseen": "2026-01-04T00:00:00Z"}}, + {"id": "domain-missing", "kinds": ["Domain"], "properties": {"name": "domain-missing"}}, + {"id": "tenant-old", "kinds": ["Tenant"], "properties": {"name": "tenant-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "meta-null", "kinds": ["Meta"], "properties": {"name": "meta-null", "lastseen": null}}, + {"id": "meta-includes-old", "kinds": ["MetaIncludes"], "properties": {"name": "meta-includes-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "migration-old", "kinds": ["MigrationData"], "properties": {"name": "migration-old", "lastseen": "2026-01-02T00:00:00Z"}}, + {"id": "multi-kind-protected", "kinds": ["CandidateNode", "Domain"], "properties": {"name": "multi-kind-protected", "lastseen": "2026-01-02T00:00:00Z"}} + ] + }, + "variants": [ + {"name": "returns only candidate missing null and old node IDs", "vars": {"projection": "id(n)"}, "assert": {"keys": ["id(n)"], "row_count": 3}}, + {"name": "exact protected multi-kind and age matrix", "vars": {"projection": "n"}, "assert": {"node_id_set": ["candidate-missing", "candidate-null", "candidate-old"]}} + ] + }, + { + "name": "PRUNE-04 orphan SID node selection", + "template": "MATCH (n) WHERE NOT (n:Domain OR n:Tenant OR n:Meta OR n:MetaIncludes OR n:MigrationData) AND n.name IS NULL AND n.objectid STARTS WITH $sid_prefix RETURN {{projection}}", + "params": {"sid_prefix": "S-1-5"}, + "fixture": { + "nodes": [ + {"id": "missing-name", "kinds": ["CandidateNode"], "properties": {"objectid": "S-1-5-100"}}, + {"id": "null-name", "kinds": ["CandidateNode"], "properties": {"name": null, "objectid": "S-1-5-101"}}, + {"id": "empty-name", "kinds": ["CandidateNode"], "properties": {"name": "", "objectid": "S-1-5-102"}}, + {"id": "named", "kinds": ["CandidateNode"], "properties": {"name": "named", "objectid": "S-1-5-103"}}, + {"id": "wrong-prefix", "kinds": ["CandidateNode"], "properties": {"objectid": "X-1-5-104"}}, + {"id": "missing-objectid", "kinds": ["CandidateNode"], "properties": {}}, + {"id": "domain", "kinds": ["Domain"], "properties": {"objectid": "S-1-5-105"}}, + {"id": "tenant", "kinds": ["Tenant"], "properties": {"name": null, "objectid": "S-1-5-106"}}, + {"id": "multi-kind-protected", "kinds": ["CandidateNode", "MigrationData"], "properties": {"objectid": "S-1-5-107"}} + ] + }, + "variants": [ + {"name": "returns only missing and null name SID node IDs", "vars": {"projection": "id(n)"}, "assert": {"keys": ["id(n)"], "row_count": 2}}, + {"name": "exact prefix name and protected-kind matrix", "vars": {"projection": "n"}, "assert": {"node_id_set": ["missing-name", "null-name"]}} + ] + } + ] +} diff --git a/integration/testdata/templates/reconciliation_shapes.json b/integration/testdata/templates/reconciliation_shapes.json new file mode 100644 index 00000000..d11ca7e6 --- /dev/null +++ b/integration/testdata/templates/reconciliation_shapes.json @@ -0,0 +1,815 @@ +{ + "families": [ + { + "name": "LOGIC-01 branch-local relationship kinds", + "template": "MATCH (s:LogicDomain)-[r]->(e:LogicDomain) WHERE (id(s) = $forward_start AND id(e) = $forward_end AND r:LogicKindA) OR (id(s) = $forward_end AND id(e) = $forward_start AND r:LogicKindB) RETURN r.marker", + "node_params": {"forward_start": "forward", "forward_end": "reverse"}, + "fixture": { + "nodes": [ + {"id": "forward", "kinds": ["LogicDomain"], "properties": {"name": "forward"}}, + {"id": "reverse", "kinds": ["LogicDomain"], "properties": {"name": "reverse"}} + ], + "edges": [ + {"start_id": "forward", "end_id": "reverse", "kind": "LogicKindA", "properties": {"marker": "valid-forward"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "LogicKindB", "properties": {"marker": "valid-reverse"}}, + {"start_id": "forward", "end_id": "reverse", "kind": "LogicKindB", "properties": {"marker": "invalid-forward-kind"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "LogicKindA", "properties": {"marker": "invalid-reverse-kind"}} + ] + }, + "variants": [ + { + "name": "both valid combinations exclude both invalid cross-combinations", + "assert": {"scalar_values": ["valid-forward", "valid-reverse"]} + } + ] + }, + { + "name": "LOGIC-02 cross-binding temporal disjunction", + "template": "MATCH (s:LogicDomain)-[r:LogicStaleTrust]->(e:LogicDomain) WHERE r.lastseen < s.lastcollected OR r.lastseen < e.lastcollected RETURN r.marker", + "fixture": { + "nodes": [ + {"id": "early-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "early-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "equal-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "equal-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "late-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-newer", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-missing-relationship", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-null-relationship", "kinds": ["LogicDomain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "missing-a", "kinds": ["LogicDomain"], "properties": {}}, + {"id": "missing-b", "kinds": ["LogicDomain"], "properties": {}}, + {"id": "null-a", "kinds": ["LogicDomain"], "properties": {"lastcollected": null}}, + {"id": "null-b", "kinds": ["LogicDomain"], "properties": {"lastcollected": null}} + ], + "edges": [ + {"start_id": "late-a", "end_id": "early-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "older-start-only"}}, + {"start_id": "early-a", "end_id": "late-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "older-end-only"}}, + {"start_id": "late-a", "end_id": "late-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "older-both"}}, + {"start_id": "equal-a", "end_id": "equal-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "equal"}}, + {"start_id": "late-a", "end_id": "late-b-newer", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-05T00:00:00Z", "marker": "newer"}}, + {"start_id": "late-a", "end_id": "late-b-missing-relationship", "kind": "LogicStaleTrust", "properties": {"marker": "missing-relationship"}}, + {"start_id": "late-a", "end_id": "late-b-null-relationship", "kind": "LogicStaleTrust", "properties": {"lastseen": null, "marker": "null-relationship"}}, + {"start_id": "missing-a", "end_id": "late-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "missing-start-valid-end"}}, + {"start_id": "null-a", "end_id": "late-a", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "null-start-valid-end"}}, + {"start_id": "late-a", "end_id": "missing-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "missing-end-valid-start"}}, + {"start_id": "late-a", "end_id": "null-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "null-end-valid-start"}}, + {"start_id": "missing-a", "end_id": "null-b", "kind": "LogicStaleTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "missing-and-null-endpoints"}} + ] + }, + "variants": [ + { + "name": "older equal newer missing and null truth table", + "assert": { + "scalar_values": [ + "older-start-only", + "older-end-only", + "older-both", + "missing-start-valid-end", + "null-start-valid-end", + "missing-end-valid-start", + "null-end-valid-start" + ] + } + } + ] + }, + { + "name": "LOGIC-04 filtered relationship delete", + "template": "MATCH (s:LogicDeleteSource)-[r:LogicDeleteEdge]->(e:LogicDeleteTarget) WHERE e.objectid = $object_id AND r.shoulddelete = $should_delete DELETE r", + "params": {"object_id": "delete-edge", "should_delete": true}, + "fixture": { + "nodes": [ + {"id": "source", "kinds": ["LogicDeleteSource"], "properties": {"name": "source"}}, + {"id": "source-property-decoy", "kinds": ["LogicDeleteSource"], "properties": {"name": "source-property-decoy"}}, + {"id": "target", "kinds": ["LogicDeleteTarget"], "properties": {"objectid": "delete-edge"}}, + {"id": "decoy-target", "kinds": ["LogicDeleteTarget"], "properties": {"objectid": "keep-edge"}} + ], + "edges": [ + {"start_id": "source", "end_id": "target", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": true, "marker": "delete"}}, + {"start_id": "source-property-decoy", "end_id": "target", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": false, "marker": "property-decoy"}}, + {"start_id": "source", "end_id": "target", "kind": "LogicSurvivorEdge", "properties": {"shoulddelete": true, "marker": "kind-decoy"}}, + {"start_id": "source", "end_id": "decoy-target", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": true, "marker": "endpoint-decoy"}}, + {"start_id": "target", "end_id": "source", "kind": "LogicDeleteEdge", "properties": {"shoulddelete": true, "marker": "direction-decoy"}} + ] + }, + "variants": [ + { + "name": "selected relationship binding is deleted and every decoy survives", + "assert": "no_error", + "post_assertions": [ + { + "cypher": "MATCH ()-[r]->() RETURN r", + "assert": { + "relationship_records": [ + {"start": "source-property-decoy", "end": "target", "kind": "LogicDeleteEdge", "props": {"shoulddelete": false, "marker": "property-decoy"}}, + {"start": "source", "end": "target", "kind": "LogicSurvivorEdge", "props": {"shoulddelete": true, "marker": "kind-decoy"}}, + {"start": "source", "end": "decoy-target", "kind": "LogicDeleteEdge", "props": {"shoulddelete": true, "marker": "endpoint-decoy"}}, + {"start": "target", "end": "source", "kind": "LogicDeleteEdge", "props": {"shoulddelete": true, "marker": "direction-decoy"}} + ] + } + } + ] + } + ] + }, + { + "name": "LOGIC-04 filtered detach node delete", + "template": "MATCH (n:LogicDeleteNode) WHERE n.objectid = $object_id DETACH DELETE n", + "params": {"object_id": "delete-node"}, + "fixture": { + "nodes": [ + {"id": "victim", "kinds": ["LogicDeleteNode"], "properties": {"objectid": "delete-node"}}, + {"id": "survivor", "kinds": ["LogicSurvivorNode"], "properties": {"objectid": "keep-node"}}, + {"id": "kind-decoy", "kinds": ["LogicSurvivorNode"], "properties": {"objectid": "delete-node"}}, + {"id": "property-decoy", "kinds": ["LogicDeleteNode"], "properties": {"objectid": "keep-node"}} + ], + "edges": [ + {"start_id": "survivor", "end_id": "victim", "kind": "LogicIncident", "properties": {"marker": "inbound"}}, + {"start_id": "victim", "end_id": "survivor", "kind": "LogicIncident", "properties": {"marker": "outbound"}}, + {"start_id": "victim", "end_id": "victim", "kind": "LogicIncident", "properties": {"marker": "self"}}, + {"start_id": "survivor", "end_id": "property-decoy", "kind": "LogicSurvivorEdge", "properties": {"marker": "survives"}} + ] + }, + "variants": [ + { + "name": "selected node binding cascades only its incident relationships", + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["survivor", "kind-decoy", "property-decoy"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r", "assert": {"relationship_records": [{"start": "survivor", "end": "property-decoy", "kind": "LogicSurvivorEdge", "props": {"marker": "survives"}}]}} + ] + } + ] + }, + { + "name": "LOGIC-05 directional projection order", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "start", "kinds": ["LogicProjectionStart"], "properties": {"name": "start"}}, + {"id": "end", "kinds": ["LogicProjectionEnd", "LogicProjectionEntity"], "properties": {"name": "end"}} + ], + "edges": [ + {"start_id": "start", "end_id": "end", "kind": "LogicProjectionEdge", "properties": {"marker": "projection"}} + ] + }, + "variants": [ + { + "name": "full opposite node plus relationship", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->(e) RETURN r, e"}, + "assert": {"keys": ["r", "e"], "row_count": 1, "contains_edge": {"start": "start", "end": "end", "kind": "LogicProjectionEdge", "props": {"marker": "projection"}}} + }, + { + "name": "opposite ID kinds and relationship ID kind", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->(e) RETURN id(e), labels(e), id(r), type(r)"}, + "assert": {"keys": ["id(e)", "labels(e)", "id(r)", "type(r)"], "row_count": 1} + }, + { + "name": "start relationship end triple", + "vars": {"query": "MATCH (s)-[r:LogicProjectionEdge]->(e) RETURN s, r, e"}, + "assert": {"keys": ["s", "r", "e"], "row_count": 1, "contains_edge": {"start": "start", "end": "end", "kind": "LogicProjectionEdge"}} + }, + { + "name": "relationship ID only", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->() RETURN id(r)"}, + "assert": {"keys": ["id(r)"], "row_count": 1} + }, + { + "name": "full relationship", + "vars": {"query": "MATCH ()-[r:LogicProjectionEdge]->() RETURN r"}, + "assert": {"keys": ["r"], "row_count": 1, "contains_edge": {"start": "start", "end": "end", "kind": "LogicProjectionEdge", "props": {"marker": "projection"}}} + } + ] + }, + { + "name": "REC-01 inbound structure reconciliation delete", + "template": "MATCH ()-[r:{{relationship_kinds}}]->(e:ADEntity) WHERE e.objectid = $object_id DELETE r", + "fixture": { + "nodes": [ + {"id": "source-a", "kinds": ["Source"], "properties": {"name": "source-a"}}, + {"id": "source-b", "kinds": ["Source"], "properties": {"name": "source-b"}}, + {"id": "target", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "target-in"}}, + {"id": "one-target", "kinds": ["ADEntity"], "properties": {"objectid": "one-in"}}, + {"id": "wrong-kind", "kinds": ["OtherEntity"], "properties": {"objectid": "target-in"}}, + {"id": "wrong-property", "kinds": ["ADEntity"], "properties": {"objectid": "other-in"}} + ], + "edges": [ + {"start_id": "source-a", "end_id": "target", "kind": "RecKind01", "properties": {"marker": "k01-a"}}, + {"start_id": "source-b", "end_id": "target", "kind": "RecKind01", "properties": {"marker": "k01-b"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind02", "properties": {"marker": "k02"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind09", "properties": {"marker": "k09"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind30", "properties": {"marker": "k30"}}, + {"start_id": "source-a", "end_id": "target", "kind": "RecKind31", "properties": {"marker": "wrong-edge"}}, + {"start_id": "source-a", "end_id": "wrong-kind", "kind": "RecKind01", "properties": {"marker": "wrong-end-kind"}}, + {"start_id": "source-a", "end_id": "wrong-property", "kind": "RecKind01", "properties": {"marker": "wrong-end-property"}}, + {"start_id": "target", "end_id": "source-a", "kind": "RecKind01", "properties": {"marker": "wrong-direction"}}, + {"start_id": "source-a", "end_id": "one-target", "kind": "RecKind01", "properties": {"marker": "one-match"}} + ] + }, + "variants": [ + { + "name": "one kind deletes many exact matches", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k02", "k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "two kinds preserve nonselected kinds and decoys", + "vars": {"relationship_kinds": "RecKind01|RecKind02"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "nine kinds include the ninth kind", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "thirty kinds include the thirtieth kind", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30"}, + "params": {"object_id": "target-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + }, + { + "name": "single exact match", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "one-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k01-a", "k01-b", "k02", "k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction"]}}] + }, + { + "name": "zero matches preserve every relationship", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "missing-in"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["k01-a", "k01-b", "k02", "k09", "k30", "wrong-edge", "wrong-end-kind", "wrong-end-property", "wrong-direction", "one-match"]}}] + } + ] + }, + { + "name": "REC-01 and REC-02 thirty-kind schema registry", + "template": "MATCH ()-[r]->() RETURN count(r)", + "fixture": { + "nodes": [ + {"id": "start", "kinds": ["RegistryStart"], "properties": {"name": "start"}}, + {"id": "end", "kinds": ["RegistryEnd"], "properties": {"name": "end"}} + ], + "edges": [ + {"start_id": "start", "end_id": "end", "kind": "RecKind03"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind04"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind05"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind06"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind07"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind08"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind10"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind11"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind12"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind13"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind14"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind15"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind16"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind17"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind18"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind19"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind20"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind21"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind22"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind23"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind24"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind25"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind26"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind27"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind28"}, + {"start_id": "start", "end_id": "end", "kind": "RecKind29"} + ] + }, + "variants": [ + {"name": "register every nonmatching relationship kind", "assert": {"exact_int": 26}} + ] + }, + { + "name": "REC-02 outbound structure reconciliation delete", + "template": "MATCH (s:ADEntity)-[r:{{relationship_kinds}}]->() WHERE s.objectid = $object_id DELETE r", + "fixture": { + "nodes": [ + {"id": "target", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "target-out"}}, + {"id": "one-target", "kinds": ["ADEntity"], "properties": {"objectid": "one-out"}}, + {"id": "wrong-kind", "kinds": ["OtherEntity"], "properties": {"objectid": "target-out"}}, + {"id": "wrong-property", "kinds": ["ADEntity"], "properties": {"objectid": "other-out"}}, + {"id": "end-a", "kinds": ["Destination"], "properties": {"name": "end-a"}}, + {"id": "end-b", "kinds": ["Destination"], "properties": {"name": "end-b"}} + ], + "edges": [ + {"start_id": "target", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-k01-a"}}, + {"start_id": "target", "end_id": "end-b", "kind": "RecKind01", "properties": {"marker": "out-k01-b"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind02", "properties": {"marker": "out-k02"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind09", "properties": {"marker": "out-k09"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind30", "properties": {"marker": "out-k30"}}, + {"start_id": "target", "end_id": "end-a", "kind": "RecKind31", "properties": {"marker": "out-wrong-edge"}}, + {"start_id": "wrong-kind", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-wrong-start-kind"}}, + {"start_id": "wrong-property", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-wrong-start-property"}}, + {"start_id": "end-a", "end_id": "target", "kind": "RecKind01", "properties": {"marker": "out-wrong-direction"}}, + {"start_id": "one-target", "end_id": "end-a", "kind": "RecKind01", "properties": {"marker": "out-one-match"}} + ] + }, + "variants": [ + { + "name": "one kind deletes many exact outbound matches", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k02", "out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "two outbound kinds", + "vars": {"relationship_kinds": "RecKind01|RecKind02"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "nine outbound kinds", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "thirty outbound kinds", + "vars": {"relationship_kinds": "RecKind01|RecKind02|RecKind03|RecKind04|RecKind05|RecKind06|RecKind07|RecKind08|RecKind09|RecKind10|RecKind11|RecKind12|RecKind13|RecKind14|RecKind15|RecKind16|RecKind17|RecKind18|RecKind19|RecKind20|RecKind21|RecKind22|RecKind23|RecKind24|RecKind25|RecKind26|RecKind27|RecKind28|RecKind29|RecKind30"}, + "params": {"object_id": "target-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + }, + { + "name": "single outbound exact match", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "one-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k01-a", "out-k01-b", "out-k02", "out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction"]}}] + }, + { + "name": "zero outbound matches preserve every relationship", + "vars": {"relationship_kinds": "RecKind01"}, + "params": {"object_id": "missing-out"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["out-k01-a", "out-k01-b", "out-k02", "out-k09", "out-k30", "out-wrong-edge", "out-wrong-start-kind", "out-wrong-start-property", "out-wrong-direction", "out-one-match"]}}] + } + ] + }, + { + "name": "REC-03 primary group reconciliation delete", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "user", "kinds": ["ADEntity", "User"], "properties": {"objectid": "user-id"}}, + {"id": "user-opposite", "kinds": ["ADEntity", "User"], "properties": {"objectid": "user-opposite-id"}}, + {"id": "user-missing", "kinds": ["ADEntity", "User"], "properties": {"objectid": "user-missing-id"}}, + {"id": "group", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "group-id"}}, + {"id": "computer", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "computer-id"}}, + {"id": "other", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "other-id"}}, + {"id": "out-opposite-target", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "out-opposite-id"}}, + {"id": "out-missing-target", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "out-missing-id"}} + ], + "edges": [ + {"start_id": "user", "end_id": "group", "kind": "MemberOf", "properties": {"isprimarygroup": false, "marker": "in-false"}}, + {"start_id": "user-opposite", "end_id": "group", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "in-opposite"}}, + {"start_id": "user-missing", "end_id": "group", "kind": "MemberOf", "properties": {"marker": "in-missing"}}, + {"start_id": "user", "end_id": "group", "kind": "OtherMembership", "properties": {"isprimarygroup": false, "marker": "in-wrong-kind"}}, + {"start_id": "computer", "end_id": "group", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "out-true-a"}}, + {"start_id": "computer", "end_id": "other", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "out-true-b"}}, + {"start_id": "computer", "end_id": "out-opposite-target", "kind": "MemberOf", "properties": {"isprimarygroup": false, "marker": "out-opposite"}}, + {"start_id": "computer", "end_id": "out-missing-target", "kind": "MemberOf", "properties": {"marker": "out-missing"}}, + {"start_id": "computer", "end_id": "group", "kind": "OtherMembership", "properties": {"isprimarygroup": true, "marker": "out-wrong-kind"}}, + {"start_id": "group", "end_id": "computer", "kind": "MemberOf", "properties": {"isprimarygroup": true, "marker": "out-wrong-direction"}} + ] + }, + "variants": [ + { + "name": "inbound false deletes only the matching MemberOf edge", + "vars": {"query": "MATCH ()-[r:MemberOf]->(e:Group) WHERE e.objectid = $object_id AND r.isprimarygroup = $flag DELETE r"}, + "params": {"object_id": "group-id", "flag": false}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["in-opposite", "in-missing", "in-wrong-kind", "out-true-a", "out-true-b", "out-opposite", "out-missing", "out-wrong-kind", "out-wrong-direction"]}}] + }, + { + "name": "outbound true deletes all exact MemberOf edges", + "vars": {"query": "MATCH (s:Computer)-[r:MemberOf]->() WHERE s.objectid = $object_id AND r.isprimarygroup = $flag DELETE r"}, + "params": {"object_id": "computer-id", "flag": true}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["in-false", "in-opposite", "in-missing", "in-wrong-kind", "out-opposite", "out-missing", "out-wrong-kind", "out-wrong-direction"]}}] + } + ] + }, + { + "name": "REC-04 endpoint object ID list reconciliation delete", + "template": "MATCH ()-[r:{{relationship_kind}}]->(e:{{entity_kind}}) WHERE e.objectid IN $object_ids DELETE r", + "fixture": { + "nodes": [ + {"id": "source", "kinds": ["Source"], "properties": {"name": "source"}}, + {"id": "source-duplicate", "kinds": ["Source"], "properties": {"name": "source-duplicate"}}, + {"id": "ad-a", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "ad-a"}}, + {"id": "ad-b", "kinds": ["ADEntity", "User"], "properties": {"objectid": "ad-b"}}, + {"id": "az-a", "kinds": ["AZEntity", "AZUser"], "properties": {"objectid": "az-a"}}, + {"id": "az-b", "kinds": ["AZEntity", "AZGroup"], "properties": {"objectid": "az-b"}}, + {"id": "wrong-kind", "kinds": ["OtherEntity"], "properties": {"objectid": "ad-a"}}, + {"id": "wrong-property", "kinds": ["ADEntity"], "properties": {"objectid": "other"}} + ], + "edges": [ + {"start_id": "source", "end_id": "ad-a", "kind": "ADReconcile", "properties": {"marker": "ad-a-1"}}, + {"start_id": "source-duplicate", "end_id": "ad-a", "kind": "ADReconcile", "properties": {"marker": "ad-a-2"}}, + {"start_id": "source", "end_id": "ad-b", "kind": "ADReconcile", "properties": {"marker": "ad-b"}}, + {"start_id": "source", "end_id": "az-a", "kind": "AZReconcile", "properties": {"marker": "az-a"}}, + {"start_id": "source", "end_id": "az-b", "kind": "AZReconcile", "properties": {"marker": "az-b"}}, + {"start_id": "source", "end_id": "wrong-kind", "kind": "ADReconcile", "properties": {"marker": "wrong-kind-end"}}, + {"start_id": "source", "end_id": "wrong-property", "kind": "ADReconcile", "properties": {"marker": "wrong-property"}}, + {"start_id": "source", "end_id": "ad-a", "kind": "OtherReconcile", "properties": {"marker": "wrong-edge"}}, + {"start_id": "ad-a", "end_id": "source", "kind": "ADReconcile", "properties": {"marker": "wrong-direction"}} + ] + }, + "variants": [ + { + "name": "empty AD list preserves every relationship", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": []}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-a-1", "ad-a-2", "ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "singleton AD list deletes all duplicate matches", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": ["ad-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "duplicate AD IDs do not widen the delete", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": ["ad-a", "ad-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "small AD list deletes both selected endpoints", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": ["ad-a", "ad-b"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "one thousand AD IDs preserve exact selection", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-ad", "count": 998, "include": ["ad-a", "ad-b"]}}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "large AD list preserves exact selection", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": {"$type": "string_list", "prefix": "large-ad", "count": 1999, "include": ["ad-a", "ad-b"]}}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "one thousand no-match IDs preserve every relationship", + "vars": {"relationship_kind": "ADReconcile", "entity_kind": "ADEntity"}, + "params": {"object_ids": {"$type": "string_list", "prefix": "no-match-ad", "count": 1000}}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-a-1", "ad-a-2", "ad-b", "az-a", "az-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + }, + { + "name": "Azure base kind and relationship kind remain isolated", + "vars": {"relationship_kind": "AZReconcile", "entity_kind": "AZEntity"}, + "params": {"object_ids": ["az-a", "az-b"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["ad-a-1", "ad-a-2", "ad-b", "wrong-kind-end", "wrong-property", "wrong-edge", "wrong-direction"]}}] + } + ] + }, + { + "name": "REC-05 delegated enrollment discovery", + "template": "MATCH (s:CertTemplate)-[r:PublishedTo]->(e) WHERE e.objectid IN $ca_ids RETURN r, s", + "fixture": { + "nodes": [ + {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"objectid": "template-a"}}, + {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"objectid": "template-b"}}, + {"id": "wrong-start", "kinds": ["OtherTemplate"], "properties": {"objectid": "wrong-start"}}, + {"id": "ca-a", "kinds": ["EnterpriseCA"], "properties": {"objectid": "ca-a"}}, + {"id": "ca-b", "kinds": ["EnterpriseCA"], "properties": {"objectid": "ca-b"}}, + {"id": "wrong-property", "kinds": ["EnterpriseCA"], "properties": {"objectid": "other-ca"}} + ], + "edges": [ + {"start_id": "template-a", "end_id": "ca-a", "kind": "PublishedTo", "properties": {"marker": "published-a"}}, + {"start_id": "template-a", "end_id": "ca-b", "kind": "PublishedTo", "properties": {"marker": "published-b"}}, + {"start_id": "template-b", "end_id": "ca-a", "kind": "PublishedTo", "properties": {"marker": "published-c"}}, + {"start_id": "wrong-start", "end_id": "ca-a", "kind": "PublishedTo", "properties": {"marker": "wrong-start"}}, + {"start_id": "template-a", "end_id": "ca-a", "kind": "OtherPublication", "properties": {"marker": "wrong-edge"}}, + {"start_id": "template-a", "end_id": "wrong-property", "kind": "PublishedTo", "properties": {"marker": "wrong-property"}} + ] + }, + "variants": [ + {"name": "empty CA list", "params": {"ca_ids": []}, "assert": {"row_count": 0}}, + { + "name": "single CA retains every raw relationship row", + "params": {"ca_ids": ["ca-a"]}, + "assert": {"row_count": 2, "node_id_set": ["template-a", "template-b"], "relationship_records": [ + {"start": "template-a", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-a"}}, + {"start": "template-b", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-c"}} + ]} + }, + { + "name": "duplicate paths retain rows and expose a deduplicated node set", + "params": {"ca_ids": ["ca-a", "ca-b"]}, + "assert": {"row_count": 3, "node_id_set": ["template-a", "template-b"], "relationship_records": [ + {"start": "template-a", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-a"}}, + {"start": "template-a", "end": "ca-b", "kind": "PublishedTo", "props": {"marker": "published-b"}}, + {"start": "template-b", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-c"}} + ]} + }, + { + "name": "large CA list retains the same exact raw rows", + "params": {"ca_ids": {"$type": "string_list", "prefix": "missing-ca", "count": 1999, "include": ["ca-a", "ca-b"]}}, + "assert": {"row_count": 3, "node_id_set": ["template-a", "template-b"], "relationship_records": [ + {"start": "template-a", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-a"}}, + {"start": "template-a", "end": "ca-b", "kind": "PublishedTo", "props": {"marker": "published-b"}}, + {"start": "template-b", "end": "ca-a", "kind": "PublishedTo", "props": {"marker": "published-c"}} + ]} + } + ] + }, + { + "name": "REC-06 delegated enrollment relationship delete", + "template": "MATCH ()-[r:DelegatedEnrollmentAgent]->(e:CertTemplate) WHERE id(e) IN $template_ids DELETE r", + "fixture": { + "nodes": [ + {"id": "agent", "kinds": ["ADEntity"], "properties": {"objectid": "agent"}}, + {"id": "agent-duplicate", "kinds": ["ADEntity"], "properties": {"objectid": "agent-duplicate"}}, + {"id": "template-a", "kinds": ["CertTemplate"], "properties": {"objectid": "template-a"}}, + {"id": "template-b", "kinds": ["CertTemplate"], "properties": {"objectid": "template-b"}}, + {"id": "wrong-end", "kinds": ["OtherTemplate"], "properties": {"objectid": "wrong-end"}} + ], + "edges": [ + {"start_id": "agent", "end_id": "template-a", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "dea-a-1"}}, + {"start_id": "agent-duplicate", "end_id": "template-a", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "dea-a-2"}}, + {"start_id": "agent", "end_id": "template-b", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "dea-b"}}, + {"start_id": "template-a", "end_id": "agent", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "wrong-direction"}}, + {"start_id": "agent", "end_id": "wrong-end", "kind": "DelegatedEnrollmentAgent", "properties": {"marker": "wrong-end-kind"}}, + {"start_id": "agent", "end_id": "template-a", "kind": "OtherDelegation", "properties": {"marker": "wrong-edge-kind"}} + ] + }, + "variants": [ + { + "name": "empty template ID list preserves every relationship", + "node_list_params": {"template_ids": []}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["dea-a-1", "dea-a-2", "dea-b", "wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + }, + { + "name": "single template ID deletes every exact relationship", + "node_list_params": {"template_ids": ["template-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["dea-b", "wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + }, + { + "name": "duplicate template IDs do not widen the delete", + "node_list_params": {"template_ids": ["template-a", "template-a"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["dea-b", "wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + }, + { + "name": "small template ID list deletes both endpoints", + "node_list_params": {"template_ids": ["template-a", "template-b"]}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["wrong-direction", "wrong-end-kind", "wrong-edge-kind"]}}] + } + ] + }, + { + "name": "REC-07 HostsCAService reconciliation delete", + "template": "MATCH ()-[r:HostsCAService]->(e:EnterpriseCA) WHERE e.objectid = $object_id DELETE r", + "fixture": { + "nodes": [ + {"id": "host-a", "kinds": ["Computer"], "properties": {"objectid": "host-a"}}, + {"id": "host-b", "kinds": ["Computer"], "properties": {"objectid": "host-b"}}, + {"id": "ca", "kinds": ["EnterpriseCA"], "properties": {"objectid": "ca-id"}}, + {"id": "wrong-kind", "kinds": ["OtherCA"], "properties": {"objectid": "ca-id"}}, + {"id": "wrong-property", "kinds": ["EnterpriseCA"], "properties": {"objectid": "other-ca"}} + ], + "edges": [ + {"start_id": "host-a", "end_id": "ca", "kind": "HostsCAService", "properties": {"marker": "hosts-a"}}, + {"start_id": "host-b", "end_id": "ca", "kind": "HostsCAService", "properties": {"marker": "hosts-b"}}, + {"start_id": "host-a", "end_id": "wrong-kind", "kind": "HostsCAService", "properties": {"marker": "wrong-ca-kind"}}, + {"start_id": "host-a", "end_id": "wrong-property", "kind": "HostsCAService", "properties": {"marker": "wrong-object-id"}}, + {"start_id": "host-a", "end_id": "ca", "kind": "OtherCAService", "properties": {"marker": "wrong-edge-kind"}} + ] + }, + "variants": [ + { + "name": "exact CA hit deletes duplicate matching edges", + "params": {"object_id": "ca-id"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["wrong-ca-kind", "wrong-object-id", "wrong-edge-kind"]}}] + }, + { + "name": "no CA hit preserves every relationship", + "params": {"object_id": "missing-ca"}, + "assert": "no_error", + "post_assertions": [{"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["hosts-a", "hosts-b", "wrong-ca-kind", "wrong-object-id", "wrong-edge-kind"]}}] + } + ] + }, + { + "name": "REC-08 AD entity detach delete", + "template": "MATCH (n:ADEntity) WHERE n.objectid IN $object_ids DETACH DELETE n", + "fixture": { + "nodes": [ + {"id": "isolated", "kinds": ["ADEntity", "User"], "properties": {"objectid": "isolated"}}, + {"id": "low", "kinds": ["ADEntity", "Computer"], "properties": {"objectid": "low"}}, + {"id": "high", "kinds": ["ADEntity", "Group"], "properties": {"objectid": "high"}}, + {"id": "kind-decoy", "kinds": ["OtherEntity"], "properties": {"objectid": "isolated"}}, + {"id": "property-decoy", "kinds": ["ADEntity"], "properties": {"objectid": "other"}}, + {"id": "neighbor-a", "kinds": ["ADEntity"], "properties": {"objectid": "neighbor-a"}}, + {"id": "neighbor-b", "kinds": ["ADEntity"], "properties": {"objectid": "neighbor-b"}}, + {"id": "neighbor-c", "kinds": ["ADEntity"], "properties": {"objectid": "neighbor-c"}} + ], + "edges": [ + {"start_id": "neighbor-a", "end_id": "low", "kind": "Incident", "properties": {"marker": "low-in"}}, + {"start_id": "low", "end_id": "neighbor-b", "kind": "Incident", "properties": {"marker": "low-out"}}, + {"start_id": "low", "end_id": "low", "kind": "Incident", "properties": {"marker": "low-self"}}, + {"start_id": "neighbor-a", "end_id": "high", "kind": "Incident", "properties": {"marker": "high-in-a"}}, + {"start_id": "neighbor-b", "end_id": "high", "kind": "Incident", "properties": {"marker": "high-in-b"}}, + {"start_id": "high", "end_id": "neighbor-a", "kind": "Incident", "properties": {"marker": "high-out-a"}}, + {"start_id": "high", "end_id": "neighbor-c", "kind": "Incident", "properties": {"marker": "high-out-c"}}, + {"start_id": "high", "end_id": "high", "kind": "Incident", "properties": {"marker": "high-self"}}, + {"start_id": "kind-decoy", "end_id": "property-decoy", "kind": "Survivor", "properties": {"marker": "survivor"}} + ] + }, + "variants": [ + { + "name": "empty object ID list preserves all nodes and relationships", + "params": {"object_ids": []}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "low", "high", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["low-in", "low-out", "low-self", "high-in-a", "high-in-b", "high-out-a", "high-out-c", "high-self", "survivor"]}} + ] + }, + { + "name": "isolated target deletes exactly one node", + "params": {"object_ids": ["isolated"]}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["low", "high", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["low-in", "low-out", "low-self", "high-in-a", "high-in-b", "high-out-a", "high-out-c", "high-self", "survivor"]}} + ] + }, + { + "name": "low degree target cascades inbound outbound and self edges", + "params": {"object_ids": ["low"]}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "high", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["high-in-a", "high-in-b", "high-out-a", "high-out-c", "high-self", "survivor"]}} + ] + }, + { + "name": "small list includes a high degree target", + "params": {"object_ids": ["low", "high"]}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["survivor"]}} + ] + }, + { + "name": "large object ID list preserves exact targets", + "params": {"object_ids": {"$type": "string_list", "prefix": "missing-node", "count": 1999, "include": ["low", "high"]}}, + "assert": "no_error", + "post_assertions": [ + {"cypher": "MATCH (n) RETURN n", "assert": {"node_id_set": ["isolated", "kind-decoy", "property-decoy", "neighbor-a", "neighbor-b", "neighbor-c"]}}, + {"cypher": "MATCH ()-[r]->() RETURN r.marker", "assert": {"scalar_values": ["survivor"]}} + ] + } + ] + }, + { + "name": "TRUST-01 and TRUST-02 stale trust temporal disjunction", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "early-a", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "early-b", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-02T00:00:00Z"}}, + {"id": "equal-a", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "equal-b", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-03T00:00:00Z"}}, + {"id": "late-a", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-newer", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-missing-relationship", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "late-b-null-relationship", "kinds": ["Domain"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "missing-a", "kinds": ["Domain"], "properties": {}}, + {"id": "missing-b", "kinds": ["Domain"], "properties": {}}, + {"id": "null-a", "kinds": ["Domain"], "properties": {"lastcollected": null}}, + {"id": "null-b", "kinds": ["Domain"], "properties": {"lastcollected": null}}, + {"id": "wrong-start", "kinds": ["Computer"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}}, + {"id": "wrong-end", "kinds": ["User"], "properties": {"lastcollected": "2026-01-04T00:00:00Z"}} + ], + "edges": [ + {"start_id": "late-a", "end_id": "early-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-older-start-only"}}, + {"start_id": "early-a", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-older-end-only"}}, + {"start_id": "late-a", "end_id": "late-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-older-both"}}, + {"start_id": "equal-a", "end_id": "equal-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-equal"}}, + {"start_id": "late-a", "end_id": "late-b-newer", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-05T00:00:00Z", "marker": "same-newer"}}, + {"start_id": "late-a", "end_id": "late-b-missing-relationship", "kind": "SameForestTrust", "properties": {"marker": "same-missing-relationship"}}, + {"start_id": "late-a", "end_id": "late-b-null-relationship", "kind": "SameForestTrust", "properties": {"lastseen": null, "marker": "same-null-relationship"}}, + {"start_id": "missing-a", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-missing-start-valid-end"}}, + {"start_id": "null-a", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-null-start-valid-end"}}, + {"start_id": "late-a", "end_id": "missing-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-missing-end-valid-start"}}, + {"start_id": "late-a", "end_id": "null-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-null-end-valid-start"}}, + {"start_id": "missing-a", "end_id": "null-b", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-missing-null-endpoints"}}, + {"start_id": "wrong-start", "end_id": "late-a", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-wrong-start-kind"}}, + {"start_id": "late-a", "end_id": "wrong-end", "kind": "SameForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "same-wrong-end-kind"}}, + {"start_id": "late-a", "end_id": "early-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-start-only"}}, + {"start_id": "early-a", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-end-only"}}, + {"start_id": "late-a", "end_id": "late-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-both"}}, + {"start_id": "equal-a", "end_id": "equal-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-equal"}}, + {"start_id": "late-a", "end_id": "late-b-newer", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-05T00:00:00Z", "marker": "cross-newer"}}, + {"start_id": "late-a", "end_id": "late-b-missing-relationship", "kind": "CrossForestTrust", "properties": {"marker": "cross-missing-relationship"}}, + {"start_id": "late-a", "end_id": "late-b-null-relationship", "kind": "CrossForestTrust", "properties": {"lastseen": null, "marker": "cross-null-relationship"}}, + {"start_id": "missing-a", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-start-valid-end"}}, + {"start_id": "null-a", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-start-valid-end"}}, + {"start_id": "late-a", "end_id": "missing-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-end-valid-start"}}, + {"start_id": "late-a", "end_id": "null-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-end-valid-start"}}, + {"start_id": "missing-a", "end_id": "null-b", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-null-endpoints"}}, + {"start_id": "wrong-start", "end_id": "late-a", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-wrong-start-kind"}}, + {"start_id": "late-a", "end_id": "wrong-end", "kind": "CrossForestTrust", "properties": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-wrong-end-kind"}} + ] + }, + "variants": [ + { + "name": "TRUST-01 returns only stale SameForestTrust relationship IDs", + "vars": {"query": "MATCH (s:Domain)-[r:SameForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN id(r)"}, + "assert": {"keys": ["id(r)"], "row_count": 7} + }, + { + "name": "TRUST-01 exact sparse truth and null matrix", + "vars": {"query": "MATCH (s:Domain)-[r:SameForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN r.marker"}, + "assert": {"scalar_values": ["same-older-start-only", "same-older-end-only", "same-older-both", "same-missing-start-valid-end", "same-null-start-valid-end", "same-missing-end-valid-start", "same-null-end-valid-start"]} + }, + { + "name": "TRUST-02 returns and hydrates only stale CrossForestTrust relationships", + "vars": {"query": "MATCH (s:Domain)-[r:CrossForestTrust]->(e:Domain) WHERE datetime(r.lastseen) < datetime(s.lastcollected) OR datetime(r.lastseen) < datetime(e.lastcollected) RETURN r"}, + "assert": {"row_count": 7, "relationship_records": [ + {"start": "late-a", "end": "early-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-start-only"}}, + {"start": "early-a", "end": "late-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-end-only"}}, + {"start": "late-a", "end": "late-b", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-older-both"}}, + {"start": "missing-a", "end": "late-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-start-valid-end"}}, + {"start": "null-a", "end": "late-a", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-start-valid-end"}}, + {"start": "late-a", "end": "missing-b", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-missing-end-valid-start"}}, + {"start": "late-a", "end": "null-b", "kind": "CrossForestTrust", "props": {"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-null-end-valid-start"}} + ]} + } + ] + }, + { + "name": "TRUST-03 directional stale trust derivation", + "template": "MATCH (s:Domain)-[r]->(e:Domain) WHERE (id(s) = $forward_start AND id(e) = $forward_end AND r:AbuseTGTDelegation) OR (id(s) = $forward_end AND id(e) = $forward_start AND r:SpoofSIDHistory) RETURN {{projection}}", + "node_params": {"forward_start": "forward", "forward_end": "reverse"}, + "fixture": { + "nodes": [ + {"id": "forward", "kinds": ["Domain"], "properties": {"name": "forward"}}, + {"id": "reverse", "kinds": ["Domain"], "properties": {"name": "reverse"}}, + {"id": "wrong-kind", "kinds": ["Computer"], "properties": {"name": "wrong-kind"}} + ], + "edges": [ + {"start_id": "forward", "end_id": "reverse", "kind": "AbuseTGTDelegation", "properties": {"marker": "valid-forward-abuse"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "SpoofSIDHistory", "properties": {"marker": "valid-reverse-spoof"}}, + {"start_id": "forward", "end_id": "reverse", "kind": "SpoofSIDHistory", "properties": {"marker": "invalid-forward-spoof"}}, + {"start_id": "reverse", "end_id": "forward", "kind": "AbuseTGTDelegation", "properties": {"marker": "invalid-reverse-abuse"}}, + {"start_id": "forward", "end_id": "wrong-kind", "kind": "AbuseTGTDelegation", "properties": {"marker": "invalid-end-kind"}} + ] + }, + "variants": [ + {"name": "relationship ID projection preserves branch-local direction and kind", "vars": {"projection": "id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 2}}, + {"name": "exact directional markers exclude both cross-combinations", "vars": {"projection": "r.marker"}, "assert": {"scalar_values": ["valid-forward-abuse", "valid-reverse-spoof"]}}, + { + "name": "reverse driving trust relationship preserves ID projection", + "vars": {"projection": "id(r)"}, + "node_params": {"forward_start": "reverse", "forward_end": "forward"}, + "assert": {"keys": ["id(r)"], "row_count": 2} + }, + { + "name": "reverse driving trust relationship swaps only the intended branch-local matches", + "vars": {"projection": "r.marker"}, + "node_params": {"forward_start": "reverse", "forward_end": "forward"}, + "assert": {"scalar_values": ["invalid-forward-spoof", "invalid-reverse-abuse"]} + } + ] + } + ] +} diff --git a/integration/testdata/templates/relationship_scan_shapes.json b/integration/testdata/templates/relationship_scan_shapes.json new file mode 100644 index 00000000..af9dec77 --- /dev/null +++ b/integration/testdata/templates/relationship_scan_shapes.json @@ -0,0 +1,119 @@ +{ + "families": [ + { + "name": "SCAN-01 through SCAN-04 wide relationship filters", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "ad-a", "kinds": ["ADBase"], "properties": {"name": "ad-a"}}, + {"id": "ad-b", "kinds": ["ADBase"], "properties": {"name": "ad-b"}}, + {"id": "az-a", "kinds": ["AZBase"], "properties": {"name": "az-a"}}, + {"id": "az-b", "kinds": ["AZBase"], "properties": {"name": "az-b"}}, + {"id": "plain-a", "kinds": ["Plain", "MissingPost"], "properties": {"name": "plain-a"}}, + {"id": "plain-b", "kinds": ["Plain"], "properties": {"name": "plain-b"}}, + {"id": "meta-start", "kinds": ["Meta", "Plain"], "properties": {"name": "meta-start"}}, + {"id": "meta-end", "kinds": ["MetaDetail", "Plain"], "properties": {"name": "meta-end"}}, + {"id": "meta-both", "kinds": ["Meta", "MetaDetail", "Plain"], "properties": {"name": "meta-both"}}, + {"id": "entity-a", "kinds": ["Entity"], "properties": {"name": "entity-a"}}, + {"id": "entity-b", "kinds": ["Entity"], "properties": {"name": "entity-b"}}, + {"id": "not-entity", "kinds": ["Other"], "properties": {"name": "not-entity"}} + ], + "edges": [ + {"start_id": "ad-a", "end_id": "ad-b", "kind": "PostProcessed", "properties": {"marker": "post-ad"}}, + {"start_id": "az-a", "end_id": "az-b", "kind": "PostProcessed", "properties": {"marker": "post-az"}}, + {"start_id": "ad-a", "end_id": "az-b", "kind": "PostProcessed", "properties": {"marker": "post-cross-a"}}, + {"start_id": "az-a", "end_id": "ad-b", "kind": "PostProcessed", "properties": {"marker": "post-cross-b"}}, + {"start_id": "plain-a", "end_id": "ad-b", "kind": "PostProcessed", "properties": {"marker": "post-wrong-start"}}, + {"start_id": "ad-a", "end_id": "plain-b", "kind": "PostProcessed", "properties": {"marker": "post-wrong-end"}}, + {"start_id": "ad-a", "end_id": "ad-b", "kind": "WrongPost", "properties": {"marker": "post-wrong-kind"}}, + {"start_id": "plain-a", "end_id": "plain-b", "kind": "TrackerA", "properties": {"marker": "tracker-a", "hydrated": true}}, + {"start_id": "plain-a", "end_id": "plain-b", "kind": "TrackerB", "properties": {"marker": "tracker-b", "hydrated": true}}, + {"start_id": "meta-start", "end_id": "plain-b", "kind": "TrackerA", "properties": {"marker": "tracker-meta-start"}}, + {"start_id": "plain-a", "end_id": "meta-end", "kind": "TrackerA", "properties": {"marker": "tracker-meta-end"}}, + {"start_id": "meta-both", "end_id": "meta-both", "kind": "TrackerB", "properties": {"marker": "tracker-meta-both"}}, + {"start_id": "plain-a", "end_id": "plain-b", "kind": "MigratedEdge", "properties": {"marker": "migration-present", "lastseen": "2026-01-03T00:00:00Z"}}, + {"start_id": "plain-b", "end_id": "plain-a", "kind": "MigratedEdge", "properties": {"marker": "migration-null", "lastseen": null}}, + {"start_id": "plain-a", "end_id": "entity-a", "kind": "MigratedEdge", "properties": {"marker": "migration-missing"}}, + {"start_id": "meta-start", "end_id": "plain-b", "kind": "MigratedEdge", "properties": {"marker": "migration-meta-start", "lastseen": "2026-01-03T00:00:00Z"}}, + {"start_id": "plain-a", "end_id": "meta-end", "kind": "MigratedEdge", "properties": {"marker": "migration-meta-end", "lastseen": "2026-01-03T00:00:00Z"}}, + {"start_id": "entity-a", "end_id": "entity-b", "kind": "OwnsRaw", "properties": {"marker": "owns", "hydrated": "yes"}}, + {"start_id": "entity-a", "end_id": "entity-b", "kind": "WriteOwnerRaw", "properties": {"marker": "write-owner", "hydrated": "yes"}}, + {"start_id": "not-entity", "end_id": "entity-b", "kind": "OwnsRaw", "properties": {"marker": "owns-wrong-start"}} + ] + }, + "variants": [ + {"name": "SCAN-01 AD and Azure bases exact relationship kind", "vars": {"query": "MATCH (s)-[r:PostProcessed]->(e) WHERE (s:ADBase OR s:AZBase) AND (e:ADBase OR e:AZBase) RETURN id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 4}}, + {"name": "SCAN-01 wrong relationship kind is empty", "vars": {"query": "MATCH (s)-[r:MissingPost]->(e) WHERE (s:ADBase OR s:AZBase) AND (e:ADBase OR e:AZBase) RETURN id(r)"}, "assert": "empty"}, + {"name": "SCAN-02 one kind excludes every Meta endpoint position", "vars": {"query": "MATCH (s)-[r:TrackerA]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND NOT (e:Meta OR e:MetaDetail) RETURN r"}, "assert": {"relationship_records": [{"start": "plain-a", "end": "plain-b", "kind": "TrackerA", "props": {"marker": "tracker-a", "hydrated": true}}]}}, + {"name": "SCAN-02 many kinds hydrate complete relationships", "vars": {"query": "MATCH (s)-[r:TrackerA|TrackerB]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND NOT (e:Meta OR e:MetaDetail) RETURN r"}, "assert": {"relationship_records": [{"start": "plain-a", "end": "plain-b", "kind": "TrackerA", "props": {"marker": "tracker-a", "hydrated": true}}, {"start": "plain-a", "end": "plain-b", "kind": "TrackerB", "props": {"marker": "tracker-b", "hydrated": true}}]}}, + {"name": "SCAN-03 present lastseen only with Meta decoys", "vars": {"query": "MATCH (s)-[r:MigratedEdge]->(e) WHERE NOT (s:Meta OR s:MetaDetail) AND r.lastseen IS NOT NULL AND NOT (e:Meta OR e:MetaDetail) RETURN id(r)"}, "assert": {"keys": ["id(r)"], "row_count": 1}}, + {"name": "SCAN-04 OwnsRaw full hydration", "vars": {"query": "MATCH (s:Entity)-[r:OwnsRaw]->() RETURN r"}, "assert": {"relationship_records": [{"start": "entity-a", "end": "entity-b", "kind": "OwnsRaw", "props": {"marker": "owns", "hydrated": "yes"}}]}}, + {"name": "SCAN-04 WriteOwnerRaw representative", "vars": {"query": "MATCH (s:Entity)-[r:WriteOwnerRaw]->() RETURN r"}, "assert": {"relationship_records": [{"start": "entity-a", "end": "entity-b", "kind": "WriteOwnerRaw", "props": {"marker": "write-owner", "hydrated": "yes"}}]}} + ] + }, + { + "name": "SCAN-05 through SCAN-08 anchored scans and projections", + "template": "{{query}}", + "fixture": { + "nodes": [ + {"id": "target", "kinds": ["Computer"], "properties": {"name": "target"}}, + {"id": "zero-target", "kinds": ["Computer", "MissingMember"], "properties": {"name": "zero-target"}}, + {"id": "wrong-end", "kinds": ["Other"], "properties": {"name": "wrong-end"}}, + {"id": "source-01", "kinds": ["Entity", "Group"], "properties": {"name": "source-01", "objectid": "S-1-5-01"}}, + {"id": "source-02", "kinds": ["Entity", "User"], "properties": {"name": "source-02", "objectid": "S-1-5-02"}}, + {"id": "source-03", "kinds": ["Entity", "Computer"], "properties": {"name": "source-03", "objectid": "S-1-5-03"}}, + {"id": "source-04", "kinds": ["Entity"], "properties": {"name": "source-04"}}, + {"id": "source-05", "kinds": ["Entity"], "properties": {"name": "source-05"}}, + {"id": "source-06", "kinds": ["Entity"], "properties": {"name": "source-06"}}, + {"id": "source-07", "kinds": ["Entity"], "properties": {"name": "source-07"}}, + {"id": "source-08", "kinds": ["Entity"], "properties": {"name": "source-08"}}, + {"id": "source-09", "kinds": ["Entity"], "properties": {"name": "source-09"}}, + {"id": "not-entity", "kinds": ["Other"], "properties": {"name": "not-entity"}}, + {"id": "victim-computer", "kinds": ["Computer"], "properties": {"name": "victim-computer"}}, + {"id": "victim-other", "kinds": ["Other"], "properties": {"name": "victim-other"}}, + {"id": "victim-unused", "kinds": ["Computer"], "properties": {"name": "victim-unused"}}, + {"id": "attacker-group", "kinds": ["Group", "Entity"], "properties": {"name": "attacker-group"}}, + {"id": "attacker-user", "kinds": ["User", "Entity"], "properties": {"name": "attacker-user"}}, + {"id": "attacker-computer", "kinds": ["Computer", "Entity"], "properties": {"name": "attacker-computer"}}, + {"id": "attacker-wrong", "kinds": ["Other"], "properties": {"name": "attacker-wrong"}} + ], + "edges": [ + {"start_id": "source-01", "end_id": "target", "kind": "ScanEdge01", "properties": {"marker": "scan-01", "hydrated": true}}, + {"start_id": "source-02", "end_id": "target", "kind": "ScanEdge02", "properties": {"marker": "scan-02"}}, + {"start_id": "source-03", "end_id": "target", "kind": "ScanEdge03", "properties": {"marker": "scan-03"}}, + {"start_id": "source-04", "end_id": "target", "kind": "ScanEdge04", "properties": {"marker": "scan-04"}}, + {"start_id": "source-05", "end_id": "target", "kind": "ScanEdge05", "properties": {"marker": "scan-05"}}, + {"start_id": "source-06", "end_id": "target", "kind": "ScanEdge06", "properties": {"marker": "scan-06"}}, + {"start_id": "source-07", "end_id": "target", "kind": "ScanEdge07", "properties": {"marker": "scan-07"}}, + {"start_id": "source-08", "end_id": "target", "kind": "ScanEdge08", "properties": {"marker": "scan-08"}}, + {"start_id": "source-09", "end_id": "target", "kind": "ScanEdge09", "properties": {"marker": "scan-09"}}, + {"start_id": "not-entity", "end_id": "target", "kind": "ScanEdge01", "properties": {"marker": "scan-wrong-start"}}, + {"start_id": "source-01", "end_id": "wrong-end", "kind": "LocalToComputer", "properties": {"marker": "local-wrong-end"}}, + {"start_id": "source-01", "end_id": "target", "kind": "LocalToComputer", "properties": {"marker": "local-valid"}}, + {"start_id": "source-01", "end_id": "target", "kind": "MemberOf", "properties": {"marker": "member-01"}}, + {"start_id": "source-01", "end_id": "target", "kind": "MemberOfLocalGroup", "properties": {"marker": "member-local-01"}}, + {"start_id": "source-02", "end_id": "target", "kind": "MemberOf", "properties": {"marker": "member-02"}}, + {"start_id": "source-02", "end_id": "target", "kind": "WrongMember", "properties": {"marker": "member-wrong"}}, + {"start_id": "attacker-group", "end_id": "victim-computer", "kind": "GenericAll", "properties": {"marker": "esc-group"}}, + {"start_id": "attacker-user", "end_id": "victim-other", "kind": "WritePublicInformation", "properties": {"marker": "esc-user-a-only"}}, + {"start_id": "attacker-computer", "end_id": "victim-computer", "kind": "WriteDACL", "properties": {"marker": "esc-computer"}}, + {"start_id": "attacker-wrong", "end_id": "victim-computer", "kind": "GenericAll", "properties": {"marker": "esc-wrong-start"}}, + {"start_id": "attacker-group", "end_id": "victim-computer", "kind": "WrongEsc", "properties": {"marker": "esc-wrong-kind"}} + ] + }, + "variants": [ + {"name": "SCAN-05 zero inbound degree", "vars": {"query": "MATCH (s:Entity)-[r:ScanEdge01]->(e) WHERE id(e) = $target RETURN r, s"}, "node_params": {"target": "zero-target"}, "assert": "empty"}, + {"name": "SCAN-05 one kind one match full hydration", "vars": {"query": "MATCH (s:Entity)-[r:ScanEdge01]->(e) WHERE id(e) = $target RETURN r, s"}, "node_params": {"target": "target"}, "assert": {"keys": ["r", "s"], "node_id_set": ["source-01"], "relationship_records": [{"start": "source-01", "end": "target", "kind": "ScanEdge01", "props": {"marker": "scan-01", "hydrated": true}}]}}, + {"name": "SCAN-05 nine kinds high inbound degree", "vars": {"query": "MATCH (s:Entity)-[r:ScanEdge01|ScanEdge02|ScanEdge03|ScanEdge04|ScanEdge05|ScanEdge06|ScanEdge07|ScanEdge08|ScanEdge09]->(e) WHERE id(e) = $target RETURN r, s"}, "node_params": {"target": "target"}, "assert": {"keys": ["r", "s"], "row_count": 9, "node_id_set": ["source-01", "source-02", "source-03", "source-04", "source-05", "source-06", "source-07", "source-08", "source-09"]}}, + {"name": "SCAN-06 exact FetchKinds projection", "vars": {"query": "MATCH (s)-[r:LocalToComputer]->(e:Computer) RETURN id(s), id(r), type(r), id(e)"}, "assert": {"keys": ["id(s)", "id(r)", "type(r)", "id(e)"], "row_count": 1}}, + {"name": "SCAN-07 one kind directed endpoint IDs", "vars": {"query": "MATCH (s)-[r:MemberOf]->(e) RETURN id(s), id(e)"}, "assert": {"keys": ["id(s)", "id(e)"], "row_count": 2}}, + {"name": "SCAN-07 many kinds retain duplicate endpoint pairs", "vars": {"query": "MATCH (s)-[r:MemberOf|MemberOfLocalGroup]->(e) RETURN id(s), id(e)"}, "assert": {"keys": ["id(s)", "id(e)"], "row_count": 3}}, + {"name": "SCAN-07 absent kind zero matches", "vars": {"query": "MATCH (s)-[r:MissingMember]->(e) RETURN id(s), id(e)"}, "assert": "empty"}, + {"name": "SCAN-08 scenario A empty victim list", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": []}, "assert": "empty"}, + {"name": "SCAN-08 scenario A single victim", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": ["victim-other"]}, "assert": {"keys": ["id(s)"], "row_count": 1}}, + {"name": "SCAN-08 scenario A thirty-two-entry victim list", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": ["victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other", "victim-unused", "victim-computer", "victim-other"]}, "assert": {"keys": ["id(s)"], "row_count": 3}}, + {"name": "SCAN-08 scenario B typed end and five kinds", "vars": {"query": "MATCH (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL]->(e:Computer) WHERE (s:Group OR s:User OR s:Computer) AND id(e) IN $victims RETURN id(s)"}, "node_list_params": {"victims": ["victim-computer", "victim-other", "victim-unused"]}, "assert": {"keys": ["id(s)"], "row_count": 2}} + ] + } + ] +} diff --git a/integration/trust_pruning_legacy_builder_test.go b/integration/trust_pruning_legacy_builder_test.go new file mode 100644 index 00000000..2f369b75 --- /dev/null +++ b/integration/trust_pruning_legacy_builder_test.go @@ -0,0 +1,854 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +//go:build manual_integration + +package integration + +import ( + "context" + "sort" + "testing" + "time" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/specterops/dawgs/ops" + "github.com/specterops/dawgs/query" + "github.com/specterops/dawgs/testutil" + "github.com/stretchr/testify/require" +) + +// TestLegacyBuilderTrustAndPruningSelectors verifies legacy trust and pruning selectors retain their filtering semantics. +func TestLegacyBuilderTrustAndPruningSelectors(t *testing.T) { + fixture := trustPruningFixture() + nodeKinds, edgeKinds := fixture.Kinds() + db, ctx := SetupDBWithKindsNoGraphCleanup(t, nodeKinds, edgeKinds) + ClearGraph(t, db, ctx) + session := &Session{ + DB: db, + Ctx: ctx, + } + threshold := regressionDay(3) + + t.Run("TRUST-01 SameForestTrust IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return trustPruningCriteria("SameForestTrust") + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + ids, err := ops.FetchRelationshipIDs(relationshipQuery) + require.NoError(t, err) + require.Len(t, ids, 1) + return nil + }) + }) + + t.Run("TRUST-02 CrossForestTrust hydration", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return trustPruningCriteria("CrossForestTrust") + }, func(relationshipQuery graph.RelationshipQuery, idMap opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 1) + require.Equal(t, idMap["late-a"], relationships[0].StartID) + require.Equal(t, idMap["early"], relationships[0].EndID) + require.Equal(t, graph.StringKind("CrossForestTrust"), relationships[0].Kind) + marker, err := relationships[0].Properties.Get("marker").String() + require.NoError(t, err) + require.Equal(t, "cross-old", marker) + return nil + }) + }) + + t.Run("TRUST-03 directional derived IDs", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(idMap opengraph.IDMap) graph.Criteria { + return directionalTrustCriteria(idMap, "late-a", "late-b") + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 2) + markers := make([]string, 0, len(relationships)) + for _, relationship := range relationships { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + sort.Strings(markers) + require.Equal(t, []string{"valid-forward-abuse", "valid-reverse-spoof"}, markers) + return nil + }) + }) + + t.Run("TRUST-03 reverse driving trust relationship", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(idMap opengraph.IDMap) graph.Criteria { + return directionalTrustCriteria(idMap, "late-b", "late-a") + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Len(t, relationships, 2) + require.Equal(t, []string{"invalid-forward-spoof", "invalid-reverse-abuse"}, trustPruningRelationshipMarkers(t, relationships)) + return nil + }) + }) + + t.Run("PRUNE-01 protected kinds and old relationships", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Relationship(), graph.StringKind("HasSession"), graph.StringKind("MetaIncludes"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Equal(t, []string{"candidate-old"}, trustPruningRelationshipMarkers(t, relationships)) + return nil + }) + }) + + t.Run("PRUNE-02 HasSession missing null or old", func(t *testing.T) { + WithLegacyRelationshipQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Kind(query.Relationship(), graph.StringKind("HasSession")), + query.Or( + query.Not(query.Exists(query.RelationshipProperty("lastseen"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ), + ) + }, func(relationshipQuery graph.RelationshipQuery, _ opengraph.IDMap) error { + relationships, err := ops.FetchRelationships(relationshipQuery) + require.NoError(t, err) + require.Equal(t, []string{"session-missing", "session-null", "session-old"}, trustPruningRelationshipMarkers(t, relationships)) + return nil + }) + }) + + t.Run("PRUNE-03 protected kinds and missing null or old nodes", func(t *testing.T) { + WithLegacyNodeQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Node(), pruningProtectedNodeKinds()...)), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + ) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + ids, err := ops.FetchNodeIDs(nodeQuery) + require.NoError(t, err) + require.Equal(t, []string{"candidate-missing", "candidate-null", "candidate-old", "orphan-empty", "orphan-missing", "orphan-null", "orphan-wrong-prefix"}, trustPruningFixtureIDs(t, idMap, ids)) + return nil + }) + }) + + t.Run("PRUNE-04 orphan SID nodes", func(t *testing.T) { + WithLegacyNodeQuery(t, session, fixture, func(opengraph.IDMap) graph.Criteria { + return query.And( + query.Not(query.KindIn(query.Node(), pruningProtectedNodeKinds()...)), + query.Not(query.Exists(query.NodeProperty("name"))), + query.StringStartsWith(query.NodeProperty("objectid"), "S-1-5"), + ) + }, func(nodeQuery graph.NodeQuery, idMap opengraph.IDMap) error { + ids, err := ops.FetchNodeIDs(nodeQuery) + require.NoError(t, err) + require.Equal(t, []string{"orphan-missing", "orphan-null"}, trustPruningFixtureIDs(t, idMap, ids)) + return nil + }) + }) +} + +// TestDirectBatchPruning verifies direct batch pruning removes selected nodes and relationships without affecting survivors. +func TestDirectBatchPruning(t *testing.T) { + fixture := batchPruningFixture(32) + nodeKinds, edgeKinds := fixture.Kinds() + db, ctx := SetupDBWithKinds(t, CleanupGraph, nodeKinds, edgeKinds) + + loadFixture := func(t *testing.T) opengraph.IDMap { + t.Helper() + ClearGraph(t, db, ctx) + idMap, err := opengraph.WriteGraph(ctx, db, fixture) + require.NoError(t, err) + return idMap + } + + t.Run("PRUNE-05 empty single and many relationships", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the relationship-pruning population. + name string + + // criteria selects relationships for deletion. + criteria graph.CriteriaProvider + + // expected is the number of accepted delete attempts. + expected int + + // remaining is the expected PruneDelete relationship count. + remaining int64 + }{ + { + name: "empty", + criteria: func() graph.Criteria { return query.Equals(query.RelationshipProperty("marker"), "absent") }, + expected: 0, + remaining: 3, + }, + { + name: "single", + criteria: func() graph.Criteria { return query.Equals(query.RelationshipProperty("marker"), "single") }, + expected: 1, + remaining: 2, + }, + { + name: "many", + criteria: func() graph.Criteria { return query.Kind(query.Relationship(), graph.StringKind("PruneDelete")) }, + expected: 3, + remaining: 0, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + loadFixture(t) + deleted, err := pruneRelationshipsInBatches(ctx, db, testCase.criteria, nil) + require.NoError(t, err) + require.Equal(t, testCase.expected, deleted) + require.Equal(t, testCase.remaining, countByCypher(t, ctx, db, "MATCH ()-[r:PruneDelete]->() RETURN count(r)")) + require.Equal(t, int64(1), countByCypher(t, ctx, db, "MATCH ()-[r:PruneSurvivor]->() RETURN count(r)")) + }) + } + }) + + t.Run("PRUNE-05 relationship absent after selection is harmless", func(t *testing.T) { + loadFixture(t) + deleted, err := pruneRelationshipsInBatches(ctx, db, func() graph.Criteria { + return query.Equals(query.RelationshipProperty("marker"), "single") + }, func(ids []graph.ID) error { + return db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.DeleteRelationship(ids[0]) + }) + }) + require.NoError(t, err) + require.Equal(t, 1, deleted, "the production workflow counts accepted delete attempts") + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH ()-[r:PruneDelete]->() RETURN count(r)")) + }) + + t.Run("PRUNE-06 empty single many and high-degree nodes", func(t *testing.T) { + for _, testCase := range []struct { + // name identifies the node-pruning population. + name string + + // criteria selects candidate nodes for deletion. + criteria graph.CriteriaProvider + + // expected is the number of accepted delete attempts. + expected int + + // expectedCandidates is the expected surviving candidate count. + expectedCandidates int64 + + // expectedIncidents is the expected surviving incident-edge count. + expectedIncidents int64 + }{ + { + name: "empty", + criteria: func() graph.Criteria { return query.Equals(query.NodeProperty("objectid"), "absent") }, + expected: 0, + expectedCandidates: 3, + expectedIncidents: 34, + }, + { + name: "single", + criteria: func() graph.Criteria { return query.Equals(query.NodeProperty("objectid"), "single") }, + expected: 1, + expectedCandidates: 2, + expectedIncidents: 34, + }, + { + name: "many including high degree", + criteria: func() graph.Criteria { return query.Equals(query.NodeProperty("remove"), true) }, + expected: 2, + expectedCandidates: 1, + expectedIncidents: 1, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + loadFixture(t) + deleted, err := pruneNodesInBatches(ctx, db, testCase.criteria, nil) + require.NoError(t, err) + require.Equal(t, testCase.expected, deleted) + require.Equal(t, testCase.expectedCandidates, countByCypher(t, ctx, db, "MATCH (n:PruneDeleteNode) RETURN count(n)")) + require.Equal(t, testCase.expectedIncidents, countByCypher(t, ctx, db, "MATCH ()-[r:PruneIncident]->() RETURN count(r)")) + }) + } + }) + + t.Run("PRUNE-06 node absent after selection is harmless", func(t *testing.T) { + loadFixture(t) + deleted, err := pruneNodesInBatches(ctx, db, func() graph.Criteria { + return query.Equals(query.NodeProperty("objectid"), "single") + }, func(ids []graph.ID) error { + return db.BatchOperation(ctx, func(batch graph.Batch) error { + return batch.DeleteNode(ids[0]) + }) + }) + require.NoError(t, err) + require.Equal(t, 1, deleted) + require.Equal(t, int64(2), countByCypher(t, ctx, db, "MATCH (n:PruneDeleteNode) RETURN count(n)")) + }) +} + +// BenchmarkDirectBatchPruning measures direct pruning across representative fixture sizes. +func BenchmarkDirectBatchPruning(b *testing.B) { + fixture := testutil.NewTrustPruningScaleFixture(2_000) + nodeKinds, edgeKinds := fixture.Kinds() + session := Open(b, Options{ + ExtraNodeKinds: nodeKinds, + ExtraEdgeKinds: edgeKinds, + CleanupMode: CloseOnly, + }) + + resetFixture := func(b *testing.B) { + b.Helper() + if err := session.DB.WriteTransaction(session.Ctx, func(tx graph.Transaction) error { + return tx.Nodes().Delete() + }); err != nil { + b.Fatalf("clear benchmark graph: %v", err) + } + if _, err := opengraph.WriteGraph(session.Ctx, session.DB, fixture); err != nil { + b.Fatalf("load benchmark fixture: %v", err) + } + } + + b.Run("PRUNE-05 relationship ID selection and batch delete", func(b *testing.B) { + b.ReportAllocs() + for idx := 0; idx < b.N; idx++ { + b.StopTimer() + resetFixture(b) + b.StartTimer() + deleted, err := pruneRelationshipsInBatches(session.Ctx, session.DB, func() graph.Criteria { + return query.Kind(query.Relationship(), graph.StringKind("PruneBatch")) + }, nil) + if err != nil { + b.Fatalf("prune relationships: %v", err) + } + if deleted != 2_000 { + b.Fatalf("deleted relationships: got %d, want 2000", deleted) + } + } + }) + + b.Run("PRUNE-06 node ID selection high-degree cascade and batch delete", func(b *testing.B) { + b.ReportAllocs() + for idx := 0; idx < b.N; idx++ { + b.StopTimer() + resetFixture(b) + b.StartTimer() + deleted, err := pruneNodesInBatches(session.Ctx, session.DB, func() graph.Criteria { + return query.Equals(query.NodeProperty("remove"), true) + }, nil) + if err != nil { + b.Fatalf("prune nodes: %v", err) + } + if deleted != 1_001 { + b.Fatalf("deleted nodes: got %d, want 1001", deleted) + } + } + }) +} + +// trustPruningCriteria selects domain-to-domain relationships of kind whose +// last-seen time predates either endpoint's collection time. +func trustPruningCriteria(kind string) graph.Criteria { + return query.And( + query.Kind(query.Start(), graph.StringKind("Domain")), + query.Kind(query.End(), graph.StringKind("Domain")), + query.KindIn(query.Relationship(), graph.StringKind(kind)), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + ) +} + +// directionalTrustCriteria selects the two directed trust kinds between the +// supplied fixture endpoints. +func directionalTrustCriteria(idMap opengraph.IDMap, forward, reverse string) graph.Criteria { + forwardID := idMap[forward] + reverseID := idMap[reverse] + return query.And( + query.Kind(query.Start(), graph.StringKind("Domain")), + query.Kind(query.End(), graph.StringKind("Domain")), + query.Or( + query.And( + query.Equals(query.StartID(), forwardID), + query.Equals(query.EndID(), reverseID), + query.KindIn(query.Relationship(), graph.StringKind("AbuseTGTDelegation")), + ), + query.And( + query.Equals(query.StartID(), reverseID), + query.Equals(query.EndID(), forwardID), + query.KindIn(query.Relationship(), graph.StringKind("SpoofSIDHistory")), + ), + ), + ) +} + +// pruningProtectedNodeKinds returns the labels whose nodes must survive trust-pruning regression queries even when their relationships are stale. +func pruningProtectedNodeKinds() graph.Kinds { + return graph.Kinds{ + graph.StringKind("Domain"), + graph.StringKind("Tenant"), + graph.StringKind("Meta"), + graph.StringKind("MetaIncludes"), + graph.StringKind("MigrationData"), + } +} + +// trustPruningRelationshipMarkers returns sorted marker properties from selected relationships. +func trustPruningRelationshipMarkers(t *testing.T, relationships []*graph.Relationship) []string { + t.Helper() + markers := make([]string, 0, len(relationships)) + for _, relationship := range relationships { + marker, err := relationship.Properties.Get("marker").String() + require.NoError(t, err) + markers = append(markers, marker) + } + sort.Strings(markers) + return markers +} + +// trustPruningFixtureIDs maps database IDs to sorted stable fixture identifiers. +func trustPruningFixtureIDs(t *testing.T, idMap opengraph.IDMap, ids []graph.ID) []string { + t.Helper() + fixtureIDs := make([]string, 0, len(ids)) + for _, id := range ids { + fixtureIDs = append(fixtureIDs, regressionFixtureID(t, idMap, id)) + } + sort.Strings(fixtureIDs) + return fixtureIDs +} + +// trustPruningFixture builds stale, current, null-timestamp, and decoy trust +// relationships for pruning regressions. +func trustPruningFixture() *opengraph.Graph { + return &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "early", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(2)}, + }, + { + ID: "late-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "late-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-equal", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-new", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-missing", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "candidate-rel-null", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-null", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-old", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-equal", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "session-new", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(4)}, + }, + { + ID: "equal-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(3)}, + }, + { + ID: "equal-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": regressionDay(3)}, + }, + { + ID: "wrong-end", + Kinds: []string{"Computer"}, + Properties: map[string]any{"lastcollected": regressionDay(4), "lastseen": regressionDay(4)}, + }, + { + ID: "candidate-missing", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{}, + }, + { + ID: "candidate-null", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": nil}, + }, + { + ID: "candidate-old", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": regressionDay(2)}, + }, + { + ID: "candidate-equal", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": regressionDay(3)}, + }, + { + ID: "candidate-new", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"lastseen": regressionDay(4)}, + }, + { + ID: "orphan-missing", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"objectid": "S-1-5-100"}, + }, + { + ID: "orphan-null", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"name": nil, "objectid": "S-1-5-101"}, + }, + { + ID: "orphan-empty", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"name": "", "objectid": "S-1-5-102"}, + }, + { + ID: "orphan-wrong-prefix", + Kinds: []string{"CandidateNode"}, + Properties: map[string]any{"objectid": "X-1-5-103"}, + }, + { + ID: "orphan-protected", + Kinds: []string{"CandidateNode", "Domain"}, + Properties: map[string]any{"objectid": "S-1-5-104"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "late-a", + EndID: "early", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "same-old"}, + }, + { + StartID: "equal-a", + EndID: "equal-b", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "same-equal"}, + }, + { + StartID: "late-a", + EndID: "wrong-end", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "same-wrong-end"}, + }, + { + StartID: "late-a", + EndID: "early", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "cross-old"}, + }, + { + StartID: "equal-a", + EndID: "equal-b", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "cross-equal"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "valid-forward-abuse"}, + }, + { + StartID: "late-b", + EndID: "late-a", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "valid-reverse-spoof"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "invalid-forward-spoof"}, + }, + { + StartID: "late-b", + EndID: "late-a", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "invalid-reverse-abuse"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": regressionDay(2), "marker": "candidate-old"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-equal", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "candidate-equal"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-new", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": regressionDay(4), "marker": "candidate-new"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-missing", + Kind: "CandidateRel", + Properties: map[string]any{"marker": "candidate-missing"}, + }, + { + StartID: "late-a", + EndID: "candidate-rel-null", + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": nil, "marker": "candidate-null"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "HasSession", + Properties: map[string]any{"marker": "session-missing"}, + }, + { + StartID: "late-a", + EndID: "session-null", + Kind: "HasSession", + Properties: map[string]any{"lastseen": nil, "marker": "session-null"}, + }, + { + StartID: "late-a", + EndID: "session-old", + Kind: "HasSession", + Properties: map[string]any{"lastseen": regressionDay(2), "marker": "session-old"}, + }, + { + StartID: "late-a", + EndID: "session-equal", + Kind: "HasSession", + Properties: map[string]any{"lastseen": regressionDay(3), "marker": "session-equal"}, + }, + { + StartID: "late-a", + EndID: "session-new", + Kind: "HasSession", + Properties: map[string]any{"lastseen": regressionDay(4), "marker": "session-new"}, + }, + { + StartID: "late-a", + EndID: "late-b", + Kind: "MetaIncludes", + Properties: map[string]any{"lastseen": regressionDay(2), "marker": "meta-includes-old"}, + }, + }, + } +} + +// batchPruningFixture builds removable relationships and a high-degree node +// population for batched pruning tests and benchmarks. +func batchPruningFixture(fanout int) *opengraph.Graph { + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "rel-a", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "rel-a"}, + }, + { + ID: "rel-b", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "rel-b"}, + }, + { + ID: "rel-c", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "rel-c"}, + }, + { + ID: "single", + Kinds: []string{"PruneDeleteNode"}, + Properties: map[string]any{"objectid": "single", "remove": true}, + }, + { + ID: "high", + Kinds: []string{"PruneDeleteNode"}, + Properties: map[string]any{"objectid": "high", "remove": true}, + }, + { + ID: "survivor", + Kinds: []string{"PruneDeleteNode"}, + Properties: map[string]any{"objectid": "survivor", "remove": false}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "rel-a", + EndID: "rel-b", + Kind: "PruneDelete", + Properties: map[string]any{"marker": "single"}, + }, + { + StartID: "rel-a", + EndID: "rel-c", + Kind: "PruneDelete", + Properties: map[string]any{"marker": "many-a"}, + }, + { + StartID: "rel-b", + EndID: "rel-a", + Kind: "PruneDelete", + Properties: map[string]any{"marker": "many-b"}, + }, + { + StartID: "rel-a", + EndID: "rel-b", + Kind: "PruneSurvivor", + Properties: map[string]any{"marker": "survivor"}, + }, + { + StartID: "survivor", + EndID: "rel-a", + Kind: "PruneIncident", + Properties: map[string]any{"marker": "survivor-incident"}, + }, + { + StartID: "high", + EndID: "high", + Kind: "PruneIncident", + Properties: map[string]any{"marker": "high-self"}, + }, + }, + } + + for idx, neighborID := range FixtureNames("neighbor", fanout) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: neighborID, + Kinds: []string{"PruneNeighbor"}, + Properties: map[string]any{"name": neighborID}, + }) + startID, endID := "high", neighborID + if idx%2 == 0 { + startID, endID = neighborID, "high" + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "PruneIncident", + Properties: map[string]any{"marker": neighborID}, + }) + } + return fixture +} + +// pruneRelationshipsInBatches snapshots matching relationship IDs, invokes the selection hook, and deletes those IDs in one batch. +func pruneRelationshipsInBatches(ctx context.Context, db graph.Database, criteria graph.CriteriaProvider, afterSelect func([]graph.ID) error) (int, error) { + var ids []graph.ID + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + ids, err = ops.FetchRelationshipIDs(tx.Relationships().Filterf(criteria)) + return err + }); err != nil { + return 0, err + } + if afterSelect != nil { + if err := afterSelect(ids); err != nil { + return 0, err + } + } + + deleted := 0 + if err := db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteRelationship(id); err != nil { + return err + } + + deleted++ + } + return nil + }); err != nil { + return 0, err + } + + return deleted, nil +} + +// pruneNodesInBatches snapshots matching node IDs, invokes the selection hook, and deletes those IDs in one batch. +func pruneNodesInBatches(ctx context.Context, db graph.Database, criteria graph.CriteriaProvider, afterSelect func([]graph.ID) error) (int, error) { + var ids []graph.ID + if err := db.ReadTransaction(ctx, func(tx graph.Transaction) error { + var err error + ids, err = ops.FetchNodeIDs(tx.Nodes().Filterf(criteria)) + return err + }); err != nil { + return 0, err + } + if afterSelect != nil { + if err := afterSelect(ids); err != nil { + return 0, err + } + } + + deleted := 0 + if err := db.BatchOperation(ctx, func(batch graph.Batch) error { + for _, id := range ids { + if err := batch.DeleteNode(id); err != nil { + return err + } + + deleted++ + } + return nil + }); err != nil { + return 0, err + } + + return deleted, nil +} + +// regressionDay returns midnight UTC on the requested January 2026 day for deterministic temporal fixtures. +func regressionDay(day int) time.Time { + return time.Date(2026, time.January, day, 0, 0, 0, 0, time.UTC) +} diff --git a/integration/wipe_graph_test.go b/integration/wipe_graph_test.go index 0e1390f2..574ccf75 100644 --- a/integration/wipe_graph_test.go +++ b/integration/wipe_graph_test.go @@ -11,16 +11,23 @@ import ( "github.com/stretchr/testify/require" ) -// WipeGraph is a Postgres-only bulk-delete primitive, so this suite is scoped to the pg driver and skips itself unless -// CONNECTION_STRING selects a Postgres backend. +// TestWipeGraph verifies the PostgreSQL-only bulk-delete primitive and skips unless CONNECTION_STRING selects that backend. func TestWipeGraph(t *testing.T) { var ( wipeNode = graph.StringKind("WipeNode") survivor = graph.StringKind("WipeSurvivor") wipeEdge = graph.StringKind("WIPE_EDGE") - defaultGraph = graph.Graph{Name: "wipe_default", Nodes: graph.Kinds{wipeNode, survivor}, Edges: graph.Kinds{wipeEdge}} - secondaryGraph = graph.Graph{Name: "wipe_secondary", Nodes: graph.Kinds{wipeNode, survivor}, Edges: graph.Kinds{wipeEdge}} + defaultGraph = graph.Graph{ + Name: "wipe_default", + Nodes: graph.Kinds{wipeNode, survivor}, + Edges: graph.Kinds{wipeEdge}, + } + secondaryGraph = graph.Graph{ + Name: "wipe_secondary", + Nodes: graph.Kinds{wipeNode, survivor}, + Edges: graph.Kinds{wipeEdge}, + } schema = graph.Schema{ Graphs: []graph.Graph{defaultGraph, secondaryGraph}, @@ -73,7 +80,7 @@ func TestWipeGraph(t *testing.T) { session.ClearGraph(t) seed(t) - require.Equal(t, int64(3), countNodes(t, ctx, db)) + require.Equal(t, int64(3), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(1), countEdges(t, ctx, db)) require.NoError(t, wiper.WipeGraph(ctx, func(tx graph.Transaction) error { @@ -81,7 +88,7 @@ func TestWipeGraph(t *testing.T) { return err })) - require.Equal(t, int64(1), countNodes(t, ctx, db)) + require.Equal(t, int64(1), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(0), countEdges(t, ctx, db)) require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { @@ -113,7 +120,7 @@ func TestWipeGraph(t *testing.T) { require.ErrorIs(t, err, errRetain) // The transaction rolled back, so the seeded graph is left untouched. - require.Equal(t, int64(3), countNodes(t, ctx, db)) + require.Equal(t, int64(3), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(1), countEdges(t, ctx, db)) }) @@ -123,25 +130,32 @@ func TestWipeGraph(t *testing.T) { require.NoError(t, wiper.WipeGraph(ctx, nil)) - require.Equal(t, int64(0), countNodes(t, ctx, db)) + require.Equal(t, int64(0), countNodes(t, ctx, db, defaultGraph, secondaryGraph)) require.Equal(t, int64(0), countEdges(t, ctx, db)) }) } -func countNodes(t *testing.T, ctx context.Context, db graph.Database) int64 { +// countNodes returns the total node count across graphs. +func countNodes(t *testing.T, ctx context.Context, db graph.Database, graphs ...graph.Graph) int64 { t.Helper() var count int64 require.NoError(t, db.ReadTransaction(ctx, func(tx graph.Transaction) error { - result, err := tx.Nodes().Count() - count = result - return err + for _, targetGraph := range graphs { + result, err := tx.WithGraph(targetGraph).Nodes().Count() + if err != nil { + return err + } + count += result + } + return nil })) return count } +// countEdges returns the relationship count in the database's current graph. func countEdges(t *testing.T, ctx context.Context, db graph.Database) int64 { t.Helper() diff --git a/query/builder_test.go b/query/builder_test.go index af2237da..091e57f2 100644 --- a/query/builder_test.go +++ b/query/builder_test.go @@ -78,6 +78,7 @@ func TestBuilderProjectionModifiersAreOrderIndependent(t *testing.T) { } } +// TestBuilderRendersRawPropertyKeys verifies the legacy builder preserves escaped property-key syntax in rendered Cypher. func TestBuilderRendersRawPropertyKeys(t *testing.T) { builder := query.NewBuilder(nil) builder.Apply(query.Returning( diff --git a/query/neo4j/neo4j_test.go b/query/neo4j/neo4j_test.go index 1305ab23..c2fa801b 100644 --- a/query/neo4j/neo4j_test.go +++ b/query/neo4j/neo4j_test.go @@ -14,27 +14,45 @@ import ( ) var ( + // SystemTags is the synthetic system-tags property used by query-builder tests. SystemTags = "system_tags" - User = graph.StringKind("User") - Domain = graph.StringKind("Domain") - Computer = graph.StringKind("Computer") - Group = graph.StringKind("Group") - HasSession = graph.StringKind("HasSession") + // User is the user node kind used by query-builder fixtures. + User = graph.StringKind("User") + + // Domain is the domain node kind used by query-builder fixtures. + Domain = graph.StringKind("Domain") + + // Computer is the computer node kind used by query-builder fixtures. + Computer = graph.StringKind("Computer") + + // Group is the group node kind used by query-builder fixtures. + Group = graph.StringKind("Group") + + // HasSession is the relationship kind used by session-path fixtures. + HasSession = graph.StringKind("HasSession") + + // GenericWrite is the relationship kind used by generic-write fixtures. GenericWrite = graph.StringKind("GenericWrite") ) +// QueryOutputAssertion contains one accepted query rendering and parameter map. type QueryOutputAssertion struct { - Query string + // Query is the expected rendered Cypher text. + Query string + + // Parameters contains the expected query parameters. Parameters map[string]any } +// expectAnalysisError returns an assertion that requires query preparation to report an analysis error. func expectAnalysisError(rawQuery *cypher.RegularQuery) func(t *testing.T) { return func(t *testing.T) { require.NotNil(t, neo4j.NewQueryBuilder(rawQuery).Prepare()) } } +// assertQueryShortestPathResult prepares a shortest-path query and compares its rendered text and optional parameters. func assertQueryShortestPathResult(rawQuery *cypher.RegularQuery, expectedOutput string, expectedParameters ...map[string]any) func(t *testing.T) { return func(t *testing.T) { builder := neo4j.NewQueryBuilder(rawQuery) @@ -53,6 +71,7 @@ func assertQueryShortestPathResult(rawQuery *cypher.RegularQuery, expectedOutput } } +// assertQueryResult prepares a query and compares its rendered text and optional parameters. func assertQueryResult(rawQuery *cypher.RegularQuery, expectedOutput string, expectedParameters ...map[string]any) func(t *testing.T) { return func(t *testing.T) { var ( @@ -76,6 +95,7 @@ func assertQueryResult(rawQuery *cypher.RegularQuery, expectedOutput string, exp } } +// assertOneOfQueryResult requires a prepared query to match one accepted rendering and parameter set. func assertOneOfQueryResult(rawQuery *cypher.RegularQuery, expectations []QueryOutputAssertion) func(t *testing.T) { return func(t *testing.T) { builder := neo4j.NewQueryBuilder(rawQuery) @@ -206,7 +226,612 @@ func TestQueryBuilderProjectionModifiersAreOrderIndependent(t *testing.T) { } } +// TestQueryBuilder_LOGIC01PreservesBranchLocalRelationshipKinds verifies disjunctive branches retain their own relationship-kind predicates. +func TestQueryBuilder_LOGIC01PreservesBranchLocalRelationshipKinds(t *testing.T) { + rawQuery := query.SinglePartQuery( + query.Where( + query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("KindA")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("KindB")), + ), + ), + ), + query.Returning(query.RelationshipID()), + ) + + assertQueryResult( + rawQuery, + "match (s)-[r]->(e) where (id(s) = $p0 and id(e) = $p1 and r:KindA or id(s) = $p2 and id(e) = $p3 and r:KindB) return id(r)", + map[string]any{ + "p0": graph.ID(101), + "p1": graph.ID(202), + "p2": graph.ID(202), + "p3": graph.ID(101), + }, + )(t) +} + +// TestQueryBuilder_LogicalForms verifies Neo4j rendering preserves supported logical expression shapes and precedence. +func TestQueryBuilder_LogicalForms(t *testing.T) { + temporalThreshold := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + + t.Run("LOGIC-02 cross-binding temporal disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where( + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + ), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r]->(e) where (r.lastseen < s.lastcollected or r.lastseen < e.lastcollected) return id(r)", + )) + + t.Run("LOGIC-03 scoped negation and null-aware age predicate", assertQueryResult( + query.SinglePartQuery( + query.Where( + query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("Protected"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), temporalThreshold), + ), + ), + ), + query.Returning(query.NodeID()), + ), + "match (n) where not (n:Protected) and (not (n.lastseen is not null) or n.lastseen < $p0) return id(n)", + map[string]any{"p0": temporalThreshold}, + )) +} + +// TestQueryBuilder_LOGIC05ProjectionOrder verifies projection ordering remains stable for the LOGIC-05 regression form. +func TestQueryBuilder_LOGIC05ProjectionOrder(t *testing.T) { + testCases := map[string]struct { + // projection is the return clause under test. + projection *cypher.Return + + // expected is the rendered Cypher query. + expected string + }{ + "full opposite node plus relationship": { + projection: query.Returning(query.Relationship(), query.End()), + expected: "match ()-[r]->(e) return r, e", + }, + "opposite ID and kinds plus relationship ID and kind": { + projection: query.Returning(query.EndID(), query.KindsOf(query.End()), query.RelationshipID(), query.KindsOf(query.Relationship())), + expected: "match ()-[r]->(e) return id(e), labels(e), id(r), type(r)", + }, + "start relationship end triple": { + projection: query.Returning(query.Start(), query.Relationship(), query.End()), + expected: "match (s)-[r]->(e) return s, r, e", + }, + "relationship ID only": { + projection: query.Returning(query.RelationshipID()), + expected: "match ()-[r]->() return id(r)", + }, + "full relationship": { + projection: query.Returning(query.Relationship()), + expected: "match ()-[r]->() return r", + }, + } + + for name, testCase := range testCases { + t.Run(name, assertQueryResult( + query.SinglePartQuery(testCase.projection), + testCase.expected, + )) + } +} + +// TestQueryBuilder_ReconciliationForms verifies reconciliation forms render the expected predicates and projections. +func TestQueryBuilder_ReconciliationForms(t *testing.T) { + reconciliationKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("ReconcileKind%02d", idx+1)) + } + return kinds + } + + for _, count := range []int{1, 2, 9, 30} { + kinds := reconciliationKinds(count) + renderedKinds := "ReconcileKind01" + for idx := 1; idx < count; idx++ { + renderedKinds += fmt.Sprintf("|ReconcileKind%02d", idx+1) + } + + t.Run(fmt.Sprintf("REC-01 inbound relationship delete with %d kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("ADEntity")), + query.Equals(query.EndProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ), + fmt.Sprintf("match ()-[r:%s]->(e) where e:ADEntity and e.objectid = $p0 delete r", renderedKinds), + map[string]any{"p0": "target-id"}, + )) + + t.Run(fmt.Sprintf("REC-02 outbound relationship delete with %d kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("ADEntity")), + query.Equals(query.StartProperty("objectid"), "target-id"), + query.KindIn(query.Relationship(), kinds...), + )), + query.Delete(query.Relationship()), + ), + fmt.Sprintf("match (s)-[r:%s]->() where s:ADEntity and s.objectid = $p0 delete r", renderedKinds), + map[string]any{"p0": "target-id"}, + )) + } + + t.Run("REC-03 inbound primary-group relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("Group")), + query.Equals(query.EndProperty("objectid"), "group-id"), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + query.Equals(query.RelationshipProperty("isprimarygroup"), false), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:MemberOf]->(e) where e:Group and e.objectid = $p0 and r.isprimarygroup = $p1 delete r", + map[string]any{"p0": "group-id", "p1": false}, + )) + + t.Run("REC-03 outbound primary-group relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("Computer")), + query.Equals(query.StartProperty("objectid"), "computer-id"), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + query.Equals(query.RelationshipProperty("isprimarygroup"), true), + )), + query.Delete(query.Relationship()), + ), + "match (s)-[r:MemberOf]->() where s:Computer and s.objectid = $p0 and r.isprimarygroup = $p1 delete r", + map[string]any{"p0": "computer-id", "p1": true}, + )) + + t.Run("REC-04 endpoint object ID list relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("ReconcileKind01")), + query.Kind(query.End(), graph.StringKind("ADEntity")), + query.In(query.EndProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:ReconcileKind01]->(e) where e:ADEntity and e.objectid in $p0 delete r", + map[string]any{"p0": []string{"target-1", "target-2"}}, + )) + + t.Run("REC-05 delegated enrollment discovery projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.In(query.EndProperty("objectid"), []string{"ca-1", "ca-2"}), + query.Kind(query.Relationship(), graph.StringKind("PublishedTo")), + query.Kind(query.Start(), graph.StringKind("CertTemplate")), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:PublishedTo]->(e) where e.objectid in $p0 and s:CertTemplate return r, s", + map[string]any{"p0": []string{"ca-1", "ca-2"}}, + )) + + t.Run("REC-06 delegated enrollment relationship delete by end IDs", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("CertTemplate")), + query.InIDs(query.EndID(), graph.ID(101), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("DelegatedEnrollmentAgent")), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:DelegatedEnrollmentAgent]->(e) where e:CertTemplate and id(e) in $p0 delete r", + map[string]any{"p0": []graph.ID{101, 202}}, + )) + + t.Run("REC-07 HostsCAService relationship delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.End(), graph.StringKind("EnterpriseCA")), + query.Equals(query.EndProperty("objectid"), "ca-id"), + query.KindIn(query.Relationship(), graph.StringKind("HostsCAService")), + )), + query.Delete(query.Relationship()), + ), + "match ()-[r:HostsCAService]->(e) where e:EnterpriseCA and e.objectid = $p0 delete r", + map[string]any{"p0": "ca-id"}, + )) + + t.Run("REC-08 AD entity detach delete", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("ADEntity")), + query.In(query.NodeProperty("objectid"), []string{"target-1", "target-2"}), + )), + query.Delete(query.Node()), + ), + "match (n) where n:ADEntity and n.objectid in $p0 detach delete n", + map[string]any{"p0": []string{"target-1", "target-2"}}, + )) +} + +// TestQueryBuilder_TrustAndPruningForms verifies trust and pruning forms preserve selector and mutation semantics. +func TestQueryBuilder_TrustAndPruningForms(t *testing.T) { + threshold := time.Date(2026, time.January, 3, 0, 0, 0, 0, time.UTC) + domain := graph.StringKind("Domain") + + t.Run("TRUST-01 SameForestTrust ID projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), domain), + query.Kind(query.End(), domain), + query.Kind(query.Relationship(), graph.StringKind("SameForestTrust")), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r:SameForestTrust]->(e) where s:Domain and e:Domain and (r.lastseen < s.lastcollected or r.lastseen < e.lastcollected) return id(r)", + )) + + t.Run("TRUST-02 CrossForestTrust full relationship projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), domain), + query.Kind(query.End(), domain), + query.KindIn(query.Relationship(), graph.StringKind("CrossForestTrust")), + query.Or( + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.StartProperty("lastcollected")), + query.BeforeGraphQuery(query.RelationshipProperty("lastseen"), query.EndProperty("lastcollected")), + ), + )), + query.Returning(query.Relationship()), + ), + "match (s)-[r:CrossForestTrust]->(e) where s:Domain and e:Domain and (r.lastseen < s.lastcollected or r.lastseen < e.lastcollected) return r", + )) + + t.Run("TRUST-03 directional derived trust disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), domain), + query.Kind(query.End(), domain), + query.Or( + query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), graph.StringKind("AbuseTGTDelegation")), + ), + query.And( + query.Equals(query.StartID(), graph.ID(202)), + query.Equals(query.EndID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("SpoofSIDHistory")), + ), + ), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r]->(e) where s:Domain and e:Domain and (id(s) = $p0 and id(e) = $p1 and r:AbuseTGTDelegation or id(s) = $p2 and id(e) = $p3 and r:SpoofSIDHistory) return id(r)", + map[string]any{"p0": graph.ID(101), "p1": graph.ID(202), "p2": graph.ID(202), "p3": graph.ID(101)}, + )) + + t.Run("PRUNE-01 relationship TTL excludes several kinds", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Relationship(), graph.StringKind("MetaIncludes"), graph.StringKind("HasSession"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + )), + query.Returning(query.RelationshipID()), + ), + "match ()-[r]->() where not ((r:MetaIncludes or r:HasSession)) and r.lastseen < $p0 return id(r)", + map[string]any{"p0": threshold}, + )) + + t.Run("PRUNE-02 HasSession missing or stale TTL", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Relationship(), graph.StringKind("HasSession")), + query.Or( + query.Not(query.Exists(query.RelationshipProperty("lastseen"))), + query.Before(query.RelationshipProperty("lastseen"), threshold), + ), + )), + query.Returning(query.RelationshipID()), + ), + "match ()-[r:HasSession]->() where (not (r.lastseen is not null) or r.lastseen < $p0) return id(r)", + map[string]any{"p0": threshold}, + )) + + t.Run("PRUNE-03 node TTL excludes several kinds", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("Domain"), graph.StringKind("Tenant"), graph.StringKind("Meta"), graph.StringKind("MetaIncludes"), graph.StringKind("MigrationData"))), + query.Or( + query.Not(query.Exists(query.NodeProperty("lastseen"))), + query.Before(query.NodeProperty("lastseen"), threshold), + ), + )), + query.Returning(query.NodeID()), + ), + "match (n) where not ((n:Domain or n:Tenant or n:Meta or n:MetaIncludes or n:MigrationData)) and (not (n.lastseen is not null) or n.lastseen < $p0) return id(n)", + map[string]any{"p0": threshold}, + )) + + t.Run("PRUNE-04 orphan SID prefix", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Node(), graph.StringKind("Domain"), graph.StringKind("Tenant"), graph.StringKind("Meta"), graph.StringKind("MetaIncludes"), graph.StringKind("MigrationData"))), + query.Not(query.Exists(query.NodeProperty("name"))), + query.StringStartsWith(query.NodeProperty("objectid"), "S-1-5"), + )), + query.Returning(query.NodeID()), + ), + "match (n) where not ((n:Domain or n:Tenant or n:Meta or n:MetaIncludes or n:MigrationData)) and not (n.name is not null) and n.objectid starts with $p0 return id(n)", + map[string]any{"p0": "S-1-5"}, + )) +} + +// TestQueryBuilder_StandaloneHopForms verifies one-hop forms preserve direction, kinds, and endpoint projections. +func TestQueryBuilder_StandaloneHopForms(t *testing.T) { + hopKinds := func(count int) graph.Kinds { + kinds := make(graph.Kinds, count) + for idx := range count { + kinds[idx] = graph.StringKind(fmt.Sprintf("HopKind%02d", idx+1)) + } + return kinds + } + + t.Run("HOP-01 outbound exact start anchor with full directional projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopKind01]->(e) where id(s) = $p0 return r, e", + map[string]any{"p0": graph.ID(101)}, + )) + + t.Run("HOP-01 outbound one-element start IN anchor", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("HopKind01")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopKind01]->(e) where id(s) in $p0 return r, e", + map[string]any{"p0": []graph.ID{101}}, + )) + + t.Run("HOP-02 inbound exact end anchor with full directional projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("HopKind01")), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:HopKind01]->(e) where id(e) = $p0 return r, s", + map[string]any{"p0": graph.ID(202)}, + )) + + for _, count := range []int{2, 5, 9, 30} { + kinds := hopKinds(count) + renderedKinds := "HopKind01" + for idx := 1; idx < count; idx++ { + renderedKinds += fmt.Sprintf("|HopKind%02d", idx+1) + } + + t.Run(fmt.Sprintf("HOP-03 outbound %d relationship kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.End()), + ), + fmt.Sprintf("match (s)-[r:%s]->(e) where id(s) in $p0 return r, e", renderedKinds), + map[string]any{"p0": []graph.ID{101}}, + )) + + t.Run(fmt.Sprintf("HOP-03 inbound %d relationship kinds", count), assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.KindIn(query.Relationship(), kinds...), + )), + query.Returning(query.Relationship(), query.Start()), + ), + fmt.Sprintf("match (s)-[r:%s]->(e) where id(e) in $p0 return r, s", renderedKinds), + map[string]any{"p0": []graph.ID{202}}, + )) + } + + t.Run("HOP-04 opposite endpoint kind disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("HopTypedEdge")), + query.KindIn(query.End(), graph.StringKind("HopEndA"), graph.StringKind("HopEndB")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopTypedEdge]->(e) where id(s) in $p0 and (e:HopEndA or e:HopEndB) return r, e", + map[string]any{"p0": []graph.ID{101}}, + )) + + t.Run("HOP-05 endpoint IDs through variable spelling", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopIDEdge")), + query.InIDs(query.End(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopIDEdge]->(e) where id(s) = $p0 and id(e) in $p1 return r, e", + map[string]any{"p0": graph.ID(101), "p1": []graph.ID{202, 303}}, + )) + + t.Run("HOP-05 endpoint IDs through identity-function spelling", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.Start(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopIDEdge")), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopIDEdge]->(e) where id(s) in $p0 and id(e) in $p1 return r, e", + map[string]any{"p0": []graph.ID{101}, "p1": []graph.ID{202, 303}}, + )) + + t.Run("HOP-06 opposite endpoint scalar properties", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopPropertyEdge")), + query.Equals(query.EndProperty("enabled"), true), + query.Equals(query.EndProperty("score"), 7), + query.Equals(query.EndProperty("name"), "target"), + query.Equals(query.EndProperty("isassignabletorole"), "true"), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopPropertyEdge]->(e) where id(s) = $p0 and e.enabled = $p1 and e.score = $p2 and e.name = $p3 and e.isassignabletorole = $p4 return r, e", + map[string]any{"p0": graph.ID(101), "p1": true, "p2": 7, "p3": "target", "p4": "true"}, + )) + + t.Run("HOP-07 nested production-style endpoint predicate", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.KindIn(query.Relationship(), graph.StringKind("HopNestedEdge")), + query.Kind(query.End(), graph.StringKind("HopTemplate")), + query.Or( + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.GreaterThan(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authorizedsignatures"), 0), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + query.And( + query.Equals(query.EndProperty("requiresmanagerapproval"), false), + query.Equals(query.EndProperty("schemaversion"), 1), + query.Equals(query.EndProperty("authenticationenabled"), true), + ), + ), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopNestedEdge]->(e) where id(s) = $p0 and e:HopTemplate and (e.requiresmanagerapproval = $p1 and e.schemaversion > $p2 and e.authorizedsignatures = $p3 and e.authenticationenabled = $p4 or e.requiresmanagerapproval = $p5 and e.schemaversion = $p6 and e.authenticationenabled = $p7) return r, e", + map[string]any{"p0": graph.ID(101), "p1": false, "p2": 1, "p3": 0, "p4": true, "p5": false, "p6": 1, "p7": true}, + )) + + t.Run("HOP-08 collection predicates nested with scalar fallback", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopCollectionEdge")), + query.Or( + query.Equals(query.EndProperty("schannelauthenticationenabled"), true), + query.Equals(query.Size(query.EndProperty("effectiveekus")), 0), + query.InInverted(query.EndProperty("effectiveekus"), "1.3.6.1.5.5.7.3.2"), + ), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopCollectionEdge]->(e) where id(s) = $p0 and (e.schannelauthenticationenabled = $p1 or size(e.effectiveekus) = $p2 or $p3 in e.effectiveekus) return r, e", + map[string]any{"p0": graph.ID(101), "p1": true, "p2": 0, "p3": "1.3.6.1.5.5.7.3.2"}, + )) + + t.Run("HOP-09 two-sided endpoint ID lists", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101), graph.ID(202)), + query.InIDs(query.EndID(), graph.ID(303), graph.ID(404)), + query.Kind(query.Relationship(), graph.StringKind("HopSetEdge")), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopSetEdge]->(e) where id(s) in $p0 and id(e) in $p1 return r, e", + map[string]any{"p0": []graph.ID{101, 202}, "p1": []graph.ID{303, 404}}, + )) + + t.Run("HOP-10 outbound endpoint kind property and start anchor", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.End(), graph.StringKind("HopProjectionEnd")), + query.Equals(query.EndProperty("active"), true), + )), + query.Returning(query.Relationship(), query.End()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(s) in $p0 and e:HopProjectionEnd and e.active = $p1 return r, e", + map[string]any{"p0": []graph.ID{101}, "p1": true}, + )) + + t.Run("HOP-10 inbound endpoint kind property and end anchor", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + query.Kind(query.Start(), graph.StringKind("HopProjectionStart")), + query.Equals(query.StartProperty("active"), true), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(e) in $p0 and s:HopProjectionStart and s.active = $p1 return r, s", + map[string]any{"p0": []graph.ID{202}, "p1": true}, + )) + + t.Run("HOP-10 explicit start-node projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + )), + query.Returning(query.Start()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(e) in $p0 return s", + map[string]any{"p0": []graph.ID{202}}, + )) + + t.Run("HOP-10 explicit end-ID and relationship projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.InIDs(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("HopProjectionEdge")), + )), + query.Returning(query.EndID(), query.Relationship()), + ), + "match (s)-[r:HopProjectionEdge]->(e) where id(s) in $p0 return id(e), r", + map[string]any{"p0": []graph.ID{101}}, + )) +} + +// TestQueryBuilder_Render verifies legacy query criteria render the expected Neo4j Cypher and parameters. func TestQueryBuilder_Render(t *testing.T) { + temporalThreshold := time.Date(2026, time.January, 2, 3, 4, 5, 0, time.UTC) + // Node Queries t.Run("Node Count", assertQueryResult(query.SinglePartQuery( query.Where( @@ -555,7 +1180,7 @@ func TestQueryBuilder_Render(t *testing.T) { t.Run("Node Datetime Before", assertQueryResult(query.SinglePartQuery( query.Where( query.And( - query.Before(query.NodeProperty("lastseen"), time.Now().UTC()), + query.Before(query.NodeProperty("lastseen"), temporalThreshold), query.In(query.NodeID(), []int{1, 2, 3, 4}), ), ), @@ -563,7 +1188,10 @@ func TestQueryBuilder_Render(t *testing.T) { query.Returning( query.Node(), ), - ), "match (n) where n.lastseen < $p0 and id(n) in $p1 return n")) + ), "match (n) where n.lastseen < $p0 and id(n) in $p1 return n", map[string]any{ + "p0": temporalThreshold, + "p1": []int{1, 2, 3, 4}, + })) t.Run("Node Datetime Before or Equal to", assertQueryResult(query.SinglePartQuery( query.Where( diff --git a/query/neo4j/relationship_scans_node_lookups_test.go b/query/neo4j/relationship_scans_node_lookups_test.go new file mode 100644 index 00000000..d84eaf82 --- /dev/null +++ b/query/neo4j/relationship_scans_node_lookups_test.go @@ -0,0 +1,419 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package neo4j_test + +import ( + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/query" +) + +// scanLookupKinds converts fixture kind names into the graph.Kinds accepted by query helpers. +func scanLookupKinds(names ...string) graph.Kinds { + kinds := make(graph.Kinds, len(names)) + for idx, name := range names { + kinds[idx] = graph.StringKind(name) + } + return kinds +} + +// TestQueryBuilder_RelationshipScans verifies relationship scan forms render kind, endpoint, property, and ordering constraints correctly. +func TestQueryBuilder_RelationshipScans(t *testing.T) { + t.Run("SCAN-01 base endpoints and relationship ID projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Start(), scanLookupKinds("ADBase", "AZBase")...), + query.Kind(query.Relationship(), graph.StringKind("PostProcessed")), + query.KindIn(query.End(), scanLookupKinds("ADBase", "AZBase")...), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r:PostProcessed]->(e) where (s:ADBase or s:AZBase) and (e:ADBase or e:AZBase) return id(r)", + )) + + t.Run("SCAN-02 excludes Meta endpoints and hydrates relationships", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupKinds("Meta", "MetaDetail")...)), + query.KindIn(query.Relationship(), scanLookupKinds("TrackerA", "TrackerB")...), + query.Not(query.KindIn(query.End(), scanLookupKinds("Meta", "MetaDetail")...)), + )), + query.Returning(query.Relationship()), + ), + "match (s)-[r:TrackerA|TrackerB]->(e) where not ((s:Meta or s:MetaDetail)) and not ((e:Meta or e:MetaDetail)) return r", + )) + + t.Run("SCAN-03 non-Meta lastseen relationship IDs", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Not(query.KindIn(query.Start(), scanLookupKinds("Meta", "MetaDetail")...)), + query.Kind(query.Relationship(), graph.StringKind("MigratedEdge")), + query.Exists(query.RelationshipProperty("lastseen")), + query.Not(query.KindIn(query.End(), scanLookupKinds("Meta", "MetaDetail")...)), + )), + query.Returning(query.RelationshipID()), + ), + "match (s)-[r:MigratedEdge]->(e) where not ((s:Meta or s:MetaDetail)) and r.lastseen is not null and not ((e:Meta or e:MetaDetail)) return id(r)", + )) + + t.Run("SCAN-04 raw ownership scan", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("OwnsRaw")), + query.Kind(query.Start(), graph.StringKind("Entity")), + )), + query.Returning(query.Relationship()), + ), + "match (s)-[r:OwnsRaw]->() where s:Entity return r", + )) + + nineKinds := scanLookupKinds("ScanEdge01", "ScanEdge02", "ScanEdge03", "ScanEdge04", "ScanEdge05", "ScanEdge06", "ScanEdge07", "ScanEdge08", "ScanEdge09") + t.Run("SCAN-05 consolidated nine-kind inbound scan", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Start(), graph.StringKind("Entity")), + query.KindIn(query.Relationship(), nineKinds...), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.Relationship(), query.Start()), + ), + "match (s)-[r:ScanEdge01|ScanEdge02|ScanEdge03|ScanEdge04|ScanEdge05|ScanEdge06|ScanEdge07|ScanEdge08|ScanEdge09]->(e) where s:Entity and id(e) = $p0 return r, s", + map[string]any{"p0": graph.ID(202)}, + )) + + t.Run("SCAN-06 FetchKinds projection order", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Kind(query.End(), graph.StringKind("Computer")), + )), + query.Returning(query.StartID(), query.RelationshipID(), query.KindsOf(query.Relationship()), query.EndID()), + ), + "match (s)-[r:LocalToComputer]->(e) where e:Computer return id(s), id(r), type(r), id(e)", + )) + + t.Run("SCAN-07 directed ID pair projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.KindIn(query.Relationship(), scanLookupKinds("MemberOf", "MemberOfLocalGroup")...)), + query.Returning(query.StartID(), query.EndID()), + ), + "match (s)-[r:MemberOf|MemberOfLocalGroup]->(e) return id(s), id(e)", + )) + + startKinds := scanLookupKinds("Group", "User", "Computer") + t.Run("SCAN-08 scenario A", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Start(), startKinds...), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + query.KindIn(query.Relationship(), scanLookupKinds("GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL", "WritePublicInformation")...), + )), + query.Returning(query.StartID()), + ), + "match (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL|WritePublicInformation]->(e) where (s:Group or s:User or s:Computer) and id(e) in $p0 return id(s)", + map[string]any{"p0": []graph.ID{202, 303}}, + )) + + t.Run("SCAN-08 scenario B", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Start(), startKinds...), + query.InIDs(query.EndID(), graph.ID(202), graph.ID(303)), + query.Kind(query.End(), graph.StringKind("Computer")), + query.KindIn(query.Relationship(), scanLookupKinds("GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL")...), + )), + query.Returning(query.StartID()), + ), + "match (s)-[r:GenericAll|GenericWrite|Owns|WriteOwner|WriteDACL]->(e) where (s:Group or s:User or s:Computer) and id(e) in $p0 and e:Computer return id(s)", + map[string]any{"p0": []graph.ID{202, 303}}, + )) +} + +// TestQueryBuilder_NodeLookups verifies node lookup forms render identifiers, kind filters, and property predicates correctly. +func TestQueryBuilder_NodeLookups(t *testing.T) { + t.Run("LOOKUP-01 kind disjunction ID projection", assertQueryResult( + query.SinglePartQuery( + query.Where(query.KindIn(query.Node(), scanLookupKinds("Group", "User")...)), + query.Returning(query.NodeID()), + ), + "match (n) where (n:Group or n:User) return id(n)", + )) + + t.Run("LOOKUP-01 exact kind full hydration", assertQueryResult( + query.SinglePartQuery( + query.Where(query.Kind(query.Node(), graph.StringKind("Tenant"))), + query.Returning(query.Node()), + ), + "match (n) where n:Tenant return n", + )) + + t.Run("LOOKUP-02 indexed equality and LIMIT 1", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("objectid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + query.Limit(1), + ), + "match (n) where n:Computer and n.objectid = $p0 return n limit 1", + map[string]any{"p0": "S-1-5-21"}, + )) + + t.Run("LOOKUP-02 no-kind two-property equality", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.NodeProperty("name"), "dc.example.test"), + query.Equals(query.NodeProperty("enabled"), true), + )), + query.Returning(query.NodeID()), + ), + "match (n) where n.name = $p0 and n.enabled = $p1 return id(n)", + map[string]any{"p0": "dc.example.test", "p1": true}, + )) + + t.Run("LOOKUP-03 boolean property projection order", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("hasura"), true), + )), + query.Returning(query.NodeID(), query.NodeProperty("hasura")), + ), + "match (n) where n:Computer and n.hasura = $p0 return id(n), n.hasura", + map[string]any{"p0": true}, + )) + + t.Run("LOOKUP-04 prefix and domain equality", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Container")), + query.StringStartsWith(query.NodeProperty("distinguishedname"), "CN=ADMINSDHOLDER,CN=SYSTEM,"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + ), + "match (n) where n:Container and n.distinguishedname starts with $p0 and n.domainsid = $p1 return n", + map[string]any{"p0": "CN=ADMINSDHOLDER,CN=SYSTEM,", "p1": "S-1-5-21"}, + )) + + t.Run("LOOKUP-04 suffix disjunction", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Group")), + query.Or( + query.StringEndsWith(query.NodeProperty("objectid"), "-S-1"), + query.StringEndsWith(query.NodeProperty("objectid"), "-S-2"), + ), + )), + query.Returning(query.NodeID()), + ), + "match (n) where n:Group and (n.objectid ends with $p0 or n.objectid ends with $p1) return id(n)", + map[string]any{"p0": "-S-1", "p1": "-S-2"}, + )) + + t.Run("LOOKUP-05 case-insensitive prefix", assertQueryResult( + query.SinglePartQuery( + query.Where(query.CaseInsensitiveStringStartsWith(query.NodeProperty("name"), "Remote Desktop Users%_")), + query.Returning(query.NodeID()), + ), + "match (n) where toLower(n.name) starts with $p0 return id(n)", + map[string]any{"p0": "remote desktop users%_"}, + )) + + t.Run("LOOKUP-05 case-insensitive contains", assertQueryResult( + query.SinglePartQuery( + query.Where(query.CaseInsensitiveStringContains(query.NodeProperty("objectid"), "Approver_GUID")), + query.Returning(query.Node()), + ), + "match (n) where toLower(n.objectid) contains $p0 return n", + map[string]any{"p0": "approver_guid"}, + )) + + t.Run("LOOKUP-06 required kind groups and suffix", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.KindIn(query.Node(), scanLookupKinds("Group", "User")...), + query.Kind(query.Node(), graph.StringKind("Entity")), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + )), + query.Returning(query.Node()), + ), + "match (n) where (n:Group or n:User) and n:Entity and n.objectid ends with $p0 and n.domainsid = $p1 return n", + map[string]any{"p0": "-512", "p1": "S-1-5-21"}, + )) + + t.Run("LOOKUP-06 required and excluded kinds", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("Entity")), + query.Not(query.KindIn(query.Node(), scanLookupKinds("Group", "LocalGroup")...)), + query.StringEndsWith(query.NodeProperty("objectid"), "-512"), + )), + query.Returning(query.Node()), + ), + "match (n) where n:Entity and not ((n:Group or n:LocalGroup)) and n.objectid ends with $p0 return n", + map[string]any{"p0": "-512"}, + )) + + t.Run("LOOKUP-07 missing name", assertQueryResult( + query.SinglePartQuery( + query.Where(query.Not(query.Exists(query.NodeProperty("name")))), + query.Returning(query.Node()), + ), + "match (n) where not (n.name is not null) return n", + )) + + t.Run("LOOKUP-08 either approver property present", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("AZRole")), + query.Equals(query.NodeProperty("tenantid"), "tenant-1"), + query.Equals(query.NodeProperty("approvalrequired"), true), + query.Or( + query.IsNotNull(query.NodeProperty("userapprovers")), + query.IsNotNull(query.NodeProperty("groupapprovers")), + ), + )), + query.Returning(query.Node()), + ), + "match (n) where n:AZRole and n.tenantid = $p0 and n.approvalrequired = $p1 and (n.userapprovers is not null or n.groupapprovers is not null) return n", + map[string]any{"p0": "tenant-1", "p1": true}, + )) + + t.Run("LOOKUP-09 ID list full hydration", assertQueryResult( + query.SinglePartQuery( + query.Where(query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202), graph.ID(101))), + query.Returning(query.Node()), + ), + "match (n) where id(n) in $p0 return n", + map[string]any{"p0": []graph.ID{101, 202, 101}}, + )) + + t.Run("LOOKUP-10 nested negated account flags", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Kind(query.Node(), graph.StringKind("User")), + query.Not(query.And( + query.Exists(query.NodeProperty("gmsa")), + query.Equals(query.NodeProperty("gmsa"), true), + )), + query.Not(query.And( + query.Exists(query.NodeProperty("msa")), + query.Equals(query.NodeProperty("msa"), true), + )), + query.InIDs(query.NodeID(), graph.ID(101), graph.ID(202)), + )), + query.Returning(query.Node()), + ), + "match (n) where n:User and not (n.gmsa is not null and n.gmsa = $p0) and not (n.msa is not null and n.msa = $p1) and id(n) in $p2 return n", + map[string]any{"p0": true, "p1": true, "p2": []graph.ID{101, 202}}, + )) + + t.Run("LOOKUP-11 tenant adjacency with endpoint list property", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Kind(query.Relationship(), graph.StringKind("Contains")), + query.KindIn(query.End(), scanLookupKinds("AZRole", "AZServicePrincipal")...), + query.In(query.EndProperty("roletemplateid"), []string{"role-a", "role-b"}), + )), + query.Returning(query.End()), + ), + "match (s)-[r:Contains]->(e) where id(s) = $p0 and (e:AZRole or e:AZServicePrincipal) and e.roletemplateid in $p1 return e", + map[string]any{"p0": graph.ID(101), "p1": []string{"role-a", "role-b"}}, + )) + + t.Run("LOOKUP-12 exact relationship key First", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.StartID(), graph.ID(101)), + query.Equals(query.EndID(), graph.ID(202)), + query.Kind(query.Relationship(), graph.StringKind("MemberOf")), + )), + query.Returning(query.Relationship()), + query.Limit(1), + ), + "match (s)-[r:MemberOf]->(e) where id(s) = $p0 and id(e) = $p1 return r limit 1", + map[string]any{"p0": graph.ID(101), "p1": graph.ID(202)}, + )) + + t.Run("LOOKUP-13 suffix and bound endpoint full start", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.Start()), + ), + "match (s)-[r:LocalToComputer]->(e) where s.objectid ends with $p0 and id(e) = $p1 return s", + map[string]any{"p0": "-555", "p1": graph.ID(202)}, + )) + + t.Run("LOOKUP-13 suffix and bound endpoint start ID", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.StringEndsWith(query.StartProperty("objectid"), "-555"), + query.Kind(query.Relationship(), graph.StringKind("LocalToComputer")), + query.Equals(query.EndID(), graph.ID(202)), + )), + query.Returning(query.StartID()), + ), + "match (s)-[r:LocalToComputer]->(e) where s.objectid ends with $p0 and id(e) = $p1 return id(s)", + map[string]any{"p0": "-555", "p1": graph.ID(202)}, + )) + + t.Run("LOOKUP-14 descending property order", assertQueryResult( + query.SinglePartQuery( + query.Where(query.Kind(query.Node(), graph.StringKind("Domain"))), + query.Returning(query.Node()), + query.OrderBy(query.Order(query.NodeProperty("name"), query.Descending())), + ), + "match (n) where n:Domain return n order by n.name desc", + )) + + ntlmCriteria := query.And( + query.Kind(query.Node(), graph.StringKind("Computer")), + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty("ldapavailable"), true), + query.Equals(query.NodeProperty("ldapsigning"), false), + ) + + t.Run("LOOKUP-16 typed NTLM ID projection", assertQueryResult( + query.SinglePartQuery(query.Where(ntlmCriteria), query.Returning(query.NodeID())), + "match (n) where n:Computer and n.domainsid = $p0 and n.isdc = $p1 and n.ldapavailable = $p2 and n.ldapsigning = $p3 return id(n)", + map[string]any{"p0": "S-1-5-21", "p1": true, "p2": true, "p3": false}, + )) + + t.Run("LOOKUP-16 untyped NTLM full hydration", assertQueryResult( + query.SinglePartQuery( + query.Where(query.And( + query.Equals(query.NodeProperty("domainsid"), "S-1-5-21"), + query.Equals(query.NodeProperty("isdc"), true), + query.Equals(query.NodeProperty("ldapsavailable"), true), + query.Equals(query.NodeProperty("epa"), false), + )), + query.Returning(query.Node()), + ), + "match (n) where n.domainsid = $p0 and n.isdc = $p1 and n.ldapsavailable = $p2 and n.epa = $p3 return n", + map[string]any{"p0": "S-1-5-21", "p1": true, "p2": true, "p3": false}, + )) +} diff --git a/query/neo4j/rewrite.go b/query/neo4j/rewrite.go index b99b06a9..27e1f272 100644 --- a/query/neo4j/rewrite.go +++ b/query/neo4j/rewrite.go @@ -20,10 +20,12 @@ func NewExpressionListRewriter() walk.Visitor[cypher.SyntaxNode] { } } +// pushExpression records a syntax node as the current ancestor during traversal. func (s *ExpressionListRewriter) pushExpression(expression cypher.SyntaxNode) { s.descentStack = append(s.descentStack, expression) } +// peekExpression returns the nearest ancestor syntax node without removing it. func (s *ExpressionListRewriter) peekExpression() (cypher.SyntaxNode, bool) { if len(s.descentStack) == 0 { return nil, false @@ -32,6 +34,7 @@ func (s *ExpressionListRewriter) peekExpression() (cypher.SyntaxNode, bool) { return s.descentStack[len(s.descentStack)-1], true } +// peekExpressionList returns the nearest ancestor when it supports list replacement operations. func (s *ExpressionListRewriter) peekExpressionList() (cypher.ExpressionList, bool) { if ancestorNode, hasPrevious := s.peekExpression(); hasPrevious { ancestorExpressionList, isExpressionList := ancestorNode.(cypher.ExpressionList) @@ -41,6 +44,7 @@ func (s *ExpressionListRewriter) peekExpressionList() (cypher.ExpressionList, bo return nil, false } +// hasNegationAncestor reports whether traversal is currently nested beneath a negation. func (s *ExpressionListRewriter) hasNegationAncestor() bool { for idx := len(s.descentStack) - 1; idx >= 0; idx-- { if _, isNegation := s.descentStack[idx].(*cypher.Negation); isNegation { @@ -51,10 +55,23 @@ func (s *ExpressionListRewriter) hasNegationAncestor() bool { return false } +// hasDisjunctionAncestor reports whether traversal is currently nested beneath a disjunction. +func (s *ExpressionListRewriter) hasDisjunctionAncestor() bool { + for idx := len(s.descentStack) - 1; idx >= 0; idx-- { + if _, isDisjunction := s.descentStack[idx].(*cypher.Disjunction); isDisjunction { + return true + } + } + + return false +} + +// popExpression removes the current node from the traversal ancestry stack. func (s *ExpressionListRewriter) popExpression() { s.descentStack = s.descentStack[:len(s.descentStack)-1] } +// unwrapParenthetical removes nested parentheses so rewrite rules can inspect the underlying syntax node. func unwrapParenthetical(expression cypher.SyntaxNode) cypher.SyntaxNode { cursor := expression @@ -71,6 +88,7 @@ func unwrapParenthetical(expression cypher.SyntaxNode) cypher.SyntaxNode { return cursor } +// rewriteStringNegation preserves null-inclusive semantics when Neo4j evaluates negated string comparisons. func (s *ExpressionListRewriter) rewriteStringNegation(negation *cypher.Negation) { if ancestorExpressionList, isExpressionList := s.peekExpressionList(); isExpressionList { switch typedNegatedExpression := unwrapParenthetical(negation.Expression).(type) { @@ -94,6 +112,7 @@ func (s *ExpressionListRewriter) rewriteStringNegation(negation *cypher.Negation } } +// peekLastMatch returns the nearest enclosing MATCH clause in the traversal stack. func (s *ExpressionListRewriter) peekLastMatch() (*cypher.Match, bool) { for idx := len(s.descentStack) - 1; idx >= 0; idx-- { if lastMatch, typeOK := s.descentStack[idx].(*cypher.Match); typeOK { @@ -109,6 +128,7 @@ func (s *ExpressionListRewriter) Enter(node cypher.SyntaxNode) { s.pushExpression(node) } +// Exit removes empty expression lists, folds eligible relationship kinds into MATCH, and normalizes negated or parenthesized expressions. func (s *ExpressionListRewriter) Exit(node cypher.SyntaxNode) { attemptSelfRemoval := func() { if ancestorNode, hasPrevious := s.peekExpression(); hasPrevious { @@ -131,7 +151,11 @@ func (s *ExpressionListRewriter) Exit(node cypher.SyntaxNode) { if variable, typeOK := typedNode.Reference.(*cypher.Variable); !typeOK { s.SetErrorf("expected a variable as the reference for a kind matcher but received: %T", node) } else if variable.Symbol == query.EdgeSymbol { - if s.hasNegationAncestor() { + // Relationship kinds can be folded into the match pattern only when + // doing so preserves their logical scope. A kind nested under a NOT or + // OR must remain in the WHERE expression; hoisting it would either + // invert the predicate or merge branch-local kinds into one pattern. + if s.hasNegationAncestor() || s.hasDisjunctionAncestor() { return } diff --git a/query/v2/backend_test.go b/query/v2/backend_test.go index f5524ff9..d8df4e10 100644 --- a/query/v2/backend_test.go +++ b/query/v2/backend_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" ) +// testKindMapper returns an in-memory mapper populated in argument order. func testKindMapper(kinds ...graph.Kind) *pgutil.InMemoryKindMapper { mapper := pgutil.NewInMemoryKindMapper() @@ -169,6 +170,7 @@ func TestBackendParityNeo4jPrepare(t *testing.T) { } } +// TestBackendParityPGTranslateTraversalDepth verifies traversal-depth controls reach PostgreSQL's recursive path translation. func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { edgeKind := graph.StringKind("MemberOf") mapper := testKindMapper(edgeKind) @@ -186,7 +188,7 @@ func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { ), expectedSQLContains: []string{ "with recursive", - "ordered_edges_to_path", + "ordered_edge_ids_to_path", "n0.id = @pi0::int8", "e0.kind_id = any (array [1]::int2[])", "depth < 2", @@ -205,7 +207,7 @@ func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { "n0.id = @pi0::int8", "e0.kind_id = any (array [1]::int2[])", "depth < 2", - "select (s0.n0).id, (s0.n1).id from s0", + "select s0.n0 as \"id(s)\", s0.n1 as \"id(e)\" from s0", }, }, } @@ -228,6 +230,7 @@ func TestBackendParityPGTranslateTraversalDepth(t *testing.T) { } } +// TestBackendParityPGTranslate verifies v2 builders produce stable PostgreSQL SQL and parameter bindings across query forms. func TestBackendParityPGTranslate(t *testing.T) { userKind := graph.StringKind("User") edgeKind := graph.StringKind("MemberOf") @@ -246,7 +249,7 @@ func TestBackendParityPGTranslate(t *testing.T) { v2.Node().ID(), v2.Node().Kinds(), ), - expectedSQL: "with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.kind_ids operator (pg_catalog.&&) array [1]::int2[] and cypher_contains((n0.properties ->> 'name'), (@pi0::text)::text)::bool)) select (s0.n0).id, (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] from s0;", + expectedSQL: "with s0 as (select (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0 from node n0 where (n0.kind_ids operator (pg_catalog.&&) array [1]::int2[] and cypher_contains((n0.properties ->> 'name'), (@pi0::text)::text)::bool)) select (s0.n0).id as \"id(n)\", (array(select _kind.name from generate_subscripts((s0.n0).kind_ids, 1) as _kind_idx, kind _kind where _kind.id = ((s0.n0).kind_ids)[_kind_idx] order by _kind_idx))::text[] as \"labels(n)\" from s0;", expectedParams: map[string]any{"pi0": "admin"}, }, "relationship read": { @@ -258,7 +261,7 @@ func TestBackendParityPGTranslate(t *testing.T) { v2.Relationship().ID(), v2.End().ID(), ), - expectedSQL: "with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, (n0.id, n0.kind_ids, n0.properties)::nodecomposite as n0, (n1.id, n1.kind_ids, n1.properties)::nodecomposite as n1 from edge e0 join node n0 on (n0.id = @pi0::int8) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [2]::int2[])) select (s0.n0).id, (s0.e0).id, (s0.n1).id from s0;", + expectedSQL: "with s0 as (select (e0.id, e0.start_id, e0.end_id, e0.kind_id, e0.properties)::edgecomposite as e0, n0.id as n0, n1.id as n1 from edge e0 join node n0 on (n0.id = @pi0::int8) and n0.id = e0.start_id join node n1 on n1.id = e0.end_id where e0.kind_id = any (array [2]::int2[])) select s0.n0 as \"id(s)\", (s0.e0).id as \"id(r)\", s0.n1 as \"id(e)\" from s0;", expectedParams: map[string]any{"pi0": 1}, }, "update node": { @@ -306,6 +309,7 @@ func TestBackendParityPGTranslate(t *testing.T) { } } +// TestBackendParityPGTranslateShortestPaths verifies shortest-path controls select the expected PostgreSQL search harness. func TestBackendParityPGTranslateShortestPaths(t *testing.T) { edgeKind := graph.StringKind("MemberOf") mapper := testKindMapper(edgeKind) @@ -332,7 +336,7 @@ func TestBackendParityPGTranslateShortestPaths(t *testing.T) { ).Return( v2.Path(), ), - expectedHarness: "bidirectional_asp_harness", + expectedHarness: "all_shortest_paths_dag", }, } @@ -347,18 +351,27 @@ func TestBackendParityPGTranslateShortestPaths(t *testing.T) { sql, err := translate.Translated(translation) require.NoError(t, err) require.Contains(t, sql, testCase.expectedHarness) - require.Contains(t, sql, "ordered_edges_to_path") - require.Contains(t, sql, "n0.id = 1") - require.Contains(t, sql, "n1.id = 2") + require.Contains(t, sql, "n0.id = @pi0::int8") + require.Contains(t, sql, "n1.id = @pi1::int8") + require.Contains(t, sql, "singleton_endpoints") - serializedHarnessQueryHasKindConstraint := false - for _, parameterValue := range translation.Parameters { - if serializedQuery, typeOK := parameterValue.(string); typeOK && strings.Contains(serializedQuery, "array [1]::int2[]") { - serializedHarnessQueryHasKindConstraint = true - break + if name == "shortest path" { + require.Contains(t, sql, "ordered_edge_ids_to_path") + serializedHarnessQueryHasKindConstraint := false + for _, parameterValue := range translation.Parameters { + if serializedQuery, typeOK := parameterValue.(string); typeOK && strings.Contains(serializedQuery, "array [1]::int2[]") { + serializedHarnessQueryHasKindConstraint = true + break + } } + require.True(t, serializedHarnessQueryHasKindConstraint, "expected serialized shortest-path harness query to contain edge kind constraint: %#v", translation.Parameters) + } else { + require.Contains(t, sql, "array [1]::int2[]") + require.Contains(t, sql, "generate_subscripts(s1.path, 1)") + require.Contains(t, sql, "m0_hydrated.hydrated_count = cardinality(s1.path)") + require.NotContains(t, sql, "ordered_edge_ids_to_path") + require.NotContains(t, sql, "bidirectional_asp_harness") } - require.True(t, serializedHarnessQueryHasKindConstraint, "expected serialized shortest-path harness query to contain edge kind constraint: %#v", translation.Parameters) }) } } diff --git a/query/v2/query.go b/query/v2/query.go index 8faf2518..3c5c0117 100644 --- a/query/v2/query.go +++ b/query/v2/query.go @@ -99,6 +99,8 @@ func (s runtimeIdentifiers) End() *cypher.Variable { return cypher.NewVariableWithSymbol(s.end) } +// Identifiers exposes the canonical variables used for path, node, start, +// relationship, and end expressions. var Identifiers = runtimeIdentifiers{ path: "p", node: "n", @@ -373,7 +375,10 @@ func Or(operands ...cypher.SyntaxNode) cypher.SyntaxNode { type SortDirection int const ( + // SortAscending orders values from least to greatest. SortAscending SortDirection = iota + + // SortDescending orders values from greatest to least. SortDescending ) @@ -652,6 +657,7 @@ func (s *entity[T]) ID() IdentityContinuation { } } +// Property returns a comparison continuation for a validated property lookup or records an invalid-key error. func (s *entity[T]) Property(propertyName string) PropertyContinuation { if err := cypher.ValidatePropertyKeyName(propertyName); err != nil { return &propertyContinuation{ @@ -783,9 +789,16 @@ type QueryBuilder interface { type updatingClauseKind int const ( + // updatingClauseSet identifies a pending SET clause. updatingClauseSet updatingClauseKind = iota + + // updatingClauseRemove identifies a pending REMOVE clause. updatingClauseRemove + + // updatingClauseDelete identifies a pending DELETE clause. updatingClauseDelete + + // updatingClauseCreate identifies a pending CREATE clause. updatingClauseCreate ) diff --git a/query/v2/query_test.go b/query/v2/query_test.go index 18102d2f..b4685eee 100644 --- a/query/v2/query_test.go +++ b/query/v2/query_test.go @@ -117,6 +117,7 @@ func TestCreateRelationshipWithExplicitEndpoints(t *testing.T) { }, preparedQuery.Parameters) } +// TestRawPropertyKeysRenderEscaped verifies raw property keys retain required Cypher escaping in prepared queries. func TestRawPropertyKeysRenderEscaped(t *testing.T) { preparedQuery, err := v2.New().Return( v2.Node().Property("a-aaa"), @@ -128,6 +129,7 @@ func TestRawPropertyKeysRenderEscaped(t *testing.T) { require.Equal(t, "match (n) return n.`a-aaa`, n.`has``tick`, n.` `", renderPrepared(t, preparedQuery)) } +// TestEmptyPropertyKeyReturnsBuildError verifies an empty raw property key fails during query construction. func TestEmptyPropertyKeyReturnsBuildError(t *testing.T) { _, err := v2.New().Return( v2.Node().Property(""), diff --git a/query/v2/util.go b/query/v2/util.go index e1bdc341..13f3d283 100644 --- a/query/v2/util.go +++ b/query/v2/util.go @@ -247,6 +247,7 @@ func variableReference(value any) (*cypher.Variable, error) { } } +// propertyLookupOrError constructs a property lookup or an error expression when its key or variable reference is invalid. func propertyLookupOrError(reference any, propertyName string) cypher.Expression { if err := cypher.ValidatePropertyKeyName(propertyName); err != nil { return invalidExpression(err) diff --git a/regression_coverage_manifest.md b/regression_coverage_manifest.md new file mode 100644 index 00000000..05f59a34 --- /dev/null +++ b/regression_coverage_manifest.md @@ -0,0 +1,353 @@ +# BloodHound Regression Coverage Manifest + +Baseline audit for the source-derived regression program, recorded when the +regression harness was established. This file is the authoritative coverage +contract and gap map for the stable query-form IDs; use the tracked +[source-parity workflow](docs/regression_source_parity.md) for the associated +audit procedure. Update a cell when a case is added, and link the exact test or +generated case that changed it. + +## Coverage contract + +The corpus represents query shapes found in reviewed BHE and BHCE source; it +does not import application business logic or reproduce complete downstream +traversal algorithms. Normalize every discovered query into this tuple: + +```text +query target ++ direction ++ start/end ID anchor ++ start/end kind constraints ++ relationship kind constraints ++ node/relationship property predicates ++ logical grouping ++ projection ++ terminal operation +``` + +Two call sites may share a stable ID only when the entire tuple is equivalent. +Add a new ID for a new operator, grouping, direction, anchor location, +projection, mutation target, or execution path. Relationship names may share a +case, but kind-list and parameter-list cardinality remain test dimensions. +Audit existing primitive coverage before adding a production composition, +builder path, projection, cardinality, or scale case. + +| ID | Layer | Contract | +| --- | --- | --- | +| `QB` | Legacy query-builder pipeline | Preserve the AST and backend forms built from reviewed criteria; raw Cypher alone is insufficient for rewrite-sensitive forms. | +| `CY` | Cypher parser/mutation cases | Preserve accepted syntax, formatting, and mutation parsing. | +| `PG` | PostgreSQL translation goldens | Preserve SQL, parameters, correlation, projection, and mutation targets. | +| `IT` | Shared integration cases | Prove backend-equivalent observations and exact mutation effects. | +| `PC` | Plan corpus | Capture translated SQL, lowering metadata, and PostgreSQL plans for comparison. | +| `PI` | PostgreSQL plan-invariant tests | Assert stable index, orientation, filter, cardinality, or mutation-target properties. | +| `SC` | Scale/runtime corpus | Exercise representative cardinality and selectivity with repeatable fixtures. | +| `DR` | Driver integration/benchmark | Exercise direct driver and batch APIs that bypass Cypher translation. | + +Coverage rules: + +1. Every active form with a Cypher equivalent requires `PG` and `IT` coverage. +2. Every legacy-builder form requires `QB`; rewrite-sensitive forms also run + through the builder API in `IT` rather than only through an equivalent raw + query. +3. Every Cypher mutation requires `CY`, `PG`, and exact `IT` post-state; + direct-driver mutations require exact `DR` post-state instead. +4. High-cardinality or join-sensitive forms require `PC`; declared + representatives additionally require `SC` and stable plan-sensitive forms + require `PI`. +5. Direct batched mutations require semantic `DR` coverage across flush + boundaries. +6. Shared integration cases remain backend-equivalent. PostgreSQL-only plan, + resource, and runtime assertions stay in PostgreSQL-scoped tests or the + scale corpus. + +Status values: + +- `E` — existing coverage is equivalent to the complete normalized tuple. +- `P` — a primitive exists, but the production composition, projection, + cardinality, mutation target, or scale dimension is missing. +- `C` — production-complete coverage added by this regression project. +- `A` — absent. +- `—` — the layer is not required by this coverage contract. + +No active production ID was complete when the audit began. The following +references are the existing primitives used by the table; they are linked here +instead of being cloned under BloodHound-specific names: + +- `QB-PRED`: [`TestQueryBuilder_Render` predicate, temporal, kind, ID, string, + null, and mutation subtests](query/neo4j/neo4j_test.go). +- `QB-PROJ`: [`TestQueryBuilder_Render` relationship projection + subtests](query/neo4j/neo4j_test.go). +- `CY-MUT`: [Cypher create/update/delete parser cases](cypher/test/cases/mutation_tests.json). +- `PG-PRED`: [PostgreSQL node/predicate translation goldens](cypher/models/pgsql/test/translation_cases/nodes.sql). +- `PG-DEL`: [PostgreSQL delete translation goldens](cypher/models/pgsql/test/translation_cases/delete.sql). +- `PG-BIND`: [PostgreSQL binding and rewrite goldens](cypher/models/pgsql/test/translation_cases/pattern_binding.sql). +- `IT-PRED`: [backend-equivalent node predicate cases](integration/testdata/cases/nodes_inline.json). +- `IT-HOP`: [backend-equivalent directed one-hop template cases](integration/testdata/templates/pattern_shapes.json). +- `IT-MUT`: [primitive mutation cases](integration/testdata/cases/delete_inline.json) and + [the initial exact post-state harness sentinel](integration/testdata/cases/mutation_post_state_inline.json). +- `SC-HOP`: [`one_hop_typed_from_bound_id`](benchmark/testdata/scale/cases/traversal.json). +- `SC-LOOKUP`: [`objectid_exact_string_anchor` and + `boolean_property_filter`](benchmark/testdata/scale/cases/lookups.json). +- `SC-COUNT`: [`all_node_count`, `typed_node_count`, and + `typed_edge_count`](benchmark/testdata/scale/cases/counts.json). +- `DR-BATCH`: [`TestBatchTransaction_NodeUpdate`](drivers/neo4j/batch_integration_test.go#L48). +- `PI-IDX`: [`TestPostgreSQLPropertyIndexPlans`](integration/pgsql_property_index_plan_test.go#L58). +- `LOGIC-QB`: [`TestQueryBuilder_LOGIC01PreservesBranchLocalRelationshipKinds`, + `TestQueryBuilder_LogicalForms`, and + `TestQueryBuilder_LOGIC05ProjectionOrder`](query/neo4j/neo4j_test.go), plus + [`TestLegacyBuilderPostgreSQL_LogicalForms` and + `TestLegacyBuilderPostgreSQL_LOGIC05ProjectionOrder`](cypher/models/pgsql/test/logical_forms_legacy_builder_test.go). +- `LOGIC-CY`: [`LOGIC-04` filtered relationship and node delete parser + cases](cypher/test/cases/mutation_tests.json). +- `LOGIC-PG`: [`reconciliation.sql`](cypher/models/pgsql/test/translation_cases/reconciliation.sql) + and [`post_processing.sql`](cypher/models/pgsql/test/translation_cases/post_processing.sql). +- `LOGIC-IT`: [`TestLegacyBuilderLogicalForms`](integration/logical_forms_legacy_builder_test.go) + and the backend-equivalent [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + and [`post_processing_shapes.json`](integration/testdata/templates/post_processing_shapes.json) corpora. +- `LOGIC-PC`: the `LOGIC-01`, `LOGIC-02`, and `LOGIC-04` families in + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json), + loaded directly by `cmd/plancorpus` with fixture-ID parameter resolution. +- `REC-QB`: [`TestQueryBuilder_ReconciliationForms`](query/neo4j/neo4j_test.go) + and [`TestLegacyBuilderPostgreSQL_ReconciliationForms`](cypher/models/pgsql/test/reconciliation_forms_legacy_builder_test.go). +- `REC-CY`: the `REC-01` through `REC-04` and `REC-06` through `REC-08` + mutation parser cases in [`mutation_tests.json`](cypher/test/cases/mutation_tests.json). +- `REC-PG`: the `REC-01` through `REC-08` PostgreSQL goldens in + [`reconciliation.sql`](cypher/models/pgsql/test/translation_cases/reconciliation.sql). +- `REC-IT`: the exact reconciliation semantic families in + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + and the [`FetchStartNodes` de-dup contract](integration/delegated_enrollment_legacy_builder_test.go). +- `REC-PC`: the `REC-01` through `REC-08` families loaded from + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + by `cmd/plancorpus`. +- `REC-SC`: the repeatable `REC-01`, `REC-02`, `REC-04`, `REC-06`, and + `REC-08` write scenarios in + [`reconciliation.json`](benchmark/testdata/scale/cases/reconciliation.json). +- `TRUST-PRUNE-QB`: [`TestQueryBuilder_TrustAndPruningForms`](query/neo4j/neo4j_test.go) + and [`TestLegacyBuilderPostgreSQL_TrustAndPruningForms`](cypher/models/pgsql/test/trust_pruning_forms_legacy_builder_test.go). +- `TRUST-PRUNE-PG`: the `TRUST-01` through `TRUST-03` and `PRUNE-01` through + `PRUNE-04` PostgreSQL goldens in + [`reconciliation.sql`](cypher/models/pgsql/test/translation_cases/reconciliation.sql) + and [`post_processing.sql`](cypher/models/pgsql/test/translation_cases/post_processing.sql). +- `TRUST-PRUNE-IT`: the exact truth/null and hydration families in + [`reconciliation_shapes.json`](integration/testdata/templates/reconciliation_shapes.json) + and [`post_processing_shapes.json`](integration/testdata/templates/post_processing_shapes.json), + plus [`TestLegacyBuilderTrustAndPruningSelectors`](integration/trust_pruning_legacy_builder_test.go). +- `TRUST-PRUNE-PC`: the `TRUST-01` through `TRUST-03` and `PRUNE-01` through + `PRUNE-04` families loaded from the shared template corpus by `cmd/plancorpus`. +- `TRUST-PRUNE-SC`: the dense trust reads, pruning selectors, and mutation-safe + batch-delete equivalents in + [`trust_pruning.json`](benchmark/testdata/scale/cases/trust_pruning.json), + backed by [`NewTrustPruningScaleFixture`](testutil/reconciliation_fixture.go). +- `PRUNE-DR`: [`TestDirectBatchPruning` and + `BenchmarkDirectBatchPruning`](integration/trust_pruning_legacy_builder_test.go), + including IDs absent at delete time and a mixed-direction high-degree cascade. +- `HOP-QB`: [`TestQueryBuilder_StandaloneHopForms`](query/neo4j/neo4j_test.go) + and [`TestLegacyBuilderPostgreSQL_StandaloneHopForms`](cypher/models/pgsql/test/standalone_hop_forms_legacy_builder_test.go). +- `HOP-PG`: the `HOP-01` through `HOP-10` PostgreSQL goldens in + [`stepwise_traversal.sql`](cypher/models/pgsql/test/translation_cases/stepwise_traversal.sql). +- `HOP-IT`: the backend-equivalent standalone-hop families in + [`post_processing_hop_shapes.json`](integration/testdata/templates/post_processing_hop_shapes.json), + plus [`TestLegacyBuilderStandaloneHops`](integration/standalone_hops_legacy_builder_test.go). +- `HOP-PC`: the `HOP-01` through `HOP-10` families loaded from + [`post_processing_hop_shapes.json`](integration/testdata/templates/post_processing_hop_shapes.json) + by `cmd/plancorpus`. +- `HOP-SC`: the repeatable standalone-hop scenarios in + [`hops.json`](benchmark/testdata/scale/cases/hops.json), backed by + [`NewHopScaleFixture`](testutil/reconciliation_fixture.go). +- `SCAN-LOOKUP-QB`: [`TestQueryBuilder_RelationshipScans` and + `TestQueryBuilder_NodeLookups`](query/neo4j/relationship_scans_node_lookups_test.go), plus + [`TestLegacyBuilderPostgreSQL_RelationshipScans` and + `TestLegacyBuilderPostgreSQL_NodeLookups`](cypher/models/pgsql/test/relationship_scans_node_lookups_legacy_builder_test.go). +- `SCAN-LOOKUP-PG`: the `SCAN-01` through `SCAN-08` and `LOOKUP-01` through + `LOOKUP-14`/`LOOKUP-16` PostgreSQL goldens in + [`relationship_scans_node_lookups.sql`](cypher/models/pgsql/test/translation_cases/relationship_scans_node_lookups.sql). +- `SCAN-LOOKUP-IT`: the backend-equivalent scan, lookup, and count families in + [`relationship_scan_shapes.json`](integration/testdata/templates/relationship_scan_shapes.json), + [`basic_lookup_shapes.json`](integration/testdata/templates/basic_lookup_shapes.json), + [`advanced_lookup_shapes.json`](integration/testdata/templates/advanced_lookup_shapes.json), + and [`count_shapes.json`](integration/testdata/templates/count_shapes.json), + plus [`TestLegacyBuilderRelationshipScansAndNodeLookups`](integration/relationship_scans_node_lookups_legacy_builder_test.go). +- `SCAN-LOOKUP-PC`: the `SCAN-*` and applicable `LOOKUP-*` families loaded from + the shared scan/lookup template corpus by `cmd/plancorpus`. +- `SCAN-LOOKUP-SC`: the required wide-scan, large-list, adjacency, count, and NTLM + scenarios in [`scans_lookups.json`](benchmark/testdata/scale/cases/scans_lookups.json), + backed by [`NewScanLookupScaleFixture`](testutil/reconciliation_fixture.go). +- `WRITE-DR`: [`TestDirectWriteDeleteRelationshipBoundariesAndSurvivors` through + `TestDirectWriteExactKeyMissThenCreateNode`](integration/direct_write_mutations_test.go), + covering direct batch and transactional APIs on the selected backend with the + shared [`NewDirectWriteScaleFixture`](testutil/reconciliation_fixture.go), plus + the PostgreSQL conflict-key/property-index regression in + [`batch_test.go`](drivers/pg/batch_test.go). +- `WRITE-IT`: the exact-key create/update, full-node update, and exact-key + miss/create workflows in [`direct_write_mutations_test.go`](integration/direct_write_mutations_test.go), + with selector and driver-operation assertions kept separate. +- `WRITE-SC`: the reset-per-iteration, post-state-checked + [`BenchmarkMutationSafeDirectWrites`](integration/direct_write_mutations_test.go) + at 1,000 items and across the 2,000-item DAWGS flush boundary. +- `SCALE-PI`: [`TestPostgreSQLScalePlanInvariants`](cmd/graphbench/postgresql_plan_invariants_integration_test.go) + executes every required Cypher scale representative through PostgreSQL with + `EXPLAIN ANALYZE`, exact read/write cardinality, rollback-isolated mutation + post-state, mutation-target, binding, and anchor-index assertions. The + backend-independent [`TestScaleCorpusRequiredRepresentativesDeclareCardinality`](cmd/graphbench/scale_corpus_contract_test.go) + prevents a required stable ID or its cardinality contract from disappearing. +- `SCALE-BASELINE`: `cmd/graphbench` captures translated SQL, lowering + metadata, plans, buffer/runtime metrics, and cardinalities for the complete + scale corpus; `cmd/plancorpus` captures the shared semantic corpus with source + metadata. Generated captures remain review artifacts under the ignored + `.coverage/` directory rather than committed machine-specific baselines. +- `DORMANT-GATE`: [`TestDormantFormsStayOutOfPlanCorpus`](cmd/plancorpus/dormant_forms_guard_test.go) + and [`TestDormantFormsStayOutOfScaleCorpus`](cmd/graphbench/dormant_forms_guard_test.go) + keep every `FUTURE-*` ID out of active semantic, plan, and scale gates. The + activation and ongoing source-review procedure is recorded in + [`regression_source_parity.md`](docs/regression_source_parity.md). +- `COMPLETION-SC`: the `SCAN-01` ID-only, `SCAN-06` shallow IDs/kind, + `SCAN-02` relationship hydration, and `LOOKUP-09` node hydration scale cases + are classified and enforced by + [`TestScaleCorpusDistinguishesProjectionClasses`](cmd/graphbench/scale_corpus_contract_test.go). +- `COMPLETION-GATE`: [`TestRegressionCoverageManifestClosesEveryActiveID`](regression_manifest_test.go) + requires all 64 stable active IDs to remain present without an `A` or `P` + layer while preserving `FUTURE-01` as non-production-complete. + +## Logical sentinels + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `LOGIC-01` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | C (`LOGIC-PC`) | C (`SCALE-PI`) | — | — | +| `LOGIC-02` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | C (`LOGIC-PC`) | C (`SCALE-PI`) | — | — | +| `LOGIC-03` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | — | — | — | — | +| `LOGIC-04` | — | C (`LOGIC-CY`) | C (`LOGIC-PG`) | C (`LOGIC-IT`) | C (`LOGIC-PC`) | C (`SCALE-PI`) | — | — | +| `LOGIC-05` | C (`LOGIC-QB`) | — | C (`LOGIC-PG`) | C (`LOGIC-IT`) | — | — | — | — | + +## Reconciliation + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `REC-01` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-02` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-03` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | — | — | — | +| `REC-04` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-05` | C (`REC-QB`) | — | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | — | — | — | +| `REC-06` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | +| `REC-07` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | — | — | — | +| `REC-08` | C (`REC-QB`) | C (`REC-CY`) | C (`REC-PG`) | C (`REC-IT`) | C (`REC-PC`) | C (`SCALE-PI`) | C (`REC-SC`) | — | + +## Trust, pruning, and aging + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `TRUST-01` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `TRUST-02` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `TRUST-03` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | — | — | +| `PRUNE-01` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-02` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-03` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-04` | C (`TRUST-PRUNE-QB`) | — | C (`TRUST-PRUNE-PG`) | C (`TRUST-PRUNE-IT`) | C (`TRUST-PRUNE-PC`) | C (`SCALE-PI`) | C (`TRUST-PRUNE-SC`) | — | +| `PRUNE-05` | — | — | — | — | — | — | C (`TRUST-PRUNE-SC`) | C (`PRUNE-DR`) | +| `PRUNE-06` | — | — | — | — | — | — | C (`TRUST-PRUNE-SC`) | C (`PRUNE-DR`) | + +## Standalone hops + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `HOP-01` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-02` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-03` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-04` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-05` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-06` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | — | — | — | +| `HOP-07` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-08` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | — | — | — | +| `HOP-09` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | C (`SCALE-PI`) | C (`HOP-SC`) | — | +| `HOP-10` | C (`HOP-QB`) | — | C (`HOP-PG`) | C (`HOP-IT`) | C (`HOP-PC`) | — | — | — | + +## Relationship scans and node lookups + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `SCAN-01` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-02` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-03` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-04` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-05` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-06` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | C (`COMPLETION-SC`) | — | +| `SCAN-07` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `SCAN-08` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-01` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-02` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-03` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | — | — | — | — | +| `LOOKUP-04` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-05` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-06` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-07` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | — | — | — | — | +| `LOOKUP-08` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-09` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-10` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-11` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-12` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-13` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-14` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | — | — | — | +| `LOOKUP-15` | — | — | — | C (`SCAN-LOOKUP-IT`) | — | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | +| `LOOKUP-16` | C (`SCAN-LOOKUP-QB`) | — | C (`SCAN-LOOKUP-PG`) | C (`SCAN-LOOKUP-IT`) | C (`SCAN-LOOKUP-PC`) | C (`SCALE-PI`) | C (`SCAN-LOOKUP-SC`) | — | + +## Direct writes + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `WRITE-01` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-02` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-03` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-04` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-05` | — | — | — | — | — | — | C (`WRITE-SC`) | C (`WRITE-DR`) | +| `WRITE-06` | — | — | — | C (`WRITE-IT`) | — | — | — | C (`WRITE-DR`) | +| `WRITE-07` | — | — | — | C (`WRITE-IT`) | — | — | — | C (`WRITE-DR`) | +| `WRITE-08` | — | — | — | C (`WRITE-IT`) | — | — | — | C (`WRITE-DR`) | + +## Dormant coverage + +`FUTURE-01` remains intentionally incomplete because its reviewed callers are +disabled. `DORMANT-GATE` protects that classification; it is not query coverage +and therefore does not change the primitive or absent cells below. + +| ID | QB | CY | PG | IT | PC | PI | SC | DR | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| `FUTURE-01` | P (`QB-PRED`) | P (`CY-MUT`) | P (`PG-DEL`) | P (`IT-MUT`) | A | — | A | — | + +## Completion audit + +The executable manifest gate and the following evidence close the coverage +contract: + +1. All 64 active stable IDs are present at their required layers without an + absent or primitive-only cell (`COMPLETION-GATE`). +2. Shared Cypher mutations and direct writes assert exact targets, survivors, + properties, and counts with rollback/reset isolation (`IT-MUT`, + `REC-IT`, `TRUST-PRUNE-IT`, and `WRITE-IT`). +3. The `LOGIC-01` branch-local direction/kind truth table executes through the + shared integration corpus on PostgreSQL and Neo4j (`LOGIC-IT`). +4. PostgreSQL translation and plan coverage exercises equality-anchored deletes + in both active endpoint orientations and every production-active list form + (`REC-PG`, `REC-PC`, and `SCALE-PI`). The only outbound tenant-list + form is disabled upstream and remains `FUTURE-01` as required. +5. Scale coverage explicitly separates ID-only, shallow IDs/kind, full + relationship, and full-node projections (`COMPLETION-SC`). +6. Direct-write coverage includes the 1,000-item application batch and the + 1,999/2,000/2,001 DAWGS flush boundary (`WRITE-DR` and `WRITE-SC`). +7. `HOP-*` semantic and scale cases remain standalone one-hop queries; no new + runner sequences BloodHound traversal behavior (`HOP-IT` and + `HOP-SC`). +8. Dormant IDs are rejected from active plan and scale corpora until their + callers are enabled (`DORMANT-GATE`). + +## Harness foundation + +These prerequisites are intentionally not marked `C` against production IDs: + +- Mutation post-state assertions: [standalone sentinel](integration/testdata/cases/mutation_post_state_inline.json) + and [template rollback/repeat sentinel](integration/testdata/templates/mutation_post_state_shapes.json). +- Reusable deterministic fixture and list/fanout generators: + [`NewReconciliationFixture`](integration/regression_fixture.go). +- Backend-selected legacy query execution: + [`WithLegacyNodeQuery` and `WithLegacyRelationshipQuery`](integration/legacy_query_harness.go). +- Mutation-safe scale execution and list-valued fixture IDs: + [`WriteScenario`](cmd/graphbench/types.go) and + [`resolveCaseParams`](cmd/graphbench/datasets.go). diff --git a/regression_manifest_test.go b/regression_manifest_test.go new file mode 100644 index 00000000..e28856c7 --- /dev/null +++ b/regression_manifest_test.go @@ -0,0 +1,97 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package dawgs + +import ( + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestRegressionCoverageManifestClosesEveryActiveID verifies that every active +// query form has complete coverage while dormant forms remain unactivated. +func TestRegressionCoverageManifestClosesEveryActiveID(t *testing.T) { + raw, err := os.ReadFile("regression_coverage_manifest.md") + require.NoError(t, err) + + var ( + rows = parseRegressionManifestRows(string(raw)) + activeFamilies = map[string]int{ + "LOGIC": 5, + "REC": 8, + "TRUST": 3, + "PRUNE": 6, + "HOP": 10, + "SCAN": 8, + "LOOKUP": 16, + "WRITE": 8, + } + ) + + for family, count := range activeFamilies { + for idx := 1; idx <= count; idx++ { + id := fmt.Sprintf("%s-%02d", family, idx) + cells, found := rows[id] + require.True(t, found, "coverage manifest is missing active query form %s", id) + for _, cell := range cells { + status := strings.Fields(cell) + if len(status) > 0 { + require.NotContains(t, []string{"A", "P"}, status[0], + "active query form %s retains an unclosed layer: %s", id, cell) + } + } + } + } + + futureCells, found := rows["FUTURE-01"] + require.True(t, found, "coverage manifest is missing dormant query form FUTURE-01") + require.Contains(t, futureCells, "A", "FUTURE-01 must retain absent activation-only layers") + for _, cell := range futureCells { + status := strings.Fields(cell) + if len(status) > 0 { + require.NotEqual(t, "C", status[0], "FUTURE-01 must remain outside production-complete coverage") + } + } +} + +// parseRegressionManifestRows indexes the coverage cells in each manifest row +// by query-form identifier. +func parseRegressionManifestRows(manifest string) map[string][]string { + rows := map[string][]string{} + for _, line := range strings.Split(manifest, "\n") { + if !strings.HasPrefix(line, "| `") { + continue + } + + columns := strings.Split(line, "|") + if len(columns) < 11 { + continue + } + + id := strings.Trim(strings.TrimSpace(columns[1]), "`") + cells := make([]string, 0, len(columns)-3) + for _, column := range columns[2 : len(columns)-1] { + cells = append(cells, strings.TrimSpace(column)) + } + rows[id] = cells + } + + return rows +} diff --git a/testutil/metadata.go b/testutil/metadata.go new file mode 100644 index 00000000..bf7522d6 --- /dev/null +++ b/testutil/metadata.go @@ -0,0 +1,57 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import "runtime/debug" + +// BaselineMetadata records build identity needed to interpret a generated benchmark baseline. +type BaselineMetadata struct { + // DAWGSVersion identifies the DAWGS build that produced the baseline. + DAWGSVersion string `json:"dawgs_version"` +} + +// ResolveBaselineMetadata returns metadata for dawgsVersion, deriving the current build identity when it is empty. +func ResolveBaselineMetadata(dawgsVersion string) BaselineMetadata { + if dawgsVersion == "" { + dawgsVersion = currentDAWGSVersion() + } + + return BaselineMetadata{ + DAWGSVersion: dawgsVersion, + } +} + +// currentDAWGSVersion derives a module version and optional VCS revision from Go build information. +func currentDAWGSVersion() string { + buildInfo, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + + version := buildInfo.Main.Version + if version == "" { + version = "(devel)" + } + + for _, setting := range buildInfo.Settings { + if setting.Key == "vcs.revision" && setting.Value != "" { + return version + "@" + setting.Value + } + } + + return version +} diff --git a/testutil/metadata_test.go b/testutil/metadata_test.go new file mode 100644 index 00000000..48106cca --- /dev/null +++ b/testutil/metadata_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestResolveBaselineMetadata verifies an explicit version is preserved in generated baseline metadata. +func TestResolveBaselineMetadata(t *testing.T) { + metadata := ResolveBaselineMetadata("dawgs") + require.Equal(t, BaselineMetadata{ + DAWGSVersion: "dawgs", + }, metadata) + + defaults := ResolveBaselineMetadata("") + require.NotEmpty(t, defaults.DAWGSVersion) +} diff --git a/testutil/params.go b/testutil/params.go new file mode 100644 index 00000000..51e49a6a --- /dev/null +++ b/testutil/params.go @@ -0,0 +1,208 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Package testutil provides reusable corpus, fixture, and baseline helpers for +// DAWGS tests and diagnostic commands. +package testutil + +import ( + "encoding/json" + "fmt" + "time" +) + +const ( + // typeKey identifies the discriminator field in a tagged test parameter. + typeKey = "$type" + + // valueKey identifies the payload field in a tagged scalar parameter. + valueKey = "value" + + // prefixKey identifies the generated value prefix in a tagged string list. + prefixKey = "prefix" + + // countKey identifies the generated value count in a tagged string list. + countKey = "count" + + // includeKey identifies literal values prepended to a tagged string list. + includeKey = "include" +) + +// Params is a query parameter map that supports tagged generated and temporal +// values. A datetime is represented in JSON as: +// +// {"$type": "datetime", "value": "2026-01-02T03:04:05Z"} +// +// A deterministic string list is represented without committing a large +// handwritten array as: +// +// {"$type": "string_list", "prefix": "missing", "count": 1000, "include": ["target-id"]} +// +// Tagged values may also appear in nested maps and lists. +type Params map[string]any + +// UnmarshalJSON decodes plain JSON parameters and expands supported tagged +// values recursively. +func (s *Params) UnmarshalJSON(raw []byte) error { + var decoded map[string]any + if err := json.Unmarshal(raw, &decoded); err != nil { + return err + } + + converted, err := convertMap(decoded) + if err != nil { + return err + } + + *s = Params(converted) + return nil +} + +// convertMap recursively converts every value in a decoded parameter map. +func convertMap(values map[string]any) (map[string]any, error) { + converted := make(map[string]any, len(values)) + for key, value := range values { + typedValue, err := convertValue(value) + if err != nil { + return nil, fmt.Errorf("parameter %q: %w", key, err) + } + + converted[key] = typedValue + } + + return converted, nil +} + +// convertValue expands tagged maps and recursively converts nested maps and +// lists while preserving scalar values. +func convertValue(value any) (any, error) { + switch typedValue := value.(type) { + case map[string]any: + if typeName, tagged := typedValue[typeKey]; tagged { + return convertTaggedValue(typeName, typedValue) + } + + return convertMap(typedValue) + + case []any: + converted := make([]any, len(typedValue)) + for idx, item := range typedValue { + next, err := convertValue(item) + if err != nil { + return nil, fmt.Errorf("list item %d: %w", idx, err) + } + converted[idx] = next + } + + return converted, nil + + default: + return value, nil + } +} + +// convertTaggedValue validates and expands one supported tagged parameter. +func convertTaggedValue(rawType any, tagged map[string]any) (any, error) { + typeName, ok := rawType.(string) + if !ok { + return nil, fmt.Errorf("%s must be a string", typeKey) + } + + switch typeName { + case "datetime": + rawValue, found := tagged[valueKey] + if !found { + return nil, fmt.Errorf("datetime is missing %q", valueKey) + } + + value, ok := rawValue.(string) + if !ok { + return nil, fmt.Errorf("datetime %q must be a string", valueKey) + } + + parsed, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return nil, fmt.Errorf("parse datetime %q: %w", value, err) + } + + if len(tagged) != 2 { + return nil, fmt.Errorf("datetime must contain only %q and %q", typeKey, valueKey) + } + + return parsed, nil + + case "string_list": + return convertStringList(tagged) + + default: + return nil, fmt.Errorf("unsupported tagged parameter type %q", typeName) + } +} + +// convertStringList expands a tagged string-list specification into its +// literal and generated values. +func convertStringList(tagged map[string]any) ([]string, error) { + rawPrefix, found := tagged[prefixKey] + if !found { + return nil, fmt.Errorf("string_list is missing %q", prefixKey) + } + prefix, ok := rawPrefix.(string) + if !ok { + return nil, fmt.Errorf("string_list %q must be a string", prefixKey) + } + + rawCount, found := tagged[countKey] + if !found { + return nil, fmt.Errorf("string_list is missing %q", countKey) + } + countValue, ok := rawCount.(float64) + if !ok || countValue < 0 || countValue != float64(int(countValue)) { + return nil, fmt.Errorf("string_list %q must be a non-negative integer", countKey) + } + count := int(countValue) + + include := make([]string, 0) + if rawInclude, found := tagged[includeKey]; found { + values, ok := rawInclude.([]any) + if !ok { + return nil, fmt.Errorf("string_list %q must be a string list", includeKey) + } + include = make([]string, len(values)) + for idx, value := range values { + stringValue, ok := value.(string) + if !ok { + return nil, fmt.Errorf("string_list %q item %d must be a string", includeKey, idx) + } + include[idx] = stringValue + } + } + + if len(tagged) != 3 && !(len(tagged) == 4 && tagged[includeKey] != nil) { + return nil, fmt.Errorf("string_list must contain only %q, %q, %q, and optional %q", typeKey, prefixKey, countKey, includeKey) + } + + width := len(fmt.Sprintf("%d", max(count-1, 0))) + if width < 2 { + width = 2 + } + + values := make([]string, 0, len(include)+count) + values = append(values, include...) + for idx := range count { + values = append(values, fmt.Sprintf("%s-%0*d", prefix, width, idx)) + } + return values, nil +} diff --git a/testutil/params_test.go b/testutil/params_test.go new file mode 100644 index 00000000..287e7669 --- /dev/null +++ b/testutil/params_test.go @@ -0,0 +1,93 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestParamsDecodesTaggedDatetime verifies tagged datetime values are parsed +// recursively with nanosecond precision. +func TestParamsDecodesTaggedDatetime(t *testing.T) { + var values Params + require.NoError(t, json.Unmarshal([]byte(`{ + "threshold": {"$type": "datetime", "value": "2026-01-02T03:04:05.123456789Z"}, + "nested": [{"$type": "datetime", "value": "2025-02-03T04:05:06Z"}] + }`), &values)) + + require.Equal(t, time.Date(2026, time.January, 2, 3, 4, 5, 123456789, time.UTC), values["threshold"]) + require.Equal(t, []any{time.Date(2025, time.February, 3, 4, 5, 6, 0, time.UTC)}, values["nested"]) +} + +// TestParamsDecodesNestedObjectsAsStandardMaps verifies untagged objects remain +// ordinary nested parameter maps. +func TestParamsDecodesNestedObjectsAsStandardMaps(t *testing.T) { + var values Params + require.NoError(t, json.Unmarshal([]byte(`{ + "properties": {"name": "node", "nested": {"enabled": true}} + }`), &values)) + + properties, ok := values["properties"].(map[string]any) + require.True(t, ok) + require.Equal(t, "node", properties["name"]) + + nested, ok := properties["nested"].(map[string]any) + require.True(t, ok) + require.Equal(t, true, nested["enabled"]) +} + +// TestParamsRejectsUnknownTaggedType verifies unsupported tagged parameter +// discriminators fail decoding. +func TestParamsRejectsUnknownTaggedType(t *testing.T) { + var values Params + err := json.Unmarshal([]byte(`{"threshold":{"$type":"timestamp","value":"2026-01-02T03:04:05Z"}}`), &values) + require.ErrorContains(t, err, `unsupported tagged parameter type "timestamp"`) +} + +// TestParamsDecodesDeterministicStringList verifies literal inclusions precede +// deterministically numbered generated values. +func TestParamsDecodesDeterministicStringList(t *testing.T) { + var values Params + require.NoError(t, json.Unmarshal([]byte(`{ + "object_ids": {"$type": "string_list", "prefix": "missing", "count": 3, "include": ["target-a", "target-b"]} + }`), &values)) + + require.Equal(t, []string{"target-a", "target-b", "missing-00", "missing-01", "missing-02"}, values["object_ids"]) +} + +// TestParamsRejectsInvalidStringList verifies malformed string-list +// specifications fail decoding. +func TestParamsRejectsInvalidStringList(t *testing.T) { + testCases := []string{ + `{"ids":{"$type":"string_list","count":1}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":-1}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":1.5}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":1,"include":[1]}}`, + `{"ids":{"$type":"string_list","prefix":"x","count":1,"extra":true}}`, + } + + for _, raw := range testCases { + t.Run(raw, func(t *testing.T) { + var values Params + require.Error(t, json.Unmarshal([]byte(raw), &values)) + }) + } +} diff --git a/testutil/perf_endpoint_seeded.go b/testutil/perf_endpoint_seeded.go new file mode 100644 index 00000000..d44e981b --- /dev/null +++ b/testutil/perf_endpoint_seeded.go @@ -0,0 +1,176 @@ +package testutil + +import ( + "fmt" + "strings" + + "github.com/specterops/dawgs/opengraph" +) + +// EndpointSeededExpansionScaleDataset identifies the generated endpoint-seeded +// expansion fixture. +const EndpointSeededExpansionScaleDataset = "generated_endpoint_seeded_expansion_v1" + +// EndpointSeededExpansionScaleConfig controls the endpoint populations and +// traversal lanes emitted by NewEndpointSeededExpansionScaleFixture. +type EndpointSeededExpansionScaleConfig struct { + // Depth sets the number of MemberOf hops in each lane. + Depth int + + // MatchingEndpoints sets the number of terminal groups whose object IDs + // satisfy the benchmark predicate. + MatchingEndpoints int + + // OtherEndpoints sets the number of terminal groups that do not satisfy the + // benchmark predicate. + OtherEndpoints int + + // MatchingEligibleLanes sets the number of session-backed lanes ending at a + // matching endpoint. + MatchingEligibleLanes int + + // OtherEligibleLanes sets the number of session-backed lanes ending at a + // nonmatching endpoint. + OtherEligibleLanes int + + // MatchingIneligibleLanes sets the number of lanes without a session edge + // that nevertheless end at a matching endpoint. + MatchingIneligibleLanes int + + // ParallelEdges sets the number of MemberOf edges emitted per lane hop. + ParallelEdges int + + // AddCycle adds a reverse MemberOf edge near the middle of every lane. + AddCycle bool + + // PropertyPayloadSize sets the length of synthetic payload properties. + PropertyPayloadSize int +} + +// ValidateEndpointSeededExpansionScaleConfig rejects fixture configurations +// that cannot produce a valid or uniquely keyed endpoint-seeded graph. +func ValidateEndpointSeededExpansionScaleConfig(config EndpointSeededExpansionScaleConfig) error { + if config.Depth < 1 || config.Depth > 64 { + return fmt.Errorf("depth must be between 1 and 64") + } + if config.MatchingEndpoints < 1 || config.OtherEndpoints < 0 || config.MatchingEligibleLanes < 1 || config.OtherEligibleLanes < 0 || config.MatchingIneligibleLanes < 0 { + return fmt.Errorf("endpoint and lane counts are invalid") + } + if config.ParallelEdges != 1 { + return fmt.Errorf("parallel edges must be exactly one because DAWGS graph storage uniquely keys edges by start, end, kind, and graph") + } + if config.PropertyPayloadSize < 0 { + return fmt.Errorf("property payload size must not be negative") + } + return nil +} + +// NewEndpointSeededExpansionScaleFixture creates terminal-selective expansion +// lanes with independently controlled productive and unproductive reverse work. +func NewEndpointSeededExpansionScaleFixture(config EndpointSeededExpansionScaleConfig) *opengraph.Graph { + if ValidateEndpointSeededExpansionScaleConfig(config) != nil { + return nil + } + payload := strings.Repeat("x", config.PropertyPayloadSize) + fixture := &opengraph.Graph{} + for idx := range config.MatchingEndpoints { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: fmt.Sprintf("ese-match-%03d", idx), + Kinds: []string{"Group"}, + Properties: map[string]any{ + "objectid": fmt.Sprintf("S-1-5-21-%03d-512", idx), + "payload": payload, + }, + }) + } + for idx := range config.OtherEndpoints { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: fmt.Sprintf("ese-other-%03d", idx), + Kinds: []string{"Group"}, + Properties: map[string]any{ + "objectid": fmt.Sprintf("S-1-5-21-%03d-513", idx), + "payload": payload, + }, + }) + } + + addLane := func(class string, lane int, endpoint string, eligible bool) { + user := fmt.Sprintf("ese-%s-user-%04d", class, lane) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: user, + Kinds: []string{"User"}, + Properties: map[string]any{"payload": payload}, + }) + if eligible { + computer := fmt.Sprintf("ese-%s-computer-%04d", class, lane) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: computer, + Kinds: []string{"Computer"}, + Properties: map[string]any{"payload": payload}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: computer, + EndID: user, + Kind: "HasSession", + Properties: map[string]any{"logical_key": computer + "-session"}, + }) + } + previous := user + for level := 1; level <= config.Depth; level++ { + next := endpoint + if level < config.Depth { + next = fmt.Sprintf("ese-%s-lane-%04d-level-%02d", class, lane, level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"Group"}, + Properties: map[string]any{"payload": payload}, + }) + } + for parallel := range config.ParallelEdges { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "MemberOf", + Properties: map[string]any{ + "logical_key": fmt.Sprintf("%s-%04d-%02d-%02d", class, lane, level, parallel), + }, + }) + } + if config.AddCycle && level == max(1, config.Depth/2) && previous != user { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: next, + EndID: previous, + Kind: "MemberOf", + Properties: map[string]any{ + "logical_key": fmt.Sprintf("%s-%04d-cycle", class, lane), + }, + }) + } + previous = next + } + } + + for lane := range config.MatchingEligibleLanes { + addLane("matching", lane, fmt.Sprintf("ese-match-%03d", lane%config.MatchingEndpoints), true) + } + for lane := range config.OtherEligibleLanes { + endpoint := "ese-other-000" + if config.OtherEndpoints > 0 { + endpoint = fmt.Sprintf("ese-other-%03d", lane%config.OtherEndpoints) + } else { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: endpoint, + Kinds: []string{"Group"}, + Properties: map[string]any{ + "objectid": "S-1-5-21-513", + "payload": payload, + }, + }) + } + addLane("other", lane, endpoint, true) + } + for lane := range config.MatchingIneligibleLanes { + addLane("ineligible", lane, fmt.Sprintf("ese-match-%03d", lane%config.MatchingEndpoints), false) + } + return fixture +} diff --git a/testutil/perf_fixtures.go b/testutil/perf_fixtures.go new file mode 100644 index 00000000..506552bb --- /dev/null +++ b/testutil/perf_fixtures.go @@ -0,0 +1,654 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "errors" + "fmt" + "strings" + + "github.com/specterops/dawgs/opengraph" +) + +const ( + // ShortestPathScaleDataset identifies the generated shortest-path fixture. + ShortestPathScaleDataset = "generated_shortest_paths" + + // FixedSuffixExpansionScaleDataset identifies the generated fixed-suffix + // expansion fixture. + FixedSuffixExpansionScaleDataset = "generated_fixed_suffix_expansion" + + // FixedSuffixExpansionScaleV3Dataset identifies the fixed-suffix fixture + // grammar with independent root, cycle, and self-loop controls. + FixedSuffixExpansionScaleV3Dataset = FixedSuffixExpansionScaleDataset + "_v3" +) + +// ShortestPathScaleConfig controls the depth and dead-end fanout of the +// generated shortest-path fixture. +type ShortestPathScaleConfig struct { + // Depth sets the length of the fixture's unique linear route. + Depth int + + // Fanout sets the number of dead ends attached to the route's start. + Fanout int +} + +// NewShortestPathScaleFixture builds deterministic linear, diamond, dead-end, +// cycle, parallel-edge, self-loop, wrong-direction, and disconnected shapes +// around a bound endpoint pair. Fanout controls parallel dead ends without +// changing the unique linear route's requested depth. +func NewShortestPathScaleFixture(config ShortestPathScaleConfig) *opengraph.Graph { + depth := max(config.Depth, 1) + fanout := max(config.Fanout, 1) + fixture := &opengraph.Graph{} + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: "sp-start", + Kinds: []string{"ShortestNode"}, + Properties: map[string]any{"role": "start"}, + }, + opengraph.Node{ + ID: "sp-end", + Kinds: []string{"ShortestNode"}, + Properties: map[string]any{"role": "end"}, + }, + opengraph.Node{ + ID: "sp-disconnected", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-wrong-direction", + Kinds: []string{"ShortestNode"}, + }, + ) + + previous := "sp-start" + for level := 1; level < depth; level++ { + next := fmt.Sprintf("sp-linear-%02d", level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"ShortestNode"}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "Traverse", + }) + previous = next + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: "sp-end", + Kind: "Traverse", + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "sp-end", + EndID: "sp-wrong-direction", + Kind: "Traverse", + }) + + for idx := range fanout { + deadEnd := fmt.Sprintf("sp-dead-%04d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: deadEnd, + Kinds: []string{"ShortestNode"}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "sp-start", + EndID: deadEnd, + Kind: "Traverse", + }) + } + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: "sp-diamond-left", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-diamond-right", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-diamond-end", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-cycle-a", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-cycle-b", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-parallel-end", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-self-loop", + Kinds: []string{"ShortestNode"}, + }, + opengraph.Node{ + ID: "sp-self-loop-exit", + Kinds: []string{"ShortestNode"}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-diamond-left", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-diamond-right", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-diamond-left", + EndID: "sp-diamond-end", + Kind: "TypedTraverse", + }, + opengraph.Edge{ + StartID: "sp-diamond-right", + EndID: "sp-diamond-end", + Kind: "TypedTraverse", + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-cycle-a", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-cycle-a", + EndID: "sp-cycle-b", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-cycle-b", + EndID: "sp-cycle-a", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-parallel-end", + Kind: "Traverse", + Properties: map[string]any{"logical_key": "sp-parallel-0"}, + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-parallel-end", + Kind: "TypedTraverse", + Properties: map[string]any{"logical_key": "sp-parallel-1"}, + }, + opengraph.Edge{ + StartID: "sp-start", + EndID: "sp-self-loop", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-self-loop", + EndID: "sp-self-loop", + Kind: "Traverse", + }, + opengraph.Edge{ + StartID: "sp-self-loop", + EndID: "sp-self-loop-exit", + Kind: "Traverse", + }, + ) + + return fixture +} + +// FixedSuffixExpansionScaleConfig controls expansion work, suffix density, +// decoys, and payload size in a fixed-suffix fixture. +type FixedSuffixExpansionScaleConfig struct { + // ExpansionDepth sets the number of Expand hops in each branch. + ExpansionDepth int + + // Fanout sets the number of expansion branches rooted at the fixture root. + Fanout int + + // ValidSuffixEvery attaches a suffix to every nth legacy branch. + ValidSuffixEvery int + + // PropertyPayloadSize sets the length of synthetic payload properties. + PropertyPayloadSize int + + // ExactReachableSuffixSources decouples reachable suffix density from the + // legacy modulus control. Nil preserves ValidSuffixEvery behavior; zero is + // an exact zero and is therefore materially different from nil. + ExactReachableSuffixSources *int + + // ReachableSuffixDepths restricts suffix attachment to the listed expansion + // depths when nonempty. + ReachableSuffixDepths []int + + // DisconnectedSuffixSources sets the number of suffix sources unreachable + // from any expansion root. + DisconnectedSuffixSources int + + // ReverseFanIn sets the number of decoy Expand edges entering a productive + // branch boundary. + ReverseFanIn int + + // SuffixPathsPerBoundary sets the number of distinct suffix paths attached + // to each selected boundary. + SuffixPathsPerBoundary int + + // RootMatchCount sets the number of roots matching the fixture root key. + RootMatchCount int + + // RootHasZeroDepthSuffix controls whether the primary root has a suffix; + // nil preserves the enabled default. + RootHasZeroDepthSuffix *bool + + // AddProductiveBoundaryCycle adds a two-edge Expand cycle at the + // deterministic productive boundary. The two physical relationships have + // distinct endpoints and logical keys, so the cycle can be traversed once + // in either relationship-distinct expansion direction. + AddProductiveBoundaryCycle bool + + // AddProductiveBoundarySelfLoop adds one Expand self-loop at the + // deterministic productive boundary. + AddProductiveBoundarySelfLoop bool +} + +// ValidateFixedSuffixExpansionScaleV3Config rejects dimensions that cannot +// describe the exact v3 fixture grammar. V3 requires every population to be +// explicit; legacy and v2 callers retain their existing defaulting behavior. +func ValidateFixedSuffixExpansionScaleV3Config(config FixedSuffixExpansionScaleConfig) error { + values := []int{ + config.ExpansionDepth, config.Fanout, config.DisconnectedSuffixSources, + config.ReverseFanIn, config.SuffixPathsPerBoundary, config.RootMatchCount, + config.PropertyPayloadSize, + } + for _, value := range values { + if value < 0 { + return errors.New("fixed-suffix v3 configuration values must not be negative") + } + } + if config.ExpansionDepth > 64 { + return errors.New("fixed-suffix v3 depth must not exceed 64") + } + if config.Fanout < 1 { + return errors.New("fixed-suffix v3 fanout must be positive") + } + if config.ExactReachableSuffixSources == nil { + return errors.New("fixed-suffix v3 reachable suffix sources must be explicit") + } + reachable := *config.ExactReachableSuffixSources + if reachable < 0 || reachable > config.Fanout { + return errors.New("fixed-suffix v3 reachable suffix sources must be between zero and fanout") + } + if config.ExpansionDepth == 0 && reachable != 0 { + return errors.New("fixed-suffix v3 depth-zero fixtures cannot have reachable branch suffixes") + } + if config.SuffixPathsPerBoundary < 1 { + return errors.New("fixed-suffix v3 suffix path multiplicity must be positive") + } + if config.RootMatchCount < 1 { + return errors.New("fixed-suffix v3 root match count must be positive") + } + if config.RootHasZeroDepthSuffix == nil { + return errors.New("fixed-suffix v3 zero-depth suffix control must be explicit") + } + if config.ValidSuffixEvery != 0 || len(config.ReachableSuffixDepths) != 0 { + return errors.New("fixed-suffix v3 cannot mix legacy suffix-density controls with exact controls") + } + + hasProductiveBoundary := *config.RootHasZeroDepthSuffix || reachable > 0 + if !hasProductiveBoundary && config.ReverseFanIn != 0 { + return errors.New("fixed-suffix v3 reverse fan-in requires a productive boundary") + } + if !hasProductiveBoundary && (config.AddProductiveBoundaryCycle || config.AddProductiveBoundarySelfLoop) { + return errors.New("fixed-suffix v3 cycle and self-loop controls require a productive boundary") + } + return nil +} + +// NewFixedSuffixExpansionScaleFixture builds a deterministic expansion fanout +// feeding a shared fixed suffix. It also emits independent wrong-kind, +// wrong-direction, wrong-endpoint-kind, and disconnected suffix decoys. +func NewFixedSuffixExpansionScaleFixture(config FixedSuffixExpansionScaleConfig) *opengraph.Graph { + if config.ExactReachableSuffixSources == nil && len(config.ReachableSuffixDepths) == 0 && config.DisconnectedSuffixSources == 0 && config.ReverseFanIn == 0 && config.SuffixPathsPerBoundary == 0 && config.RootMatchCount == 0 && config.RootHasZeroDepthSuffix == nil && !config.AddProductiveBoundaryCycle && !config.AddProductiveBoundarySelfLoop { + return newLegacyFixedSuffixExpansionScaleFixture(config) + } + depth := max(config.ExpansionDepth, 0) + fanout := max(config.Fanout, 1) + validEvery := max(config.ValidSuffixEvery, 1) + reachableSources := -1 + if config.ExactReachableSuffixSources != nil { + reachableSources = min(max(*config.ExactReachableSuffixSources, 0), fanout) + } + suffixPaths := max(config.SuffixPathsPerBoundary, 1) + rootCount := max(config.RootMatchCount, 1) + rootHasSuffix := true + if config.RootHasZeroDepthSuffix != nil { + rootHasSuffix = *config.RootHasZeroDepthSuffix + } + payload := strings.Repeat("x", max(config.PropertyPayloadSize, 0)) + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "fse-terminal", + Kinds: []string{"SuffixTerminal"}, + }, + { + ID: "fse-wrong-endpoint", + Kinds: []string{"ExpansionNode"}, + }, + }, + } + for rootIdx := range rootCount { + rootID := "fse-root" + if rootIdx > 0 { + rootID = fmt.Sprintf("fse-root-%02d", rootIdx) + } + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: rootID, + Kinds: []string{"ExpansionRoot"}, + Properties: map[string]any{"root_key": "generated-fse-root", "payload": payload}, + }) + } + addSuffix := func(source, key string) { + for pathIdx := range suffixPaths { + headID := fmt.Sprintf("fse-head-%s-%02d", key, pathIdx) + middleID := fmt.Sprintf("fse-middle-%s-%02d", key, pathIdx) + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: headID, + Kinds: []string{"SuffixHead"}, + Properties: map[string]any{"payload": payload}, + }, + opengraph.Node{ + ID: middleID, + Kinds: []string{"SuffixMiddle"}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: source, + EndID: headID, + Kind: "EnterSuffix", + Properties: map[string]any{"payload": payload, "logical_key": key + ":enter"}, + }, + opengraph.Edge{ + StartID: headID, + EndID: middleID, + Kind: "ContinueSuffix", + Properties: map[string]any{"logical_key": key + ":continue"}, + }, + opengraph.Edge{ + StartID: middleID, + EndID: "fse-terminal", + Kind: "CompleteSuffix", + Properties: map[string]any{"logical_key": key + ":complete"}, + }, + ) + } + } + if rootHasSuffix { + addSuffix("fse-root", "root") + } + + productiveBoundary := "fse-root" + if depth > 0 { + for branch := range fanout { + previous := "fse-root" + for level := 1; level <= depth; level++ { + next := fmt.Sprintf("fse-branch-%04d-level-%02d", branch, level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"ExpansionNode"}, + Properties: map[string]any{"payload": payload}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "Expand", + Properties: map[string]any{"logical_key": fmt.Sprintf("branch-%04d-level-%02d", branch, level)}, + }) + previous = next + } + reachable := branch%validEvery == 0 + if reachableSources >= 0 { + reachable = branch < reachableSources + } + if reachable && (len(config.ReachableSuffixDepths) == 0 || containsInt(config.ReachableSuffixDepths, depth)) { + addSuffix(previous, fmt.Sprintf("branch-%04d-depth-%02d", branch, depth)) + if branch == 0 { + productiveBoundary = previous + } + } + } + } + for idx := range max(config.DisconnectedSuffixSources, 0) { + source := fmt.Sprintf("fse-disconnected-%05d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: source, + Kinds: []string{"ExpansionNode"}, + }) + addSuffix(source, fmt.Sprintf("disconnected-%05d", idx)) + } + for idx := range max(config.ReverseFanIn, 0) { + source := fmt.Sprintf("fse-fanin-%05d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: source, + Kinds: []string{"ExpansionNode"}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: source, + EndID: productiveBoundary, + Kind: "Expand", + Properties: map[string]any{"logical_key": fmt.Sprintf("fanin-%05d", idx)}, + }) + } + if config.AddProductiveBoundaryCycle { + const cycleNode = "fse-productive-boundary-cycle" + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: cycleNode, + Kinds: []string{"ExpansionNode"}, + }) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: productiveBoundary, + EndID: cycleNode, + Kind: "Expand", + Properties: map[string]any{ + "logical_key": "productive-boundary-cycle-enter", + }, + }, + opengraph.Edge{ + StartID: cycleNode, + EndID: productiveBoundary, + Kind: "Expand", + Properties: map[string]any{ + "logical_key": "productive-boundary-cycle-return", + }, + }, + ) + } + if config.AddProductiveBoundarySelfLoop { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: productiveBoundary, + EndID: productiveBoundary, + Kind: "Expand", + Properties: map[string]any{ + "logical_key": "productive-boundary-self-loop", + }, + }) + } + + decoySource := "fse-root" + if depth > 0 { + decoySource = "fse-branch-0000-level-01" + } + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: "fse-decoy-head", + Kinds: []string{"SuffixHead"}, + }, + opengraph.Node{ + ID: "fse-decoy-middle", + Kinds: []string{"SuffixMiddle"}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-decoy-head", + Kind: "WrongEnterSuffix", + }, + opengraph.Edge{ + StartID: "fse-decoy-head", + EndID: decoySource, + Kind: "EnterSuffix", + }, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-wrong-endpoint", + Kind: "EnterSuffix", + }, + ) + + return fixture +} + +// newLegacyFixedSuffixExpansionScaleFixture builds the original shared-suffix +// topology used when no independent population controls are configured. +func newLegacyFixedSuffixExpansionScaleFixture(config FixedSuffixExpansionScaleConfig) *opengraph.Graph { + depth := max(config.ExpansionDepth, 0) + fanout := max(config.Fanout, 1) + validEvery := max(config.ValidSuffixEvery, 1) + payload := strings.Repeat("x", max(config.PropertyPayloadSize, 0)) + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "fse-root", + Kinds: []string{"ExpansionRoot"}, + Properties: map[string]any{"root_key": "generated-fse-root", "payload": payload}, + }, + { + ID: "fse-head", + Kinds: []string{"SuffixHead"}, + Properties: map[string]any{"payload": payload}, + }, + { + ID: "fse-middle", + Kinds: []string{"SuffixMiddle"}, + }, + { + ID: "fse-terminal", + Kinds: []string{"SuffixTerminal"}, + }, + { + ID: "fse-wrong-endpoint", + Kinds: []string{"ExpansionNode"}, + }, + { + ID: "fse-disconnected", + Kinds: []string{"ExpansionNode"}, + }, + }, + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "fse-root", + EndID: "fse-head", + Kind: "EnterSuffix", + Properties: map[string]any{"payload": payload}, + }, + opengraph.Edge{ + StartID: "fse-head", + EndID: "fse-middle", + Kind: "ContinueSuffix", + }, + opengraph.Edge{ + StartID: "fse-middle", + EndID: "fse-terminal", + Kind: "CompleteSuffix", + }, + ) + if depth > 0 { + for branch := range fanout { + previous := "fse-root" + for level := 1; level <= depth; level++ { + next := fmt.Sprintf("fse-branch-%04d-level-%02d", branch, level) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: next, + Kinds: []string{"ExpansionNode"}, + Properties: map[string]any{"payload": payload}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: next, + Kind: "Expand", + }) + previous = next + } + if branch%validEvery == 0 { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: previous, + EndID: "fse-head", + Kind: "EnterSuffix", + }) + } + } + } + decoySource := "fse-root" + if depth > 0 { + decoySource = "fse-branch-0000-level-01" + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-head", + Kind: "WrongEnterSuffix", + }, + opengraph.Edge{ + StartID: "fse-head", + EndID: decoySource, + Kind: "EnterSuffix", + }, + opengraph.Edge{ + StartID: decoySource, + EndID: "fse-wrong-endpoint", + Kind: "EnterSuffix", + }, + opengraph.Edge{ + StartID: "fse-disconnected", + EndID: "fse-head", + Kind: "EnterSuffix", + }, + ) + return fixture +} + +// containsInt reports whether target occurs in values. +func containsInt(values []int, target int) bool { + for _, value := range values { + if value == target { + return true + } + } + return false +} diff --git a/testutil/perf_fixtures_test.go b/testutil/perf_fixtures_test.go new file mode 100644 index 00000000..16daf1af --- /dev/null +++ b/testutil/perf_fixtures_test.go @@ -0,0 +1,342 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "encoding/json" + "slices" + "strings" + "testing" + + "github.com/specterops/dawgs/opengraph" + "github.com/stretchr/testify/require" +) + +// TestShortestPathScaleFixtureIsDeterministicAndCardinalityExact verifies the +// legacy shortest-path fixture is stable and emits the expected topology. +func TestShortestPathScaleFixtureIsDeterministicAndCardinalityExact(t *testing.T) { + config := ShortestPathScaleConfig{ + Depth: 16, + Fanout: 10, + } + first := NewShortestPathScaleFixture(config) + second := NewShortestPathScaleFixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.Len(t, first.Nodes, 4+(config.Depth-1)+config.Fanout+8) + require.Len(t, first.Edges, config.Depth+1+config.Fanout+12) + + var parallel, selfLoops int + for _, edge := range first.Edges { + if edge.StartID == "sp-start" && edge.EndID == "sp-parallel-end" { + parallel++ + } + if edge.StartID == "sp-self-loop" && edge.EndID == "sp-self-loop" { + selfLoops++ + } + } + require.Equal(t, 2, parallel) + require.Equal(t, 1, selfLoops) +} + +// TestEndpointSeededExpansionFixtureIsDeterministicAndSeparatesWorkClasses verifies productive, nonmatching, and ineligible lanes remain distinct. +func TestEndpointSeededExpansionFixtureIsDeterministicAndSeparatesWorkClasses(t *testing.T) { + config := EndpointSeededExpansionScaleConfig{ + Depth: 3, + MatchingEndpoints: 2, + OtherEndpoints: 1, + MatchingEligibleLanes: 2, + OtherEligibleLanes: 1, + MatchingIneligibleLanes: 1, + ParallelEdges: 1, + AddCycle: true, + PropertyPayloadSize: 8, + } + first := NewEndpointSeededExpansionScaleFixture(config) + second := NewEndpointSeededExpansionScaleFixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.NotEmpty(t, first.Nodes) + require.NotEmpty(t, first.Edges) + require.NoError(t, ValidateEndpointSeededExpansionScaleConfig(config)) + require.Error(t, ValidateEndpointSeededExpansionScaleConfig(EndpointSeededExpansionScaleConfig{})) + config.ParallelEdges = 2 + require.ErrorContains(t, ValidateEndpointSeededExpansionScaleConfig(config), "uniquely keys edges") +} + +// TestShortestPathScaleV2FixtureIsDeterministicAndTopologyExact verifies the +// configurable fixture is stable and assigns unique logical edge keys. +func TestShortestPathScaleV2FixtureIsDeterministicAndTopologyExact(t *testing.T) { + config := ShortestPathScaleV2Config{ + Depth: 3, + ForwardRootFanOut: 2, + ReverseRootFanIn: 2, + IntermediateFanOut: 1, + IntermediateReverseFanIn: 4, + FanInLevel: 2, + ParallelKindCount: 3, + ParallelTargetCount: 2, + DiamondWidth: 2, + DisconnectedWidth: 3, + PropertyPayloadSize: 8, + AddCycle: true, + AddSelfLoop: true, + } + first := NewShortestPathScaleV2Fixture(config) + second := NewShortestPathScaleV2Fixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.Len(t, first.Nodes, 32) + require.Len(t, first.Edges, 33) + + logicalKeys := map[string]bool{} + for _, edge := range first.Edges { + key, ok := edge.Properties["logical_key"].(string) + require.True(t, ok) + require.NotEmpty(t, key) + require.False(t, logicalKeys[key], key) + logicalKeys[key] = true + } +} + +// TestShortestPathScaleV2ConfigurationRejectsImpossibleShapes verifies invalid +// dimensions and inconsistent fan-in controls are rejected. +func TestShortestPathScaleV2ConfigurationRejectsImpossibleShapes(t *testing.T) { + for _, config := range []ShortestPathScaleV2Config{ + { + Depth: -1, + }, + { + Depth: 65, + }, + { + Depth: 3, + FanInLevel: 2, + }, + { + Depth: 3, + IntermediateReverseFanIn: 1, + FanInLevel: 3, + }, + { + ParallelKindCount: 1, + }, + { + ParallelTargetCount: 1, + }, + } { + require.Error(t, ValidateShortestPathScaleV2Config(config)) + } + require.NoError(t, ValidateShortestPathScaleV2Config(ShortestPathScaleV2Config{})) +} + +// TestFixedSuffixExpansionScaleFixtureIsDeterministicAndCoversDecoys verifies +// the legacy suffix topology remains stable and includes wrong-kind edges. +func TestFixedSuffixExpansionScaleFixtureIsDeterministicAndCoversDecoys(t *testing.T) { + config := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 4, + Fanout: 10, + ValidSuffixEvery: 2, + PropertyPayloadSize: 32, + } + first := NewFixedSuffixExpansionScaleFixture(config) + second := NewFixedSuffixExpansionScaleFixture(config) + firstJSON, err := json.Marshal(first) + require.NoError(t, err) + secondJSON, err := json.Marshal(second) + require.NoError(t, err) + require.Equal(t, firstJSON, secondJSON) + require.Len(t, first.Nodes, 6+config.ExpansionDepth*config.Fanout) + require.Len(t, first.Edges, 3+config.ExpansionDepth*config.Fanout+5+4) + + _, edgeKinds := first.Kinds() + require.Contains(t, edgeKinds.Strings(), "WrongEnterSuffix") +} + +// TestFixedSuffixExpansionScaleFixtureV2ControlsSuffixPopulationsIndependently verifies reachable, disconnected, and reverse-fan-in populations can vary +// without changing one another. +func TestFixedSuffixExpansionScaleFixtureV2ControlsSuffixPopulationsIndependently(t *testing.T) { + reachable := 0 + zeroDepth := false + fixture := NewFixedSuffixExpansionScaleFixture(FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 4, + ExactReachableSuffixSources: &reachable, + DisconnectedSuffixSources: 3, + ReverseFanIn: 2, + SuffixPathsPerBoundary: 2, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &zeroDepth, + }) + + var enterSuffix, expand int + for _, edge := range fixture.Edges { + switch edge.Kind { + case "EnterSuffix": + enterSuffix++ + case "Expand": + expand++ + } + } + require.Equal(t, 8, enterSuffix) + require.Equal(t, 10, expand) + nodeIDs := make([]string, 0, len(fixture.Nodes)) + for _, node := range fixture.Nodes { + nodeIDs = append(nodeIDs, node.ID) + } + require.NotContains(t, nodeIDs, "fse-disconnected") + require.Contains(t, nodeIDs, "fse-disconnected-00002") +} + +// TestFixedSuffixExpansionScaleFixtureV3ControlsRootMultiplicity verifies that +// matching root rows vary without multiplying the primary root's fanout or +// suffix population. +func TestFixedSuffixExpansionScaleFixtureV3ControlsRootMultiplicity(t *testing.T) { + reachable := 1 + zeroDepth := false + config := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 2, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 3, + RootHasZeroDepthSuffix: &zeroDepth, + } + require.NoError(t, ValidateFixedSuffixExpansionScaleV3Config(config)) + + fixture := NewFixedSuffixExpansionScaleFixture(config) + matchingRoots := 0 + for _, node := range fixture.Nodes { + if slices.Contains(node.Kinds, "ExpansionRoot") && node.Properties["root_key"] == "generated-fse-root" { + matchingRoots++ + } + } + require.Equal(t, 3, matchingRoots) + + rootExpandEdges := 0 + for _, edge := range fixture.Edges { + if edge.Kind == "Expand" && edge.StartID == "fse-root" { + rootExpandEdges++ + } + } + require.Equal(t, 2, rootExpandEdges) +} + +// TestFixedSuffixExpansionScaleFixtureV3ProductiveBoundaryControls verifies +// all cycle/self-loop combinations and the stable, relationship-distinct +// topology emitted for each enabled control. +func TestFixedSuffixExpansionScaleFixtureV3ProductiveBoundaryControls(t *testing.T) { + for _, testCase := range []struct { + // name retains the name while anonymous record is assembled or evaluated. + name string + // cycle indicates whether cycle applies. + cycle bool + // selfLoop indicates whether self loop applies. + selfLoop bool + // wantEdges retains the want edges while anonymous record is assembled or evaluated. + wantEdges int + }{ + {name: "neither"}, + { + name: "cycle", + cycle: true, + wantEdges: 2, + }, + { + name: "self-loop", + selfLoop: true, + wantEdges: 1, + }, + { + name: "both", + cycle: true, + selfLoop: true, + wantEdges: 3, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + reachable := 0 + zeroDepth := true + config := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 1, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &zeroDepth, + AddProductiveBoundaryCycle: testCase.cycle, + AddProductiveBoundarySelfLoop: testCase.selfLoop, + } + require.NoError(t, ValidateFixedSuffixExpansionScaleV3Config(config)) + + fixture := NewFixedSuffixExpansionScaleFixture(config) + controlEdges := map[string]opengraph.Edge{} + for _, edge := range fixture.Edges { + logicalKey, _ := edge.Properties["logical_key"].(string) + if strings.HasPrefix(logicalKey, "productive-boundary-") { + controlEdges[logicalKey] = edge + } + } + require.Len(t, controlEdges, testCase.wantEdges) + if testCase.cycle { + require.Equal(t, "fse-productive-boundary-cycle", controlEdges["productive-boundary-cycle-enter"].EndID) + require.Equal(t, "fse-root", controlEdges["productive-boundary-cycle-return"].EndID) + } + if testCase.selfLoop { + selfLoop := controlEdges["productive-boundary-self-loop"] + require.Equal(t, "fse-root", selfLoop.StartID) + require.Equal(t, selfLoop.StartID, selfLoop.EndID) + } + }) + } +} + +// TestFixedSuffixExpansionScaleV3ConfigurationRejectsUnproductiveControls +// verifies that topology and fan-in controls cannot be attached to a boundary +// with no generated suffix. +func TestFixedSuffixExpansionScaleV3ConfigurationRejectsUnproductiveControls(t *testing.T) { + reachable := 0 + zeroDepth := false + base := FixedSuffixExpansionScaleConfig{ + ExpansionDepth: 2, + Fanout: 1, + ExactReachableSuffixSources: &reachable, + SuffixPathsPerBoundary: 1, + RootMatchCount: 1, + RootHasZeroDepthSuffix: &zeroDepth, + } + require.NoError(t, ValidateFixedSuffixExpansionScaleV3Config(base)) + + withCycle := base + withCycle.AddProductiveBoundaryCycle = true + require.Error(t, ValidateFixedSuffixExpansionScaleV3Config(withCycle)) + withSelfLoop := base + withSelfLoop.AddProductiveBoundarySelfLoop = true + require.Error(t, ValidateFixedSuffixExpansionScaleV3Config(withSelfLoop)) + withFanIn := base + withFanIn.ReverseFanIn = 1 + require.Error(t, ValidateFixedSuffixExpansionScaleV3Config(withFanIn)) +} diff --git a/testutil/perf_shortest_v2.go b/testutil/perf_shortest_v2.go new file mode 100644 index 00000000..d6fce0aa --- /dev/null +++ b/testutil/perf_shortest_v2.go @@ -0,0 +1,248 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "errors" + "fmt" + "strings" + + "github.com/specterops/dawgs/opengraph" +) + +// ShortestPathScaleV2Dataset identifies the second-generation generated +// shortest-path fixture. +const ShortestPathScaleV2Dataset = ShortestPathScaleDataset + "_v2" + +// ShortestPathScaleV2Config controls independent path, decoy, fanout, and +// payload dimensions in the second-generation shortest-path fixture. +type ShortestPathScaleV2Config struct { + // Depth sets the number of edges in each primary path. + Depth int + + // ForwardRootFanOut sets the number of forward dead ends at the primary + // start node. + ForwardRootFanOut int + + // ReverseRootFanIn sets the number of reverse dead ends entering the inbound + // root node. + ReverseRootFanIn int + + // IntermediateFanOut sets the number of forward dead ends at FanInLevel. + IntermediateFanOut int + + // IntermediateReverseFanIn sets the number of reverse dead ends entering + // the inbound path at FanInLevel. + IntermediateReverseFanIn int + + // FanInLevel selects the intermediate level used for fanout and reverse + // fan-in decoys. + FanInLevel int + + // ParallelKindCount sets the number of distinct relationship kinds between + // each parallel start and target pair. + ParallelKindCount int + + // ParallelTargetCount sets the number of targets in the parallel-edge + // subgraph. + ParallelTargetCount int + + // DiamondWidth sets the number of equal-length branches in the diamond + // subgraph. + DiamondWidth int + + // DisconnectedWidth sets the number of intermediate nodes in the + // disconnected path. + DisconnectedWidth int + + // PropertyPayloadSize sets the length of synthetic payload properties. + PropertyPayloadSize int + + // AddCycle includes a reachable two-node cycle. + AddCycle bool + + // AddSelfLoop includes a reachable self-loop. + AddSelfLoop bool +} + +// ValidateShortestPathScaleV2Config rejects negative, inconsistent, or +// unsupported fixture dimensions. +func ValidateShortestPathScaleV2Config(config ShortestPathScaleV2Config) error { + values := []int{ + config.Depth, config.ForwardRootFanOut, config.ReverseRootFanIn, + config.IntermediateFanOut, config.IntermediateReverseFanIn, + config.FanInLevel, config.ParallelKindCount, config.ParallelTargetCount, + config.DiamondWidth, config.DisconnectedWidth, config.PropertyPayloadSize, + } + for _, value := range values { + if value < 0 { + return errors.New("shortest-path v2 configuration values must not be negative") + } + } + if config.Depth > 64 { + return errors.New("shortest-path v2 depth must not exceed 64") + } + if config.IntermediateFanOut == 0 && config.IntermediateReverseFanIn == 0 { + if config.FanInLevel != 0 { + return errors.New("shortest-path v2 fan-in level must be zero without intermediate fanout or fan-in") + } + } else if config.FanInLevel < 1 || config.FanInLevel >= config.Depth { + return errors.New("shortest-path v2 fan-in level must identify an intermediate path level") + } + if (config.ParallelKindCount == 0) != (config.ParallelTargetCount == 0) { + return errors.New("shortest-path v2 parallel kind and target counts must both be zero or both be positive") + } + return nil +} + +// NewShortestPathScaleV2Fixture builds independent deterministic anchors for +// a primary path, hidden fan-in/fan-out, parallel kinds, diamonds, cycles, +// self-loops, and disconnected exhaustion. Every relationship has a stable +// logical_key so backend physical IDs are never required for path comparison. +func NewShortestPathScaleV2Fixture(config ShortestPathScaleV2Config) *opengraph.Graph { + if err := ValidateShortestPathScaleV2Config(config); err != nil { + panic(err) + } + + payload := strings.Repeat("x", config.PropertyPayloadSize) + fixture := &opengraph.Graph{} + addNode := func(id string, properties map[string]any) { + if properties == nil { + properties = map[string]any{} + } + if payload != "" { + properties["payload"] = payload + } + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: id, + Kinds: []string{"ShortestNode"}, + Properties: properties, + }) + } + addEdge := func(start, end, kind, key string) { + properties := map[string]any{"logical_key": key} + if payload != "" { + properties["payload"] = payload + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: start, + EndID: end, + Kind: kind, + Properties: properties, + }) + } + + addNode("sp-v2-start", map[string]any{"role": "start", "level": 0}) + addNode("sp-v2-end", map[string]any{"role": "end", "level": config.Depth}) + pathNodes := []string{"sp-v2-start"} + for level := 1; level < config.Depth; level++ { + id := fmt.Sprintf("sp-v2-linear-%02d", level) + addNode(id, map[string]any{"role": "path", "level": level}) + pathNodes = append(pathNodes, id) + } + if config.Depth > 0 { + pathNodes = append(pathNodes, "sp-v2-end") + for level := 1; level < len(pathNodes); level++ { + addEdge(pathNodes[level-1], pathNodes[level], "Traverse", fmt.Sprintf("primary-%02d", level)) + } + } + inboundPathNodes := []string{"sp-v2-inbound-end"} + addNode("sp-v2-inbound-end", map[string]any{"role": "inbound_terminal", "level": config.Depth}) + addNode("sp-v2-inbound-root", map[string]any{"role": "inbound_root", "level": 0}) + for level := config.Depth - 1; level >= 1; level-- { + id := fmt.Sprintf("sp-v2-inbound-linear-%02d", level) + addNode(id, map[string]any{"role": "inbound_path", "level": level}) + inboundPathNodes = append(inboundPathNodes, id) + } + if config.Depth > 0 { + inboundPathNodes = append(inboundPathNodes, "sp-v2-inbound-root") + for level := 1; level < len(inboundPathNodes); level++ { + addEdge(inboundPathNodes[level-1], inboundPathNodes[level], "Traverse", fmt.Sprintf("inbound-primary-%02d", level)) + } + } + + for idx := range config.ForwardRootFanOut { + id := fmt.Sprintf("sp-v2-root-out-%06d", idx) + addNode(id, map[string]any{"role": "root_forward_dead_end"}) + addEdge("sp-v2-start", id, "Traverse", fmt.Sprintf("root-out-%06d", idx)) + } + for idx := range config.ReverseRootFanIn { + id := fmt.Sprintf("sp-v2-root-in-%06d", idx) + addNode(id, map[string]any{"role": "root_reverse_dead_end"}) + addEdge(id, "sp-v2-inbound-root", "Traverse", fmt.Sprintf("root-in-%06d", idx)) + } + if config.FanInLevel > 0 { + boundary := pathNodes[config.FanInLevel] + for idx := range config.IntermediateFanOut { + id := fmt.Sprintf("sp-v2-level-%02d-out-%06d", config.FanInLevel, idx) + addNode(id, map[string]any{"role": "intermediate_forward_dead_end", "level": config.FanInLevel + 1}) + addEdge(boundary, id, "Traverse", fmt.Sprintf("level-%02d-out-%06d", config.FanInLevel, idx)) + } + for idx := range config.IntermediateReverseFanIn { + id := fmt.Sprintf("sp-v2-level-%02d-in-%06d", config.FanInLevel, idx) + addNode(id, map[string]any{"role": "intermediate_reverse_dead_end", "level": config.FanInLevel - 1}) + inboundBoundary := fmt.Sprintf("sp-v2-inbound-linear-%02d", config.FanInLevel) + addEdge(id, inboundBoundary, "Traverse", fmt.Sprintf("level-%02d-in-%06d", config.FanInLevel, idx)) + } + } + + if config.ParallelKindCount > 0 { + addNode("sp-v2-parallel-start", map[string]any{"role": "parallel_start"}) + for target := range config.ParallelTargetCount { + targetID := fmt.Sprintf("sp-v2-parallel-target-%06d", target) + addNode(targetID, map[string]any{"role": "parallel_target"}) + for kind := range config.ParallelKindCount { + addEdge("sp-v2-parallel-start", targetID, fmt.Sprintf("ParallelKind%02d", kind), fmt.Sprintf("parallel-k%02d-t%06d", kind, target)) + } + } + } + + if config.DiamondWidth > 0 { + addNode("sp-v2-diamond-start", map[string]any{"role": "diamond_start"}) + addNode("sp-v2-diamond-end", map[string]any{"role": "diamond_end"}) + for idx := range config.DiamondWidth { + middle := fmt.Sprintf("sp-v2-diamond-%06d", idx) + addNode(middle, map[string]any{"role": "diamond_middle"}) + addEdge("sp-v2-diamond-start", middle, "DiamondTraverse", fmt.Sprintf("diamond-%06d-a", idx)) + addEdge(middle, "sp-v2-diamond-end", "DiamondTraverse", fmt.Sprintf("diamond-%06d-b", idx)) + } + } + + addNode("sp-v2-disconnected-start", map[string]any{"role": "disconnected_start"}) + addNode("sp-v2-disconnected-end", map[string]any{"role": "disconnected_end"}) + previous := "sp-v2-disconnected-start" + for idx := range config.DisconnectedWidth { + next := fmt.Sprintf("sp-v2-disconnected-%06d", idx) + addNode(next, map[string]any{"role": "disconnected_state"}) + addEdge(previous, next, "Traverse", fmt.Sprintf("disconnected-%06d", idx)) + previous = next + } + if config.AddCycle { + addNode("sp-v2-cycle-a", map[string]any{"role": "cycle"}) + addNode("sp-v2-cycle-b", map[string]any{"role": "cycle"}) + addEdge("sp-v2-start", "sp-v2-cycle-a", "Traverse", "cycle-entry") + addEdge("sp-v2-cycle-a", "sp-v2-cycle-b", "Traverse", "cycle-a-b") + addEdge("sp-v2-cycle-b", "sp-v2-cycle-a", "Traverse", "cycle-b-a") + } + if config.AddSelfLoop { + addNode("sp-v2-self-loop", map[string]any{"role": "self_loop"}) + addEdge("sp-v2-start", "sp-v2-self-loop", "Traverse", "self-loop-entry") + addEdge("sp-v2-self-loop", "sp-v2-self-loop", "Traverse", "self-loop") + } + + return fixture +} diff --git a/testutil/reconciliation_fixture.go b/testutil/reconciliation_fixture.go new file mode 100644 index 00000000..ca93400d --- /dev/null +++ b/testutil/reconciliation_fixture.go @@ -0,0 +1,1051 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "fmt" + + "github.com/specterops/dawgs/opengraph" +) + +const ( + // ReconciliationScaleDataset identifies the generated reconciliation + // fixture. + ReconciliationScaleDataset = "generated_reconciliation" + + // TrustPruningScaleDataset identifies the generated trust-pruning fixture. + TrustPruningScaleDataset = "generated_trust_pruning" + + // HopScaleDataset identifies the generated relationship-hop fixture. + HopScaleDataset = "generated_hops" + + // ScanLookupScaleDataset identifies the generated scan-and-lookup fixture. + ScanLookupScaleDataset = "generated_scan_lookups" +) + +// GeneratedNodeListParam resolves optional fixture IDs followed by a +// deterministic prefix/count sequence. It keeps high-cardinality database-ID +// parameters out of handwritten JSON. +type GeneratedNodeListParam struct { + // Prefix is prepended to each generated node identifier. + Prefix string `json:"prefix"` + + // Count is the number of sequential identifiers to generate. + Count int `json:"count"` + + // Include lists literal identifiers to place before generated identifiers. + Include []string `json:"include,omitempty"` +} + +// FixtureNames returns stable, zero-padded fixture IDs. +func FixtureNames(prefix string, count int) []string { + if count < 0 { + count = 0 + } + + width := len(fmt.Sprintf("%d", max(count-1, 0))) + if width < 2 { + width = 2 + } + + values := make([]string, count) + for idx := range count { + values[idx] = fmt.Sprintf("%s-%0*d", prefix, width, idx) + } + return values +} + +// NewDirectWriteScaleFixture returns a deterministic graph for direct batch +// mutation tests. The requested number of target nodes is used exactly so that +// callers can exercise batch-flush boundaries without fixture rounding. +// +// Every target has one deletable relationship and one relationship-upsert +// baseline. Deletion directions alternate, while the first two targets (when +// present) provide self-connected and high-degree cascade shapes. Separate +// root-to-survivor relationships are never incident to a target, including a +// same-kind survivor for exact delete-set assertions. +func NewDirectWriteScaleFixture(targets int) *opengraph.Graph { + if targets < 0 { + targets = 0 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "write-root", + Kinds: []string{"WriteEndpoint"}, + Properties: map[string]any{"objectid": "write-root", "role": "root"}, + }, + { + ID: "write-survivor", + Kinds: []string{"WriteEndpoint"}, + Properties: map[string]any{"objectid": "write-survivor", "role": "survivor"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "write-root", + EndID: "write-survivor", + Kind: "WriteSurvivor", + Properties: map[string]any{"marker": "survivor"}, + }, + { + StartID: "write-root", + EndID: "write-survivor", + Kind: "WriteDeleteRelationship", + Properties: map[string]any{"deletebatch": false, "marker": "same-kind-survivor"}, + }, + }, + } + + targetIDs := FixtureNames("write-target", targets) + for idx, targetID := range targetIDs { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: targetID, + Kinds: []string{"WriteDeleteNode", "WriteUpdateNode"}, + Properties: map[string]any{ + "objectid": targetID, + "deletebatch": true, + "lastseen": "2026-01-01T00:00:00Z", + "ordinal": idx, + }, + }) + + startID, endID := "write-root", targetID + if idx%2 == 1 { + startID, endID = targetID, "write-root" + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "WriteDeleteRelationship", + Properties: map[string]any{ + "deletebatch": true, + "marker": targetID, + }, + }, + opengraph.Edge{ + StartID: "write-root", + EndID: targetID, + Kind: "WriteUpdateRelationship", + Properties: map[string]any{ + "lastseen": "2026-01-01T00:00:00Z", + "marker": targetID, + }, + }, + ) + } + + if targets > 0 { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: targetIDs[0], + EndID: targetIDs[0], + Kind: "WriteIncident", + Properties: map[string]any{"marker": "self"}, + }) + } + if targets > 1 { + for idx, targetID := range targetIDs { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: targetIDs[1], + EndID: targetID, + Kind: "WriteIncident", + Properties: map[string]any{"marker": fmt.Sprintf("high-%04d", idx)}, + }) + } + } + + return fixture +} + +// NewReconciliationScaleFixture returns the deterministic graphbench fixture +// for the ingestion reconciliation forms. fanout controls the degree of the +// REC-08 detach-delete target. +func NewReconciliationScaleFixture(fanout int) *opengraph.Graph { + if fanout < 1 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "source", + Kinds: []string{"Source"}, + Properties: map[string]any{"objectid": "source"}, + }, + { + ID: "list-source-duplicate", + Kinds: []string{"Source"}, + Properties: map[string]any{"objectid": "list-source-duplicate"}, + }, + { + ID: "sink", + Kinds: []string{"Destination"}, + Properties: map[string]any{"objectid": "sink"}, + }, + { + ID: "inbound-target", + Kinds: []string{"ADEntity", "Group"}, + Properties: map[string]any{"objectid": "rec-in"}, + }, + { + ID: "outbound-target", + Kinds: []string{"ADEntity", "Computer"}, + Properties: map[string]any{"objectid": "rec-out"}, + }, + { + ID: "list-target", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "rec-list"}, + }, + { + ID: "template", + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": "template"}, + }, + { + ID: "agent", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "agent"}, + }, + { + ID: "agent-duplicate", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "agent-duplicate"}, + }, + { + ID: "delete-target", + Kinds: []string{"ADEntity", "Group"}, + Properties: map[string]any{"objectid": "delete-target"}, + }, + { + ID: "survivor", + Kinds: []string{"ADEntity", "User"}, + Properties: map[string]any{"objectid": "survivor"}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "source", + EndID: "inbound-target", + Kind: "RecKind01", + Properties: map[string]any{"marker": "rec-01-a"}, + }, + { + StartID: "source", + EndID: "inbound-target", + Kind: "RecKind30", + Properties: map[string]any{"marker": "rec-01-b"}, + }, + { + StartID: "outbound-target", + EndID: "sink", + Kind: "RecKind01", + Properties: map[string]any{"marker": "rec-02-a"}, + }, + { + StartID: "outbound-target", + EndID: "sink", + Kind: "RecKind30", + Properties: map[string]any{"marker": "rec-02-b"}, + }, + { + StartID: "source", + EndID: "list-target", + Kind: "ADReconcile", + Properties: map[string]any{"marker": "rec-04-a"}, + }, + { + StartID: "list-source-duplicate", + EndID: "list-target", + Kind: "ADReconcile", + Properties: map[string]any{"marker": "rec-04-b"}, + }, + { + StartID: "agent", + EndID: "template", + Kind: "DelegatedEnrollmentAgent", + Properties: map[string]any{"marker": "rec-06-a"}, + }, + { + StartID: "agent-duplicate", + EndID: "template", + Kind: "DelegatedEnrollmentAgent", + Properties: map[string]any{"marker": "rec-06-b"}, + }, + { + StartID: "source", + EndID: "survivor", + Kind: "Survivor", + Properties: map[string]any{"marker": "survivor"}, + }, + }, + } + + for _, templateID := range FixtureNames("scale-template", 2_000) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: templateID, + Kinds: []string{"CertTemplate"}, + Properties: map[string]any{"objectid": templateID}, + }) + } + + // Ensure every relationship kind in the 30-kind disjunction is registered, + // while anchoring each decoy away from the REC-01/REC-02 target endpoints. + for idx := 2; idx < 30; idx++ { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "source", + EndID: "survivor", + Kind: fmt.Sprintf("RecKind%02d", idx), + Properties: map[string]any{"marker": fmt.Sprintf("kind-decoy-%02d", idx)}, + }) + } + + for idx := range fanout { + neighborID := fmt.Sprintf("detach-neighbor-%04d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: neighborID, + Kinds: []string{"ADEntity"}, + Properties: map[string]any{"objectid": neighborID}, + }) + + startID, endID := "delete-target", neighborID + if idx%2 == 0 { + startID, endID = neighborID, "delete-target" + } + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "Incident", + Properties: map[string]any{"marker": fmt.Sprintf("incident-%04d", idx)}, + }) + } + + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "delete-target", + EndID: "delete-target", + Kind: "Incident", + Properties: map[string]any{"marker": "incident-self"}, + }) + return fixture +} + +// NewTrustPruningScaleFixture returns deterministic dense trust and pruning +// shapes without changing the cardinalities of the reconciliation fixture. +func NewTrustPruningScaleFixture(fanout int) *opengraph.Graph { + if fanout < 1 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "trust-early", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-02T00:00:00Z"}, + }, + { + ID: "trust-late-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-04T00:00:00Z"}, + }, + { + ID: "trust-late-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-04T00:00:00Z"}, + }, + { + ID: "trust-equal-a", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-03T00:00:00Z"}, + }, + { + ID: "trust-equal-b", + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-03T00:00:00Z"}, + }, + { + ID: "prune-a", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "a"}, + }, + { + ID: "prune-b", + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": "b"}, + }, + { + ID: "prune-missing", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": "missing"}, + }, + { + ID: "prune-null", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": "null", "lastseen": nil}, + }, + { + ID: "prune-protected", + Kinds: []string{"PruneCandidate", "Domain"}, + Properties: map[string]any{"name": "protected", "lastseen": "2026-01-02T00:00:00Z"}, + }, + { + ID: "orphan-missing", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"objectid": "S-1-5-100"}, + }, + { + ID: "orphan-null", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": nil, "objectid": "S-1-5-101"}, + }, + { + ID: "orphan-named", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": "named", "objectid": "S-1-5-102"}, + }, + { + ID: "orphan-wrong-prefix", + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"objectid": "X-1-5-103"}, + }, + { + ID: "prune-batch-high", + Kinds: []string{"PruneBatchNode"}, + Properties: map[string]any{"remove": true}, + }, + { + ID: "prune-batch-survivor", + Kinds: []string{"PruneBatchNode"}, + Properties: map[string]any{"remove": false}, + }, + }, + Edges: []opengraph.Edge{ + { + StartID: "trust-equal-a", + EndID: "trust-equal-b", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "same-equal"}, + }, + { + StartID: "trust-equal-a", + EndID: "trust-equal-b", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-equal"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": "2026-01-05T00:00:00Z", "marker": "same-new"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": "2026-01-05T00:00:00Z", "marker": "cross-new"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "valid-forward-abuse"}, + }, + { + StartID: "trust-late-b", + EndID: "trust-late-a", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "valid-reverse-spoof"}, + }, + { + StartID: "trust-late-a", + EndID: "trust-late-b", + Kind: "SpoofSIDHistory", + Properties: map[string]any{"marker": "invalid-forward-spoof"}, + }, + { + StartID: "trust-late-b", + EndID: "trust-late-a", + Kind: "AbuseTGTDelegation", + Properties: map[string]any{"marker": "invalid-reverse-abuse"}, + }, + { + StartID: "prune-a", + EndID: "prune-b", + Kind: "PruneBatchSurvivor", + Properties: map[string]any{"remove": false}, + }, + { + StartID: "prune-a", + EndID: "prune-b", + Kind: "MetaIncludes", + Properties: map[string]any{"lastseen": "2026-01-02T00:00:00Z", "marker": "protected-meta-includes"}, + }, + }, + } + + for idx := range fanout { + var ( + suffix = fmt.Sprintf("%04d", idx) + trustEarlyID = "trust-early-" + suffix + oldNodeID = "prune-old-" + suffix + newNodeID = "prune-new-" + suffix + orphanNodeID = "orphan-scale-" + suffix + batchNodeID = "prune-batch-" + suffix + neighborID = "prune-neighbor-" + suffix + candidateOldTargetID = "candidate-old-target-" + suffix + candidateNewTargetID = "candidate-new-target-" + suffix + sessionMissingTargetID = "session-missing-target-" + suffix + sessionOldTargetID = "session-old-target-" + suffix + sessionEqualTargetID = "session-equal-target-" + suffix + batchEdgeTargetID = "prune-batch-edge-target-" + suffix + ) + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: trustEarlyID, + Kinds: []string{"Domain"}, + Properties: map[string]any{"lastcollected": "2026-01-02T00:00:00Z"}, + }, + opengraph.Node{ + ID: oldNodeID, + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": oldNodeID, "lastseen": "2026-01-02T00:00:00Z"}, + }, + opengraph.Node{ + ID: newNodeID, + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"name": newNodeID, "lastseen": "2026-01-04T00:00:00Z"}, + }, + opengraph.Node{ + ID: orphanNodeID, + Kinds: []string{"PruneCandidate"}, + Properties: map[string]any{"objectid": "S-1-5-" + suffix}, + }, + opengraph.Node{ + ID: batchNodeID, + Kinds: []string{"PruneBatchNode"}, + Properties: map[string]any{"remove": idx%2 == 0}, + }, + opengraph.Node{ + ID: neighborID, + Kinds: []string{"PruneNeighbor"}, + Properties: map[string]any{"name": neighborID}, + }, + opengraph.Node{ + ID: candidateOldTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": candidateOldTargetID}, + }, + opengraph.Node{ + ID: candidateNewTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": candidateNewTargetID}, + }, + opengraph.Node{ + ID: sessionMissingTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": sessionMissingTargetID}, + }, + opengraph.Node{ + ID: sessionOldTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": sessionOldTargetID}, + }, + opengraph.Node{ + ID: sessionEqualTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": sessionEqualTargetID}, + }, + opengraph.Node{ + ID: batchEdgeTargetID, + Kinds: []string{"PruneEndpoint"}, + Properties: map[string]any{"name": batchEdgeTargetID}, + }, + ) + + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "trust-late-a", + EndID: trustEarlyID, + Kind: "SameForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "same-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "trust-late-a", + EndID: trustEarlyID, + Kind: "CrossForestTrust", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "cross-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: candidateOldTargetID, + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": "2026-01-02T00:00:00Z", "marker": "candidate-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: candidateNewTargetID, + Kind: "CandidateRel", + Properties: map[string]any{"lastseen": "2026-01-04T00:00:00Z", "marker": "candidate-new-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: sessionMissingTargetID, + Kind: "HasSession", + Properties: map[string]any{"marker": "session-missing-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: sessionOldTargetID, + Kind: "HasSession", + Properties: map[string]any{"lastseen": "2026-01-02T00:00:00Z", "marker": "session-old-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: sessionEqualTargetID, + Kind: "HasSession", + Properties: map[string]any{"lastseen": "2026-01-03T00:00:00Z", "marker": "session-equal-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-a", + EndID: batchEdgeTargetID, + Kind: "PruneBatch", + Properties: map[string]any{"remove": true, "marker": "batch-" + suffix}, + }, + opengraph.Edge{ + StartID: "prune-batch-high", + EndID: neighborID, + Kind: "PruneIncident", + Properties: map[string]any{"marker": "incident-" + suffix}, + }, + ) + } + + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "prune-batch-high", + EndID: "prune-batch-high", + Kind: "PruneIncident", + Properties: map[string]any{"marker": "incident-self"}, + }) + return fixture +} + +// NewHopScaleFixture returns deterministic one-hop fanout, kind-cardinality, +// endpoint-list, predicate-selectivity, and two-sided set shapes. +func NewHopScaleFixture(fanout int) *opengraph.Graph { + if fanout < 30 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "hop-out-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-out-root"}, + }, + { + ID: "hop-in-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-in-root"}, + }, + { + ID: "hop-kind-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-kind-root"}, + }, + { + ID: "hop-decoy-root", + Kinds: []string{"HopAnchor"}, + Properties: map[string]any{"name": "hop-decoy-root"}, + }, + }, + } + + for idx := range fanout { + var ( + suffix = fmt.Sprintf("%04d", idx) + peerID = "hop-peer-" + suffix + sourceID = "hop-source-" + suffix + properties = map[string]any{ + "name": peerID, + "requiresmanagerapproval": false, + "authenticationenabled": true, + } + ) + + switch idx % 4 { + case 0: + properties["schemaversion"] = 2 + properties["authorizedsignatures"] = 0 + case 1: + properties["schemaversion"] = 1 + properties["authorizedsignatures"] = 9 + case 2: + properties["schemaversion"] = 2 + properties["authorizedsignatures"] = 1 + case 3: + properties["schemaversion"] = 2 + properties["authorizedsignatures"] = 0 + properties["authenticationenabled"] = false + } + + peerKinds := []string{"HopEndpoint", "HopEndA", "HopTemplate"} + if idx%2 == 0 { + peerKinds = append(peerKinds, "HopEndB") + } + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: peerID, + Kinds: peerKinds, + Properties: properties, + }, + opengraph.Node{ + ID: sourceID, + Kinds: []string{"HopSource"}, + Properties: map[string]any{"name": sourceID}, + }, + ) + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "hop-out-root", + EndID: peerID, + Kind: "HopKind01", + Properties: map[string]any{"marker": "out-" + suffix}, + }, + opengraph.Edge{ + StartID: sourceID, + EndID: "hop-in-root", + Kind: "HopKind01", + Properties: map[string]any{"marker": "in-" + suffix}, + }, + opengraph.Edge{ + StartID: "hop-kind-root", + EndID: peerID, + Kind: fmt.Sprintf("HopKind%02d", idx%30+1), + Properties: map[string]any{"marker": "kind-" + suffix}, + }, + opengraph.Edge{ + StartID: "hop-out-root", + EndID: peerID, + Kind: "HopTypedEdge", + Properties: map[string]any{"marker": "typed-" + suffix}, + }, + opengraph.Edge{ + StartID: "hop-out-root", + EndID: peerID, + Kind: "HopNestedEdge", + Properties: map[string]any{"marker": "nested-" + suffix}, + }, + ) + } + + for idx, targetID := range FixtureNames("hop-id-target", 1_000) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: targetID, + Kinds: []string{"HopIDEndpoint"}, + Properties: map[string]any{"name": targetID}, + }) + if idx < fanout { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "hop-out-root", + EndID: targetID, + Kind: "HopIDEdge", + Properties: map[string]any{"marker": fmt.Sprintf("id-%04d", idx)}, + }) + } + } + + setStarts := FixtureNames("hop-set-start", 32) + setEnds := FixtureNames("hop-set-end", 32) + for _, startID := range setStarts { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: startID, + Kinds: []string{"HopSetStart"}, + Properties: map[string]any{"name": startID}, + }) + } + for _, endID := range setEnds { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: endID, + Kinds: []string{"HopSetEnd"}, + Properties: map[string]any{"name": endID}, + }) + } + for _, startID := range setStarts { + for _, endID := range setEnds { + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: startID, + EndID: endID, + Kind: "HopSetEdge", + Properties: map[string]any{"marker": startID + "-" + endID}, + }) + } + } + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "hop-decoy-root", + EndID: "hop-peer-0000", + Kind: "HopTypedEdge", + Properties: map[string]any{"marker": "wrong-root"}, + }, + opengraph.Edge{ + StartID: "hop-peer-0000", + EndID: "hop-out-root", + Kind: "HopTypedEdge", + Properties: map[string]any{"marker": "wrong-direction"}, + }, + opengraph.Edge{ + StartID: setStarts[0], + EndID: setEnds[0], + Kind: "HopWrongSetEdge", + Properties: map[string]any{"marker": "wrong-set-kind"}, + }, + opengraph.Edge{ + StartID: setEnds[0], + EndID: setStarts[0], + Kind: "HopSetEdge", + Properties: map[string]any{"marker": "wrong-set-direction"}, + }, + ) + return fixture +} + +// NewScanLookupScaleFixture returns deterministic wide-scan, large lookup, +// adjacency, ordering, and count shapes for the scan/lookup regression corpus. +func NewScanLookupScaleFixture(fanout int) *opengraph.Graph { + if fanout < 9 { + fanout = 128 + } + + fixture := &opengraph.Graph{ + Nodes: []opengraph.Node{ + { + ID: "scan-base-root", + Kinds: []string{"ADBase"}, + Properties: map[string]any{"name": "scan-base-root"}, + }, + { + ID: "scan-tracker-root", + Kinds: []string{"Plain"}, + Properties: map[string]any{"name": "scan-tracker-root"}, + }, + { + ID: "scan-nine-kind-target", + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": "scan-nine-kind-target"}, + }, + { + ID: "scan-local-target", + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": "scan-local-target"}, + }, + { + ID: "lookup-tenant", + Kinds: []string{"Tenant"}, + Properties: map[string]any{"name": "lookup-tenant", "objectid": "tenant-scale"}, + }, + // The extra isolated labels make negative Meta/MetaDetail predicates + // translatable without changing any fixture cardinality. + { + ID: "lookup-local-target", + Kinds: []string{"Computer", "Meta", "MetaDetail"}, + Properties: map[string]any{"name": "lookup-local-target"}, + }, + }, + } + + escalationKinds := []string{"GenericAll", "GenericWrite", "Owns", "WriteOwner", "WriteDACL", "WritePublicInformation"} + victimIDs := FixtureNames("scan-victim", max(1_000, fanout)) + for idx := range fanout { + suffix := fmt.Sprintf("%04d", idx) + scanEndID := "scan-end-" + suffix + scanEntityID := "scan-entity-" + suffix + lookupObjectID := "lookup-object-" + suffix + lookupStringID := "lookup-string-" + suffix + lookupLocalID := "lookup-local-" + suffix + ntlmID := "lookup-ntlm-" + suffix + + entityKinds := []string{"Entity"} + switch idx % 3 { + case 0: + entityKinds = append(entityKinds, "Group") + case 1: + entityKinds = append(entityKinds, "User") + case 2: + entityKinds = append(entityKinds, "Computer") + } + + lookupObjectSuffix := "-513" + if idx%2 == 0 { + lookupObjectSuffix = "-512" + } + lookupName := fmt.Sprintf("Remote Desktop Users %04d", idx) + if idx%2 == 1 { + lookupName = fmt.Sprintf("rEmOtE dEsKtOp UsErS %04d", idx) + } + + fixture.Nodes = append(fixture.Nodes, + opengraph.Node{ + ID: scanEndID, + Kinds: []string{"AZBase", "Plain"}, + Properties: map[string]any{"name": scanEndID}, + }, + opengraph.Node{ + ID: scanEntityID, + Kinds: entityKinds, + Properties: map[string]any{"name": scanEntityID}, + }, + opengraph.Node{ + ID: lookupObjectID, + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": lookupObjectID, "objectid": "S-1-5-21-scale", "enabled": true}, + }, + opengraph.Node{ + ID: lookupStringID, + Kinds: []string{"Group", "Entity"}, + Properties: map[string]any{"name": lookupName, "objectid": "S-1-5-21" + lookupObjectSuffix, "domainsid": "S-1-5-21"}, + }, + opengraph.Node{ + ID: lookupLocalID, + Kinds: []string{"LocalGroup", "Entity"}, + Properties: map[string]any{"name": lookupLocalID, "objectid": "S-1-5-21-555"}, + }, + opengraph.Node{ + ID: ntlmID, + Kinds: []string{"Computer"}, + Properties: map[string]any{"name": ntlmID, "domainsid": "S-1-5-21", "isdc": true, "ldapavailable": true, "ldapsigning": false}, + }, + ) + + migrationProperties := map[string]any{"marker": "migration-" + suffix} + if idx%2 == 0 { + migrationProperties["lastseen"] = "2026-01-03T00:00:00Z" + } else if idx%4 == 1 { + migrationProperties["lastseen"] = nil + } + + victimID := victimIDs[idx] + fixture.Edges = append(fixture.Edges, + opengraph.Edge{ + StartID: "scan-base-root", + EndID: scanEndID, + Kind: "ScanPostProcessed", + Properties: map[string]any{"marker": "post-" + suffix}, + }, + opengraph.Edge{ + StartID: "scan-tracker-root", + EndID: scanEndID, + Kind: "TrackerA", + Properties: map[string]any{"marker": "tracker-a-" + suffix}, + }, + opengraph.Edge{ + StartID: "scan-tracker-root", + EndID: scanEndID, + Kind: "TrackerB", + Properties: map[string]any{"marker": "tracker-b-" + suffix}, + }, + opengraph.Edge{ + StartID: "scan-tracker-root", + EndID: scanEndID, + Kind: "MigratedEdge", + Properties: migrationProperties, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: scanEndID, + Kind: "OwnsRaw", + Properties: map[string]any{"marker": "owns-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: "scan-nine-kind-target", + Kind: fmt.Sprintf("ScanEdge%02d", idx%9+1), + Properties: map[string]any{"marker": "scan-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: "scan-local-target", + Kind: "LocalToComputer", + Properties: map[string]any{"marker": "scan-local-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: scanEndID, + Kind: "MemberOf", + Properties: map[string]any{"marker": "member-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: scanEndID, + Kind: "MemberOfLocalGroup", + Properties: map[string]any{"marker": "member-local-" + suffix}, + }, + opengraph.Edge{ + StartID: scanEntityID, + EndID: victimID, + Kind: escalationKinds[idx%len(escalationKinds)], + Properties: map[string]any{"marker": "esc-" + suffix}, + }, + opengraph.Edge{ + StartID: lookupLocalID, + EndID: "lookup-local-target", + Kind: "LocalToComputer", + Properties: map[string]any{"marker": "lookup-local-" + suffix}, + }, + ) + } + + for idx, victimID := range victimIDs { + victimKinds := []string{"Other"} + if idx%2 == 0 { + victimKinds = []string{"Computer"} + } + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: victimID, + Kinds: victimKinds, + Properties: map[string]any{"name": victimID}, + }) + } + + for _, targetID := range FixtureNames("lookup-id-target", 1_000) { + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: targetID, + Kinds: []string{"Hydrate"}, + Properties: map[string]any{"name": targetID}, + }) + } + + for idx, roleID := range FixtureNames("lookup-role", 1_000) { + roleTemplateID := fmt.Sprintf("role-template-%03d", idx) + fixture.Nodes = append(fixture.Nodes, opengraph.Node{ + ID: roleID, + Kinds: []string{"AZRole"}, + Properties: map[string]any{"name": roleID, "roletemplateid": roleTemplateID, "enabled": true}, + }) + fixture.Edges = append(fixture.Edges, opengraph.Edge{ + StartID: "lookup-tenant", + EndID: roleID, + Kind: "Contains", + Properties: map[string]any{"marker": roleID}, + }) + } + + return fixture +} diff --git a/testutil/reconciliation_fixture_test.go b/testutil/reconciliation_fixture_test.go new file mode 100644 index 00000000..62a10e59 --- /dev/null +++ b/testutil/reconciliation_fixture_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Specter Ops, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package testutil + +import ( + "fmt" + "testing" + + "github.com/specterops/dawgs/graph" + "github.com/specterops/dawgs/opengraph" + "github.com/stretchr/testify/require" +) + +// requireUniqueScaleEdgeKeys verifies a fixture does not contain duplicate +// start, end, and kind tuples rejected by PostgreSQL storage. +func requireUniqueScaleEdgeKeys(t *testing.T, fixture *opengraph.Graph) { + t.Helper() + + keys := map[string]struct{}{} + for _, edge := range fixture.Edges { + key := edge.StartID + "\x00" + edge.EndID + "\x00" + edge.Kind + require.NotContains(t, keys, key, "duplicate PostgreSQL edge key %s -> %s [%s]", edge.StartID, edge.EndID, edge.Kind) + keys[key] = struct{}{} + } +} + +// TestNewReconciliationScaleFixture verifies the reconciliation fixture's +// cardinality, edge-key uniqueness, and complete kind range. +func TestNewReconciliationScaleFixture(t *testing.T) { + fixture := NewReconciliationScaleFixture(8) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 2_019) + require.Len(t, fixture.Edges, 46) + requireUniqueScaleEdgeKeys(t, fixture) + require.Contains(t, nodeKinds, graph.StringKind("ADEntity")) + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("RecKind%02d", idx))) + } +} + +// TestFixtureNamesAreDeterministic verifies generated fixture identifiers are +// stable, padded, and empty for negative counts. +func TestFixtureNamesAreDeterministic(t *testing.T) { + require.Equal(t, []string{"item-00", "item-01", "item-02"}, FixtureNames("item", 3)) + require.Equal(t, FixtureNames("item", 2_000), FixtureNames("item", 2_000)) + require.Empty(t, FixtureNames("item", -1)) +} + +// TestNewDirectWriteScaleFixtureUsesExactBoundaryAndCascadeShape verifies exact +// target counts and the intended delete, update, and incident edge populations. +func TestNewDirectWriteScaleFixtureUsesExactBoundaryAndCascadeShape(t *testing.T) { + empty := NewDirectWriteScaleFixture(0) + require.Len(t, empty.Nodes, 2) + require.Len(t, empty.Edges, 2) + + fixture := NewDirectWriteScaleFixture(3) + require.Len(t, fixture.Nodes, 5) + require.Len(t, fixture.Edges, 12) + + var ( + deleteEdges int + updateEdges int + incidentEdges int + ) + for _, edge := range fixture.Edges { + switch edge.Kind { + case "WriteDeleteRelationship": + deleteEdges++ + case "WriteUpdateRelationship": + updateEdges++ + case "WriteIncident": + incidentEdges++ + } + } + require.Equal(t, 4, deleteEdges) + require.Equal(t, 3, updateEdges) + require.Equal(t, 4, incidentEdges) + + require.Equal(t, "write-target-00", fixture.Nodes[2].ID) + require.Equal(t, "write-target-02", fixture.Nodes[4].ID) + require.Equal(t, "write-root", fixture.Edges[2].StartID) + require.Equal(t, "write-target-01", fixture.Edges[4].StartID) +} + +// TestNewTrustPruningScaleFixtureIncludesDenseAndDecoyShapes verifies the +// fixture includes all node and relationship categories used by pruning cases. +func TestNewTrustPruningScaleFixtureIncludesDenseAndDecoyShapes(t *testing.T) { + fixture := NewTrustPruningScaleFixture(8) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 112) + require.Len(t, fixture.Edges, 83) + requireUniqueScaleEdgeKeys(t, fixture) + require.Contains(t, nodeKinds, graph.StringKind("Domain")) + require.Contains(t, nodeKinds, graph.StringKind("PruneCandidate")) + require.Contains(t, nodeKinds, graph.StringKind("PruneBatchNode")) + require.Contains(t, edgeKinds, graph.StringKind("SameForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("CrossForestTrust")) + require.Contains(t, edgeKinds, graph.StringKind("HasSession")) + require.Contains(t, edgeKinds, graph.StringKind("PruneBatch")) + require.Contains(t, edgeKinds, graph.StringKind("MetaIncludes")) +} + +// TestNewHopScaleFixtureIncludesDenseAndLargeListShapes verifies dense hop +// topology, broad kind coverage, and large-list endpoints are present. +func TestNewHopScaleFixtureIncludesDenseAndLargeListShapes(t *testing.T) { + fixture := NewHopScaleFixture(32) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 1_132) + require.Len(t, fixture.Edges, 1_220) + require.Contains(t, nodeKinds, graph.StringKind("HopTemplate")) + require.Contains(t, nodeKinds, graph.StringKind("HopIDEndpoint")) + for idx := 1; idx <= 30; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("HopKind%02d", idx))) + } + require.Contains(t, edgeKinds, graph.StringKind("HopSetEdge")) +} + +// TestNewScanLookupScaleFixtureIncludesWideAndLargeListShapes verifies the +// fixture contains all scan, lookup, hydration, and relationship categories. +func TestNewScanLookupScaleFixtureIncludesWideAndLargeListShapes(t *testing.T) { + fixture := NewScanLookupScaleFixture(32) + nodeKinds, edgeKinds := fixture.Kinds() + + require.Len(t, fixture.Nodes, 3_198) + require.Len(t, fixture.Edges, 1_352) + require.Contains(t, nodeKinds, graph.StringKind("ADBase")) + require.Contains(t, nodeKinds, graph.StringKind("AZRole")) + require.Contains(t, nodeKinds, graph.StringKind("Hydrate")) + require.Contains(t, nodeKinds, graph.StringKind("Meta")) + require.Contains(t, nodeKinds, graph.StringKind("MetaDetail")) + require.Contains(t, edgeKinds, graph.StringKind("ScanPostProcessed")) + require.Contains(t, edgeKinds, graph.StringKind("Contains")) + for idx := 1; idx <= 9; idx++ { + require.Contains(t, edgeKinds, graph.StringKind(fmt.Sprintf("ScanEdge%02d", idx))) + } +} diff --git a/tools/dawgrun/pkg/commands/cypher.go b/tools/dawgrun/pkg/commands/cypher.go index 92c3755e..57f9bfe8 100644 --- a/tools/dawgrun/pkg/commands/cypher.go +++ b/tools/dawgrun/pkg/commands/cypher.go @@ -8,7 +8,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/kanmu/go-sqlfmt/sqlfmt" cypherFormat "github.com/specterops/dawgs/cypher/models/cypher/format" - pgFormat "github.com/specterops/dawgs/cypher/models/pgsql/format" + "github.com/specterops/dawgs/cypher/models/pgsql/format" "github.com/specterops/dawgs/cypher/models/pgsql/optimize" "github.com/specterops/dawgs/cypher/models/pgsql/translate" "github.com/specterops/dawgs/drivers/pg" @@ -19,10 +19,14 @@ import ( ) const ( + // queryCypherOutputFormatTable selects tabular rendering for fetched rows. queryCypherOutputFormatTable = "table" - queryCypherOutputFormatJSON = "json" + + // queryCypherOutputFormatJSON selects JSON rendering for fetched rows. + queryCypherOutputFormatJSON = "json" ) +// parseCmd describes the command that parses Cypher and prints its AST. func parseCmd() CommandDesc { return CommandDesc{ args: []string{"<...query>"}, @@ -40,6 +44,8 @@ func parseCmd() CommandDesc { } } +// translateToPsqlCmd describes the command that translates Cypher into +// formatted PostgreSQL SQL. func translateToPsqlCmd() CommandDesc { flagSet := flag.NewFlagSet("translate-psql", flag.ContinueOnError) @@ -94,12 +100,12 @@ func translateToPsqlCmd() CommandDesc { // Certain queries will materialize parameters into the output when translated, so we need to build // an OutputBuilder so we can carry forward those params. - queryBuilder := pgFormat.NewOutputBuilder() + queryBuilder := format.NewOutputBuilder().WithTargetGraph(result.GraphID) if result.Parameters != nil { queryBuilder.WithMaterializedParameters(result.Parameters) } - sqlQuery, err := pgFormat.Statement(result.Statement, queryBuilder) + sqlQuery, err := format.Statement(result.Statement, queryBuilder) if err != nil { return fmt.Errorf("could not format translated statement into a string query: %w", err) } @@ -118,6 +124,7 @@ func translateToPsqlCmd() CommandDesc { } } +// explainAsPsqlCmd defines the interactive command that translates Cypher and asks PostgreSQL to explain the resulting SQL. func explainAsPsqlCmd() CommandDesc { return CommandDesc{ args: []string{"", "<...query>"}, @@ -155,12 +162,12 @@ func explainAsPsqlCmd() CommandDesc { // Certain queries will materialize parameters into the output when translated, so we need to build // an OutputBuilder so we can carry forward those params. - queryBuilder := pgFormat.NewOutputBuilder() + queryBuilder := format.NewOutputBuilder().WithTargetGraph(result.GraphID) if result.Parameters != nil { queryBuilder.WithMaterializedParameters(result.Parameters) } - sqlQuery, err := pgFormat.Statement(result.Statement, queryBuilder) + sqlQuery, err := format.Statement(result.Statement, queryBuilder) if err != nil { return fmt.Errorf("could not format translated statement into a string query: %w", err) } @@ -202,6 +209,8 @@ func explainAsPsqlCmd() CommandDesc { } } +// defaultGraphID returns a connection's configured default graph or the +// translator fallback when no PostgreSQL default is available. func defaultGraphID(ctx *CommandContext, connName string) int32 { if connName == "" { return translate.DefaultGraphID @@ -224,6 +233,8 @@ func defaultGraphID(ctx *CommandContext, connName string) int32 { return translate.DefaultGraphID } +// queryCypherCmd describes the command that executes Cypher and renders fetched +// rows as a table or JSON. func queryCypherCmd() CommandDesc { flagSet := flag.NewFlagSet("query-cypher", flag.ContinueOnError) @@ -313,16 +324,15 @@ func optimizeCypherCmd() CommandDesc { return fmt.Errorf("could not format optimized query: %w", err) } - fmt.Fprintf(ctx.output, "Original Query:\n") + fmt.Fprint(ctx.output, "Original Query:\n") ctx.output.WriteHighlighted(originalQuery, "cypher") - fmt.Fprintf(ctx.output, "\n\n") - - fmt.Fprintf(ctx.output, "Optimized Query:\n") + fmt.Fprint(ctx.output, "\n\n") + fmt.Fprint(ctx.output, "Optimized Query:\n") ctx.output.WriteHighlighted(optimizedQuery, "cypher") - fmt.Fprintf(ctx.output, "\n\n") + fmt.Fprint(ctx.output, "\n\n") includePredicateAttachments := false - fmt.Fprintf(ctx.output, "Optimization rules considered:\n") + fmt.Fprint(ctx.output, "Optimization rules considered:\n") for _, ruleResult := range optimizationPlan.Rules { isApplied := "not applied" if ruleResult.Applied { @@ -335,20 +345,20 @@ func optimizeCypherCmd() CommandDesc { fmt.Fprintf(ctx.output, " - %s: %s\n", ruleResult.Name, isApplied) } - fmt.Fprintf(ctx.output, "\n") + fmt.Fprint(ctx.output, "\n") - fmt.Fprintf(ctx.output, "Analysis:\n") + fmt.Fprint(ctx.output, "Analysis:\n") ctx.output.WriteHighlighted(spew.Sdump(optimizationPlan.Analysis), "golang") - fmt.Fprintf(ctx.output, "\n") + fmt.Fprint(ctx.output, "\n") - fmt.Fprintf(ctx.output, "Lowering Plan:\n") + fmt.Fprint(ctx.output, "Lowering Plan:\n") ctx.output.WriteHighlighted(spew.Sdump(optimizationPlan.LoweringPlan), "golang") - fmt.Fprintf(ctx.output, "\n") + fmt.Fprint(ctx.output, "\n") if includePredicateAttachments { - fmt.Fprintf(ctx.output, "Predicate Attachments:\n") + fmt.Fprint(ctx.output, "Predicate Attachments:\n") ctx.output.WriteHighlighted(spew.Sdump(optimizationPlan.PredicateAttachments), "golang") - fmt.Fprintf(ctx.output, "\n") + fmt.Fprint(ctx.output, "\n") } return nil