feat!: remove virtual data environments and validate models at compile time

.github/workflows/verify.yml

@@ -76,7 +76,7 @@ jobs:
                 performance=true
                 rules_performance=true
                 ;;
-              src/sqlbuild/cli/commands/_helpers/compile/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|tests/e2e/src/sqlbuild/cli/commands/main/compile/*)
+              src/sqlbuild/cli/commands/_helpers/compile/*|src/sqlbuild/executor/testing/*|scripts/cold_compile_performance/*|scripts/compile_performance_ratio/*|scripts/compare_compile_performance.py|tests/e2e/src/sqlbuild/cli/commands/main/compile/*)
                 broad=true
                 performance=true
                 ;;
@@ -90,7 +90,7 @@ jobs:
                 duckdb=true
                 format_performance=true
                 ;;
-              tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/virtual/state/classes/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*)
+              tests/e2e/src/sqlbuild/cli/commands/main/postgres/*|tests/integration/src/sqlbuild/adapters/postgres/*|tests/integration/src/sqlbuild/runtime/execution_history/*)
                 postgres=true
                 ;;
               tests/e2e/*)
@@ -195,14 +195,12 @@ jobs:
             target: test-e2e-duckdb-build-core
           - name: build incremental
             target: test-e2e-duckdb-build-incremental
-          - name: build virtual
-            target: test-e2e-duckdb-build-virtual
           - name: CLI data
             target: test-e2e-duckdb-cli-data
           - name: CLI commands
             target: test-e2e-duckdb-cli
-          - name: virtual environments
-            target: test-e2e-duckdb-virtual
+          - name: CLI inspection
+            target: test-e2e-duckdb-inspection
           - name: integrations
             target: test-e2e-duckdb-integrations
     env:
@@ -390,6 +388,62 @@ jobs:
             make test-e2e-dense-compile-performance \
               SQLBUILD_BENCHMARK_MODELS="${BENCHMARK_MODELS}"
 
+      - name: Run diagnostic-heavy compile guards
+        if: matrix.models == 1000
+        run: make test-e2e-diagnostic-compile-performance
+
+  compile_performance_ratio:
+    name: Same-runner compile ratio (${{ matrix.kind }} 3000 models)
+    needs: [changes, lockfile]
+    if: needs.changes.outputs.performance == 'true'
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - kind: dense
+            max_ratio: "1.10"
+          - kind: fresh
+            max_ratio: "1.30"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check out the comparison base
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.base.sha || 'main' }}
+          path: base
+      - uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.12.3"
+          enable-cache: true
+          cache-dependency-glob: |
+            uv.lock
+            Cargo.lock
+            base/uv.lock
+            base/Cargo.lock
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      - name: Install Python
+        run: uv python install 3.12
+      - name: Install head dependencies
+        run: uv sync --locked --all-groups --all-extras
+      - name: Install base dependencies
+        working-directory: base
+        run: uv sync --locked --all-extras
+      - name: Compare base and head cold compiles
+        env:
+          KIND: ${{ matrix.kind }}
+          MAX_RATIO: ${{ matrix.max_ratio }}
+        run: |
+          uv run python -m scripts.compare_compile_performance \
+            --kind "${KIND}" \
+            --max-ratio "${MAX_RATIO}" \
+            --models 3000 \
+            --base-python base/.venv/bin/python \
+            --head-python .venv/bin/python
+
   cache_compile_performance:
     name: Fresh-process compile cache (${{ matrix.models }} models)
     needs: [changes, lockfile]
@@ -551,6 +605,7 @@ jobs:
         performance,
         cold_compile_performance,
         dense_compile_performance,
+        compile_performance_ratio,
         cache_compile_performance,
         rules_performance,
         postgres,
@@ -568,6 +623,7 @@ jobs:
           PERFORMANCE_RESULT: ${{ needs.performance.result }}
           COLD_COMPILE_PERFORMANCE_RESULT: ${{ needs.cold_compile_performance.result }}
           DENSE_COMPILE_PERFORMANCE_RESULT: ${{ needs.dense_compile_performance.result }}
+          COMPILE_PERFORMANCE_RATIO_RESULT: ${{ needs.compile_performance_ratio.result }}
           CACHE_COMPILE_PERFORMANCE_RESULT: ${{ needs.cache_compile_performance.result }}
           RULES_PERFORMANCE_RESULT: ${{ needs.rules_performance.result }}
           POSTGRES_RESULT: ${{ needs.postgres.result }}
@@ -587,6 +643,7 @@ jobs:
             "${PERFORMANCE_RESULT}" \
             "${COLD_COMPILE_PERFORMANCE_RESULT}" \
             "${DENSE_COMPILE_PERFORMANCE_RESULT}" \
+            "${COMPILE_PERFORMANCE_RATIO_RESULT}" \
             "${CACHE_COMPILE_PERFORMANCE_RESULT}" \
             "${RULES_PERFORMANCE_RESULT}" \
             "${POSTGRES_RESULT}"; do

AGENTS.md

@@ -18,6 +18,56 @@
   explicitly requests it; rely on CI for those suites.
 - Before pushing, run the exact static CI target with all optional dependencies available: `uv sync --all-extras` followed by `make check-ci`.
 
+## Real-Project Verification and Performance Guards
+
+- Synthetic fixtures and clean benchmark projects do not exercise every path. Before releasing
+  changes to compilation, SQL analysis, diagnostics, formatting, Rules, or planning, run the built
+  branch against at least one large real project available locally outside this repository, and
+  compare it with the latest release. Compare diagnostic counts by code, cold compile wall time and
+  peak memory, and, for formatter changes, that formatting preserves compiled dependencies, lineage,
+  and query semantics.
+- Treat every new diagnostic on a real project as a false positive until warehouse evidence proves
+  otherwise, for example a model that has built successfully since its SQL last changed.
+- A material slowdown or new false positive is a release blocker, not a follow-up.
+- Real-project names, SQL, identifiers, file paths, and results never enter this repository, commits,
+  pull requests, or CI output. Record only neutral aggregate conclusions, and rebuild every
+  reproduction synthetically.
+- Performance guards must cover worst-case paths, not only clean projects: very large models,
+  diagnostic-heavy compiles with many errors and warnings, wide queries, and deep macro expansion.
+  Any per-diagnostic, per-reference, or per-token work must be bounded by a test with a strict time
+  limit. A path that is only slow when something is wrong is still a regression.
+
+## Performance Work
+
+- Never compare timings across different machines, runners, or sessions. Every performance claim
+  must come from a same-machine comparison: base and candidate built from the same toolchain, run
+  alternately at least three times each, reporting medians of both wall time and CPU time
+  (user+sys) plus the per-phase `compile_timings`.
+- On shared local machines, confirm no concurrent builds or benchmarks are running and record the
+  load average. If load exceeds the core count, treat wall times as unreliable and report CPU time.
+- Hosted CI runners vary by up to ~60%. Do not diagnose regressions from absolute CI budgets or
+  from runs on different runners; use the same-runner ratio guard or a same-job paired comparison.
+- Before attributing a regression to a cause, prove it with an on/off experiment (a temporary
+  kill switch or a revert) on the same machine. Profiles identify suspects; experiments convict.
+  Remove experiment switches before committing.
+- Measure the modes users run: cold without cache, cold writing the cache, and warm. A change that
+  affects analysis, validation, or caching must report all three.
+- Performance fixes must not reduce coverage. Show that diagnostics are unchanged or a strict,
+  validated superset on the dense benchmark and a real-project run.
+- Do not raise performance budgets or tolerances to make a regression pass.
+
+## Native SQL (Polyglot) Changes
+
+- Treat every Polyglot change as a compiler change: before adopting a new Polyglot version, run the
+  same-machine dense and fresh 3,000-model comparisons against the current version and report the
+  ratio and per-phase timings.
+- Polyglot diagnostics adopted by SQLBuild are enforceable errors or dropped. Never surface a new
+  native finding as a non-blocking warning; map it to a SQLBuild error code with a mechanical remedy.
+- Coverage conditions must not depend on incidental query shape (for example, only checking
+  statements with a set operation). Checks apply uniformly or the gap is documented and ticketed.
+- Run the Polyglot feature-gate checks (`make test-rust-feature-gates`) before pushing Polyglot
+  changes; cross-module helpers must respect feature gates.
+
 ## Public Repository Hygiene
 
 - Treat every tracked file, generated artifact, fixture, benchmark, filename, commit, branch, pull
@@ -68,6 +118,9 @@
   requests local verification or CI is unavailable.
 - State the expected verification scope in subagent prompts and explicitly prohibit unnecessary full-suite runs.
 - Do not delay committing and pushing a focused fix solely to repeat checks already completed successfully by another agent or CI.
+- Subagents reporting performance results must include wall time, CPU time, run count, machine
+  load, and whether base and candidate ran on the same machine. Results lacking these are
+  hypotheses, not evidence.
 
 ## State and Source of Truth
 

Cargo.toml

@@ -15,7 +15,7 @@ fensu-policy = "=0.22.0"
 globset = "0.4"
 proc-macro2 = { version = "1", features = ["span-locations"] }
 pyo3 = { version = "0.26", features = ["abi3-py312"] }
-polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "f3774102f518f716cd648e4fdebb9d8558a0b63a", default-features = false, features = [
+polyglot-sql = { git = "https://github.com/chio-labs/polyglot.git", rev = "25ad011c49f2c7cd48f4427abb8fc0a6af2e4120", default-features = false, features = [
     "ast-tools",
     "builder",
     "dialect-bigquery",

Makefile

@@ -2,8 +2,8 @@ SHELL := /bin/bash
 
 .PHONY: verify verify-quick verify-pg coverage cli-preview rust-check \
 	check-e2e-shards test-e2e-duckdb test-e2e-duckdb-build-core \
-	test-e2e-duckdb-build-incremental test-e2e-duckdb-build-virtual \
-	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-virtual \
+	test-e2e-duckdb-build-incremental \
+	test-e2e-duckdb-cli-data test-e2e-duckdb-cli test-e2e-duckdb-inspection \
 	test-e2e-duckdb-integrations test-e2e-performance \
 	test-e2e-cold-compile-performance test-e2e-cache-compile-performance \
 	test-e2e-dense-compile-performance test-e2e-varied-cache-performance
@@ -102,24 +102,17 @@ E2E_DUCKDB_BUILD_INCREMENTAL_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_seed_watermark_build.py \
 	tests/e2e/src/sqlbuild/cli/commands/main/build/test_snapshot_build.py
 
-E2E_DUCKDB_BUILD_VIRTUAL_PATHS := \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_build_state.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_incremental_build.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_microbatch_lifecycle.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_mode_guard.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_promote.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_python_build.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_rollback.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_seed_build.py \
-	tests/e2e/src/sqlbuild/cli/commands/main/build/test_virtual_source_freshness_build.py
 
 E2E_DUCKDB_CLI_DATA_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/load \
 	tests/e2e/src/sqlbuild/cli/commands/main/providers \
-	tests/e2e/src/sqlbuild/cli/commands/main/scenario
+	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_scenario.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_capture.py \
+	tests/e2e/src/sqlbuild/cli/commands/main/scenario/test_namespace.py
 
 E2E_DUCKDB_CLI_PATHS := \
 	tests/e2e/scripts/cli_preview \
+	tests/e2e/scripts/compile_performance_ratio \
 	tests/e2e/src/sqlbuild/cli/commands/main/adapters \
 	tests/e2e/src/sqlbuild/cli/commands/main/audit \
 	tests/e2e/src/sqlbuild/cli/commands/main/bigquery \
@@ -144,23 +137,20 @@ E2E_DUCKDB_CLI_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/sqlserver \
 	tests/e2e/src/sqlbuild/cli/commands/main/test
 
-E2E_DUCKDB_VIRTUAL_PATHS := \
+E2E_DUCKDB_INSPECTION_PATHS := \
 	tests/e2e/src/sqlbuild/cli/commands/main/clone \
 	tests/e2e/src/sqlbuild/cli/commands/main/diff \
 	tests/e2e/src/sqlbuild/cli/commands/main/janitor \
-	tests/e2e/src/sqlbuild/cli/commands/main/plan \
-	tests/e2e/src/sqlbuild/cli/commands/main/reconcile \
-	tests/e2e/src/sqlbuild/cli/commands/main/state
+	tests/e2e/src/sqlbuild/cli/commands/main/plan
 
 E2E_DUCKDB_INTEGRATIONS_PATHS := tests/e2e/src/sqlbuild/integrations
 
 E2E_DUCKDB_PATHS := \
 	$(E2E_DUCKDB_BUILD_CORE_PATHS) \
 	$(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS) \
-	$(E2E_DUCKDB_BUILD_VIRTUAL_PATHS) \
 	$(E2E_DUCKDB_CLI_DATA_PATHS) \
 	$(E2E_DUCKDB_CLI_PATHS) \
-	$(E2E_DUCKDB_VIRTUAL_PATHS) \
+	$(E2E_DUCKDB_INSPECTION_PATHS) \
 	$(E2E_DUCKDB_INTEGRATIONS_PATHS)
 
 define run_e2e_duckdb
@@ -199,17 +189,15 @@ test-e2e-duckdb-build-core:
 test-e2e-duckdb-build-incremental:
 	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS))
 
-test-e2e-duckdb-build-virtual:
-	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_BUILD_VIRTUAL_PATHS))
 
 test-e2e-duckdb-cli-data:
 	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_CLI_DATA_PATHS))
 
 test-e2e-duckdb-cli:
 	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_CLI_PATHS))
 
-test-e2e-duckdb-virtual:
-	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_VIRTUAL_PATHS))
+test-e2e-duckdb-inspection:
+	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_INSPECTION_PATHS))
 
 test-e2e-duckdb-integrations:
 	$(call run_e2e_duckdb,$@,$(E2E_DUCKDB_INTEGRATIONS_PATHS))
@@ -218,10 +206,9 @@ check-e2e-shards:
 	bash scripts/check_e2e_shards.sh \
 		--shard build-core $(E2E_DUCKDB_BUILD_CORE_PATHS) \
 		--shard build-incremental $(E2E_DUCKDB_BUILD_INCREMENTAL_PATHS) \
-		--shard build-virtual $(E2E_DUCKDB_BUILD_VIRTUAL_PATHS) \
 		--shard cli-data $(E2E_DUCKDB_CLI_DATA_PATHS) \
 		--shard cli $(E2E_DUCKDB_CLI_PATHS) \
-		--shard virtual $(E2E_DUCKDB_VIRTUAL_PATHS) \
+		--shard inspection $(E2E_DUCKDB_INSPECTION_PATHS) \
 		--shard integrations $(E2E_DUCKDB_INTEGRATIONS_PATHS)
 
 
@@ -244,6 +231,13 @@ test-e2e-cold-compile-performance:
 		-m cold_compile_performance -k "models_$(SQLBUILD_BENCHMARK_MODELS)" \
 		-vv --log-cli-level=INFO --color=yes
 
+.PHONY: test-e2e-diagnostic-compile-performance
+test-e2e-diagnostic-compile-performance:
+	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
+		tests/e2e/src/sqlbuild/cli/commands/main/compile/test_diagnostic_compile_performance.py \
+		-n auto --dist loadfile -m cold_compile_performance \
+		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
+
 test-e2e-dense-compile-performance:
 	test -n "$(SQLBUILD_BENCHMARK_MODELS)"
 	env PYTHONUNBUFFERED=1 SQLBUILD_CONCURRENCY=4 uv run pytest \
@@ -266,17 +260,6 @@ test-e2e-varied-cache-performance:
 		-n auto --dist loadfile -m cache_compile_performance \
 		-vv -rP --log-level=INFO --log-cli-level=INFO --color=yes
 
-test-virtual:
-	@mkdir -p /tmp/opencode
-	@log="/tmp/opencode/test-virtual-$$(date +%Y%m%d-%H%M%S).log"; \
-	echo "Logging to $$log"; \
-	env PYTHONUNBUFFERED=1 TESTCONTAINERS_RYUK_DISABLED=true SQLBUILD_CONCURRENCY=$(SQLBUILD_CONCURRENCY) uv run pytest \
-		$(VIRTUAL_TEST_ROOTS) \
-		-k "$(VIRTUAL_TEST_KEYWORD)" \
-		-m "not dbt and not performance" -vv --color=yes -n auto --dist loadfile 2>&1 | tee "$$log"; \
-	status=$${PIPESTATUS[0]}; \
-	echo "TEST_VIRTUAL_EXIT=$$status (log: $$log)" | tee -a "$$log"; \
-	exit $$status
 
 
 skills:
@@ -290,8 +273,6 @@ export DBT_EXECUTABLE
 
 SQLBUILD_CONCURRENCY ?= 8
 
-VIRTUAL_TEST_ROOTS ?= tests/e2e/src/sqlbuild/cli/commands/main tests/e2e/src/sqlbuild/integrations/dagster
-VIRTUAL_TEST_KEYWORD ?= virtual or reconcile or diff or janitor or snapshot or dagster
 
 
 test-dbt: