Skip to content

Commit 97e890e

Browse files
committed
fix: retain parallel analysis for lighter large projects
1 parent a608f34 commit 97e890e

2 files changed

Lines changed: 10 additions & 3 deletions

File tree

‎docs/dense-compile-performance.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,9 @@ analysis results in batches of 64. Smaller requests retain a single batch. Both
4747
use up to four workers; local resolution views avoid copying CTE definitions
4848
that are already available through the source catalogue.
4949
These limits change scheduling, not analysis or fallback semantics.
50-
Projects with at least 10,000 model projections use one native analysis worker to
51-
leave memory headroom for the isolated custom-rule host after analysis.
50+
Projects with at least 10,000 model projections and 48MiB of prepared SQL use one
51+
native analysis worker to leave memory headroom for the isolated custom-rule host
52+
after analysis. Smaller SQL payloads retain parallel analysis.
5253
The 10k cold ceiling includes headroom for this memory-bounded scheduling; warm
5354
cache hits and small edit batches keep their existing scheduling.
5455

‎src/sqlbuild/compiler/compile/_helpers/analysis/compact.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@
171171

172172
_DEBUG_LOGGER: logging.Logger = logging.getLogger("sqlbuild.compile")
173173
_LARGE_COMPACT_PROJECT_MODELS: int = 10_000
174+
_LARGE_COMPACT_SQL_BYTES: int = 48 * 1024 * 1024
174175
_NATIVE_LEGACY_FALLBACK: str = "native project type recovery requires legacy fallback"
175176
_REF_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.REF)
176177
_SEED_PATTERN: re.Pattern[str] = quoted_reference_call_pattern(SqlReferenceKind.SEED)
@@ -626,7 +627,12 @@ def _prepare_compact_analysis_batch(
626627
def _run_compact_analysis_batch(*, preparation: CompactBatchPreparation) -> object:
627628
"""Bound resident analysis heaps before the isolated custom-rule host starts."""
628629

629-
workers: int = 1 if len(preparation.projections) >= _LARGE_COMPACT_PROJECT_MODELS else 4
630+
large_project: bool = (
631+
len(preparation.projections) >= _LARGE_COMPACT_PROJECT_MODELS
632+
and sum(len(sql.encode("utf-8")) for sql in preparation.cleaned_sql)
633+
>= _LARGE_COMPACT_SQL_BYTES
634+
)
635+
workers: int = 1 if large_project else 4
630636
return orjson.loads(
631637
cast(NativeQueryAnalysisModule, _native).analyze_project_queries_compact_json(
632638
orjson.dumps(

0 commit comments

Comments
 (0)