fix: preserve CPU headroom during background indexing

src/daemon/application.c

@@ -2126,6 +2126,9 @@ static bool application_index_args_normalize_defaults(yyjson_mut_val *root) {
     if (name && (!yyjson_mut_is_str(name) || yyjson_mut_get_len(name) == 0)) {
         (void)yyjson_mut_obj_remove_key(root, "name");
     }
+    /* Automatic jobs carry _cbm_background, which picks the worker count and
+     * never what is indexed, so an explicit request joins them. */
+    (void)yyjson_mut_obj_remove_key(root, "_cbm_background");
     return true;
 }
 
@@ -2539,10 +2542,10 @@ static char *application_auto_index_args(cbm_daemon_application_t *application,
         return NULL;
     }
     yyjson_mut_doc_set_root(document, root);
-    char *args = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
-                         application_index_args_add_policy(application, document, root)
-                     ? yyjson_mut_write(document, 0, NULL)
-                     : NULL;
+    bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", root_path) &&
+                   application_index_args_add_policy(application, document, root) &&
+                   yyjson_mut_obj_add_bool(document, root, "_cbm_background", true);
+    char *args = encoded ? yyjson_mut_write(document, 0, NULL) : NULL;
     yyjson_mut_doc_free(document);
     return args;
 }
@@ -4366,7 +4369,8 @@ static int application_background_index(cbm_daemon_application_t *application,
     }
     yyjson_mut_doc_set_root(document, root);
     bool encoded = yyjson_mut_obj_add_strcpy(document, root, "repo_path", canonical_root) &&
-                   application_index_args_add_policy(application, document, root);
+                   application_index_args_add_policy(application, document, root) &&
+                   yyjson_mut_obj_add_bool(document, root, "_cbm_background", true);
     char *default_project = cbm_project_name_from_path(canonical_root);
     bool custom_project =
         project_name[0] && (!default_project || strcmp(default_project, project_name) != 0);

src/mcp/mcp.c

@@ -12435,7 +12435,8 @@ static char *index_run_supervised_path(cbm_mcp_server_t *srv, const char *root_p
      * worker reads no config); unset, the worker's index mode decides. */
     if (!yyjson_mut_obj_add_strcpy(doc, root, "repo_path", root_path) ||
         !cbm_mcp_index_policy_add_to_args(doc, root, &policy) ||
-        !cbm_mcp_index_doc_candidates_add_to_args(doc, root, srv ? srv->config : NULL)) {
+        !cbm_mcp_index_doc_candidates_add_to_args(doc, root, srv ? srv->config : NULL) ||
+        !yyjson_mut_obj_add_bool(doc, root, "_cbm_background", true)) {
         yyjson_mut_doc_free(doc);
         return NULL;
     }
@@ -12738,6 +12739,22 @@ static bool index_root_owner_resolve(const char *repo_path, char **owner_out, ch
     return false;
 }
 
+/* Background CPU headroom is for a refresh of a graph that already serves.
+ * A project's first index keeps every core, even when it was started
+ * automatically: the caller is waiting on a graph that does not exist yet.
+ * An empty or unreadable file is not a committed index. Both sites that
+ * honour automatic indexing — handle_index_repository's _cbm_background
+ * flag and the in-process autoindex_thread — go through this so they
+ * cannot drift. */
+static bool project_has_committed_index(const char *project) {
+    if (!project || !project[0]) {
+        return false;
+    }
+    char db_path[CBM_SZ_1K];
+    project_db_path(project, db_path, sizeof(db_path));
+    return db_path[0] && project_db_is_servable(project, db_path);
+}
+
 /* The three heap strings handle_index_repository owns from
  * cbm_mcp_get_string_arg / resolved_repo_path_from_project_arg. One release
  * point keeps the dozen early-return paths in step; free(NULL) is a no-op, so
@@ -12867,6 +12884,7 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
     char *repo_path = cbm_mcp_get_string_arg(args, "repo_path");
     char *mode_str = cbm_mcp_get_string_arg(args, "mode");
     char *name_override = cbm_mcp_get_string_arg(args, "name");
+    bool background = cbm_mcp_get_bool_arg(args, "_cbm_background");
     cbm_normalize_path_sep(repo_path);
 
     if (!repo_path) {
@@ -13039,6 +13057,10 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
         return cbm_mcp_text_result("invalid project name", true);
     }
     free(name_override);
+    /* Headroom follows the final project name, decided before artifact
+     * bootstrap can create the first local copy of this index. */
+    cbm_pipeline_set_background(p, background &&
+                                       project_has_committed_index(cbm_pipeline_project_name(p)));
     cbm_pipeline_set_persistence(p, persistence);
     cbm_pipeline_set_resource_policy(p, &resource_policy);
     index_doc_candidates_apply(p, doc_candidates);
@@ -21388,6 +21410,7 @@ static void *autoindex_thread(void *arg) {
         return NULL;
     }
     index_doc_candidates_apply(p, index_doc_candidates_choice(srv, NULL));
+    cbm_pipeline_set_background(p, project_has_committed_index(cbm_pipeline_project_name(p)));
 
     /* Block until any concurrent pipeline finishes */
     cbm_pipeline_lock();

src/pipeline/lsp_surface.c

@@ -486,10 +486,12 @@ typedef struct {
     const CBMLSPDef *all_defs;
     const int *def_starts;
     cbm_lsp_surface_row_t *rows;
+    int file_count;
+    _Atomic int next_file;
     _Atomic bool failed;
 } surface_job_t;
 
-static void surface_row_one(int i, void *arg) {
+static void surface_row_build(int i, void *arg) {
     surface_job_t *job = (surface_job_t *)arg;
     if (atomic_load_explicit(&job->failed, memory_order_relaxed)) {
         return;
@@ -529,6 +531,45 @@ static void surface_row_one(int i, void *arg) {
     }
 }
 
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+static cbm_lsp_surface_row_observer_fn g_row_observer;
+static void *g_row_observer_ctx;
+
+void cbm_lsp_surface_test_set_row_observer(cbm_lsp_surface_row_observer_fn observer, void *ctx) {
+    g_row_observer = observer;
+    g_row_observer_ctx = ctx;
+}
+#endif
+
+static void surface_row_one(int i, void *arg) {
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+    if (g_row_observer) {
+        g_row_observer(i, true, g_row_observer_ctx);
+    }
+#endif
+    surface_row_build(i, arg);
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+    if (g_row_observer) {
+        g_row_observer(i, false, g_row_observer_ctx);
+    }
+#endif
+}
+
+/* The pool also runs work on its caller. Dispatch at most one coarse job per
+ * allowed participant, each stealing files, so the total active row builders
+ * cannot exceed the pipeline budget even when the caller takes a job. */
+static void surface_rows_worker(int worker, void *arg) {
+    (void)worker;
+    surface_job_t *job = arg;
+    for (;;) {
+        int i = atomic_fetch_add_explicit(&job->next_file, 1, memory_order_relaxed);
+        if (i >= job->file_count) {
+            break;
+        }
+        surface_row_one(i, job);
+    }
+}
+
 int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *project,
                                CBMFileResult **cache, const cbm_file_info_t *files, int file_count,
                                const CBMLSPDef *all_defs, const int *def_starts,
@@ -550,10 +591,16 @@ int cbm_lsp_surface_build_rows(const cbm_pipeline_ctx_t *ctx, const char *projec
         .all_defs = all_defs,
         .def_starts = def_starts,
         .rows = rows,
+        .file_count = file_count,
     };
+    atomic_init(&job.next_file, 0);
     atomic_init(&job.failed, false);
-    cbm_parallel_for(file_count, surface_row_one, &job,
-                     (cbm_parallel_for_opts_t){.max_workers = 0, .force_pthreads = false});
+    int workers = cbm_pipeline_worker_count(ctx ? ctx->pipeline : NULL);
+    if (workers > file_count) {
+        workers = file_count;
+    }
+    cbm_parallel_for(workers, surface_rows_worker, &job,
+                     (cbm_parallel_for_opts_t){.max_workers = workers, .force_pthreads = false});
     if (atomic_load_explicit(&job.failed, memory_order_relaxed)) {
         cbm_store_free_lsp_surfaces(rows, file_count); /* untouched rows are all NULL */
         return -1;

src/pipeline/lsp_surface.h

@@ -68,4 +68,11 @@ int cbm_lsp_surface_python_namespace_from_json(CBMArena *arena, const char *json
  */
 int cbm_lsp_surface_python_change(const char *old_json, const char *new_json);
 
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+/* Bracket each file callback, including skipped/error rows. Install only with
+ * no surface build running; clear after the observed build returns. */
+typedef void (*cbm_lsp_surface_row_observer_fn)(int index, bool entering, void *ctx);
+void cbm_lsp_surface_test_set_row_observer(cbm_lsp_surface_row_observer_fn observer, void *ctx);
+#endif
+
 #endif /* CBM_PIPELINE_LSP_SURFACE_H */

src/pipeline/pipeline.c

@@ -81,6 +81,18 @@ static inline void *intptr_to_ptr(intptr_t v) {
  * Atomic spinlock: 0 = free, 1 = locked. */
 static atomic_int g_pipeline_busy = 0;
 
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+static atomic_int g_last_worker_count = 0;
+
+void cbm_pipeline_worker_count_test_reset(void) {
+    atomic_store(&g_last_worker_count, 0);
+}
+
+int cbm_pipeline_worker_count_test_last(void) {
+    return atomic_load(&g_last_worker_count);
+}
+#endif
+
 #if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
 static atomic_bool g_persist_test_fail_after_stage_dump = false;
 static atomic_bool g_persist_test_cancel_after_predump = false;
@@ -210,6 +222,7 @@ struct cbm_pipeline {
     bool persistence; /* write .codebase-memory/graph.db.zst after indexing */
     cbm_index_resource_policy_t resource_policy;
     cbm_index_resource_violation_t resource_violation;
+    bool background; /* reserve CPU headroom when no user is waiting */
 
     /* Snapshot of the artifact export failure of THIS run (set only by
      * export_after_publish failure, zeroed at run start, cleared on success).
@@ -965,6 +978,12 @@ const char *cbm_pipeline_export_error(const cbm_pipeline_t *p) {
     return p ? p->export_error : "";
 }
 
+void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background) {
+    if (p) {
+        p->background = background;
+    }
+}
+
 bool cbm_pipeline_set_project_name(cbm_pipeline_t *p, const char *name) {
     if (!p || !name || !name[0]) {
         return false;
@@ -1829,17 +1848,42 @@ void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges)
     }
 }
 
+/* Two workers is the smallest width that takes the parallel route. */
+enum { PL_PARALLEL_MIN_WORKERS = 2 };
+
+/* Background refreshes leave CPU headroom, but never drop below two workers
+ * while the foreground width is two or more. WHY: one worker sends the indexer
+ * down the sequential route, which still builds a different graph than the
+ * parallel route (fewer Route and HANDLES edges, churned CALLS). Until the two
+ * routes agree, an automatic refresh must not change the graph only because
+ * the host has two or fewer performance cores. */
+int cbm_pipeline_background_worker_width(int foreground, int headroom) {
+    if (foreground >= PL_PARALLEL_MIN_WORKERS && headroom < PL_PARALLEL_MIN_WORKERS) {
+        return PL_PARALLEL_MIN_WORKERS;
+    }
+    return headroom;
+}
+
 /* Effective worker count. The crash supervisor re-runs its worker single-
  * threaded (CBM_INDEX_SINGLE_THREAD=1) so a per-file marker can pin the EXACT
  * crasher; a parallel re-run would race the marker. Honour that override
  * everywhere the worker count drives the parallel/sequential decision, so the
  * whole extraction phase collapses to the deterministic sequential path. */
-static int effective_worker_count(bool initial) {
+int cbm_pipeline_worker_count(const cbm_pipeline_t *p) {
     const char *st = getenv("CBM_INDEX_SINGLE_THREAD");
+    int workers;
     if (st && st[0] == '1') {
-        return 1;
+        workers = 1;
+    } else if (p && p->background) {
+        workers = cbm_pipeline_background_worker_width(cbm_default_worker_count(true),
+                                                       cbm_default_worker_count(false));
+    } else {
+        workers = cbm_default_worker_count(true);
     }
-    return cbm_default_worker_count(initial);
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+    atomic_store(&g_last_worker_count, workers);
+#endif
+    return workers;
 }
 
 /* Resolve the DB path for this pipeline. Caller must free(). */
@@ -4186,7 +4230,7 @@ static int run_githistory(cbm_pipeline_t *p, cbm_pipeline_ctx_t *ctx) {
     gh_compute_arg_t gh_arg = {.repo_path = ctx->repo_path, .result = &gh_result};
 
     if (p->mode != CBM_MODE_FAST) {
-        if (effective_worker_count(true) > SKIP_ONE) {
+        if (cbm_pipeline_worker_count(p) > SKIP_ONE) {
             if (cbm_thread_create(&gh_thread, 0, gh_compute_thread_fn, &gh_arg) == 0) {
                 gh_threaded = true;
             }
@@ -4286,7 +4330,7 @@ static int run_structure_and_extraction(cbm_pipeline_t *p, cbm_pipeline_ctx_t *c
         return CBM_NOT_FOUND;
     }
 
-    int worker_count = p->frozen ? 1 : effective_worker_count(true);
+    int worker_count = p->frozen ? 1 : cbm_pipeline_worker_count(p);
     if (p->frozen)
         frozen_observe(p, 0, 1);
     CBM_PROF_START(t_extract_total);
@@ -4417,7 +4461,7 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) {
          : mode_promoted
              ? cbm_pipeline_build_fresh_semantic_manifest(p, p->project_name, &baseline_manifest,
                                                           &baseline_count)
-             : cbm_pipeline_build_semantic_manifest(p->project_name, p->repo_path, files,
+             : cbm_pipeline_build_semantic_manifest(p, p->project_name, p->repo_path, files,
                                                     file_count, p->excluded_dirs, p->excluded_count,
                                                     &p->git_ctx, p->userconfig, &baseline_manifest,
                                                     &baseline_count);

src/pipeline/pipeline.h

@@ -71,6 +71,23 @@ bool cbm_pipeline_doc_candidates_enabled(const cbm_pipeline_t *p);
 /* Copy the exact discovery violation from the most recent run. */
 void cbm_pipeline_get_resource_violation(const cbm_pipeline_t *p,
                                          cbm_index_resource_violation_t *violation);
+/* Mark work that runs without a waiting user. Background pipelines reserve
+ * CPU headroom; foreground pipelines retain the initial-index all-core policy. */
+void cbm_pipeline_set_background(cbm_pipeline_t *p, bool background);
+
+/* Resolve the worker policy for this pipeline, including environment and
+ * crash-recovery overrides. */
+int cbm_pipeline_worker_count(const cbm_pipeline_t *p);
+
+/* Background width from the foreground width and the headroom width: the
+ * headroom, floored at two whenever the foreground width is two or more so an
+ * automatic refresh keeps the parallel route. */
+int cbm_pipeline_background_worker_width(int foreground, int headroom);
+
+#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API
+void cbm_pipeline_worker_count_test_reset(void);
+int cbm_pipeline_worker_count_test_last(void);
+#endif
 
 /* Snapshot of the artifact export failure of the last cbm_pipeline_run, or ""
  * when the run succeeded / did not reach post-publish export. Used to