fix(watcher): pace re-indexes by the cost of the last successful run

src/watcher/watcher.c

@@ -154,6 +154,15 @@ struct cbm_watcher {
  * daemon log names the stuck project instead of only repeating the warning. */
 #define INDEX_FAIL_SUSTAINED 10
 
+/* Success cooldown (#2835): after a successful re-index the next poll waits
+ * INDEX_COOLDOWN_FACTOR x the index's own duration, counted from when the
+ * index finished, so short rebuilds spend at most ~1/(K+1) of wall time
+ * indexing one project. Sub-second indexes stay at the plain cadence. The
+ * cooldown is capped at INDEX_FAIL_CEILING_MS (5 min) so that, under
+ * continuous writes, a project never falls more than one rebuild plus five
+ * minutes behind. */
+#define INDEX_COOLDOWN_FACTOR 3
+
 /* Stale-root pruning (#286): a watched project whose root directory stays
  * missing is pruned — its cached DB is deleted and the watch entry removed.
  * Deletion is destructive (the DB can hold user-authored data such as the
@@ -175,7 +184,17 @@ struct cbm_watcher {
 
 /* ── Time helper ────────────────────────────────────────────────── */
 
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+/* Test seam: nonzero freezes the watcher's scheduling clock at this value. */
+static int64_t test_clock_ns = 0;
+#endif
+
 static int64_t now_ns(void) {
+#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
+    if (test_clock_ns != 0) {
+        return test_clock_ns;
+    }
+#endif
     struct timespec ts;
     cbm_clock_gettime(CLOCK_MONOTONIC, &ts);
     return ((int64_t)ts.tv_sec * NS_PER_SEC) + ts.tv_nsec;
@@ -204,6 +223,20 @@ int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) {
     return (int)(delay < interval_ms ? interval_ms : delay);
 }
 
+int cbm_watcher_success_delay_ms(int interval_ms, int64_t index_elapsed_ms) {
+    if (interval_ms < 0) {
+        interval_ms = 0;
+    }
+    if (index_elapsed_ms <= 0) {
+        return interval_ms;
+    }
+    /* Divide instead of multiply so a huge elapsed cannot overflow. */
+    int64_t cooldown = index_elapsed_ms > INDEX_FAIL_CEILING_MS / INDEX_COOLDOWN_FACTOR
+                           ? INDEX_FAIL_CEILING_MS
+                           : index_elapsed_ms * INDEX_COOLDOWN_FACTOR;
+    return (int)(cooldown > interval_ms ? cooldown : interval_ms);
+}
+
 int cbm_watcher_poll_interval_ms(int file_count) {
     int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * CBM_MSEC_PER_SEC);
     if (ms > POLL_MAX_MS) {
@@ -1693,12 +1726,22 @@ static void poll_project(const char *key, void *val, void *ud) {
     }
     cbm_log_info("watcher.changed", "project", s->project_name, "strategy",
                  s->is_git ? "git" : "tree");
+    int delay_ms = -1;            /* < 0: use the failure-backoff cadence below */
+    int64_t anchor_ns = ctx->now; /* success re-anchors to when the index finished */
     if (ctx->w->index_fn) {
+        int64_t index_start = now_ns();
         int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data);
         if (rc == 0) {
             ctx->reindexed++;
             s->index_failures = 0;
             commit_baselines(ctx->w, s);
+            anchor_ns = now_ns();
+            delay_ms =
+                cbm_watcher_success_delay_ms(s->interval_ms, (anchor_ns - index_start) / US_PER_MS);
+            if (delay_ms > s->interval_ms) {
+                cbm_log_info("watcher.cooldown", "project", s->project_name, "delay_ms",
+                             itoa_buf(delay_ms));
+            }
         } else if (rc > 0) {
             /* Busy-skip: baseline stays uncommitted, next poll retries. */
             cbm_log_info("watcher.index.retry", "project", s->project_name);
@@ -1721,10 +1764,12 @@ static void poll_project(const char *key, void *val, void *ud) {
         }
     }
 
-    /* Failures back off; success and busy-skip keep the adaptive cadence. */
-    s->next_poll_ns =
-        ctx->now +
-        ((int64_t)cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures) * US_PER_MS);
+    /* Success pays back the index's cost; failures back off; busy-skip keeps
+     * the adaptive cadence. */
+    if (delay_ms < 0) {
+        delay_ms = cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures);
+    }
+    s->next_poll_ns = anchor_ns + ((int64_t)delay_ms * US_PER_MS);
 }
 
 /* Callback to snapshot project state pointers into an array. */
@@ -1876,4 +1921,19 @@ int cbm_watcher_test_pending_free_count(cbm_watcher_t *w) {
     cbm_mutex_unlock(&w->projects_lock);
     return count;
 }
+
+void cbm_watcher_test_set_clock_ns(int64_t ns) {
+    test_clock_ns = ns;
+}
+
+int64_t cbm_watcher_test_next_poll_ns(cbm_watcher_t *w, const char *project) {
+    if (!w || !project) {
+        return -1;
+    }
+    cbm_mutex_lock(&w->projects_lock);
+    project_state_t *s = cbm_ht_get(w->projects, project);
+    int64_t next = s ? s->next_poll_ns : -1;
+    cbm_mutex_unlock(&w->projects_lock);
+    return next;
+}
 #endif

src/watcher/watcher.h

@@ -145,6 +145,15 @@ int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name);
  * otherwise unobservable — cbm_watcher_free frees the same list, so a poll
  * that stopped draining would neither crash nor leak. Test builds only. */
 int cbm_watcher_test_pending_free_count(cbm_watcher_t *w);
+
+/* Test seam: freeze the watcher's monotonic scheduling clock at `ns`
+ * (nonzero), or restore the real clock (0). Process-global; single-threaded
+ * tests only. Lets a test advance time inside index_fn without sleeping. */
+void cbm_watcher_test_set_clock_ns(int64_t ns);
+
+/* Test seam: a project's scheduled next poll (monotonic ns), read under
+ * projects_lock; -1 when it is not watched. */
+int64_t cbm_watcher_test_next_poll_ns(cbm_watcher_t *w, const char *project);
 #endif
 
 /* Per-project watch status, for index_status (#2167). */
@@ -195,6 +204,15 @@ int cbm_watcher_poll_interval_ms(int file_count);
  * Negative inputs are treated as zero. */
 int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures);
 
+/* Return the delay (ms) before the next poll after a SUCCESSFUL re-index that
+ * took `index_elapsed_ms`: max(interval_ms, INDEX_COOLDOWN_FACTOR (3) x
+ * index_elapsed_ms), with the cooldown term capped at INDEX_FAIL_CEILING_MS
+ * (5 min). The caller counts it from when the index finished, so short
+ * rebuilds spend at most ~1/4 of wall time indexing a project; sub-second
+ * indexes keep `interval_ms`.
+ * Never below `interval_ms`; overflow-safe; negative inputs act as zero. */
+int cbm_watcher_success_delay_ms(int interval_ms, int64_t index_elapsed_ms);
+
 /* Classify a stat() errno observed on a watched project root: returns true
  * only for values that mean the root itself is gone (ENOENT, ENOTDIR) and
  * may count toward stale-root pruning (#286). Any other failure (EACCES,

tests/test_watcher.c

@@ -2260,6 +2260,127 @@ TEST(watcher_sustained_failure_logs_once_issue2015) {
     PASS();
 }
 
+/* ── Success cooldown (#2835) ─────────────────────────────────────── */
+
+TEST(success_delay_fast_index_keeps_interval) {
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 0), 5000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 1), 5000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 1666), 5000); /* 3x < interval */
+    ASSERT_EQ(cbm_watcher_success_delay_ms(60000, 20000), 60000);
+    PASS();
+}
+
+TEST(success_delay_slow_index_scales_by_factor) {
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 1667), 5001);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 10000), 30000);
+    /* 90 s rebuild → next poll 4.5 min after it finishes. */
+    ASSERT_EQ(cbm_watcher_success_delay_ms(60000, 90000), 270000);
+    PASS();
+}
+
+TEST(success_delay_clamped_and_degenerate) {
+    /* Capped at 5 min (INDEX_FAIL_CEILING_MS) from 100 s of indexing up. */
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 100000), 300000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 100001), 300000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(60000, 300000), 300000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, INT64_MAX), 300000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, -1), 5000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(5000, INT64_MIN), 5000);
+    ASSERT_EQ(cbm_watcher_success_delay_ms(-1, 0), 0);
+    /* Never below the caller's own cadence, even above the ceiling. */
+    ASSERT_EQ(cbm_watcher_success_delay_ms(INT_MAX, INT64_MAX), INT_MAX);
+    PASS();
+}
+
+TEST(success_delay_is_monotonic) {
+    int previous = cbm_watcher_success_delay_ms(5000, 0);
+    for (int64_t e = 0; e <= 2000000; e += 997) {
+        int delay = cbm_watcher_success_delay_ms(5000, e);
+        ASSERT_TRUE(delay >= previous);
+        ASSERT_TRUE(delay >= 5000 && delay <= 300000);
+        previous = delay;
+    }
+    PASS();
+}
+
+/* Wire-up: the cooldown must be measured around the real index_fn call and
+ * counted from when the index finished. The watcher's clock is frozen via a
+ * test seam and the callback advances it, so an index "takes" a chosen time
+ * with no sleep and the scheduled poll time can be asserted exactly. */
+#define COOL_NS_PER_MS 1000000LL
+static int64_t cool_clock_ns = 0;
+static int64_t cool_index_ms = 0;
+static int cool_index_calls = 0;
+static int cool_index_callback(const char *name, const char *path, void *ud) {
+    (void)name;
+    (void)path;
+    (void)ud;
+    cool_index_calls++;
+    cool_clock_ns += cool_index_ms * COOL_NS_PER_MS;
+    cbm_watcher_test_set_clock_ns(cool_clock_ns);
+    return 0;
+}
+
+/* Re-index once (touch + poll until the callback runs; a transient git probe
+ * failure makes a poll a no-op, see the sustained-failure test) and return
+ * the scheduled next poll relative to when the index finished, in ms. */
+static int64_t cool_reindex_delay_ms(cbm_watcher_t *w, const char *file, const char *line,
+                                     int64_t index_ms) {
+    cool_index_ms = index_ms;
+    th_append_file(file, line);
+    int target = cool_index_calls + 1;
+    while (cool_index_calls < target) {
+        cbm_watcher_touch(w, "cool-repo");
+        cbm_watcher_poll_once(w);
+    }
+    return (cbm_watcher_test_next_poll_ns(w, "cool-repo") - cool_clock_ns) / COOL_NS_PER_MS;
+}
+
+TEST(watcher_success_cooldown_measures_index_issue2835) {
+    char tmpdir[256];
+    snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_cool_XXXXXX");
+    if (!cbm_mkdtemp(tmpdir))
+        FAIL("cbm_mkdtemp failed");
+    if (wt_git(tmpdir, "init -q") != 0) {
+        th_rmtree(tmpdir);
+        FAIL("git init failed");
+    }
+    char p[300];
+    th_write_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "hello\n");
+    wt_git(tmpdir, "add file.txt");
+    wt_git(tmpdir, "commit -q -m init");
+
+    cool_clock_ns = 1000LL * 1000 * COOL_NS_PER_MS; /* any nonzero start */
+    cbm_watcher_test_set_clock_ns(cool_clock_ns);
+    cool_index_calls = 0;
+
+    cbm_store_t *store = cbm_store_open_memory();
+    cbm_watcher_t *w = cbm_watcher_new(store, cool_index_callback, NULL);
+    cbm_watcher_watch(w, "cool-repo", tmpdir);
+    cbm_watcher_poll_once(w); /* baseline */
+
+    wt_path(p, sizeof(p), tmpdir, "file.txt");
+    /* Instant index: plain cadence. */
+    int64_t fast = cool_reindex_delay_ms(w, p, "a\n", 0);
+    /* 60 s index → 3 x 60 s, counted from the END of the index (anchoring at
+     * the start would give 120 s here). */
+    int64_t slow = cool_reindex_delay_ms(w, p, "b\n", 60000);
+    /* 200 s index → 600 s, capped at 5 min. */
+    int64_t capped = cool_reindex_delay_ms(w, p, "c\n", 200000);
+
+    /* Restore the real clock before any assert can return early. */
+    cbm_watcher_test_set_clock_ns(0);
+
+    ASSERT_EQ(fast, cbm_watcher_poll_interval_ms(1));
+    ASSERT_EQ(slow, 180000);
+    ASSERT_EQ(capped, 300000);
+
+    cbm_watcher_free(w);
+    cbm_store_close(store);
+    th_rmtree(tmpdir);
+    PASS();
+}
+
 TEST(watcher_index_failure_count_unknown_project) {
     cbm_store_t *store = cbm_store_open_memory();
     cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL);
@@ -4043,6 +4164,11 @@ SUITE(watcher) {
     RUN_TEST(index_backoff_never_schedules_sooner_than_the_interval);
     RUN_TEST(watcher_index_failure_streak_increments_and_resets_issue2015);
     RUN_TEST(watcher_index_failure_backoff_gates_repolling_issue2015);
+    RUN_TEST(success_delay_fast_index_keeps_interval);
+    RUN_TEST(success_delay_slow_index_scales_by_factor);
+    RUN_TEST(success_delay_clamped_and_degenerate);
+    RUN_TEST(success_delay_is_monotonic);
+    RUN_TEST(watcher_success_cooldown_measures_index_issue2835);
     RUN_TEST(watcher_sustained_failure_logs_once_issue2015);
     RUN_TEST(watcher_index_failure_count_unknown_project);
     RUN_TEST(watcher_project_info_publishes_scan_issue2167);