Reference-chain cache: rebuild improved-path chains instead of erasing them (PROF-16316)

ddprof-lib/src/main/cpp/referenceChainEvents.cpp

@@ -150,12 +150,13 @@ void ReferenceChainTracker::pollWatchedTargets(jvmtiEnv *jvmti, JNIEnv *jni) {
     return;
   }
 
-  // Don't rebuild chains against cache entries the heap callback asked to drop.
-  drainPendingChainInvalidations();
-
   // Stamp every entry this poll refreshes with the current search generation.
   const u64 current_search_ns = load(_search_start_ns);
 
+  // Don't rebuild chains against cache entries the heap callback asked to drop, and refresh the
+  // ones whose path changed (the only site that applies REBUILD invalidations).
+  applyPendingChainInvalidations(jvmti, jni, current_search_ns);
+
   // klass_ids resolved (and therefore already pruned-if-dead) by the candidate loop below, so the
   // prune pass afterwards skips re-resolving them - it only needs to cover cached klasses that are
   // no longer flagged.
@@ -572,7 +573,7 @@ int ReferenceChainTracker::snapshotResolvedChainTags(jlong *out) {
 
 void ReferenceChainTracker::deferResolvedChainInvalidation(
     jlong source_tag, const jlong *resolved_chain_tags,
-    int resolved_chain_tag_count) {
+    int resolved_chain_tag_count, ChainInvalidationKind kind) {
   bool was_cached = false;
   for (int i = 0; i < resolved_chain_tag_count; i++) {
     if (resolved_chain_tags[i] == source_tag) {
@@ -583,32 +584,61 @@ void ReferenceChainTracker::deferResolvedChainInvalidation(
   if (!was_cached) {
     return;
   }
+  // Cache keys are frontier tags (>= 0); a negative key would alias another tag's REBUILD
+  // encoding, so it is never queued.
+  assert(source_tag >= 0);
+  if (source_tag < 0) {
+    return;
+  }
+  const jlong evict = encodePendingChainInvalidation(source_tag, ChainInvalidationKind::EVICT);
+  const jlong rebuild = encodePendingChainInvalidation(source_tag, ChainInvalidationKind::REBUILD);
 
   _pending_chain_invalidations_lock.lock();
-  for (jlong tag : _pending_chain_invalidations) {
-    if (tag == source_tag) {
+  for (jlong &pending : _pending_chain_invalidations) {
+    if (pending == evict) {
+      _pending_chain_invalidations_lock.unlock();
+      return; // EVICT already queued - it wins over any REBUILD
+    }
+    if (pending == rebuild) {
+      if (kind == ChainInvalidationKind::EVICT) {
+        pending = evict; // the source was rejected after its path changed: erase, don't rebuild
+      }
       _pending_chain_invalidations_lock.unlock();
       return;
     }
   }
   // Cache insertion is serialized with traversals on the engine thread and dumps can only remove
-  // keys, so one pre-traversal snapshot bounds distinct eligible tags to the cache cap.
+  // keys, so one pre-traversal snapshot bounds distinct eligible tags to the cache cap. REBUILD
+  // entries the dump path leaves pending do not break the bound: only pollWatchedTargets() inserts
+  // into the cache, and it drains every pending entry first.
   assert((int)_pending_chain_invalidations.size() <
          MAX_PENDING_CHAIN_INVALIDATIONS);
   if ((int)_pending_chain_invalidations.size() <
       MAX_PENDING_CHAIN_INVALIDATIONS) {
-    _pending_chain_invalidations.push_back(source_tag);
+    _pending_chain_invalidations.push_back(
+        kind == ChainInvalidationKind::REBUILD ? rebuild : evict);
   }
   _pending_chain_invalidations_lock.unlock();
 }
 
 void ReferenceChainTracker::drainPendingChainInvalidations() {
   // Serialize drainers and scratch-buffer use with the cache. The producer lock is held only for
-  // the O(1) handoff, never for unordered_map erase or node destruction.
+  // a bounded (<= MAX_PENDING_CHAIN_INVALIDATIONS) in-place partition - EVICT entries move to the
+  // reserved scratch buffer, REBUILD entries stay pending for the BFS thread - never for
+  // unordered_map erase or node destruction. Neither step reallocates.
   ExclusiveLockGuard guard(&_resolved_chains_lock);
   _pending_chain_invalidations_lock.lock();
   assert(_draining_chain_invalidations.empty());
-  _pending_chain_invalidations.swap(_draining_chain_invalidations);
+  size_t kept = 0;
+  for (size_t i = 0; i < _pending_chain_invalidations.size(); i++) {
+    jlong pending = _pending_chain_invalidations[i];
+    if (isPendingChainRebuild(pending)) {
+      _pending_chain_invalidations[kept++] = pending;
+    } else {
+      _draining_chain_invalidations.push_back(pending);
+    }
+  }
+  _pending_chain_invalidations.resize(kept);
   _pending_chain_invalidations_lock.unlock();
 
   for (jlong tag : _draining_chain_invalidations) {
@@ -620,6 +650,76 @@ void ReferenceChainTracker::drainPendingChainInvalidations() {
   _draining_chain_invalidations.clear();
 }
 
+void ReferenceChainTracker::applyPendingChainInvalidations(
+    jvmtiEnv *jvmti, JNIEnv *jni, u64 current_search_ns) {
+  // REBUILD tags whose cached chain is current-generation. A chain cached by an earlier search
+  // under the same numeric tag describes a different object (restartSearch() resets the tag
+  // namespace), so a path change on the new object says nothing about it: leave it alone.
+  jlong rebuild_tags[MAX_PENDING_CHAIN_INVALIDATIONS];
+  int rebuild_count = 0;
+  {
+    ExclusiveLockGuard guard(&_resolved_chains_lock);
+    _pending_chain_invalidations_lock.lock();
+    assert(_draining_chain_invalidations.empty());
+    _pending_chain_invalidations.swap(_draining_chain_invalidations);
+    _pending_chain_invalidations_lock.unlock();
+
+    for (jlong pending : _draining_chain_invalidations) {
+      jlong tag = pendingChainInvalidationTag(pending);
+      if (!isPendingChainRebuild(pending)) {
+        if (_resolved_chains.erase(tag) > 0) {
+          TEST_LOG("ReferenceChainTracker::drainPendingChainInvalidations source_tag=%lld",
+                   (long long)tag);
+        }
+        continue;
+      }
+      auto it = _resolved_chains.find(tag);
+      if (it != _resolved_chains.end() &&
+          it->second.source_search_ns == current_search_ns &&
+          rebuild_count < MAX_PENDING_CHAIN_INVALIDATIONS) {
+        rebuild_tags[rebuild_count++] = tag;
+      }
+    }
+    _draining_chain_invalidations.clear();
+  }
+
+  for (int i = 0; i < rebuild_count; i++) {
+    const jlong tag = rebuild_tags[i];
+    ReferenceChainEvent event;
+    // The frontier entry stays intact until restartSearch(), so this does not depend on the
+    // candidate/discovered slots runPass()'s terminal branch may already have zeroed.
+    if (!buildChainEvent(jvmti, jni, tag, &event)) {
+      TEST_LOG("ReferenceChainTracker::applyPendingChainInvalidations rebuild failed, "
+               "keeping cached chain source_tag=%lld",
+               (long long)tag);
+      continue;
+    }
+    if (suppressChainEvent(event)) {
+      // The new path turned the chain into a filtered shape: same outcome as every other build
+      // site (see buildDiscoveredInstanceChains()).
+      invalidateResolvedChain(tag);
+      continue;
+    }
+    if (isLeakTag((jlong)event._target_tag) && !chainTargetsCurrentLeakOwner(event)) {
+      // Dead-owner rebuild: keep the old chain; the next dump evicts it if it is dead too.
+      continue;
+    }
+    event._start_time = TSC::ticks();
+    ExclusiveLockGuard guard(&_resolved_chains_lock);
+    auto it = _resolved_chains.find(tag);
+    // Replace only: a dump may have evicted the entry (dead owner) since the scan above, and a
+    // rebuild must never resurrect it.
+    if (it != _resolved_chains.end() &&
+        it->second.source_search_ns == current_search_ns) {
+      it->second.event = std::move(event);
+      it->second.source_tag = tag;
+      TEST_LOG("ReferenceChainTracker::applyPendingChainInvalidations rebuilt "
+               "source_tag=%lld depth=%u",
+               (long long)tag, it->second.event._depth);
+    }
+  }
+}
+
 void ReferenceChainTracker::clearPendingChainInvalidations() {
   ExclusiveLockGuard cache_guard(&_resolved_chains_lock);
   _pending_chain_invalidations_lock.lock();
@@ -759,7 +859,8 @@ void ReferenceChainTracker::recordDiscoveredInstance(
         _candidate_discovered_tags[s][d] = frontier_tag;
         if (invalidation_mode == CacheInvalidationMode::DEFERRED) {
           deferResolvedChainInvalidation(victim, resolved_chain_tags,
-                                         resolved_chain_tag_count);
+                                         resolved_chain_tag_count,
+                                         ChainInvalidationKind::EVICT);
         } else {
           invalidateResolvedChain(victim);
         }

ddprof-lib/src/main/cpp/referenceChainTraversal.cpp

@@ -129,7 +129,8 @@ jvmtiIterationControl JNICALL ReferenceChainTracker::heapRootCallback(
             ctx->frontier, *tag_ptr, translated_root_kind)) {
       ctx->tracker->deferResolvedChainInvalidation(
           *tag_ptr, ctx->resolved_chain_tags,
-          ctx->resolved_chain_tag_count);
+          ctx->resolved_chain_tag_count,
+          ReferenceChainTracker::ChainInvalidationKind::REBUILD);
     }
     break;
   default:

ddprof-lib/src/main/cpp/referenceChainWalk.cpp

@@ -457,12 +457,13 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback(
       if (ctx->frontier->improveChain(*tag_ptr, parent_tag, referrer_klass,
                                        depth, 0, edge_field_index,
                                        (u8)reference_kind)) {
-        // Chain was improved — invalidate any cached chain for this tag so pollWatchedTargets
-        // rebuilds it with the deeper path.
+        // Chain was improved — queue a REBUILD so pollWatchedTargets replaces any cached chain for
+        // this tag with the deeper path (never erased: the source is still live).
         // Deferred: we're inside the FollowReferences pause (see deferResolvedChainInvalidation()).
         ctx->tracker->deferResolvedChainInvalidation(
             *tag_ptr, ctx->resolved_chain_tags,
-            ctx->resolved_chain_tag_count);
+            ctx->resolved_chain_tag_count,
+            ReferenceChainTracker::ChainInvalidationKind::REBUILD);
         if (was_root_attached_durable) {
           // Demotion push: the replaced entry's static/JNI-global attribution was its only
           // anchor-tier eligibility, and it is gone now.
@@ -477,7 +478,8 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback(
         // durable root.
         ctx->tracker->deferResolvedChainInvalidation(
             *tag_ptr, ctx->resolved_chain_tags,
-            ctx->resolved_chain_tag_count);
+            ctx->resolved_chain_tag_count,
+            ReferenceChainTracker::ChainInvalidationKind::REBUILD);
       }
     } else {
       // Already-admitted entry reached via a NEW root-like edge (parent_tag == 0): the static-field
@@ -489,7 +491,8 @@ jint JNICALL ReferenceChainTracker::heapReferenceCallback(
                                                           (u8)reference_kind)) {
         ctx->tracker->deferResolvedChainInvalidation(
             *tag_ptr, ctx->resolved_chain_tags,
-            ctx->resolved_chain_tag_count);
+            ctx->resolved_chain_tag_count,
+            ReferenceChainTracker::ChainInvalidationKind::REBUILD);
       } else if (reference_kind == JVMTI_HEAP_REFERENCE_STATIC_FIELD) {
         // The upgrade refused (maybeUpgradeRootAttachedRootKind returns false for parent_tag != 0
         // by design), so this STATIC_FIELD edge just proved an at-risk static attachment the anchor

ddprof-lib/src/main/cpp/referenceChains.h

@@ -586,10 +586,13 @@ class ReferenceChainTracker {
   std::unordered_map<jlong, CachedChain> _resolved_chains;
   SpinLock _resolved_chains_lock;
 
-  // Chain evictions deferred by heap callbacks. Only tags present in the immutable pre-traversal
-  // resolved-cache snapshot are eligible, so distinct pending tags are bounded exactly by the
-  // cache cap. Both buffers are reserved up front: drains swap the active producer buffer into the
-  // scratch buffer under the pending lock, then erase cache entries after releasing that lock.
+  // Chain invalidations deferred by heap callbacks. Only tags present in the immutable
+  // pre-traversal resolved-cache snapshot are eligible, so distinct pending tags are bounded
+  // exactly by the cache cap. Both buffers are reserved up front: drains move entries into the
+  // scratch buffer under the pending lock, then touch cache entries after releasing that lock.
+  // The ChainInvalidationKind is stored in-band (see encodePendingChainInvalidation()): EVICT
+  // entries hold the source tag, REBUILD entries its bitwise complement. Every read and write of
+  // both buffers happens under _pending_chain_invalidations_lock (full-barrier acquire/release).
   static constexpr int MAX_PENDING_CHAIN_INVALIDATIONS = MAX_RESOLVED_CHAINS;
   std::vector<jlong> _pending_chain_invalidations;
   std::vector<jlong> _draining_chain_invalidations;
@@ -1010,16 +1013,46 @@ class ReferenceChainTracker {
   // Copies the current cache keys into fixed caller-owned storage before a synchronous traversal.
   int snapshotResolvedChainTags(jlong *out);
 
-  // Queues an eviction from a heap callback only when the tag was cached in the immutable
+  // What a deferred invalidation does to the cached chain.
+  // EVICT: the source was rejected (noise eviction) - the cached chain is erased, at dump time or
+  // on the next poll, whichever drains first.
+  // REBUILD: the source is still live and its frontier entry intact, only its path changed
+  // (improveChain()/reparentToDurableRoot()/root-kind upgrade). Never erased at dump time: only
+  // the BFS-thread poll applies it, rebuilding the chain from the frontier and keeping the old one
+  // if the rebuild fails (restartSearch()'s keep-across-restart contract). EVICT wins over
+  // REBUILD for the same pending tag.
+  enum class ChainInvalidationKind : u8 { EVICT, REBUILD };
+
+  // In-band kind encoding for the pending buffers. Cache keys are frontier tags (>= 0), so the
+  // complement of a REBUILD tag is always negative and never collides with an EVICT entry.
+  static jlong encodePendingChainInvalidation(jlong source_tag,
+                                              ChainInvalidationKind kind) {
+    return kind == ChainInvalidationKind::REBUILD ? ~source_tag : source_tag;
+  }
+  static bool isPendingChainRebuild(jlong encoded) { return encoded < 0; }
+  static jlong pendingChainInvalidationTag(jlong encoded) {
+    return encoded < 0 ? ~encoded : encoded;
+  }
+
+  // Queues an invalidation from a heap callback only when the tag was cached in the immutable
   // pre-traversal snapshot. This never takes _resolved_chains_lock or allocates.
   void deferResolvedChainInvalidation(jlong source_tag,
                                       const jlong *resolved_chain_tags,
-                                      int resolved_chain_tag_count);
+                                      int resolved_chain_tag_count,
+                                      ChainInvalidationKind kind);
 
-  // Applies queued exact-tag evictions. Called by the engine thread and by dump threads, which may
-  // overlap a traversal; the resolved-cache/pending locks provide synchronization.
+  // Dump-path drain: applies only the queued EVICT invalidations and leaves REBUILD entries
+  // pending for the BFS thread (see ChainInvalidationKind). May overlap a traversal; the
+  // resolved-cache/pending locks provide synchronization.
   void drainPendingChainInvalidations();
 
+  // BFS-thread drain (pollWatchedTargets()): applies EVICT entries like the dump path and rebuilds
+  // each REBUILD entry's current-generation cached chain from the frontier via buildChainEvent(),
+  // overwriting it on success and keeping it on failure. Independent of candidate/discovered
+  // slots, so it still works after runPass()'s terminal branch zeroed them.
+  void applyPendingChainInvalidations(jvmtiEnv *jvmti, JNIEnv *jni,
+                                      u64 current_search_ns);
+
   // Clears both pending buffers at cache/tag-generation boundaries without releasing capacity.
   void clearPendingChainInvalidations();