From b515fb553368ea2ed920b61aad444813c7c22d57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:12:15 +0000 Subject: [PATCH 01/14] Bump taiki-e/install-action Bumps the github-actions group with 1 update in the / directory: [taiki-e/install-action](https://github.com/taiki-e/install-action). Updates `taiki-e/install-action` from 2.86.1 to 2.86.2 - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/288e746965032cfcc232e09af2daf5f23c14d780...b6b84cf49ebfe0176417bdce007c624f0db37f20) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.86.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/zjit-macos.yml | 2 +- .github/workflows/zjit-ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index 82e1d911eb5e62..4e574b6be167f8 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -98,7 +98,7 @@ jobs: rustup install ${{ matrix.rust_version }} --profile minimal rustup default ${{ matrix.rust_version }} - - uses: taiki-e/install-action@288e746965032cfcc232e09af2daf5f23c14d780 # v2.86.1 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index c14fceb2437092..2f323a61f4a78b 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -152,7 +152,7 @@ jobs: ruby-version: '3.1' bundler: none - - uses: taiki-e/install-action@288e746965032cfcc232e09af2daf5f23c14d780 # v2.86.1 + - uses: taiki-e/install-action@b6b84cf49ebfe0176417bdce007c624f0db37f20 # v2.86.2 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} From 7d5aaefc01e9a34484b50b388baa4fdb3283d407 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Wed, 19 Aug 2026 19:54:38 +0000 Subject: [PATCH 02/14] Ractor: keep a marshaled copy payload off-heap An unshareable object that the native copier cannot handle is sent as a Marshal dump. That dump was a String in the sender's objspace, kept alive by an in-flight pin until the receiver materialized it. When the message is never received the pin never lifts, so the page holding it stays allocated for the life of the process: GC.start, GC.compact and Port#close all leave it. With per-Ractor GC that is one 64KB heap page per sending Ractor, and it grows without bound. Carry the dump in an xmalloc'd buffer instead, the way a move courier already carries its payload (design_v2.md 4.5). An in-flight payload that is not a GC object needs no pin, so nothing of the sender's heap is held while the message waits, and an unreceived message costs only the buffer. Measured with 3000 send-and-never-receive rounds, each from a Ractor that then exits: payload before after Time 79.9 KB/Ractor 13.7 KB/Ractor (1.00 -> 0.01 page) Set 79.5 KB/Ractor 13.3 KB/Ractor Random 83.6 KB/Ractor 15.7 KB/Ractor 13 KB/Ractor is what a Ractor that sends nothing at all retains, and that part plateaus. Send/receive throughput is unchanged (20k messages, n=9, median us/message: Time 8.75 -> 8.79, Set 4.88 -> 4.84). Natively copied payloads (String, Array, Hash, Object, Struct, MatchData) still build a snapshot in the sender's objspace and still pin it; they keep the old behaviour for now. Marshal does not mark its source (mark_load_arg), and the basket is off the queue while it materializes, so the rebuilt String is rooted from the receiving frame's stack slot. Co-Authored-By: Claude Opus 5 (1M context) --- ractor_sync.c | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/ractor_sync.c b/ractor_sync.c index 13cbc6ad3f136a..e01658da89db23 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -232,6 +232,11 @@ struct ractor_basket { /* The off-heap (xmalloc) courier of a basket_type_move. A move basket does * not use v. */ struct rb_ractor_move_courier *move_courier; + /* The marshaled bytes of a copy payload, off-heap like a move courier. When + * set, v is unused: an in-flight payload that is not a GC object needs no + * in-flight pin, so it never keeps a page of the sender's heap alive. */ + char *mbuf; + size_t mlen; /* Every node of a native copy snapshot, collected while building it (raw * malloc). The global GC's re-pin walks this list, since traversing the graph * in-GC would need generic-ivar lookups. NULL: only the root (p.v) is pinned. */ @@ -262,7 +267,7 @@ ractor_basket_mark(const struct ractor_basket *b) /* A move courier lives off-heap, and the shareable REFs it carries are marked and * pinned as a global GC root by the in-flight registry (ractor.c). Nothing to do * here. */ - if (b->type != basket_type_move) { + if (b->type != basket_type_move && b->p.mbuf == NULL) { rb_gc_mark(b->p.v); } } @@ -279,6 +284,9 @@ ractor_basket_free(struct ractor_basket *b) free(b->p.pinned); b->p.pinned = NULL; b->p.pinned_cnt = 0; + ruby_xfree(b->p.mbuf); + b->p.mbuf = NULL; + b->p.mlen = 0; if (b->type == basket_type_move && b->p.move_courier) { /* A move courier that was never consumed (a queue being torn down, say). */ rb_ractor_move_courier_free(b->p.move_courier); @@ -777,7 +785,7 @@ ractor_sync_mark(rb_ractor_t *r) static void ractor_basket_repin_in_flight(const struct ractor_basket *b) { - if (b->type != basket_type_copy) return; + if (b->type != basket_type_copy || b->p.mbuf != NULL) return; rb_gc_pin_in_flight_message(b->p.v); for (size_t i = 0; i < b->p.pinned_cnt; i++) { rb_gc_pin_in_flight_message(b->p.pinned[i]); @@ -1073,6 +1081,8 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type volatile VALUE v = Qfalse; bool marshaled = false; struct rb_ractor_move_courier *courier = NULL; + char *mbuf = NULL; + size_t mlen = 0; struct ractor_basket *b; if (type == basket_type_move) { @@ -1098,10 +1108,18 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { + /* Take the dump off-heap before the basket exists, so an alloc raise below + * frees it through mbuf rather than orphaning it. */ + if (type == basket_type_copy && marshaled) { + mlen = (size_t)RSTRING_LEN(v); + mbuf = ALLOC_N(char, mlen > 0 ? mlen : 1); + memcpy(mbuf, RSTRING_PTR(v), mlen); + } b = ractor_basket_alloc(); } EC_POP_TAG(); if (state != TAG_NONE) { + ruby_xfree(mbuf); /* Drop the pin list, or every global GC re-pins the dead snapshot from it * forever (rb_ractor_repin_in_flight walks it unconditionally). The nodes * stay shref-pinned only until the next global GC clears the bits. */ @@ -1111,12 +1129,11 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type cr->pin_capture_cnt = cr->pin_capture_capa = 0; EC_JUMP_TAG(ec, state); } - /* copy_enter pinned every node at construction with cr->pin_capture as the - * re-pin source; hand it to the basket only after basket_alloc (which may GC) - * so the cover never lapses. A marshaled String is pinned here, after the - * alloc, so an alloc raise leaves no stale pin. */ - if (type == basket_type_copy && marshaled) { - rb_gc_pin_in_flight_message(v); + /* The dump is off-heap now, so the sender's copy of it is ordinary garbage: + * nothing to pin. A native snapshot still lives in the sender's objspace and + * is pinned through cr->pin_capture below. */ + if (mbuf != NULL) { + v = Qundef; } } @@ -1125,6 +1142,8 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type b->p.v = v; b->p.marshaled = marshaled; b->p.move_courier = courier; + b->p.mbuf = mbuf; + b->p.mlen = mlen; b->p.pinned = NULL; b->p.pinned_cnt = 0; if (type == basket_type_copy) { @@ -1186,7 +1205,15 @@ ractor_basket_value(struct ractor_basket *b) enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - if (b->p.marshaled) { + if (b->p.mbuf != NULL) { + /* Rebuild the marshaled bytes in this Ractor's objspace. Marshal does + * not mark its source (mark_load_arg) and the basket is off the queue, + * so this frame's stack slot is the String's only root for the load. */ + VALUE bin = rb_str_new(b->p.mbuf, (long)b->p.mlen); + result = rb_marshal_load(bin); + RB_GC_GUARD(bin); + } + else if (b->p.marshaled) { result = rb_marshal_load(b->p.v); } else { @@ -1644,6 +1671,8 @@ ractor_basket_new_ref(VALUE shareable) b->p.exception = false; b->p.marshaled = false; b->p.move_courier = NULL; + b->p.mbuf = NULL; + b->p.mlen = 0; b->p.pinned = NULL; b->p.pinned_cnt = 0; From cd2ea9df6c38fd9225ff54f38bf0973f32e7c2b7 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Wed, 19 Aug 2026 19:55:04 +0000 Subject: [PATCH 03/14] Ractor: carry a natively copied payload off-heap too Step 2 took the marshaled copy payload off-heap; this does the same for the natively copied one, so an unreceived copy no longer pins a page of the sender's heap for any payload type. move_capture gains a copy mode: it reads the sources instead of taking them apart (no husk, no buffer hand-over, no freeing the source's internals), and copy_courier_supported_p decides up front whether the courier can carry the whole graph. MatchData, IO, any other T_DATA and a singleton class are rejected there and keep using the on-heap snapshot path, which handles or rejects them exactly as before. Retention with 3000 send-and-never-receive rounds (KB per sending Ractor, before -> after): String 79.5 -> 14.0 Array 79.5 -> 14.0 Object 79.5 -> 14.0 Hash 79.5 -> 14.0 14 KB is what a Ractor that sends nothing retains, and that part plateaus. A courier is an in-process format, so a shareable payload can be embedded as the VALUE itself rather than costing a whole move_node: child slots now hold either a node id or, with the top bit set, an index into a compact c->refs array that the registry marks. Without that, an array of immediates paid a 72-byte node, a hash lookup and an insert per element, and the message path cost 2.5x what the native copier did (it skipped shareable children). Message throughput (20k messages, n=9, interleaved, median us/message): 100-element Integer array 5.1 -> 4.7 4KB String 16.6 -> 9.3 (one memcpy, not two copies) 20-key Hash 12.0 -> 9.4 nested Hash/Array/String 10.1 -> 7.2 40B String, bare Object unchanged It also restores a behaviour master lost: a String or Array subclass now survives the copy again (4.0.2 copies with #clone and keeps the class; master's native copier builds a base-class object). MyStr < String arrives as MyStr, as it does under Marshal and #clone. move_capture keeps testing the dedup table before shareability: move husks each source as it goes, and a husk is a frozen field-less object that rb_ractor_shareable_p answers true for, so the other order embedded the husk instead of resolving a second occurrence to the first one's node (bootstraptest/test_ractor.rb:661). An immediate skips the lookup, since only captured objects are ever inserted. Co-Authored-By: Claude Opus 5 (1M context) --- ractor.c | 215 ++++++++++++++++++++++++++++++++++++++++++-------- ractor_sync.c | 26 ++++-- 2 files changed, 204 insertions(+), 37 deletions(-) diff --git a/ractor.c b/ractor.c index c6b6c32b8242b6..f12f2928fc6c8c 100644 --- a/ractor.c +++ b/ractor.c @@ -2412,10 +2412,18 @@ struct move_node { } u; }; +/* A child slot holds a node id, or -- with this bit set -- an index into c->refs. + * The courier is in-process, so a shareable payload can travel as the VALUE itself + * instead of costing a whole move_node; the registry marks and pins c->refs. */ +#define MOVE_ID_REF_BIT 0x80000000u + struct rb_ractor_move_courier { struct move_node *nodes; uint32_t count; uint32_t capa; + VALUE *refs; /* shareable payloads, embedded by value */ + uint32_t refs_count; + uint32_t refs_capa; uint32_t root; struct ccan_list_node reg_node; /* in-flight courier registry (a GC root while it lives) */ }; @@ -2460,6 +2468,9 @@ rb_ractor_move_courier_registry_mark(void) struct move_build { struct rb_ractor_move_courier *c; st_table *seen; /* src VALUE -> (node id + 1) */ + /* Copy mode: read the sources instead of taking them apart. No husk, no buffer + * hand-over, no freeing of the source's internals. */ + bool copy; }; static uint32_t move_capture(struct move_build *b, VALUE obj); @@ -2483,6 +2494,29 @@ move_alloc_node(struct rb_ractor_move_courier *c) return id; } +/* Embed a shareable payload by value and return its tagged child id. No dedup: a REF + * is the same word however often it appears, and an array of immediates would otherwise + * pay a lookup and an insert per element. */ +static uint32_t +move_alloc_ref(struct rb_ractor_move_courier *c, VALUE v) +{ + if (c->refs_count == c->refs_capa) { + c->refs_capa = c->refs_capa ? c->refs_capa * 2 : 8; + REALLOC_N(c->refs, VALUE, c->refs_capa); + } + uint32_t idx = c->refs_count++; + c->refs[idx] = v; + return MOVE_ID_REF_BIT | idx; +} + +/* Resolve a child slot to the object it names. */ +static VALUE +move_child(const struct rb_ractor_move_courier *c, VALUE shells, uint32_t id) +{ + if (id & MOVE_ID_REF_BIT) return c->refs[id & ~MOVE_ID_REF_BIT]; + return RARRAY_AREF(shells, id); +} + /* Turn a moved source into a valid RactorMovedObject without passing through flags==0, * so a concurrent foreign marker always sees either the original object or the shell. */ static void @@ -2591,24 +2625,28 @@ move_capture_ivars(struct move_build *b, VALUE obj, uint32_t id) static uint32_t move_capture(struct move_build *b, VALUE obj) { + /* An immediate is never in seen (only captured objects are inserted), so it can + * skip the lookup entirely: that is the whole cost of an array of numbers. */ + if (RB_SPECIAL_CONST_P(obj)) { + return move_alloc_ref(b->c, obj); + } + + /* Seen first, and only then shareable: move husks each source as it goes, and a + * husk is a frozen field-less object, which rb_ractor_shareable_p answers true for. + * Testing shareable first would embed the husk instead of resolving the second + * occurrence to the node the first one built. */ st_data_t existing; if (st_lookup(b->seen, (st_data_t)obj, &existing)) { return (uint32_t)existing - 1; } + if (rb_ractor_shareable_p(obj)) { + return move_alloc_ref(b->c, obj); + } + uint32_t id = move_alloc_node(b->c); st_insert(b->seen, (st_data_t)obj, (st_data_t)(uintptr_t)(id + 1)); - if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) { - b->c->nodes[id].kind = MOVE_KIND_REF; - b->c->nodes[id].frozen = false; - b->c->nodes[id].niv = 0; - b->c->nodes[id].iv_ids = NULL; - b->c->nodes[id].iv_vals = NULL; - b->c->nodes[id].u.ref = obj; - return id; - } - /* Reject an unmovable object before anything is mutated. */ if (BUILTIN_TYPE(obj) == T_FILE && RFILE(obj)->fptr == NULL) { rb_raise(rb_eRactorError, "can not move an uninitialized IO"); @@ -2623,11 +2661,11 @@ move_capture(struct move_build *b, VALUE obj) /* Give the source its own buffer (drop sharing, copy a static STR_NOFREE one). * Safe even when frozen: it changes ownership, not content. Afterwards a string * is embedded, owns a private heap buffer, or is a shared ROOT (a no-op). */ - rb_str_make_independent(obj); + if (!b->copy) rb_str_make_independent(obj); long len = RSTRING_LEN(obj); int encidx = ENCODING_GET(obj); char *ptr; - if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) { + if (!b->copy && !STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) { /* Owns a private heap buffer: carry the pointer over (zero-copy) and leave * the source as a shell that does not free it. */ ptr = RSTRING(obj)->as.heap.ptr; @@ -2661,7 +2699,7 @@ move_capture(struct move_build *b, VALUE obj) /* Free the source's heap buffer now that the children were read, but only when it * is private: a sharer's belongs to its root, a root's to its sharers -- and a * frozen array is a root without carrying the flag. */ - if (!ARY_EMBED_P(obj) && !ARY_SHARED_P(obj) && !ARY_SHARED_ROOT_P(obj) && !OBJ_FROZEN(obj)) { + if (!b->copy && !ARY_EMBED_P(obj) && !ARY_SHARED_P(obj) && !ARY_SHARED_ROOT_P(obj) && !OBJ_FROZEN(obj)) { ruby_xfree((void *)RARRAY_CONST_PTR(obj)); } break; @@ -2681,7 +2719,7 @@ move_capture(struct move_build *b, VALUE obj) b->c->nodes[id].u.hash.compare_by_id = RTEST(rb_hash_compare_by_id_p(obj)); b->c->nodes[id].u.hash.proc_default = FL_TEST_RAW(obj, RHASH_PROC_DEFAULT) != 0; /* Free the source's st-table internals (an ar table lives in the slot) */ - rb_hash_free(obj); + if (!b->copy) rb_hash_free(obj); break; } @@ -2704,7 +2742,7 @@ move_capture(struct move_build *b, VALUE obj) b->c->nodes[id].u.strct.elems = elems; b->c->nodes[id].u.strct.klass = RBASIC_CLASS(obj); /* Free the source's private heap buffer (an embedded struct has none) */ - if (RSTRUCT_EMBED_LEN(obj) == 0) { + if (!b->copy && RSTRUCT_EMBED_LEN(obj) == 0) { ruby_xfree((void *)RSTRUCT_CONST_PTR(obj)); } break; @@ -2713,6 +2751,7 @@ move_capture(struct move_build *b, VALUE obj) case T_MATCH: { /* The regexp and the matched string travel as ordinary children; re.c dumps the * registers (freeing the source's onig and char_offset). */ + VM_ASSERT(!b->copy); /* copy_courier_supported_p rejects it */ VALUE re, st; int nregs; void *regs = rb_match_move_dump(obj, &re, &st, &nregs); @@ -2729,6 +2768,7 @@ move_capture(struct move_build *b, VALUE obj) case T_FILE: { + VM_ASSERT(!b->copy); /* copy_courier_supported_p rejects it */ /* Carry the whole fptr (fd included) by pointer; the source shell does not * close it. fptr's VALUE members lose their root once the source is T_MOVED, * so capture them as ordinary child nodes, detached; rebuild writes them back. */ @@ -2764,7 +2804,7 @@ move_capture(struct move_build *b, VALUE obj) rb_class_name(rb_obj_class(obj))); } - move_neutralize_source(obj); + if (!b->copy) move_neutralize_source(obj); return id; } @@ -2848,6 +2888,115 @@ move_preflight(VALUE obj, st_table *seen) rb_ivar_foreach(obj, move_preflight_ivar_i, (st_data_t)seen); } +struct copy_support_ctx { + st_table *seen; + bool ok; +}; + +static bool copy_courier_supported_p(VALUE obj, st_table *seen); + +static int +copy_support_val_i(st_data_t val, st_data_t arg) +{ + struct copy_support_ctx *ctx = (struct copy_support_ctx *)arg; + if (!copy_courier_supported_p((VALUE)val, ctx->seen)) { + ctx->ok = false; + return ST_STOP; + } + return ST_CONTINUE; +} + +static int +copy_support_ivar_i(ID name, VALUE val, st_data_t arg) +{ + return copy_support_val_i((st_data_t)val, arg); +} + +static int +copy_support_hash_i(st_data_t key, st_data_t val, st_data_t arg) +{ + if (copy_support_val_i(key, arg) == ST_STOP) return ST_STOP; + return copy_support_val_i(val, arg); +} + +/* Read-only walk: can the copy courier carry obj's whole graph? Everything it says no + * to (MatchData, IO, any other T_DATA, a singleton class) stays on the older on-heap + * snapshot path, which keeps handling or rejecting it exactly as before. */ +static bool +copy_courier_supported_p(VALUE obj, st_table *seen) +{ + if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) return true; + if (st_lookup(seen, (st_data_t)obj, NULL)) return true; /* cycle */ + st_insert(seen, (st_data_t)obj, 0); + + /* A singleton class is a send error today (the native copier refuses it and Marshal + * then raises); the courier would happily carry it, so keep it off this path. */ + VALUE klass = RBASIC_CLASS(obj); + if (klass == 0 || FL_TEST_RAW(klass, FL_SINGLETON)) return false; + + struct copy_support_ctx ctx = { seen, true }; + + switch (BUILTIN_TYPE(obj)) { + case T_STRING: + case T_OBJECT: + break; /* children are ivars only (below) */ + case T_ARRAY: + for (long i = 0; i < RARRAY_LEN(obj); i++) { + if (!copy_courier_supported_p(RARRAY_AREF(obj, i), seen)) return false; + } + break; + case T_HASH: + rb_hash_stlike_foreach(obj, copy_support_hash_i, (st_data_t)&ctx); + if (!ctx.ok) return false; + if (!copy_courier_supported_p(RHASH_IFNONE(obj), seen)) return false; + break; + case T_STRUCT: + for (long i = 0; i < RSTRUCT_LEN(obj); i++) { + if (!copy_courier_supported_p(RSTRUCT_GET(obj, (int)i), seen)) return false; + } + break; + default: + return false; + } + + rb_ivar_foreach(obj, copy_support_ivar_i, (st_data_t)&ctx); + return ctx.ok; +} + +/* Build a courier holding a copy of obj's graph, leaving the sources untouched. + * Returns NULL when the graph has a type only the on-heap snapshot path handles. */ +struct rb_ractor_move_courier * +rb_ractor_copy_courier_build(VALUE obj) +{ + { + st_table *seen = st_init_numtable(); + bool ok = copy_courier_supported_p(obj, seen); + st_free_table(seen); + if (!ok) return NULL; + } + + struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); + struct move_build b = { c, st_init_numtable(), true }; + + /* Same registry cover as a move courier: the shareable REFs it carries need a root + * for its whole lifetime. */ + move_courier_registry_add(c); + + enum ruby_tag_type state; + rb_execution_context_t *ec = GET_EC(); + EC_PUSH_TAG(ec); + if ((state = EC_EXEC_TAG()) == TAG_NONE) { + c->root = move_capture(&b, obj); + } + EC_POP_TAG(); + st_free_table(b.seen); + if (state != TAG_NONE) { + rb_ractor_move_courier_free(c); + EC_JUMP_TAG(ec, state); + } + return c; +} + /* Build a move courier from obj and turn every captured source into a * RactorMovedObject (move semantics). Returns the xmalloc'd courier. */ struct rb_ractor_move_courier * @@ -2869,7 +3018,7 @@ rb_ractor_move_courier_build(VALUE obj) } struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); - struct move_build b = { c, st_init_numtable() }; + struct move_build b = { c, st_init_numtable(), false }; /* Between send and materialization the courier's shareable REFs pass through * windows where nothing else roots them; register it for its whole lifetime so the @@ -2970,7 +3119,7 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) switch (n->kind) { case MOVE_KIND_ARRAY: for (long j = 0; j < n->u.ary.len; j++) { - rb_ary_push(shell, RARRAY_AREF(shells, n->u.ary.elems[j])); + rb_ary_push(shell, move_child(c, shells, n->u.ary.elems[j])); } break; case MOVE_KIND_HASH: @@ -2980,23 +3129,23 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) break; case MOVE_KIND_STRUCT: for (long j = 0; j < n->u.strct.len; j++) { - RSTRUCT_SET(shell, (int)j, RARRAY_AREF(shells, n->u.strct.elems[j])); + RSTRUCT_SET(shell, (int)j, move_child(c, shells, n->u.strct.elems[j])); } break; case MOVE_KIND_MATCH: - rb_match_move_load(shell, RARRAY_AREF(shells, n->u.match.regexp_id), - RARRAY_AREF(shells, n->u.match.str_id), + rb_match_move_load(shell, move_child(c, shells, n->u.match.regexp_id), + move_child(c, shells, n->u.match.str_id), n->u.match.num_regs, n->u.match.regs); break; case MOVE_KIND_IO: { /* Write the rebuilt VALUE members back into fptr (capture detached them). * write_lock and wakeup_mutex stay nil; io.c recreates them lazily. */ struct rb_io *fptr = RFILE(shell)->fptr; - RB_OBJ_WRITE(shell, &fptr->pathv, RARRAY_AREF(shells, n->u.io.pathv_id)); - RB_OBJ_WRITE(shell, &fptr->encs.ecopts, RARRAY_AREF(shells, n->u.io.ecopts_id)); - RB_OBJ_WRITE(shell, &fptr->writeconv_pre_ecopts, RARRAY_AREF(shells, n->u.io.wc_pre_ecopts_id)); - RB_OBJ_WRITE(shell, &fptr->writeconv_asciicompat, RARRAY_AREF(shells, n->u.io.wc_asciicompat_id)); - RB_OBJ_WRITE(shell, &fptr->timeout, RARRAY_AREF(shells, n->u.io.timeout_id)); + RB_OBJ_WRITE(shell, &fptr->pathv, move_child(c, shells, n->u.io.pathv_id)); + RB_OBJ_WRITE(shell, &fptr->encs.ecopts, move_child(c, shells, n->u.io.ecopts_id)); + RB_OBJ_WRITE(shell, &fptr->writeconv_pre_ecopts, move_child(c, shells, n->u.io.wc_pre_ecopts_id)); + RB_OBJ_WRITE(shell, &fptr->writeconv_asciicompat, move_child(c, shells, n->u.io.wc_asciicompat_id)); + RB_OBJ_WRITE(shell, &fptr->timeout, move_child(c, shells, n->u.io.timeout_id)); break; } default: @@ -3004,7 +3153,7 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) } /* Restore instance and generic ivars (any non-REF node can have them) */ for (uint32_t j = 0; j < n->niv; j++) { - rb_ivar_set(shell, n->iv_ids[j], RARRAY_AREF(shells, n->iv_vals[j])); + rb_ivar_set(shell, n->iv_ids[j], move_child(c, shells, n->iv_vals[j])); } } @@ -3016,11 +3165,11 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) if (n->kind != MOVE_KIND_HASH) continue; VALUE shell = RARRAY_AREF(shells, i - 1); for (long j = 0; j < n->u.hash.size; j++) { - rb_hash_aset(shell, RARRAY_AREF(shells, n->u.hash.kv[2 * j]), - RARRAY_AREF(shells, n->u.hash.kv[2 * j + 1])); + rb_hash_aset(shell, move_child(c, shells, n->u.hash.kv[2 * j]), + move_child(c, shells, n->u.hash.kv[2 * j + 1])); } /* Restore the default value and default proc (before freezing) */ - VALUE ifnone = RARRAY_AREF(shells, n->u.hash.ifnone_id); + VALUE ifnone = move_child(c, shells, n->u.hash.ifnone_id); if (n->u.hash.proc_default) { rb_hash_set_default_proc(shell, ifnone); } @@ -3037,7 +3186,7 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) } } - VALUE root = c->count ? RARRAY_AREF(shells, c->root) : Qnil; + VALUE root = (c->count || c->refs_count) ? move_child(c, shells, c->root) : Qnil; RB_GC_GUARD(shells); return root; } @@ -3080,6 +3229,7 @@ rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) } move_courier_registry_remove(c); ruby_xfree(c->nodes); + ruby_xfree(c->refs); ruby_xfree(c); } @@ -3090,6 +3240,9 @@ void rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c) { if (!c) return; + for (uint32_t i = 0; i < c->refs_count; i++) { + rb_gc_mark(c->refs[i]); + } for (uint32_t i = 0; i < c->count; i++) { struct move_node *n = &c->nodes[i]; if (n->kind == MOVE_KIND_REF) { diff --git a/ractor_sync.c b/ractor_sync.c index e01658da89db23..62a341ed1123a4 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -23,6 +23,7 @@ static void ractor_add_port(rb_ractor_t *r, st_data_t id); struct rb_ractor_move_courier *rb_ractor_move_courier_build(VALUE obj); VALUE rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c); void rb_ractor_move_courier_free(struct rb_ractor_move_courier *c); +struct rb_ractor_move_courier *rb_ractor_copy_courier_build(VALUE obj); static void ractor_port_mark(void *ptr) @@ -267,7 +268,7 @@ ractor_basket_mark(const struct ractor_basket *b) /* A move courier lives off-heap, and the shareable REFs it carries are marked and * pinned as a global GC root by the in-flight registry (ractor.c). Nothing to do * here. */ - if (b->type != basket_type_move && b->p.mbuf == NULL) { + if (b->type != basket_type_move && b->p.mbuf == NULL && b->p.move_courier == NULL) { rb_gc_mark(b->p.v); } } @@ -287,7 +288,7 @@ ractor_basket_free(struct ractor_basket *b) ruby_xfree(b->p.mbuf); b->p.mbuf = NULL; b->p.mlen = 0; - if (b->type == basket_type_move && b->p.move_courier) { + if (b->p.move_courier) { /* A move courier that was never consumed (a queue being torn down, say). */ rb_ractor_move_courier_free(b->p.move_courier); b->p.move_courier = NULL; @@ -785,7 +786,7 @@ ractor_sync_mark(rb_ractor_t *r) static void ractor_basket_repin_in_flight(const struct ractor_basket *b) { - if (b->type != basket_type_copy || b->p.mbuf != NULL) return; + if (b->type != basket_type_copy || b->p.mbuf != NULL || b->p.move_courier != NULL) return; rb_gc_pin_in_flight_message(b->p.v); for (size_t i = 0; i < b->p.pinned_cnt; i++) { rb_gc_pin_in_flight_message(b->p.pinned[i]); @@ -1025,7 +1026,8 @@ ractor_marshal_dump_rescue(VALUE obj, VALUE errinfo) } static VALUE -ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type *ptype, bool *pmarshaled) +ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type *ptype, bool *pmarshaled, + struct rb_ractor_move_courier **pcourier) { switch (*ptype) { case basket_type_ref: @@ -1040,6 +1042,13 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket * #clone: core types are deep-copied natively and anything else is * marshaled here, so its user hooks run on the sender. */ *ptype = basket_type_copy; + /* An off-heap courier first: an in-flight payload that is not a GC object + * needs no pin, so nothing of the sender's heap stays alive while the + * message waits (design_v2.md 4.5). NULL means the graph holds a type only + * the on-heap snapshot path below handles. */ + *pcourier = rb_ractor_copy_courier_build(obj); + if (*pcourier != NULL) return Qundef; + /* During a native copy, copy_enter collects every snapshot node into the * pin list that covers construction, enqueue and materialization. */ rb_ractor_t *cr = rb_ec_ractor_ptr(ec); @@ -1104,7 +1113,7 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type } } else { - v = ractor_prepare_payload(ec, obj, &type, &marshaled); + v = ractor_prepare_payload(ec, obj, &type, &marshaled, &courier); enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { @@ -1120,6 +1129,7 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type EC_POP_TAG(); if (state != TAG_NONE) { ruby_xfree(mbuf); + if (courier != NULL) rb_ractor_move_courier_free(courier); /* Drop the pin list, or every global GC re-pins the dead snapshot from it * forever (rb_ractor_repin_in_flight walks it unconditionally). The nodes * stay shref-pinned only until the next global GC clears the bits. */ @@ -1182,6 +1192,9 @@ ractor_basket_value(struct ractor_basket *b) case basket_type_ref: break; case basket_type_copy: { + /* An off-heap copy courier rebuilds exactly like a move one; only the sources + * differ (still alive here, already shells there). */ + if (b->p.move_courier != NULL) goto materialize_courier; /* Materialize the sender's snapshot into the receiving Ractor's objspace. * Passing the sender-resident graph by reference would create an unshareable * cross-objspace edge that neither local GC can follow. The snapshot stays @@ -1241,7 +1254,8 @@ ractor_basket_value(struct ractor_basket *b) RB_GC_GUARD(result); break; } - case basket_type_move: { + case basket_type_move: + materialize_courier: { /* Rebuild the moved graph from the off-heap courier into this Ractor's * objspace. The sources are already RactorMovedObject (set when the courier * was built), so move's snapshot semantics hold. The courier is xmalloc'd From 68447b1559e220ffa1b36fda8f21d2ac2f7e52f3 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Wed, 19 Aug 2026 19:55:21 +0000 Subject: [PATCH 04/14] Ractor: retire the on-heap copy snapshot and its in-flight pins With MatchData and an exception's backtrace on the courier, every copy payload is off-heap: the courier carries the core types and anything else travels as marshaled bytes. The sender-resident snapshot is gone, and with it all the machinery that kept one alive across a global GC. MatchData: rb_match_move_dump takes a release_source flag, since only a move takes the source's onig region and char_offset apart. Backtrace: rb_backtrace_blob_dump / _load / _mark copy the frames into an off-heap blob. A frame only references shareable iseq and method-entry imemos, so the blob can carry them as they are; it has no compaction update hook, so the mark pins them (rb_gc_mark, not _movable). Removed, all of it unreachable once no payload lives in the sender's heap: rb_ractor_t pin_capture / _cnt / _capa, sending_basket, gen_fields_capturing, ractor_pin_capture_push basket p.pinned / p.pinned_cnt, ractor_basket_repin_in_flight, rb_ractor_repin_in_flight and its queue walks receiver struct ractor_materialize_frame, ec->materialize_frames and its mark loop, sync.materializing_copies, rb_ractor_materializing_p gc rb_gc_pin_in_flight_message and the whole rb_gc_impl_pin_in_flight_message hook (default, mmtk, the modular function table), plus the two verifier relaxations that existed for a snapshot being materialized shref bits stay: the write barrier still records them for other reasons. ractor_copy_native_try and copy_enter stay too, since Ractor.make_shareable(obj, copy: true) still deep-copies within one objspace. Message cost, 20k messages per trial, 9 trials, median us per message: 4.0.2 master here 4KB String 3.8 19.4 9.4 20-key Hash 3.3 11.9 7.5 nested Hash/Array/Str 5.3 7.1 5.1 100-elem Integer Array 4.4 5.7 6.9 40B String, Object same across all three master is well behind 4.0.2 on multi-KB payloads; this recovers about half of that and puts nested graphs back at 4.0.2's level. Known issue, already in master and not introduced here: the courier's registry mark does not traverse a shareable payload, so its children are collected if a global GC runs while the message is in flight. Unpatched master crashes on both repros (a dynamic Symbol loses its fstr, a frozen Array loses its elements) and 4.0.2 does not. It only shows up for move today; routing copy through the courier makes it reachable from a common path, so it wants fixing before this ships. Co-Authored-By: Claude Opus 5 (1M context) --- gc.c | 22 +---- gc/default/default.c | 25 ------ gc/gc.h | 1 - gc/gc_impl.h | 1 - gc/mmtk/mmtk.c | 6 -- internal/gc.h | 1 - internal/re.h | 2 +- internal/vm.h | 3 + ractor.c | 70 +++++++-------- ractor_core.h | 29 ------ ractor_sync.c | 210 +++++-------------------------------------- re.c | 23 ++--- vm.c | 17 ---- vm_backtrace.c | 45 ++++++++++ vm_core.h | 6 -- 15 files changed, 117 insertions(+), 344 deletions(-) diff --git a/gc.c b/gc.c index 9e7410a404e264..3c5f9d9e3e1fc9 100644 --- a/gc.c +++ b/gc.c @@ -686,7 +686,6 @@ typedef struct gc_function_map { void (*writebarrier_unprotect)(void *objspace_ptr, VALUE obj); void (*writebarrier_remember)(void *objspace_ptr, VALUE obj); void (*obj_became_shareable)(void *objspace_ptr, VALUE obj); - void (*pin_in_flight_message)(void *objspace_ptr, VALUE obj); // Heap walking void (*each_objects)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); void (*each_objects_shareable)(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); @@ -882,7 +881,6 @@ ruby_modular_gc_init(void) load_modular_gc_func(writebarrier_unprotect); load_modular_gc_func(writebarrier_remember); load_modular_gc_func(obj_became_shareable); - load_modular_gc_func(pin_in_flight_message); // Heap walking load_modular_gc_func(each_objects); load_modular_gc_func(each_objects_shareable); @@ -987,7 +985,6 @@ ruby_modular_gc_init(void) # define rb_gc_impl_writebarrier_unprotect rb_gc_functions.writebarrier_unprotect # define rb_gc_impl_writebarrier_remember rb_gc_functions.writebarrier_remember # define rb_gc_impl_obj_became_shareable rb_gc_functions.obj_became_shareable -# define rb_gc_impl_pin_in_flight_message rb_gc_functions.pin_in_flight_message // Heap walking # define rb_gc_impl_each_objects rb_gc_functions.each_objects # define rb_gc_impl_each_objects_shareable rb_gc_functions.each_objects_shareable @@ -3267,14 +3264,12 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) !rb_gc_impl_multi_objspace_p(); /* Mark the current Ractor's roots from its C structs (a local GC must not depend on - * heap wrapper traversal). A global GC does the same for every Ractor and re-pins - * the in-flight payloads whose shrefs its clear pass dropped. */ + * heap wrapper traversal). A global GC does the same for every Ractor. */ MARK_CHECKPOINT("ractor"); if (global_gc) { rb_ractor_t *r; ccan_list_for_each(&vm->ractor.set, r, vmlr_node) { rb_ractor_mark_local_roots(r); - rb_ractor_repin_in_flight(r); } /* Early in boot (before rb_ractor_main_setup) main is not in vm->ractor.set @@ -3757,14 +3752,6 @@ rb_gc_obj_became_shareable(VALUE obj) /* Pin an in-flight message payload in its owner's (the sender's) objspace, so the * sender's local GC keeps it alive while it sits in a queue the sender does not walk. */ -void -rb_gc_pin_in_flight_message(VALUE obj) -{ - if (RB_SPECIAL_CONST_P(obj)) return; - - rb_gc_impl_pin_in_flight_message(rb_gc_get_objspace(), obj); -} - void rb_gc_copy_attributes(VALUE dest, VALUE obj) { @@ -4954,13 +4941,6 @@ rb_gc_vm_generic_fields_drain_dead(bool (*is_dead)(VALUE key)) rb_generic_fields_tables_foreach(gf_drain_table_cb, &ctx); } -/* A wrapper exported from gc.c so a modular build's gc-impl can call it. */ -bool -rb_gc_current_ractor_materializing_p(void) -{ - return rb_ractor_materializing_p(); -} - VALUE rb_gc_vm_top_self(void) { diff --git a/gc/default/default.c b/gc/default/default.c index cefcf9d508838c..b9ee9f06c5e3e1 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -6423,7 +6423,6 @@ check_children_i(const VALUE child, void *ptr) !MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) && !MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(child), child) && !rb_gc_impl_during_global_gc_p(data->objspace) && - !rb_gc_current_ractor_materializing_p() && !global_objspace->during_absorb) { fprintf(stderr, "check_children_i: containment violation: " "unshareable %s (objspace %p) -> foreign unshareable %s (objspace %p)\n", @@ -6500,10 +6499,6 @@ root_scope_check_i(const char *category, VALUE obj, void *ptr) if (MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(obj), obj)) return; if (MARKED_IN_BITMAP(GET_HEAP_SHREF_BITS(obj), obj)) return; if (obj == rb_gc_vm_top_self()) return; /* VM-permanent (see check_children_i) */ - /* A sender-resident snapshot being materialized by a receive is rooted through - * sync.materializing_copies: a foreign-unshareable root that is valid only while - * the copy runs (see check_children_i). */ - if (rb_gc_current_ractor_materializing_p()) return; fprintf(stderr, "root_scope_check_i: root category \"%s\" names a foreign " "unshareable without a shref record: %s (owner %p, self %p)\n", @@ -7843,26 +7838,6 @@ rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj) } } -void -rb_gc_impl_pin_in_flight_message(void *objspace_ptr, VALUE obj) -{ - if (RB_FL_TEST_RAW(obj, RUBY_FL_SHAREABLE)) return; /* pinned anyway */ - - /* The payload's pages belong to the sender, so a plain store is enough. */ - struct heap_page *page = GET_HEAP_PAGE(obj); - if (!_MARKED_IN_BITMAP(page->shref_bits, page, obj)) { - _MARK_IN_BITMAP(page->shref_bits, page, obj); - page->flags.has_shref_objects = TRUE; - } - /* A shref bit only makes the object a root for the next local GC; it does not affect an - * in-progress global compaction's move decision (pinned_bits). Moving a payload node - * would break the address-keyed maps, dedup tables and pin lists, so pin it as well. */ - rb_objspace_t *objspace = objspace_ptr; - if (objspace->flags.during_global_gc) { - gc_pin(objspace, obj); - } -} - void rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj) { diff --git a/gc/gc.h b/gc/gc.h index 0ff503a222df3e..656f403d8c4d79 100644 --- a/gc/gc.h +++ b/gc/gc.h @@ -90,7 +90,6 @@ MODULAR_GC_FN void rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func ca MODULAR_GC_FN void rb_gc_vm_generic_fields_mark_foreach(int (*cb)(VALUE key, VALUE val, void *arg), void *arg); MODULAR_GC_FN void rb_gc_vm_generic_fields_drain_dead(bool (*is_dead)(VALUE key)); /* Exemptions for the shareable containment verifier (called from a gc-impl). */ -MODULAR_GC_FN bool rb_gc_current_ractor_materializing_p(void); MODULAR_GC_FN VALUE rb_gc_vm_top_self(void); MODULAR_GC_FN void rb_gc_update_object_references(void *objspace, VALUE obj); MODULAR_GC_FN void rb_gc_update_vm_references(void *objspace); diff --git a/gc/gc_impl.h b/gc/gc_impl.h index ef5bc5a5f2894c..76634e62992664 100644 --- a/gc/gc_impl.h +++ b/gc/gc_impl.h @@ -129,7 +129,6 @@ GC_IMPL_FN void rb_gc_impl_writebarrier(void *objspace_ptr, VALUE a, VALUE b); GC_IMPL_FN void rb_gc_impl_writebarrier_unprotect(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj); -GC_IMPL_FN void rb_gc_impl_pin_in_flight_message(void *objspace_ptr, VALUE obj); // Heap walking GC_IMPL_FN void rb_gc_impl_each_objects(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); GC_IMPL_FN void rb_gc_impl_each_objects_shareable(void *objspace_ptr, int (*callback)(void *, void *, size_t, void *), void *data); diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index a7aec6e68338e7..8e0c7a72e88e69 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -1265,12 +1265,6 @@ rb_gc_impl_obj_became_shareable(void *objspace_ptr, VALUE obj) /* MMTk has no per-page shareable bits. */ } -void -rb_gc_impl_pin_in_flight_message(void *objspace_ptr, VALUE obj) -{ - /* With a single objspace there is nothing to pin. */ -} - void rb_gc_impl_writebarrier_remember(void *objspace_ptr, VALUE obj) { diff --git a/internal/gc.h b/internal/gc.h index 23392d36230e66..f79e39ae952133 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -299,7 +299,6 @@ rb_obj_atomic_write( int rb_ec_stack_check(struct rb_execution_context_struct *ec); void rb_gc_writebarrier_remember(VALUE obj); void rb_gc_obj_became_shareable(VALUE obj); -void rb_gc_pin_in_flight_message(VALUE obj); bool rb_gc_multi_objspace_p(void); bool rb_gc_obj_foreign_p(VALUE obj); void *rb_gc_objspace_alloc(void); diff --git a/internal/re.h b/internal/re.h index 0d4bc43ad40acb..c5dfe2341397b5 100644 --- a/internal/re.h +++ b/internal/re.h @@ -66,7 +66,7 @@ VALUE rb_reg_match_p(VALUE re, VALUE str, long pos); VALUE rb_reg_regsub_match(VALUE str, VALUE src, VALUE match); VALUE rb_match_init_copy(VALUE copy, VALUE orig); /* MatchData transfer for the move courier (ractor.c). */ -void *rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out); +void *rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source); VALUE rb_match_move_alloc(VALUE klass, int num_regs); void rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob); void rb_match_move_free(void *blob); diff --git a/internal/vm.h b/internal/vm.h index 1820a4e69f2d2a..560c51d703435d 100644 --- a/internal/vm.h +++ b/internal/vm.h @@ -115,6 +115,9 @@ VALUE rb_make_backtrace(void); void rb_backtrace_print_as_bugreport(FILE*); int rb_backtrace_p(VALUE obj); VALUE rb_backtrace_dup(VALUE btobj); +void *rb_backtrace_blob_dump(VALUE btobj, int *size_out); +VALUE rb_backtrace_blob_load(const void *blob, int size); +void rb_backtrace_blob_mark(const void *blob, int size); VALUE rb_backtrace_to_str_ary(VALUE obj); VALUE rb_backtrace_to_location_ary(VALUE obj); VALUE rb_location_ary_to_backtrace(VALUE ary); diff --git a/ractor.c b/ractor.c index f12f2928fc6c8c..b476570044e287 100644 --- a/ractor.c +++ b/ractor.c @@ -442,10 +442,6 @@ ractor_free(void *ptr) r->registered_marks = NULL; r->registered_marks_cnt = r->registered_marks_capa = 0; - free(r->pin_capture); - r->pin_capture = NULL; - r->pin_capture_cnt = r->pin_capture_capa = 0; - if (!r->main_ractor) { SIZED_FREE(r); } @@ -777,10 +773,6 @@ static void ractor_init(rb_ractor_t *r, VALUE name, VALUE loc) { ractor_sync_init(r); - r->gen_fields_capturing = false; - r->pin_capture = NULL; - r->pin_capture_cnt = r->pin_capture_capa = 0; - r->sending_basket = NULL; st_init_existing_numtable_with_size(&r->pub.targeted_hooks, 0); r->pub.hooks.type = hook_list_type_ractor_local; @@ -2376,6 +2368,7 @@ rb_obj_traverse_replace(VALUE obj, enum move_node_kind { MOVE_KIND_REF, /* an immediate or a shareable object: carried by value */ + MOVE_KIND_BACKTRACE, /* an exception's backtrace: frames copied into an off-heap blob */ MOVE_KIND_STRING, MOVE_KIND_ARRAY, MOVE_KIND_HASH, @@ -2401,6 +2394,7 @@ struct move_node { struct { VALUE klass; } obj; struct { long len; uint32_t *elems; VALUE klass; } strct; /* owns elems */ struct { uint32_t regexp_id, str_id; int num_regs; void *regs; VALUE klass; } match; /* owns regs */ + struct { void *blob; int size; } bt; /* the courier owns blob */ struct { struct rb_io *fptr; /* carried by pointer (it owns the fd) */ VALUE klass; @@ -2751,10 +2745,9 @@ move_capture(struct move_build *b, VALUE obj) case T_MATCH: { /* The regexp and the matched string travel as ordinary children; re.c dumps the * registers (freeing the source's onig and char_offset). */ - VM_ASSERT(!b->copy); /* copy_courier_supported_p rejects it */ VALUE re, st; int nregs; - void *regs = rb_match_move_dump(obj, &re, &st, &nregs); + void *regs = rb_match_move_dump(obj, &re, &st, &nregs, !b->copy); uint32_t rid = move_capture(b, re); uint32_t sid = move_capture(b, st); b->c->nodes[id].kind = MOVE_KIND_MATCH; @@ -2799,6 +2792,18 @@ move_capture(struct move_build *b, VALUE obj) break; } + case T_DATA: + /* Only an exception's backtrace, and only for a copy: move still refuses every + * T_DATA (its source would have to be taken apart). */ + if (b->copy && rb_backtrace_p(obj)) { + int size; + void *blob = rb_backtrace_blob_dump(obj, &size); + b->c->nodes[id].kind = MOVE_KIND_BACKTRACE; + b->c->nodes[id].u.bt.blob = blob; + b->c->nodes[id].u.bt.size = size; + break; + } + /* fall through */ default: rb_raise(rb_eRactorError, "can not move a %"PRIsVALUE" object", rb_class_name(rb_obj_class(obj))); @@ -2940,6 +2945,16 @@ copy_courier_supported_p(VALUE obj, st_table *seen) case T_STRING: case T_OBJECT: break; /* children are ivars only (below) */ + case T_MATCH: { + struct RMatch *rm = RMATCH(obj); + if (!copy_courier_supported_p(rm->regexp, seen)) return false; + if (!copy_courier_supported_p(rm->str, seen)) return false; + break; + } + case T_DATA: + /* An exception's backtrace is the one T_DATA the courier carries. */ + if (!rb_backtrace_p(obj)) return false; + break; case T_ARRAY: for (long i = 0; i < RARRAY_LEN(obj); i++) { if (!copy_courier_supported_p(RARRAY_AREF(obj, i), seen)) return false; @@ -3100,6 +3115,9 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) shell = rb_match_move_alloc(rb_class_real(n->u.match.klass), n->u.match.num_regs); move_apply_moved_klass(shell, n->u.match.klass); break; + case MOVE_KIND_BACKTRACE: + shell = rb_backtrace_blob_load(n->u.bt.blob, n->u.bt.size); + break; case MOVE_KIND_IO: shell = rb_obj_alloc(rb_class_real(n->u.io.klass)); move_apply_moved_klass(shell, n->u.io.klass); @@ -3214,6 +3232,9 @@ rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) case MOVE_KIND_MATCH: rb_match_move_free(n->u.match.regs); break; + case MOVE_KIND_BACKTRACE: + ruby_xfree(n->u.bt.blob); + break; case MOVE_KIND_IO: /* A delivered IO left fptr == NULL (the rebuilt IO owns it). An * undelivered one still owns the fd and its source is already a @@ -3263,6 +3284,9 @@ rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c) else if (n->kind == MOVE_KIND_STRING) { rb_gc_mark(n->u.str.klass); } + else if (n->kind == MOVE_KIND_BACKTRACE) { + rb_backtrace_blob_mark(n->u.bt.blob, n->u.bt.size); + } else if (n->kind == MOVE_KIND_ARRAY) { rb_gc_mark(n->u.ary.klass); } @@ -3337,21 +3361,6 @@ ractor_native_shallow_copy(VALUE obj) return copy; } -/* Add a node of the snapshot under construction to the pin list and pin it now. */ -static void -ractor_pin_capture_push(rb_ractor_t *cr, VALUE v) -{ - if (cr->pin_capture_cnt == cr->pin_capture_capa) { - size_t nc = cr->pin_capture_capa ? cr->pin_capture_capa * 2 : 16; - VALUE *p = realloc(cr->pin_capture, nc * sizeof(VALUE)); - if (!p) rb_bug("ractor_pin_capture_push: out of memory"); - cr->pin_capture = p; - cr->pin_capture_capa = nc; - } - cr->pin_capture[cr->pin_capture_cnt++] = v; - rb_gc_pin_in_flight_message(v); -} - static enum obj_traverse_iterator_result copy_enter(VALUE obj, struct obj_traverse_replace_data *data) { @@ -3363,17 +3372,6 @@ copy_enter(VALUE obj, struct obj_traverse_replace_data *data) VALUE copy = ractor_native_shallow_copy(obj); if (UNDEF_P(copy)) return traverse_stop; /* no native copy for this type */ data->replacement = copy; - /* Collect every node into the pin list as the snapshot is built: the global - * GC's re-pin must cover all nodes, not just the root (moving one breaks the - * address-keyed dedup table). fields_obj is not included: the global - * generic_fields table reaches it and compaction updates that. */ - rb_ractor_t *cr = GET_RACTOR(); - if (cr->gen_fields_capturing) { - /* Pin from birth (shref bit, plus the pin bit during a global compaction). - * rb_ractor_repin_in_flight re-pins via cr->pin_capture, so the cover runs - * unbroken from construction through enqueue to materialization. */ - ractor_pin_capture_push(cr, copy); - } return traverse_cont; } } diff --git a/ractor_core.h b/ractor_core.h index 6ab440a680969c..8003899e96a627 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -46,23 +46,10 @@ struct rb_ractor_sync { VALUE legacy; bool legacy_exc; bool legacy_taken; /* Ractor#value already returned the value */ - - /* Number of receives currently materializing a copy (only the owner's threads - * update it, under the GVL). */ - int materializing_copies; }; struct ractor_basket; -/* One in-flight copy payload being rebuilt (lives on the receiver's machine - * stack) */ -struct ractor_materialize_frame { - VALUE snapshot; /* the sender-side snapshot */ - const VALUE *pinned; /* pin list of every snapshot node (owned by the basket) */ - size_t pinned_cnt; - struct ractor_materialize_frame *prev; -}; - // created // | ready to run // ====================== inserted to vm->ractor @@ -162,28 +149,12 @@ struct rb_ractor_struct { * still enumerates it. */ void *creating_child_objspace; - /* True while Ractor#send builds a native copy snapshot; copy_enter then collects - * every snapshot node into pin_capture below. Owner thread only. */ - bool gen_fields_capturing; - - /* Pin list collecting every node while a copy snapshot is built (basket_new - * hands it over to the basket). A global GC clears every shref, so the re-pin - * has to cover all nodes, not just the root. */ - VALUE *pin_capture; - size_t pin_capture_cnt, pin_capture_capa; - /* The in-flight copy basket between basket_new and the enqueue, so the re-pin - * covers that window too */ - struct ractor_basket *sending_basket; }; // rb_ractor_t is defined in vm_core.h /* Mark the GC roots held in Ractor r's C structs (from the root scan in gc.c). */ void rb_ractor_mark_local_roots(rb_ractor_t *r); void rb_ractor_mark_terminated_join_value(rb_ractor_t *r); -void rb_ractor_repin_in_flight(rb_ractor_t *r); void rb_ractor_mark_in_flight_for_single_objspace(rb_ractor_t *r); -/* True while the current Ractor is materializing an arriving copy (see the - * definition in ractor_sync.c). */ -bool rb_ractor_materializing_p(void); /* Move src's registered_marks to dst and leave src empty (on join or when an orphan * is absorbed). An absorb can run during a GC sweep, so the implementation uses raw diff --git a/ractor_sync.c b/ractor_sync.c index 62a341ed1123a4..cacd6fe455006f 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -238,11 +238,6 @@ struct ractor_basket { * in-flight pin, so it never keeps a page of the sender's heap alive. */ char *mbuf; size_t mlen; - /* Every node of a native copy snapshot, collected while building it (raw - * malloc). The global GC's re-pin walks this list, since traversing the graph - * in-GC would need generic-ivar lookups. NULL: only the root (p.v) is pinned. */ - VALUE *pinned; - size_t pinned_cnt; } p; // payload struct ccan_list_node node; @@ -276,15 +271,6 @@ ractor_basket_mark(const struct ractor_basket *b) static void ractor_basket_free(struct ractor_basket *b) { - /* A basket that dies before being enqueued clears the sender's re-pin slot; a - * free by the Ractor tearing the queue down does not match and is a no-op. */ - rb_ractor_t *cr = rb_current_ractor_raw(false); - if (cr != NULL && cr->sending_basket == b) { - cr->sending_basket = NULL; - } - free(b->p.pinned); - b->p.pinned = NULL; - b->p.pinned_cnt = 0; ruby_xfree(b->p.mbuf); b->p.mbuf = NULL; b->p.mlen = 0; @@ -782,57 +768,6 @@ ractor_sync_mark(rb_ractor_t *r) } } -/* Re-pin a copy basket's payload: the root and every collected node. */ -static void -ractor_basket_repin_in_flight(const struct ractor_basket *b) -{ - if (b->type != basket_type_copy || b->p.mbuf != NULL || b->p.move_courier != NULL) return; - rb_gc_pin_in_flight_message(b->p.v); - for (size_t i = 0; i < b->p.pinned_cnt; i++) { - rb_gc_pin_in_flight_message(b->p.pinned[i]); - } -} - -static void -ractor_queue_repin_in_flight(const struct ractor_queue *rq) -{ - const struct ractor_basket *b; - ccan_list_for_each(&rq->set, b, node) { - /* A move basket carries an off-heap courier, so it has no shref to re-pin; - * ractor_basket_mark marks the shareable VALUEs it carries instead. */ - ractor_basket_repin_in_flight(b); - } -} - -static int -ractor_repin_ports_i(st_data_t key, st_data_t val, st_data_t data) -{ - ractor_queue_repin_in_flight((struct ractor_queue *)val); - return ST_CONTINUE; -} - -/* A global GC clears every shref bit, so all in-flight payloads have to be re-pinned - * before the unified mark. Runs on the driver, under the barrier. */ -void -rb_ractor_repin_in_flight(rb_ractor_t *r) -{ - if (r->sync.ports) { - ractor_queue_repin_in_flight(r->sync.recv_queue); - st_foreach(r->sync.ports, ractor_repin_ports_i, 0); - } - /* Baskets already built but not enqueued yet (in flight on the send path). */ - if (r->sending_basket != NULL) { - ractor_basket_repin_in_flight(r->sending_basket); - } - /* A snapshot still being built (from prepare_payload's walk until it moves into - * the basket). */ - for (size_t i = 0; i < r->pin_capture_cnt; i++) { - rb_gc_pin_in_flight_message(r->pin_capture[i]); - } - /* Snapshots being materialized are re-pinned from the EC frame chains instead - * (rb_execution_context_mark, which also covers a suspended fiber's EC). */ -} - /* A single-objspace impl (mmtk) has no pin or shref bits and no zombie_objspaces, so * plain marking from the wrapper keeps these alive; the default GC covers the same set * with its pins and its zombie scan. */ @@ -840,12 +775,6 @@ void rb_ractor_mark_in_flight_for_single_objspace(rb_ractor_t *r) { rb_gc_mark(r->sync.legacy); - if (r->sending_basket != NULL) { - ractor_basket_mark(r->sending_basket); - } - for (size_t i = 0; i < r->pin_capture_cnt; i++) { - rb_gc_mark(r->pin_capture[i]); - } } static int @@ -909,7 +838,6 @@ ractor_sync_init(rb_ractor_t *r) r->sync.legacy = Qundef; // no receive is rebuilding a payload yet - r->sync.materializing_copies = 0; #ifndef RUBY_THREAD_PTHREAD_H rb_native_cond_initialize(&r->sync.wakeup_cond); @@ -1039,44 +967,19 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket } else { /* Snapshot the object on the sender side without calling the user-visible - * #clone: core types are deep-copied natively and anything else is - * marshaled here, so its user hooks run on the sender. */ + * #clone. Both forms are off-heap, so an in-flight payload is never a GC + * object and needs no pin: nothing of the sender's heap stays alive while + * the message waits (design_v2.md 4.5). The courier carries the core + * types; anything else is marshaled here, so its user hooks run on the + * sender, and the dump travels as plain bytes. */ *ptype = basket_type_copy; - /* An off-heap courier first: an in-flight payload that is not a GC object - * needs no pin, so nothing of the sender's heap stays alive while the - * message waits (design_v2.md 4.5). NULL means the graph holds a type only - * the on-heap snapshot path below handles. */ *pcourier = rb_ractor_copy_courier_build(obj); if (*pcourier != NULL) return Qundef; - /* During a native copy, copy_enter collects every snapshot node into the - * pin list that covers construction, enqueue and materialization. */ - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - VM_ASSERT(!cr->gen_fields_capturing); - cr->gen_fields_capturing = true; - VALUE snapshot = Qundef; - /* A native copy can raise (allocation, async interrupt). Leaving the - * capturing flag set would fail the next send's assert and leak a stale - * pin_capture list into that basket. */ - enum ruby_tag_type state; - EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG()) == TAG_NONE) { - snapshot = ractor_copy_native_try(obj); - } - EC_POP_TAG(); - cr->gen_fields_capturing = false; - if (state != TAG_NONE) { - cr->pin_capture_cnt = 0; - EC_JUMP_TAG(ec, state); - } - if (UNDEF_P(snapshot)) { - cr->pin_capture_cnt = 0; - snapshot = rb_rescue2(ractor_marshal_dump_body, obj, - ractor_marshal_dump_rescue, obj, - rb_eTypeError, (VALUE)0); - *pmarshaled = true; - } - return snapshot; + *pmarshaled = true; + return rb_rescue2(ractor_marshal_dump_body, obj, + ractor_marshal_dump_rescue, obj, + rb_eTypeError, (VALUE)0); } } } @@ -1130,18 +1033,9 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type if (state != TAG_NONE) { ruby_xfree(mbuf); if (courier != NULL) rb_ractor_move_courier_free(courier); - /* Drop the pin list, or every global GC re-pins the dead snapshot from it - * forever (rb_ractor_repin_in_flight walks it unconditionally). The nodes - * stay shref-pinned only until the next global GC clears the bits. */ - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - free(cr->pin_capture); - cr->pin_capture = NULL; - cr->pin_capture_cnt = cr->pin_capture_capa = 0; EC_JUMP_TAG(ec, state); } - /* The dump is off-heap now, so the sender's copy of it is ordinary garbage: - * nothing to pin. A native snapshot still lives in the sender's objspace and - * is pinned through cr->pin_capture below. */ + /* The payload is off-heap, so the sender's dump is ordinary garbage now. */ if (mbuf != NULL) { v = Qundef; } @@ -1154,37 +1048,9 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type b->p.move_courier = courier; b->p.mbuf = mbuf; b->p.mlen = mlen; - b->p.pinned = NULL; - b->p.pinned_cnt = 0; - if (type == basket_type_copy) { - /* Hand the pin list to the basket, moving the re-pin cover from - * cr->pin_capture to cr->sending_basket with no safepoint in between. */ - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - b->p.pinned = cr->pin_capture; - b->p.pinned_cnt = cr->pin_capture_cnt; - VM_ASSERT(cr->sending_basket == NULL); - cr->sending_basket = b; - cr->pin_capture = NULL; - cr->pin_capture_cnt = cr->pin_capture_capa = 0; - } return b; } -/* True while this Ractor materializes an arriving copy: the half-built result - * legitimately points at the sender-resident (pinned) snapshot, so a local GC's - * verifier must not report containment violations, and the copy's own allocations can - * start that GC. */ -bool -rb_ractor_materializing_p(void) -{ - const rb_ractor_t *cr = rb_current_ractor_raw(false); - if (cr == NULL) return false; - /* Only a COPY materialization sets this: move shells reference other shells in - * this objspace, never the sender's graph. The count is per Ractor, so a fiber - * switch keeps it exact. */ - return cr->sync.materializing_copies > 0; -} - static VALUE ractor_basket_value(struct ractor_basket *b) { @@ -1195,48 +1061,23 @@ ractor_basket_value(struct ractor_basket *b) /* An off-heap copy courier rebuilds exactly like a move one; only the sources * differ (still alive here, already shells there). */ if (b->p.move_courier != NULL) goto materialize_courier; - /* Materialize the sender's snapshot into the receiving Ractor's objspace. - * Passing the sender-resident graph by reference would create an unshareable - * cross-objspace edge that neither local GC can follow. The snapshot stays - * pinned in the sender's objspace and becomes garbage there once this copy - * finishes. Marshal.load allocates through this Ractor's normal newobj and - * write-barrier paths. - * - * Rebuilding can raise (marshal load hooks and autoload run user code and an - * async interrupt can arrive anywhere), and those hooks can run a nested - * Ractor.receive. The frame is pushed on the machine stack and popped under a - * TAG, so the chain never leaks a dead materialization or drops an outer one. */ + /* The payload is the marshaled bytes. Marshal.load allocates through this + * Ractor's normal newobj and write-barrier paths, and can raise (load hooks and + * autoload run user code, an async interrupt can arrive anywhere), so it runs + * under a TAG. */ rb_execution_context_t *ec = rb_current_ec_noinline(); - rb_ractor_t *cr = rb_ec_ractor_ptr(ec); - struct ractor_materialize_frame frame = { - .snapshot = b->p.v, .pinned = b->p.pinned, .pinned_cnt = b->p.pinned_cnt, - .prev = ec->materialize_frames, - }; - ec->materialize_frames = &frame; - cr->sync.materializing_copies++; VALUE result = Qundef; enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - if (b->p.mbuf != NULL) { - /* Rebuild the marshaled bytes in this Ractor's objspace. Marshal does - * not mark its source (mark_load_arg) and the basket is off the queue, - * so this frame's stack slot is the String's only root for the load. */ - VALUE bin = rb_str_new(b->p.mbuf, (long)b->p.mlen); - result = rb_marshal_load(bin); - RB_GC_GUARD(bin); - } - else if (b->p.marshaled) { - result = rb_marshal_load(b->p.v); - } - else { - result = ractor_copy_native_try(b->p.v); - if (UNDEF_P(result)) rb_bug("ractor_basket_value: native snapshot not natively copyable"); - } + /* Rebuild the byte string in this Ractor's objspace. Marshal does not mark + * its source (mark_load_arg) and the basket is off the queue, so this + * frame's stack slot is the String's only root for the load. */ + VALUE bin = rb_str_new(b->p.mbuf, (long)b->p.mlen); + result = rb_marshal_load(bin); + RB_GC_GUARD(bin); } EC_POP_TAG(); - ec->materialize_frames = frame.prev; - cr->sync.materializing_copies--; /* rb_copy_generic_ivar left the sender-resident snapshot host and fields_obj in * this EC's gen_fields_cache; the snapshot is garbage on the sender now, and a * stale cache hit on a reused address would deref a freed foreign fields_obj. @@ -1643,15 +1484,6 @@ ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, str else { b->port_id = ractor_port_id(rp); ractor_queue_enq(rp->r, rp->r->sync.recv_queue, b); - /* From basket_new to the enqueue the sender's sending_basket slot covers - * the re-pin; from here the queue walk does, so drop the slot (no safepoint - * or malloc-triggered GC inside the lock, so the cover never lapses). */ - if (b->type == basket_type_copy) { - rb_ractor_t *scr = rb_current_ractor_raw(false); - if (scr != NULL && scr->sending_basket == b) { - scr->sending_basket = NULL; - } - } } } RACTOR_UNLOCK(rp->r); @@ -1687,8 +1519,6 @@ ractor_basket_new_ref(VALUE shareable) b->p.move_courier = NULL; b->p.mbuf = NULL; b->p.mlen = 0; - b->p.pinned = NULL; - b->p.pinned_cnt = 0; return b; } diff --git a/re.c b/re.c index 332b850d10287c..35d675f71aa117 100644 --- a/re.c +++ b/re.c @@ -1084,7 +1084,7 @@ match_set_regs(VALUE match, int num_regs, const OnigPosition *beg, const OnigPos * registers are written out to an onig-independent blob so the original malloc'd area can be * freed, leaving an empty shell behind, and rebuilt from the blob on the receiving side. */ void * -rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out) +rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source) { struct RMatch *rm = RMATCH(match); int n = rm->num_regs; @@ -1100,15 +1100,18 @@ rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs blob[2 * i + 1] = end[i]; } - if (FL_TEST_RAW(match, RMATCH_ONIG)) { - onig_region_free(&rm->as.onig, 0); - memset(&rm->as.onig, 0, sizeof(rm->as.onig)); - FL_UNSET_RAW(match, RMATCH_ONIG); - } - if (rm->char_offset) { - ruby_xfree(rm->char_offset); - rm->char_offset = NULL; - rm->char_offset_num_allocated = 0; + /* A copy leaves the source usable; only a move takes its internals apart. */ + if (release_source) { + if (FL_TEST_RAW(match, RMATCH_ONIG)) { + onig_region_free(&rm->as.onig, 0); + memset(&rm->as.onig, 0, sizeof(rm->as.onig)); + FL_UNSET_RAW(match, RMATCH_ONIG); + } + if (rm->char_offset) { + ruby_xfree(rm->char_offset); + rm->char_offset = NULL; + rm->char_offset_num_allocated = 0; + } } return blob; } diff --git a/vm.c b/vm.c index ff1ec24b93138e..16146c3108def3 100644 --- a/vm.c +++ b/vm.c @@ -3905,23 +3905,6 @@ rb_execution_context_mark(const rb_execution_context_t *ec) rb_gc_mark(ec->local_storage_recursive_hash_for_trace); rb_gc_mark(ec->private_const_reference); - /* Snapshots of copy receives being materialized; off the queue, this is their only - * root. A snapshot is sender-resident, skipped as foreign by our local GC; the - * global GC marks it and re-pins its shrefs (its clear pass dropped all). Move - * couriers are covered by the in-flight registry instead (ractor.c). */ - for (const struct ractor_materialize_frame *f = ec->materialize_frames; f != NULL; f = f->prev) { - rb_gc_mark(f->snapshot); - if (f->snapshot && !RB_SPECIAL_CONST_P(f->snapshot) && rb_gc_during_global_gc_p()) { - /* Every node, not just the root: if compaction moved a snapshot node, - * the address-keyed generic_fields entries and the dedup table would - * break. */ - rb_gc_pin_in_flight_message(f->snapshot); - for (size_t i = 0; i < f->pinned_cnt; i++) { - rb_gc_pin_in_flight_message(f->pinned[i]); - } - } - } - rb_gc_mark_movable(ec->storage); } diff --git a/vm_backtrace.c b/vm_backtrace.c index 573d671de1a88a..1c7ff0dad5eef4 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -867,6 +867,51 @@ rb_backtrace_dup(VALUE btobj) } +/* Copy a backtrace's frames into an off-heap blob for a Ractor copy courier. A frame + * only references shareable iseq / method-entry imemos, so the blob can carry them as + * they are. It has no compaction update hook, so rb_backtrace_blob_mark pins them + * (rb_gc_mark, not _movable) for as long as the message is in flight. */ +void * +rb_backtrace_blob_dump(VALUE btobj, int *size_out) +{ + rb_backtrace_t *bt; + TypedData_Get_Struct(btobj, rb_backtrace_t, &backtrace_data_type, bt); + + int size = bt->backtrace_size; + *size_out = size; + rb_backtrace_location_t *blob = ALLOC_N(rb_backtrace_location_t, size > 0 ? size : 1); + MEMCPY(blob, bt->backtrace, rb_backtrace_location_t, size); + return blob; +} + +VALUE +rb_backtrace_blob_load(const void *blob_, int size) +{ + const rb_backtrace_location_t *blob = blob_; + rb_backtrace_t *dst; + VALUE btobj = backtrace_alloc_capa(size, &dst); + + dst->backtrace_size = size; + MEMCPY(dst->backtrace, blob, rb_backtrace_location_t, size); + for (int i = 0; i < size; i++) { + const rb_backtrace_location_t *fi = &dst->backtrace[i]; + if (fi->cme) RB_OBJ_WRITTEN(btobj, Qundef, (VALUE)fi->cme); + if (fi->iseq) RB_OBJ_WRITTEN(btobj, Qundef, (VALUE)fi->iseq); + } + /* strary / locary stay unset: the receiver rebuilds them lazily. */ + return btobj; +} + +void +rb_backtrace_blob_mark(const void *blob_, int size) +{ + const rb_backtrace_location_t *blob = blob_; + for (int i = 0; i < size; i++) { + if (blob[i].cme) rb_gc_mark((VALUE)blob[i].cme); + if (blob[i].iseq) rb_gc_mark((VALUE)blob[i].iseq); + } +} + static long backtrace_size(const rb_execution_context_t *ec) { diff --git a/vm_core.h b/vm_core.h index 47c1382b26135f..482a63c6d27605 100644 --- a/vm_core.h +++ b/vm_core.h @@ -1125,7 +1125,6 @@ struct rb_waiting_list { struct rb_fiber_struct *fiber; }; -struct ractor_materialize_frame; struct rb_execution_context_struct { /* execution information */ @@ -1178,11 +1177,6 @@ struct rb_execution_context_struct { VALUE fields_obj; } gen_fields_cache; - /* Chain of receive frames being materialized on this EC (LIFO; the frames live - * on the C stack). A thread or fiber switch cannot corrupt it, since each EC's - * chain only contains that EC's own nesting. */ - struct ractor_materialize_frame *materialize_frames; - /* for GC */ struct { VALUE *stack_start; From 28f119c96f4cb834d4ac21441deb6fb117f10676 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Wed, 19 Aug 2026 19:55:37 +0000 Subject: [PATCH 05/14] Ractor: hand the courier's buffer to the rebuilt String Materializing a string node called rb_enc_str_new, which copies the bytes a second time: once into the courier when the node was built, once out of it when the message arrives. Give the buffer to the String instead. rb_str_new_owned takes an xmalloc'd buffer as a heap String's body, the way rb_str_new_static does for a static one, except the String owns it and frees it like any other. The courier's node hands its pointer over and forgets it, so courier_free no longer frees what the String now owns. Message cost, 20k messages per trial, 9 trials, median us per message: payload 4.0.2 master before here 1KB 2.97 4.11 3.4 3.10 4KB 3.52 8.75 9.4 4.22 8KB 4.26 12.39 12.0 5.08 16KB 5.43 16.24 16.5 6.53 master pulls away from 4.0.2 as the payload grows; this tracks it within about 20% across the range. Small strings are embedded and never had a buffer, so they are unchanged. A heap String is freed by its size (STR_HEAP_SIZE = capa + terminator), so the node carries the capacity of the allocation rather than the bytes in use, and reserves the encoding's terminator length instead of a single NUL. Co-Authored-By: Claude Opus 5 (1M context) --- internal/string.h | 1 + ractor.c | 19 +++++++++++++++---- string.c | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/internal/string.h b/internal/string.h index a8893a42f1e10a..67216f574879de 100644 --- a/internal/string.h +++ b/internal/string.h @@ -124,6 +124,7 @@ bool rb_obj_is_fstring_table(VALUE obj); void Init_fstring_table(); VALUE rb_obj_as_string_result(VALUE str, VALUE obj); VALUE rb_str_opt_plus(VALUE x, VALUE y); +VALUE rb_str_new_owned(char *ptr, long len, long capa, int encindex); VALUE rb_str_concat_literals(size_t num, const VALUE *strary); VALUE rb_str_eql(VALUE str1, VALUE str2); VALUE rb_id_quote_unprintable(ID); diff --git a/ractor.c b/ractor.c index b476570044e287..8e3b1e4d6a5dcd 100644 --- a/ractor.c +++ b/ractor.c @@ -2388,7 +2388,7 @@ struct move_node { uint32_t *iv_vals; /* owned by the courier; node ids */ union { VALUE ref; - struct { char *ptr; long len; int encidx; VALUE klass; } str; /* the courier owns ptr */ + struct { char *ptr; long len, capa; int encidx; VALUE klass; } str; /* the courier owns ptr */ struct { long len; uint32_t *elems; VALUE klass; } ary; /* the courier owns elems */ struct { long size; uint32_t *kv; uint32_t ifnone_id; bool compare_by_id; bool proc_default; VALUE klass; } hash; /* owns kv (2*size) */ struct { VALUE klass; } obj; @@ -2658,24 +2658,31 @@ move_capture(struct move_build *b, VALUE obj) if (!b->copy) rb_str_make_independent(obj); long len = RSTRING_LEN(obj); int encidx = ENCODING_GET(obj); + /* The receiver adopts this buffer as a String body, which is freed by size: + * capa has to describe the allocation exactly (capa + terminator bytes). */ + const int termlen = rb_enc_mbminlen(rb_enc_from_index(encidx)); char *ptr; + long capa; if (!b->copy && !STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) { /* Owns a private heap buffer: carry the pointer over (zero-copy) and leave * the source as a shell that does not free it. */ ptr = RSTRING(obj)->as.heap.ptr; + capa = RSTRING(obj)->as.heap.aux.capa; } else { /* Embedded or a shared root: copy the bytes into a courier-owned buffer. * Taking a root's buffer would dangle its copy-on-write children, so leave * it (the same reason T_ARRAY excludes ARY_SHARED_ROOT_P below). */ - ptr = ALLOC_N(char, len + 1); + ptr = ALLOC_N(char, len + termlen); if (len) memcpy(ptr, RSTRING_PTR(obj), len); - ptr[len] = '\0'; + memset(ptr + len, 0, termlen); + capa = len; } b->c->nodes[id].kind = MOVE_KIND_STRING; b->c->nodes[id].u.str.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.str.ptr = ptr; b->c->nodes[id].u.str.len = len; + b->c->nodes[id].u.str.capa = capa; b->c->nodes[id].u.str.encidx = encidx; break; } @@ -3090,7 +3097,11 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) shell = n->u.ref; break; case MOVE_KIND_STRING: - shell = rb_enc_str_new(n->u.str.ptr, n->u.str.len, rb_enc_from_index(n->u.str.encidx)); + /* Hand the courier's buffer to the String instead of copying it again: the + * bytes were already copied (or taken from the source) when the node was + * built. */ + shell = rb_str_new_owned(n->u.str.ptr, n->u.str.len, n->u.str.capa, n->u.str.encidx); + n->u.str.ptr = NULL; /* consumed: the new String owns it now */ move_apply_moved_klass(shell, n->u.str.klass); break; case MOVE_KIND_ARRAY: diff --git a/string.c b/string.c index 5481705cf8b3b4..09384e4470b1c7 100644 --- a/string.c +++ b/string.c @@ -1210,6 +1210,23 @@ rb_str_new_static(const char *ptr, long len) return str_new_static(rb_cString, ptr, len, 0); } +/* Take an xmalloc'd buffer as the String's body without copying it; the String owns it + * from here and frees it like any other heap string. ptr must hold capa bytes plus the + * terminator for encindex, which is what a Ractor courier's string node carries. */ +VALUE +rb_str_new_owned(char *ptr, long len, long capa, int encindex) +{ + RUBY_DTRACE_CREATE_HOOK(STRING, len); + VALUE str = str_alloc_heap(rb_cString); + RSTRING(str)->len = len; + RSTRING(str)->as.heap.ptr = ptr; + /* Freed by size (STR_HEAP_SIZE = capa + terminator), so capa must describe the + * allocation the caller made, not just the bytes in use. */ + RSTRING(str)->as.heap.aux.capa = capa; + rb_enc_associate_index(str, encindex); + return str; +} + VALUE rb_usascii_str_new_static(const char *ptr, long len) { From 4fa95746e41978d2cfb950286bfbd33ebda54e49 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 01:27:15 +0000 Subject: [PATCH 06/14] Ractor: root a courier from the basket that holds it A basket is rooted by whoever holds it, and nothing else: the port queue it waits on, or -- while it is on no queue -- its holder's off_queue_baskets list. rb_ractor_sync grows that list and the basket a second ccan_list_node, so it can sit on one or the other. send alloc the basket, put it on the sender's list, then build the payload into it: rb_ractor_{move,copy}_courier_build publish the courier into the basket right after ZALLOC, so even a half-built one is rooted. A move needs this -- it husks the sources as it captures them, so what the courier has collected loses its other root as the walk proceeds. enqueue off the sender's list, onto the receiver's queue receive off the queue, onto the receiver's list until the basket is freed The publish is the handover of ownership too: when the capture raises, the builder re-raises and leaves the courier where it is, and the basket frees it exactly once as ractor_basket_new unwinds. (The raise path was pointed out by the Copilot review on ruby/ruby#18392.) Before this, a courier was the one payload ractor_basket_mark skipped: it left the whole job to vm->ractor.move_courier_registry, which only a global GC walks. Between the send and the next global GC nothing rooted the payload, and a shareable object reachable from the courier alone was collected -- a dynamic Symbol loses its fstr, a frozen Array its elements, and reading it crashes. A shareable String survived only by being an fstring, rooted elsewhere. That is already broken in master for move; it becomes reachable from an ordinary send once copy travels by courier. Two reproducers, both crashing unpatched master and fine on 4.0.2, now pass: ports = 5.times.map do |i| port = Ractor::Port.new r = Ractor.new(port, i) { |p, n| p.send([:"sym_#{n}"], move: true) } r.join port end 5.times { GC.start } ports.each { |p| puts p.receive[0].inspect } (the symbol has to be interpolated: a literal one is held by the iseq) The second sends Ractor.make_shareable([1, "child"]) the same way. ractor_sync_mark walks the list next to the queues it already walks, so this needs no new root pass, and it walks both in every collection. "What a courier holds is shareable, and only a global GC frees a shareable" does not hold: pinned_roots_mark, which roots a shareable from its page bit, is skipped once the process is back to a single Ractor (rb_gc_single_objspace_p), and then an ordinary local GC frees one that nothing else names. A payload in flight is named by its basket and nothing else. The list needs no sync lock even so -- only its owner touches it, unlike the queues a foreign sender writes. The one caller that does skip it is the single-objspace root pass, where ractor_mark_unshareable_parts has just walked the same baskets through ractor_sync_mark. This retires the registry, its lock, its fork re-initialization and rb_ractor_move_courier_registry_mark. Walking the registry in every collection instead of the basket would not do: a receiver's local GC, which materialize's own allocation can trigger, would then walk a courier another Ractor is inside move_alloc_ref for -- observed as a crash. move_alloc_node and move_alloc_ref grow by swapping in a fresh array rather than realloc, for the same reason the nodes are initialized mark-safe: the courier is a GC root while it is built, and a realloc can leave c->nodes pointing at a block it has already freed. Co-Authored-By: Claude Opus 5 (1M context) --- gc.c | 9 --- gc/default/default.c | 9 ++- ractor.c | 108 +++++++++++-------------------- ractor_core.h | 5 ++ ractor_sync.c | 151 ++++++++++++++++++++++++++++--------------- vm.c | 2 - vm_core.h | 7 +- 7 files changed, 149 insertions(+), 142 deletions(-) diff --git a/gc.c b/gc.c index 3c5f9d9e3e1fc9..f3731acd5702f8 100644 --- a/gc.c +++ b/gc.c @@ -3341,15 +3341,6 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) if (vm_mark_needs_lock) vm_mark_lock_lev = RB_GC_VM_LOCK_NO_BARRIER(); rb_vm_mark(vm); - if (global_gc) { - /* Mark and pin the shareable REFs of in-flight (off-heap) move couriers, - * covering the transient window between queue and materialize frame. Only - * a global GC frees shareable objects, so only it needs this pass. */ - MARK_CHECKPOINT("move_couriers"); - void rb_ractor_move_courier_registry_mark(void); - rb_ractor_move_courier_registry_mark(); - } - MARK_CHECKPOINT("global_tbl"); rb_gc_mark_global_tbl(); diff --git a/gc/default/default.c b/gc/default/default.c index b9ee9f06c5e3e1..a0e2cfd0a64206 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -6415,9 +6415,8 @@ check_children_i(const VALUE child, void *ptr) * unshareable parent holding an unrecorded foreign unshareable child would be * invisible to both local GCs. The exception is a box's top_self, which every * thread's th->top_self points at and which is VM-permanent. Skipped during a - * global GC: it clears every shref bit and keeps in-flight payloads alive by - * re-pinning, so the shref exemption would not fire, and its unified exact - * stop-the-world mark makes the invariant itself moot. */ + * global GC: it clears every shref bit, so the shref exemption would not fire, + * and its unified exact stop-the-world mark makes the invariant itself moot. */ if (!data->parent_shareable && child != rb_gc_vm_top_self() && !MARKED_IN_BITMAP(GET_HEAP_SHAREABLE_BITS(child), child) && @@ -9044,8 +9043,8 @@ gc_start_global(rb_objspace_t *driver, unsigned int reason, bool compact, bool a } } - /* steps 6-7: every Ractor's roots (gc.c walks them all and re-pins in-flight payloads), - * then one unified precise mark. A global GC does not go through gc_marks, so the marking + /* steps 6-7: every Ractor's roots (gc.c walks them all), then one unified precise + * mark. A global GC does not go through gc_marks, so the marking * phase is opened here instead; it closes after rb_ractor_finish_marking below, which is * where gc_marks_finish ends for a local collection. */ gc_marking_enter(driver); diff --git a/ractor.c b/ractor.c index 8e3b1e4d6a5dcd..b74f2af0352ca4 100644 --- a/ractor.c +++ b/ractor.c @@ -723,10 +723,6 @@ rb_ractor_atfork(rb_vm_t *vm, rb_thread_t *th) // initialize as a main ractor vm->ractor.cnt = 0; vm->ractor.blocking_cnt = 0; - /* Another thread may have held the lock at fork, so rebuild it in the child (the - * same reason generic_fields_lock is re-initialized at fork). The registry's list - * head is left alone: the nodes of surviving couriers are still linked into it. */ - rb_native_mutex_initialize(&vm->ractor.move_courier_registry_lock); /* Only main survives a fork: the holds of dead Ractors and of critical sections are * gone, leaving main's own disable. */ rb_gc_disable_holders_atfork(); @@ -2408,7 +2404,7 @@ struct move_node { /* A child slot holds a node id, or -- with this bit set -- an index into c->refs. * The courier is in-process, so a shareable payload can travel as the VALUE itself - * instead of costing a whole move_node; the registry marks and pins c->refs. */ + * instead of costing a whole move_node; the basket holding the courier marks c->refs. */ #define MOVE_ID_REF_BIT 0x80000000u struct rb_ractor_move_courier { @@ -2419,46 +2415,8 @@ struct rb_ractor_move_courier { uint32_t refs_count; uint32_t refs_capa; uint32_t root; - struct ccan_list_node reg_node; /* in-flight courier registry (a GC root while it lives) */ }; -/* VM-global list of move couriers in flight (vm->ractor.move_courier_registry). A - * courier is off-heap and carries shareable REFs as raw pointers; in some windows only - * a transient (a stack-local message queue, say) reaches it, so a global GC could - * collect the REFs. Registered from build to free, marked and pinned by the global - * GC's root pass (only a global GC frees shareable objects, so only it needs this). - * add/remove run concurrently and take the lock; stop-the-world marking does not, which - * is sound only because add/remove contain no safepoint (none may be added: a mark - * could then see a half-linked list across the barrier). */ - -static void -move_courier_registry_add(struct rb_ractor_move_courier *c) -{ - rb_native_mutex_lock(&GET_VM()->ractor.move_courier_registry_lock); - ccan_list_add(&GET_VM()->ractor.move_courier_registry, &c->reg_node); - rb_native_mutex_unlock(&GET_VM()->ractor.move_courier_registry_lock); -} - -static void -move_courier_registry_remove(struct rb_ractor_move_courier *c) -{ - rb_native_mutex_lock(&GET_VM()->ractor.move_courier_registry_lock); - ccan_list_del(&c->reg_node); - rb_native_mutex_unlock(&GET_VM()->ractor.move_courier_registry_lock); -} - -void rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c); - -/* Called from the global GC's root pass; stop-the-world, so no lock. */ -void -rb_ractor_move_courier_registry_mark(void) -{ - struct rb_ractor_move_courier *c; - ccan_list_for_each(&GET_VM()->ractor.move_courier_registry, c, reg_node) { - rb_ractor_move_courier_mark(c); - } -} - struct move_build { struct rb_ractor_move_courier *c; st_table *seen; /* src VALUE -> (node id + 1) */ @@ -2472,9 +2430,18 @@ static uint32_t move_capture(struct move_build *b, VALUE obj); static uint32_t move_alloc_node(struct rb_ractor_move_courier *c) { + /* Swap a fresh array in rather than realloc: the courier is a GC root while it is + * being built, and a realloc leaves c->nodes pointing at a block it may already + * have freed. (The count bump below is safe: nothing between it and the field + * stores can start a GC.) */ if (c->count == c->capa) { - c->capa = c->capa ? c->capa * 2 : 8; - REALLOC_N(c->nodes, struct move_node, c->capa); + uint32_t capa = c->capa ? c->capa * 2 : 8; + struct move_node *nodes = ALLOC_N(struct move_node, capa); + if (c->count > 0) MEMCPY(nodes, c->nodes, struct move_node, c->count); + struct move_node *old_nodes = c->nodes; + c->nodes = nodes; + c->capa = capa; + ruby_xfree(old_nodes); } uint32_t id = c->count++; /* Initialize to a harmless REF/Qnil so the courier mark (a GC root while sending) @@ -2494,13 +2461,21 @@ move_alloc_node(struct rb_ractor_move_courier *c) static uint32_t move_alloc_ref(struct rb_ractor_move_courier *c, VALUE v) { + /* The courier is a GC root while it is being built, so it must never be walkable + * in a half-written state. Grow by allocating a new array and swapping it in: a + * realloc would leave c->refs pointing at a block it may already have freed. The + * count is bumped only after the slot holds a real VALUE. */ if (c->refs_count == c->refs_capa) { - c->refs_capa = c->refs_capa ? c->refs_capa * 2 : 8; - REALLOC_N(c->refs, VALUE, c->refs_capa); + uint32_t capa = c->refs_capa ? c->refs_capa * 2 : 8; + VALUE *refs = ALLOC_N(VALUE, capa); + if (c->refs_count > 0) MEMCPY(refs, c->refs, VALUE, c->refs_count); + VALUE *old_refs = c->refs; + c->refs = refs; + c->refs_capa = capa; + ruby_xfree(old_refs); } - uint32_t idx = c->refs_count++; - c->refs[idx] = v; - return MOVE_ID_REF_BIT | idx; + c->refs[c->refs_count] = v; + return MOVE_ID_REF_BIT | c->refs_count++; } /* Resolve a child slot to the object it names. */ @@ -2988,7 +2963,7 @@ copy_courier_supported_p(VALUE obj, st_table *seen) /* Build a courier holding a copy of obj's graph, leaving the sources untouched. * Returns NULL when the graph has a type only the on-heap snapshot path handles. */ struct rb_ractor_move_courier * -rb_ractor_copy_courier_build(VALUE obj) +rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) { { st_table *seen = st_init_numtable(); @@ -3000,9 +2975,9 @@ rb_ractor_copy_courier_build(VALUE obj) struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); struct move_build b = { c, st_init_numtable(), true }; - /* Same registry cover as a move courier: the shareable REFs it carries need a root - * for its whole lifetime. */ - move_courier_registry_add(c); + /* Publish it into the caller's basket before capturing anything: from here the + * shareable payloads it collects are rooted by the basket's holder. */ + *slot = c; enum ruby_tag_type state; rb_execution_context_t *ec = GET_EC(); @@ -3012,17 +2987,15 @@ rb_ractor_copy_courier_build(VALUE obj) } EC_POP_TAG(); st_free_table(b.seen); - if (state != TAG_NONE) { - rb_ractor_move_courier_free(c); - EC_JUMP_TAG(ec, state); - } + /* Published above, so the basket owns it even half-built: it frees it. */ + if (state != TAG_NONE) EC_JUMP_TAG(ec, state); return c; } /* Build a move courier from obj and turn every captured source into a * RactorMovedObject (move semantics). Returns the xmalloc'd courier. */ struct rb_ractor_move_courier * -rb_ractor_move_courier_build(VALUE obj) +rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) { /* Two phases, preflight then commit, so an unmovable object is raised from the * read-only walk while the graph is still intact. */ @@ -3042,11 +3015,10 @@ rb_ractor_move_courier_build(VALUE obj) struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); struct move_build b = { c, st_init_numtable(), false }; - /* Between send and materialization the courier's shareable REFs pass through - * windows where nothing else roots them; register it for its whole lifetime so the - * registry root pass marks and pins them. Registering before the sources become - * T_MOVED is safe: partial nodes are initialized mark-safe. */ - move_courier_registry_add(c); + /* Publish it into the caller's basket before the sources become T_MOVED: from here + * the basket's holder roots what the courier carries, and partial nodes are + * initialized mark-safe. */ + *slot = c; enum ruby_tag_type state; rb_execution_context_t *ec = GET_EC(); @@ -3057,10 +3029,9 @@ rb_ractor_move_courier_build(VALUE obj) EC_POP_TAG(); st_free_table(b.seen); if (state != TAG_NONE) { - /* move_capture raised (an unmovable type, an interrupt). Remove the courier - * from the registry and free it before re-raising; the partial nodes are - * mark-safe and safe to free. */ - rb_ractor_move_courier_free(c); + /* move_capture raised (an unmovable type, an interrupt). The courier belongs to + * the basket from the publish above, so leave it there and re-raise: the basket + * frees it, once, on the way out. */ EC_JUMP_TAG(ec, state); } return c; @@ -3259,7 +3230,6 @@ rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) break; } } - move_courier_registry_remove(c); ruby_xfree(c->nodes); ruby_xfree(c->refs); ruby_xfree(c); diff --git a/ractor_core.h b/ractor_core.h index 8003899e96a627..d5833d34e721d3 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -38,6 +38,11 @@ struct rb_ractor_sync { struct st_table *ports; size_t next_port_id; + /* The baskets this Ractor holds that are on no queue: one it is building to send, + * and one it has taken off a queue and is materializing. A queued basket is rooted + * by its queue instead. Only the owner touches this list. */ + struct ccan_list_head off_queue_baskets; + // monitors struct ccan_list_head monitors; diff --git a/ractor_sync.c b/ractor_sync.c index cacd6fe455006f..b28f6feba2a8c5 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -20,10 +20,13 @@ static void ractor_send_basket(rb_execution_context_t *ec, const struct ractor_p static void ractor_add_port(rb_ractor_t *r, st_data_t id); // The off-heap courier used for moves. It is defined in ractor.c. -struct rb_ractor_move_courier *rb_ractor_move_courier_build(VALUE obj); +struct rb_ractor_move_courier *rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot); VALUE rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c); void rb_ractor_move_courier_free(struct rb_ractor_move_courier *c); -struct rb_ractor_move_courier *rb_ractor_copy_courier_build(VALUE obj); +static void ractor_off_queue_add(rb_ractor_t *cr, struct ractor_basket *b); +static void ractor_off_queue_remove(struct ractor_basket *b); +void rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c); +struct rb_ractor_move_courier *rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot); static void ractor_port_mark(void *ptr) @@ -240,7 +243,8 @@ struct ractor_basket { size_t mlen; } p; // payload - struct ccan_list_node node; + struct ccan_list_node node; /* the port queue it waits on */ + struct ccan_list_node off_queue_node; /* or sync.off_queue_baskets, when on none */ }; #if 0 @@ -260,10 +264,15 @@ ractor_basket_none_p(const struct ractor_basket *b) static void ractor_basket_mark(const struct ractor_basket *b) { - /* A move courier lives off-heap, and the shareable REFs it carries are marked and - * pinned as a global GC root by the in-flight registry (ractor.c). Nothing to do - * here. */ - if (b->type != basket_type_move && b->p.mbuf == NULL && b->p.move_courier == NULL) { + if (b->p.move_courier != NULL) { + /* The payload became this Ractor's to root the moment the message was enqueued + * here: the sender's own roots stop at the send. Before and after the queue the + * basket is on its holder's off_queue_baskets instead, so a courier is rooted + * from the moment it is allocated to the moment it is freed. */ + rb_ractor_move_courier_mark(b->p.move_courier); + } + else if (b->p.mbuf == NULL) { + /* Marshaled bytes are off-heap and hold nothing to mark. */ rb_gc_mark(b->p.v); } } @@ -271,6 +280,7 @@ ractor_basket_mark(const struct ractor_basket *b) static void ractor_basket_free(struct ractor_basket *b) { + ractor_off_queue_remove(b); ruby_xfree(b->p.mbuf); b->p.mbuf = NULL; b->p.mlen = 0; @@ -286,9 +296,47 @@ static struct ractor_basket * ractor_basket_alloc(void) { struct ractor_basket *b = ALLOC(struct ractor_basket); + + /* Empty and mark-safe from the start: a basket goes on its holder's in-flight list + * before it has a payload, so a GC can walk it while it is still being filled. */ + b->type = basket_type_none; + b->sender = Qnil; + b->port_id = 0; + b->p.v = Qnil; + b->p.exception = false; + b->p.marshaled = false; + b->p.move_courier = NULL; + b->p.mbuf = NULL; + b->p.mlen = 0; + ccan_list_node_init(&b->off_queue_node); + return b; } +/* A basket is rooted by whoever holds it: a port queue while it waits there, and its + * holder's off-queue list while it is being built or materialized. */ +static void +ractor_off_queue_add(rb_ractor_t *cr, struct ractor_basket *b) +{ + VM_ASSERT(cr == rb_current_ractor_raw(false)); + ccan_list_add_tail(&cr->sync.off_queue_baskets, &b->off_queue_node); +} + +static void +ractor_off_queue_remove(struct ractor_basket *b) +{ + ccan_list_del_init(&b->off_queue_node); +} + +static void +ractor_mark_off_queue_baskets(rb_ractor_t *r) +{ + struct ractor_basket *b; + ccan_list_for_each(&r->sync.off_queue_baskets, b, off_queue_node) { + ractor_basket_mark(b); + } +} + // ractor-internal - ractor_queue struct ractor_queue { @@ -746,8 +794,6 @@ ractor_sync_mark(rb_ractor_t *r) rb_gc_mark(r->sync.default_port_value); - /* (A copy snapshot being materialized is not marked here: each EC's frame - * chain roots it in rb_execution_context_mark, which also re-pins it.) */ /* Until the value is absorbed this is its only reliable root (Qundef while the * Ractor still runs); after Ractor#value returns it, the Ruby side roots it. */ rb_gc_mark(r->sync.legacy); @@ -765,6 +811,17 @@ ractor_sync_mark(rb_ractor_t *r) ractor_mark_monitors(r); } if (!world_stopped) RACTOR_UNLOCK_SELF(r); + + /* The baskets on no queue: one being built to send, one being materialized. + * Walked in every collection, like the queues. What they hold is shareable, but + * "only a global GC frees a shareable" does not hold: pinned_roots_mark, which + * roots a shareable from its page bit, is skipped once the process is back to a + * single Ractor (rb_gc_single_objspace_p), and then an ordinary local GC frees + * one that nothing else names. A payload in flight is named by its basket and + * nothing else, so this list has to be a root whenever the queues are. No sync + * lock, though: only the owner touches it (the lock above guards the queues, + * which a foreign sender writes). */ + ractor_mark_off_queue_baskets(r); } } @@ -820,6 +877,7 @@ ractor_sync_init(rb_ractor_t *r) rb_native_mutex_initialize(&r->sync.lock); // monitors + ccan_list_head_init(&r->sync.off_queue_baskets); ccan_list_head_init(&r->sync.monitors); // waiters @@ -973,8 +1031,7 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket * types; anything else is marshaled here, so its user hooks run on the * sender, and the dump travels as plain bytes. */ *ptype = basket_type_copy; - *pcourier = rb_ractor_copy_courier_build(obj); - if (*pcourier != NULL) return Qundef; + if (rb_ractor_copy_courier_build(obj, pcourier) != NULL) return Qundef; *pmarshaled = true; return rb_rescue2(ractor_marshal_dump_body, obj, @@ -987,65 +1044,51 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket static struct ractor_basket * ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type type, bool exc) { - /* A copy payload's preparation can raise (an uncopyable object), so it runs before - * the basket is allocated and cannot leak one; the move branch allocates first, - * since an alloc raise must not orphan an already built courier. */ + rb_ractor_t *cr = rb_ec_ractor_ptr(ec); + /* Allocate and list the basket before anything is built into it: from here the + * courier it is about to hold is rooted by this Ractor's in-flight list, even + * half-built, and every raise below frees it through one path. */ + struct ractor_basket *b = ractor_basket_alloc(); + ractor_off_queue_add(cr, b); + volatile VALUE v = Qfalse; bool marshaled = false; - struct rb_ractor_move_courier *courier = NULL; char *mbuf = NULL; size_t mlen = 0; - struct ractor_basket *b; - if (type == basket_type_move) { - /* Allocate the basket first: its xmalloc can raise NoMemoryError, and a courier - * already built (sources destroyed, registry entry live) would be orphaned. */ - b = ractor_basket_alloc(); - enum ruby_tag_type state; - EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG()) == TAG_NONE) { + enum ruby_tag_type state; + EC_PUSH_TAG(ec); + if ((state = EC_EXEC_TAG()) == TAG_NONE) { + if (type == basket_type_move) { /* Serialize the graph into an off-heap courier; the sources become * RactorMovedObject. While in flight there is no GC object left for the - * sender's GC to mark, sweep or move. */ - courier = rb_ractor_move_courier_build(obj); - } - EC_POP_TAG(); - if (state != TAG_NONE) { - SIZED_FREE(b); - EC_JUMP_TAG(ec, state); + * sender's GC to mark, sweep or move. The build publishes the courier into + * the basket as soon as it exists. */ + rb_ractor_move_courier_build(obj, &b->p.move_courier); } - } - else { - v = ractor_prepare_payload(ec, obj, &type, &marshaled, &courier); - enum ruby_tag_type state; - EC_PUSH_TAG(ec); - if ((state = EC_EXEC_TAG()) == TAG_NONE) { - /* Take the dump off-heap before the basket exists, so an alloc raise below - * frees it through mbuf rather than orphaning it. */ + else { + v = ractor_prepare_payload(ec, obj, &type, &marshaled, &b->p.move_courier); if (type == basket_type_copy && marshaled) { + /* Take the dump off-heap: the sender's copy of it is ordinary garbage + * from here, so nothing of its heap is held while the message waits. */ mlen = (size_t)RSTRING_LEN(v); mbuf = ALLOC_N(char, mlen > 0 ? mlen : 1); memcpy(mbuf, RSTRING_PTR(v), mlen); + v = Qundef; } - b = ractor_basket_alloc(); - } - EC_POP_TAG(); - if (state != TAG_NONE) { - ruby_xfree(mbuf); - if (courier != NULL) rb_ractor_move_courier_free(courier); - EC_JUMP_TAG(ec, state); - } - /* The payload is off-heap, so the sender's dump is ordinary garbage now. */ - if (mbuf != NULL) { - v = Qundef; } } + EC_POP_TAG(); + if (state != TAG_NONE) { + ruby_xfree(mbuf); + ractor_basket_free(b); /* leaves the list and frees a courier already built */ + EC_JUMP_TAG(ec, state); + } b->type = type; b->p.exception = exc; b->p.v = v; b->p.marshaled = marshaled; - b->p.move_courier = courier; b->p.mbuf = mbuf; b->p.mlen = mlen; return b; @@ -1101,8 +1144,8 @@ ractor_basket_value(struct ractor_basket *b) * objspace. The sources are already RactorMovedObject (set when the courier * was built), so move's snapshot semantics hold. The courier is xmalloc'd * rather than a GC object, so the sender's concurrent local GC never touches - * it; the VALUEs it carries are shareable or immediates, marked and pinned as - * a global GC root by the in-flight registry (ractor.c). + * it; the shareable VALUEs it carries are marked through this basket, which is + * on this Ractor's off-queue list until it is freed. * * Rebuilding can raise here too (rb_hash_aset on a moved key with a custom * #hash runs user code, and an async interrupt can arrive). On a raise the @@ -1434,6 +1477,8 @@ ractor_try_receive(rb_execution_context_t *ec, rb_ractor_t *cr, const struct rac } struct ractor_basket *b = ractor_queue_deq(cr, rq); + /* Off the queue and not yet freed: this Ractor roots it while it materializes. */ + if (b) ractor_off_queue_add(cr, b); if (rq->closed && ractor_queue_empty_p(cr, rq)) { ractor_delete_port(cr, ractor_port_id(rp), false); @@ -1483,6 +1528,8 @@ ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, str } else { b->port_id = ractor_port_id(rp); + /* The receiver's queue roots it from here; drop it from ours. */ + ractor_off_queue_remove(b); ractor_queue_enq(rp->r, rp->r->sync.recv_queue, b); } } diff --git a/vm.c b/vm.c index 16146c3108def3..eed553494a38fc 100644 --- a/vm.c +++ b/vm.c @@ -4826,8 +4826,6 @@ Init_BareVM(void) rb_native_mutex_initialize(&vm->ractor.sync.lock); rb_native_cond_initialize(&vm->ractor.sync.terminate_cond); rb_native_mutex_initialize(&vm->ractor.generic_fields_lock); - rb_native_mutex_initialize(&vm->ractor.move_courier_registry_lock); - ccan_list_head_init(&vm->ractor.move_courier_registry); rb_native_mutex_initialize(&vm->gc.registered_globals.lock); vm->gc.orphan_merge_pjob = POSTPONED_JOB_HANDLE_INVALID; diff --git a/vm_core.h b/vm_core.h index 482a63c6d27605..4c851bccac3d7a 100644 --- a/vm_core.h +++ b/vm_core.h @@ -735,12 +735,9 @@ typedef struct rb_vm_struct { #endif } sync; - /* VM-wide locks for the Ractor transfer/inheritance machinery, plus the - * registry of in-flight move couriers. All of them are leaf locks: no - * safepoint inside a critical section. */ + /* VM-wide locks for the Ractor transfer/inheritance machinery. All of them + * are leaf locks: no safepoint inside a critical section. */ rb_nativethread_lock_t generic_fields_lock; /* the shared generic-fields table in variable.c */ - struct ccan_list_head move_courier_registry; /* couriers in flight (ractor.c); the global GC marks them */ - rb_nativethread_lock_t move_courier_registry_lock; #ifdef RUBY_THREAD_PTHREAD_H // ractor scheduling From 2e53b0fd83feae8e1fc623ebbffbdd2a9b8031ed Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 01:27:15 +0000 Subject: [PATCH 07/14] Ractor: size the courier from the preflight, and fill an array by index Two costs on the courier path that the on-heap snapshot did not have. The node and ref arrays were grown by doubling while the graph was captured, and growing them is not a realloc: the courier is a GC root while it is being built, so a realloc would leave the old pointer live across a window where it may already have been freed. Copying into a fresh array instead is correct but not free, and it also made move_alloc_node/move_alloc_ref too big to inline, which costs once per captured object. Both walks that run before capture -- copy_courier_supported_p and move_preflight -- already visit the whole graph, so let them count what capture will allocate: one node per distinct unshareable object, one ref per occurrence of a shareable one. move_courier_reserve then sizes both arrays once and the growth path never runs (verified: it does not fire for any of the benchmark payloads). It stays in place, out of line, in case a count ever comes out short. Materializing an array pushed its elements one at a time, through the capacity check, although the length is known and the shell was allocated with it. Set the length once and write the slots. Message cost, 20k messages per trial, 9 trials, median us per message, measured against master: 40B String 1.40 -> 1.04 4KB String 17.28 -> 8.66 bare Object 1.15 -> 1.05 20-key Hash 12.35 -> 7.61 nested 5.95 -> 5.48 100-elem Integer array 5.70 -> 6.46 The array of immediates is the one shape still behind master. Copy walks the graph twice there (the preflight, then the capture) where the on-heap snapshot walked it once; dropping the preflight would mean capture bailing out on an unsupported type and falling back to Marshal, which also gives up the sizing above. Co-Authored-By: Claude Opus 5 (1M context) --- ractor.c | 189 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 121 insertions(+), 68 deletions(-) diff --git a/ractor.c b/ractor.c index b74f2af0352ca4..9979515cbb1ca5 100644 --- a/ractor.c +++ b/ractor.c @@ -2427,22 +2427,41 @@ struct move_build { static uint32_t move_capture(struct move_build *b, VALUE obj); +/* Off the hot path: the preflight sizes both arrays, so this only runs if its count + * came out short. Swap a fresh array in rather than realloc -- the courier is a GC + * root while it is being built, and a realloc leaves the old pointer live over a + * window where it may already have been freed. */ +NOINLINE(static void move_grow_nodes(struct rb_ractor_move_courier *c)); +NOINLINE(static void move_grow_refs(struct rb_ractor_move_courier *c)); + +static void +move_grow_nodes(struct rb_ractor_move_courier *c) +{ + uint32_t capa = c->capa ? c->capa * 2 : 8; + struct move_node *nodes = ALLOC_N(struct move_node, capa); + if (c->count > 0) MEMCPY(nodes, c->nodes, struct move_node, c->count); + struct move_node *old_nodes = c->nodes; + c->nodes = nodes; + c->capa = capa; + ruby_xfree(old_nodes); +} + +static void +move_grow_refs(struct rb_ractor_move_courier *c) +{ + uint32_t capa = c->refs_capa ? c->refs_capa * 2 : 8; + VALUE *refs = ALLOC_N(VALUE, capa); + if (c->refs_count > 0) MEMCPY(refs, c->refs, VALUE, c->refs_count); + VALUE *old_refs = c->refs; + c->refs = refs; + c->refs_capa = capa; + ruby_xfree(old_refs); +} + static uint32_t move_alloc_node(struct rb_ractor_move_courier *c) { - /* Swap a fresh array in rather than realloc: the courier is a GC root while it is - * being built, and a realloc leaves c->nodes pointing at a block it may already - * have freed. (The count bump below is safe: nothing between it and the field - * stores can start a GC.) */ - if (c->count == c->capa) { - uint32_t capa = c->capa ? c->capa * 2 : 8; - struct move_node *nodes = ALLOC_N(struct move_node, capa); - if (c->count > 0) MEMCPY(nodes, c->nodes, struct move_node, c->count); - struct move_node *old_nodes = c->nodes; - c->nodes = nodes; - c->capa = capa; - ruby_xfree(old_nodes); - } + if (RB_UNLIKELY(c->count == c->capa)) move_grow_nodes(c); uint32_t id = c->count++; /* Initialize to a harmless REF/Qnil so the courier mark (a GC root while sending) * is safe even mid-construction; a captured node overwrites it later. */ @@ -2455,25 +2474,30 @@ move_alloc_node(struct rb_ractor_move_courier *c) return id; } +/* Size the arrays from the preflight's count, so capture never grows them. A count + * that turns out short is not a problem: the growth path below still works. */ +static void +move_courier_reserve(struct rb_ractor_move_courier *c, uint32_t nodes, uint32_t refs) +{ + if (nodes > 0) { + c->nodes = ALLOC_N(struct move_node, nodes); + c->capa = nodes; + } + if (refs > 0) { + c->refs = ALLOC_N(VALUE, refs); + c->refs_capa = refs; + } +} + /* Embed a shareable payload by value and return its tagged child id. No dedup: a REF * is the same word however often it appears, and an array of immediates would otherwise * pay a lookup and an insert per element. */ static uint32_t move_alloc_ref(struct rb_ractor_move_courier *c, VALUE v) { - /* The courier is a GC root while it is being built, so it must never be walkable - * in a half-written state. Grow by allocating a new array and swapping it in: a - * realloc would leave c->refs pointing at a block it may already have freed. The - * count is bumped only after the slot holds a real VALUE. */ - if (c->refs_count == c->refs_capa) { - uint32_t capa = c->refs_capa ? c->refs_capa * 2 : 8; - VALUE *refs = ALLOC_N(VALUE, capa); - if (c->refs_count > 0) MEMCPY(refs, c->refs, VALUE, c->refs_count); - VALUE *old_refs = c->refs; - c->refs = refs; - c->refs_capa = capa; - ruby_xfree(old_refs); - } + /* The count is bumped only after the slot holds a real VALUE: the courier is a GC + * root while it is being built and must never be walkable half-written. */ + if (RB_UNLIKELY(c->refs_count == c->refs_capa)) move_grow_refs(c); c->refs[c->refs_count] = v; return MOVE_ID_REF_BIT | c->refs_count++; } @@ -2795,20 +2819,26 @@ move_capture(struct move_build *b, VALUE obj) return id; } -static void move_preflight(VALUE obj, st_table *seen); +/* Like the copy walk, this also sizes the courier: see copy_support_ctx. */ +struct move_preflight_ctx { + st_table *seen; + uint32_t nodes, refs; +}; + +static void move_preflight(VALUE obj, struct move_preflight_ctx *ctx); static int move_preflight_ivar_i(ID name, VALUE val, st_data_t arg) { - move_preflight(val, (st_table *)arg); + move_preflight(val, (struct move_preflight_ctx *)arg); return ST_CONTINUE; } static int move_preflight_hash_i(st_data_t key, st_data_t val, st_data_t arg) { - move_preflight((VALUE)key, (st_table *)arg); - move_preflight((VALUE)val, (st_table *)arg); + move_preflight((VALUE)key, (struct move_preflight_ctx *)arg); + move_preflight((VALUE)val, (struct move_preflight_ctx *)arg); return ST_CONTINUE; } @@ -2816,11 +2846,17 @@ move_preflight_hash_i(st_data_t key, st_data_t val, st_data_t arg) * T_MOVED as it goes, so an unmovable object midway would leave the graph broken beyond * repair; every "can not move" error is raised here, before anything is mutated. */ static void -move_preflight(VALUE obj, st_table *seen) +move_preflight(VALUE obj, struct move_preflight_ctx *ctx) { - if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) return; + st_table *const seen = ctx->seen; + + if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) { + ctx->refs++; + return; + } if (st_lookup(seen, (st_data_t)obj, NULL)) return; /* cycle */ st_insert(seen, (st_data_t)obj, 0); + ctx->nodes++; switch (BUILTIN_TYPE(obj)) { case T_STRING: @@ -2828,22 +2864,22 @@ move_preflight(VALUE obj, st_table *seen) break; /* children are ivars only (below) */ case T_MATCH: { struct RMatch *rm = RMATCH(obj); - move_preflight(rm->regexp, seen); - move_preflight(rm->str, seen); + move_preflight(rm->regexp, ctx); + move_preflight(rm->str, ctx); break; } case T_ARRAY: for (long i = 0; i < RARRAY_LEN(obj); i++) { - move_preflight(RARRAY_AREF(obj, i), seen); + move_preflight(RARRAY_AREF(obj, i), ctx); } break; case T_HASH: - rb_hash_stlike_foreach(obj, move_preflight_hash_i, (st_data_t)seen); - move_preflight(RHASH_IFNONE(obj), seen); + rb_hash_stlike_foreach(obj, move_preflight_hash_i, (st_data_t)ctx); + move_preflight(RHASH_IFNONE(obj), ctx); break; case T_STRUCT: for (long i = 0; i < RSTRUCT_LEN(obj); i++) { - move_preflight(RSTRUCT_GET(obj, (int)i), seen); + move_preflight(RSTRUCT_GET(obj, (int)i), ctx); } break; case T_FILE: { @@ -2860,11 +2896,11 @@ move_preflight(VALUE obj, st_table *seen) /* A close is in progress: a thread is blocked on this IO. */ rb_raise(rb_eRactorError, "can not move an IO that is being closed"); } - move_preflight(fptr->pathv, seen); - move_preflight(fptr->encs.ecopts, seen); - move_preflight(fptr->writeconv_pre_ecopts, seen); - move_preflight(fptr->writeconv_asciicompat, seen); - move_preflight(fptr->timeout, seen); + move_preflight(fptr->pathv, ctx); + move_preflight(fptr->encs.ecopts, ctx); + move_preflight(fptr->writeconv_pre_ecopts, ctx); + move_preflight(fptr->writeconv_asciicompat, ctx); + move_preflight(fptr->timeout, ctx); break; } default: @@ -2872,21 +2908,25 @@ move_preflight(VALUE obj, st_table *seen) rb_class_name(rb_obj_class(obj))); } - rb_ivar_foreach(obj, move_preflight_ivar_i, (st_data_t)seen); + rb_ivar_foreach(obj, move_preflight_ivar_i, (st_data_t)ctx); } +/* The walk also sizes the courier: one node per distinct unshareable object, one ref + * per occurrence of a shareable one -- exactly what move_capture allocates, so the + * arrays never have to grow while the graph is being captured. */ struct copy_support_ctx { st_table *seen; + uint32_t nodes, refs; bool ok; }; -static bool copy_courier_supported_p(VALUE obj, st_table *seen); +static bool copy_courier_supported_p(VALUE obj, struct copy_support_ctx *ctx); static int copy_support_val_i(st_data_t val, st_data_t arg) { struct copy_support_ctx *ctx = (struct copy_support_ctx *)arg; - if (!copy_courier_supported_p((VALUE)val, ctx->seen)) { + if (!copy_courier_supported_p((VALUE)val, ctx)) { ctx->ok = false; return ST_STOP; } @@ -2910,27 +2950,31 @@ copy_support_hash_i(st_data_t key, st_data_t val, st_data_t arg) * to (MatchData, IO, any other T_DATA, a singleton class) stays on the older on-heap * snapshot path, which keeps handling or rejecting it exactly as before. */ static bool -copy_courier_supported_p(VALUE obj, st_table *seen) +copy_courier_supported_p(VALUE obj, struct copy_support_ctx *ctx) { - if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) return true; + st_table *const seen = ctx->seen; + + if (RB_SPECIAL_CONST_P(obj) || rb_ractor_shareable_p(obj)) { + ctx->refs++; + return true; + } if (st_lookup(seen, (st_data_t)obj, NULL)) return true; /* cycle */ st_insert(seen, (st_data_t)obj, 0); + ctx->nodes++; /* A singleton class is a send error today (the native copier refuses it and Marshal * then raises); the courier would happily carry it, so keep it off this path. */ VALUE klass = RBASIC_CLASS(obj); if (klass == 0 || FL_TEST_RAW(klass, FL_SINGLETON)) return false; - struct copy_support_ctx ctx = { seen, true }; - switch (BUILTIN_TYPE(obj)) { case T_STRING: case T_OBJECT: break; /* children are ivars only (below) */ case T_MATCH: { struct RMatch *rm = RMATCH(obj); - if (!copy_courier_supported_p(rm->regexp, seen)) return false; - if (!copy_courier_supported_p(rm->str, seen)) return false; + if (!copy_courier_supported_p(rm->regexp, ctx)) return false; + if (!copy_courier_supported_p(rm->str, ctx)) return false; break; } case T_DATA: @@ -2939,25 +2983,25 @@ copy_courier_supported_p(VALUE obj, st_table *seen) break; case T_ARRAY: for (long i = 0; i < RARRAY_LEN(obj); i++) { - if (!copy_courier_supported_p(RARRAY_AREF(obj, i), seen)) return false; + if (!copy_courier_supported_p(RARRAY_AREF(obj, i), ctx)) return false; } break; case T_HASH: - rb_hash_stlike_foreach(obj, copy_support_hash_i, (st_data_t)&ctx); - if (!ctx.ok) return false; - if (!copy_courier_supported_p(RHASH_IFNONE(obj), seen)) return false; + rb_hash_stlike_foreach(obj, copy_support_hash_i, (st_data_t)ctx); + if (!ctx->ok) return false; + if (!copy_courier_supported_p(RHASH_IFNONE(obj), ctx)) return false; break; case T_STRUCT: for (long i = 0; i < RSTRUCT_LEN(obj); i++) { - if (!copy_courier_supported_p(RSTRUCT_GET(obj, (int)i), seen)) return false; + if (!copy_courier_supported_p(RSTRUCT_GET(obj, (int)i), ctx)) return false; } break; default: return false; } - rb_ivar_foreach(obj, copy_support_ivar_i, (st_data_t)&ctx); - return ctx.ok; + rb_ivar_foreach(obj, copy_support_ivar_i, (st_data_t)ctx); + return ctx->ok; } /* Build a courier holding a copy of obj's graph, leaving the sources untouched. @@ -2965,14 +3009,15 @@ copy_courier_supported_p(VALUE obj, st_table *seen) struct rb_ractor_move_courier * rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) { + struct copy_support_ctx scan = { st_init_numtable(), 0, 0, true }; { - st_table *seen = st_init_numtable(); - bool ok = copy_courier_supported_p(obj, seen); - st_free_table(seen); + bool ok = copy_courier_supported_p(obj, &scan); + st_free_table(scan.seen); if (!ok) return NULL; } struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); + move_courier_reserve(c, scan.nodes, scan.refs); struct move_build b = { c, st_init_numtable(), true }; /* Publish it into the caller's basket before capturing anything: from here the @@ -2999,20 +3044,21 @@ rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) { /* Two phases, preflight then commit, so an unmovable object is raised from the * read-only walk while the graph is still intact. */ + struct move_preflight_ctx scan = { st_init_numtable(), 0, 0 }; { - st_table *pf_seen = st_init_numtable(); enum ruby_tag_type state; rb_execution_context_t *ec = GET_EC(); EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - move_preflight(obj, pf_seen); + move_preflight(obj, &scan); } EC_POP_TAG(); - st_free_table(pf_seen); + st_free_table(scan.seen); if (state != TAG_NONE) EC_JUMP_TAG(ec, state); } struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); + move_courier_reserve(c, scan.nodes, scan.refs); struct move_build b = { c, st_init_numtable(), false }; /* Publish it into the caller's basket before the sources become T_MOVED: from here @@ -3117,11 +3163,18 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) struct move_node *n = &c->nodes[i]; VALUE shell = RARRAY_AREF(shells, i); switch (n->kind) { - case MOVE_KIND_ARRAY: - for (long j = 0; j < n->u.ary.len; j++) { - rb_ary_push(shell, move_child(c, shells, n->u.ary.elems[j])); + case MOVE_KIND_ARRAY: { + /* The length is known, so set it once and write the slots, rather than + * pushing each element through the capacity check. */ + const long len = n->u.ary.len; + if (len > 0) { + rb_ary_resize(shell, len); + for (long j = 0; j < len; j++) { + RARRAY_ASET(shell, j, move_child(c, shells, n->u.ary.elems[j])); + } } break; + } case MOVE_KIND_HASH: /* Entry insertion is deferred to a third pass: insertion calls the key's * #hash / #eql?, and a content-based #hash would collide on every key while From d94c7ee6c6975d43df1b52c3f7c925e3bc773f37 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 01:27:34 +0000 Subject: [PATCH 08/14] Ractor: name the courier machinery after the courier, not after move The off-heap courier started out as the move path's own structure, so every part of it was named move_*. Copy now travels in the same structure and through the same capture walk, with courier_build.copy selecting whether the source is read or taken apart, which left the prefix claiming something untrue: move_capture is what a copy goes through too. Rename the shared machinery to courier_*: the struct and its build/ materialize/mark/free entry points, the node kinds, the capture and materialize walks, and the array allocators. The two builders become rb_ractor_courier_build_copy and rb_ractor_courier_build_move so the pair reads as one family. What is still move-only keeps the move_ prefix -- move_preflight (copy has its own walk, copy_courier_supported_p) and move_neutralize_source, which is exactly the step copy skips. In re.c the MatchData transfer helpers become rb_match_blob_*, matching the neighbouring rb_backtrace_blob_* it sits beside in the same node. No behaviour change. Co-Authored-By: Claude Opus 5 (1M context) --- internal/re.h | 10 +- ractor.c | 329 +++++++++++++++++++++++++------------------------- ractor_core.h | 2 +- ractor_sync.c | 56 ++++----- re.c | 8 +- 5 files changed, 203 insertions(+), 202 deletions(-) diff --git a/internal/re.h b/internal/re.h index c5dfe2341397b5..49d82d5c209727 100644 --- a/internal/re.h +++ b/internal/re.h @@ -65,11 +65,11 @@ long rb_reg_search0(VALUE, VALUE, long, int, int, VALUE *); VALUE rb_reg_match_p(VALUE re, VALUE str, long pos); VALUE rb_reg_regsub_match(VALUE str, VALUE src, VALUE match); VALUE rb_match_init_copy(VALUE copy, VALUE orig); -/* MatchData transfer for the move courier (ractor.c). */ -void *rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source); -VALUE rb_match_move_alloc(VALUE klass, int num_regs); -void rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob); -void rb_match_move_free(void *blob); +/* MatchData transfer for the Ractor courier (ractor.c). */ +void *rb_match_blob_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source); +VALUE rb_match_blob_alloc(VALUE klass, int num_regs); +void rb_match_blob_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob); +void rb_match_blob_free(void *blob); bool rb_reg_start_with_p(VALUE re, VALUE str); VALUE rb_reg_hash(VALUE re); VALUE rb_reg_equal(VALUE re1, VALUE re2); diff --git a/ractor.c b/ractor.c index 9979515cbb1ca5..f30c3dbe262e20 100644 --- a/ractor.c +++ b/ractor.c @@ -2357,25 +2357,26 @@ rb_obj_traverse_replace(VALUE obj, } } -/* Move courier: serializes the payload of Ractor#send(move: true) into an xmalloc'd +/* Courier: serializes a Ractor message payload -- copied or moved -- into an xmalloc'd * structure that belongs to no objspace, so no sender GC can mark, sweep, compact or * race with it. A node array with id references handles sharing and cycles, and the - * receiver rebuilds it in its own objspace in two passes. */ - -enum move_node_kind { - MOVE_KIND_REF, /* an immediate or a shareable object: carried by value */ - MOVE_KIND_BACKTRACE, /* an exception's backtrace: frames copied into an off-heap blob */ - MOVE_KIND_STRING, - MOVE_KIND_ARRAY, - MOVE_KIND_HASH, - MOVE_KIND_OBJECT, - MOVE_KIND_STRUCT, - MOVE_KIND_MATCH, - MOVE_KIND_IO, + * receiver rebuilds it in its own objspace in two passes. Copy and move differ only in + * whether the source is read or taken apart: see courier_build.copy. */ + +enum courier_node_kind { + COURIER_KIND_REF, /* an immediate or a shareable object: carried by value */ + COURIER_KIND_BACKTRACE, /* an exception's backtrace: frames copied into an off-heap blob */ + COURIER_KIND_STRING, + COURIER_KIND_ARRAY, + COURIER_KIND_HASH, + COURIER_KIND_OBJECT, + COURIER_KIND_STRUCT, + COURIER_KIND_MATCH, + COURIER_KIND_IO, }; -struct move_node { - enum move_node_kind kind; +struct courier_node { + enum courier_node_kind kind; bool frozen; /* The instance and generic ivars every non-REF node can have (a String or Array * can hold generic ivars too) */ @@ -2404,11 +2405,11 @@ struct move_node { /* A child slot holds a node id, or -- with this bit set -- an index into c->refs. * The courier is in-process, so a shareable payload can travel as the VALUE itself - * instead of costing a whole move_node; the basket holding the courier marks c->refs. */ -#define MOVE_ID_REF_BIT 0x80000000u + * instead of costing a whole courier_node; the basket holding the courier marks c->refs. */ +#define COURIER_ID_REF_BIT 0x80000000u -struct rb_ractor_move_courier { - struct move_node *nodes; +struct rb_ractor_courier { + struct courier_node *nodes; uint32_t count; uint32_t capa; VALUE *refs; /* shareable payloads, embedded by value */ @@ -2417,37 +2418,37 @@ struct rb_ractor_move_courier { uint32_t root; }; -struct move_build { - struct rb_ractor_move_courier *c; +struct courier_build { + struct rb_ractor_courier *c; st_table *seen; /* src VALUE -> (node id + 1) */ /* Copy mode: read the sources instead of taking them apart. No husk, no buffer * hand-over, no freeing of the source's internals. */ bool copy; }; -static uint32_t move_capture(struct move_build *b, VALUE obj); +static uint32_t courier_capture(struct courier_build *b, VALUE obj); /* Off the hot path: the preflight sizes both arrays, so this only runs if its count * came out short. Swap a fresh array in rather than realloc -- the courier is a GC * root while it is being built, and a realloc leaves the old pointer live over a * window where it may already have been freed. */ -NOINLINE(static void move_grow_nodes(struct rb_ractor_move_courier *c)); -NOINLINE(static void move_grow_refs(struct rb_ractor_move_courier *c)); +NOINLINE(static void courier_grow_nodes(struct rb_ractor_courier *c)); +NOINLINE(static void courier_grow_refs(struct rb_ractor_courier *c)); static void -move_grow_nodes(struct rb_ractor_move_courier *c) +courier_grow_nodes(struct rb_ractor_courier *c) { uint32_t capa = c->capa ? c->capa * 2 : 8; - struct move_node *nodes = ALLOC_N(struct move_node, capa); - if (c->count > 0) MEMCPY(nodes, c->nodes, struct move_node, c->count); - struct move_node *old_nodes = c->nodes; + struct courier_node *nodes = ALLOC_N(struct courier_node, capa); + if (c->count > 0) MEMCPY(nodes, c->nodes, struct courier_node, c->count); + struct courier_node *old_nodes = c->nodes; c->nodes = nodes; c->capa = capa; ruby_xfree(old_nodes); } static void -move_grow_refs(struct rb_ractor_move_courier *c) +courier_grow_refs(struct rb_ractor_courier *c) { uint32_t capa = c->refs_capa ? c->refs_capa * 2 : 8; VALUE *refs = ALLOC_N(VALUE, capa); @@ -2459,13 +2460,13 @@ move_grow_refs(struct rb_ractor_move_courier *c) } static uint32_t -move_alloc_node(struct rb_ractor_move_courier *c) +courier_alloc_node(struct rb_ractor_courier *c) { - if (RB_UNLIKELY(c->count == c->capa)) move_grow_nodes(c); + if (RB_UNLIKELY(c->count == c->capa)) courier_grow_nodes(c); uint32_t id = c->count++; /* Initialize to a harmless REF/Qnil so the courier mark (a GC root while sending) * is safe even mid-construction; a captured node overwrites it later. */ - c->nodes[id].kind = MOVE_KIND_REF; + c->nodes[id].kind = COURIER_KIND_REF; c->nodes[id].frozen = false; c->nodes[id].niv = 0; c->nodes[id].iv_ids = NULL; @@ -2477,10 +2478,10 @@ move_alloc_node(struct rb_ractor_move_courier *c) /* Size the arrays from the preflight's count, so capture never grows them. A count * that turns out short is not a problem: the growth path below still works. */ static void -move_courier_reserve(struct rb_ractor_move_courier *c, uint32_t nodes, uint32_t refs) +courier_reserve(struct rb_ractor_courier *c, uint32_t nodes, uint32_t refs) { if (nodes > 0) { - c->nodes = ALLOC_N(struct move_node, nodes); + c->nodes = ALLOC_N(struct courier_node, nodes); c->capa = nodes; } if (refs > 0) { @@ -2493,20 +2494,20 @@ move_courier_reserve(struct rb_ractor_move_courier *c, uint32_t nodes, uint32_t * is the same word however often it appears, and an array of immediates would otherwise * pay a lookup and an insert per element. */ static uint32_t -move_alloc_ref(struct rb_ractor_move_courier *c, VALUE v) +courier_alloc_ref(struct rb_ractor_courier *c, VALUE v) { /* The count is bumped only after the slot holds a real VALUE: the courier is a GC * root while it is being built and must never be walkable half-written. */ - if (RB_UNLIKELY(c->refs_count == c->refs_capa)) move_grow_refs(c); + if (RB_UNLIKELY(c->refs_count == c->refs_capa)) courier_grow_refs(c); c->refs[c->refs_count] = v; - return MOVE_ID_REF_BIT | c->refs_count++; + return COURIER_ID_REF_BIT | c->refs_count++; } /* Resolve a child slot to the object it names. */ static VALUE -move_child(const struct rb_ractor_move_courier *c, VALUE shells, uint32_t id) +courier_child(const struct rb_ractor_courier *c, VALUE shells, uint32_t id) { - if (id & MOVE_ID_REF_BIT) return c->refs[id & ~MOVE_ID_REF_BIT]; + if (id & COURIER_ID_REF_BIT) return c->refs[id & ~COURIER_ID_REF_BIT]; return RARRAY_AREF(shells, id); } @@ -2557,25 +2558,25 @@ move_neutralize_source(VALUE obj) } } -struct move_hash_ctx { - struct move_build *b; +struct courier_hash_ctx { + struct courier_build *b; uint32_t *kv; long i; }; static int -move_capture_hash_i(st_data_t key, st_data_t val, st_data_t arg) +courier_capture_hash_i(st_data_t key, st_data_t val, st_data_t arg) { - struct move_hash_ctx *hc = (struct move_hash_ctx *)arg; - uint32_t kid = move_capture(hc->b, (VALUE)key); - uint32_t vid = move_capture(hc->b, (VALUE)val); + struct courier_hash_ctx *hc = (struct courier_hash_ctx *)arg; + uint32_t kid = courier_capture(hc->b, (VALUE)key); + uint32_t vid = courier_capture(hc->b, (VALUE)val); hc->kv[hc->i++] = kid; hc->kv[hc->i++] = vid; return ST_CONTINUE; } -struct move_obj_ctx { - struct move_build *b; +struct courier_obj_ctx { + struct courier_build *b; ID *ids; uint32_t *vals; long n; @@ -2583,15 +2584,15 @@ struct move_obj_ctx { }; static int -move_capture_ivar_i(ID name, VALUE val, st_data_t arg) +courier_capture_ivar_i(ID name, VALUE val, st_data_t arg) { - struct move_obj_ctx *oc = (struct move_obj_ctx *)arg; + struct courier_obj_ctx *oc = (struct courier_obj_ctx *)arg; if (oc->n == oc->capa) { oc->capa = oc->capa ? oc->capa * 2 : 4; REALLOC_N(oc->ids, ID, oc->capa); REALLOC_N(oc->vals, uint32_t, oc->capa); } - uint32_t vid = move_capture(oc->b, val); + uint32_t vid = courier_capture(oc->b, val); oc->ids[oc->n] = name; oc->vals[oc->n] = vid; oc->n++; @@ -2602,10 +2603,10 @@ move_capture_ivar_i(ID name, VALUE val, st_data_t arg) * Handles both a T_OBJECT's inline ivars and the generic ivars of a String, Array and * so on. */ static void -move_capture_ivars(struct move_build *b, VALUE obj, uint32_t id) +courier_capture_ivars(struct courier_build *b, VALUE obj, uint32_t id) { - struct move_obj_ctx oc = { b, NULL, NULL, 0, 0 }; - rb_ivar_foreach_buffered(obj, move_capture_ivar_i, (st_data_t)&oc); + struct courier_obj_ctx oc = { b, NULL, NULL, 0, 0 }; + rb_ivar_foreach_buffered(obj, courier_capture_ivar_i, (st_data_t)&oc); b->c->nodes[id].niv = (uint32_t)oc.n; b->c->nodes[id].iv_ids = oc.ids; b->c->nodes[id].iv_vals = oc.vals; @@ -2613,15 +2614,15 @@ move_capture_ivars(struct move_build *b, VALUE obj, uint32_t id) /* Capture obj into the courier, recurse into its children, return its node id. The id * is registered before recursing (a cycle back resolves to the same node); node fields - * are written after (recursion can realloc c->nodes); the source is neutralized exactly - * once after the switch. */ + * are written after (recursion can realloc c->nodes); a move neutralizes the source + * exactly once after the switch. */ static uint32_t -move_capture(struct move_build *b, VALUE obj) +courier_capture(struct courier_build *b, VALUE obj) { /* An immediate is never in seen (only captured objects are inserted), so it can * skip the lookup entirely: that is the whole cost of an array of numbers. */ if (RB_SPECIAL_CONST_P(obj)) { - return move_alloc_ref(b->c, obj); + return courier_alloc_ref(b->c, obj); } /* Seen first, and only then shareable: move husks each source as it goes, and a @@ -2634,10 +2635,10 @@ move_capture(struct move_build *b, VALUE obj) } if (rb_ractor_shareable_p(obj)) { - return move_alloc_ref(b->c, obj); + return courier_alloc_ref(b->c, obj); } - uint32_t id = move_alloc_node(b->c); + uint32_t id = courier_alloc_node(b->c); st_insert(b->seen, (st_data_t)obj, (st_data_t)(uintptr_t)(id + 1)); /* Reject an unmovable object before anything is mutated. */ @@ -2647,7 +2648,7 @@ move_capture(struct move_build *b, VALUE obj) bool frozen = OBJ_FROZEN(obj); b->c->nodes[id].frozen = frozen; - move_capture_ivars(b, obj, id); /* shared: instance and generic ivars */ + courier_capture_ivars(b, obj, id); /* shared: instance and generic ivars */ switch (BUILTIN_TYPE(obj)) { case T_STRING: { @@ -2677,7 +2678,7 @@ move_capture(struct move_build *b, VALUE obj) memset(ptr + len, 0, termlen); capa = len; } - b->c->nodes[id].kind = MOVE_KIND_STRING; + b->c->nodes[id].kind = COURIER_KIND_STRING; b->c->nodes[id].u.str.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.str.ptr = ptr; b->c->nodes[id].u.str.len = len; @@ -2690,9 +2691,9 @@ move_capture(struct move_build *b, VALUE obj) long len = RARRAY_LEN(obj); uint32_t *elems = len ? ALLOC_N(uint32_t, len) : NULL; for (long i = 0; i < len; i++) { - elems[i] = move_capture(b, RARRAY_AREF(obj, i)); + elems[i] = courier_capture(b, RARRAY_AREF(obj, i)); } - b->c->nodes[id].kind = MOVE_KIND_ARRAY; + b->c->nodes[id].kind = COURIER_KIND_ARRAY; b->c->nodes[id].u.ary.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.ary.len = len; b->c->nodes[id].u.ary.elems = elems; @@ -2706,12 +2707,12 @@ move_capture(struct move_build *b, VALUE obj) } case T_HASH: { - uint32_t ifnone_id = move_capture(b, RHASH_IFNONE(obj)); + uint32_t ifnone_id = courier_capture(b, RHASH_IFNONE(obj)); long size = RHASH_SIZE(obj); uint32_t *kv = size ? ALLOC_N(uint32_t, size * 2) : NULL; - struct move_hash_ctx hc = { b, kv, 0 }; - rb_hash_stlike_foreach(obj, move_capture_hash_i, (st_data_t)&hc); - b->c->nodes[id].kind = MOVE_KIND_HASH; + struct courier_hash_ctx hc = { b, kv, 0 }; + rb_hash_stlike_foreach(obj, courier_capture_hash_i, (st_data_t)&hc); + b->c->nodes[id].kind = COURIER_KIND_HASH; b->c->nodes[id].u.hash.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.hash.size = size; b->c->nodes[id].u.hash.kv = kv; @@ -2724,7 +2725,7 @@ move_capture(struct move_build *b, VALUE obj) } case T_OBJECT: - b->c->nodes[id].kind = MOVE_KIND_OBJECT; + b->c->nodes[id].kind = COURIER_KIND_OBJECT; /* Keep the real class: even a singleton class is shareable, so a cross-objspace * reference is safe. rebuild re-attaches it after allocating with a * non-singleton class. */ @@ -2735,9 +2736,9 @@ move_capture(struct move_build *b, VALUE obj) long len = RSTRUCT_LEN(obj); uint32_t *elems = len ? ALLOC_N(uint32_t, len) : NULL; for (long i = 0; i < len; i++) { - elems[i] = move_capture(b, RSTRUCT_GET(obj, (int)i)); + elems[i] = courier_capture(b, RSTRUCT_GET(obj, (int)i)); } - b->c->nodes[id].kind = MOVE_KIND_STRUCT; + b->c->nodes[id].kind = COURIER_KIND_STRUCT; b->c->nodes[id].u.strct.len = len; b->c->nodes[id].u.strct.elems = elems; b->c->nodes[id].u.strct.klass = RBASIC_CLASS(obj); @@ -2753,10 +2754,10 @@ move_capture(struct move_build *b, VALUE obj) * registers (freeing the source's onig and char_offset). */ VALUE re, st; int nregs; - void *regs = rb_match_move_dump(obj, &re, &st, &nregs, !b->copy); - uint32_t rid = move_capture(b, re); - uint32_t sid = move_capture(b, st); - b->c->nodes[id].kind = MOVE_KIND_MATCH; + void *regs = rb_match_blob_dump(obj, &re, &st, &nregs, !b->copy); + uint32_t rid = courier_capture(b, re); + uint32_t sid = courier_capture(b, st); + b->c->nodes[id].kind = COURIER_KIND_MATCH; b->c->nodes[id].u.match.regexp_id = rid; b->c->nodes[id].u.match.str_id = sid; b->c->nodes[id].u.match.num_regs = nregs; @@ -2773,11 +2774,11 @@ move_capture(struct move_build *b, VALUE obj) * so capture them as ordinary child nodes, detached; rebuild writes them back. */ struct rb_io *fptr = RFILE(obj)->fptr; VM_ASSERT(!RTEST(fptr->tied_io_for_writing) && !RTEST(fptr->wakeup_mutex)); - uint32_t pathv_id = move_capture(b, fptr->pathv); - uint32_t ecopts_id = move_capture(b, fptr->encs.ecopts); - uint32_t wc_pre_id = move_capture(b, fptr->writeconv_pre_ecopts); - uint32_t wc_ac_id = move_capture(b, fptr->writeconv_asciicompat); - uint32_t timeout_id = move_capture(b, fptr->timeout); + uint32_t pathv_id = courier_capture(b, fptr->pathv); + uint32_t ecopts_id = courier_capture(b, fptr->encs.ecopts); + uint32_t wc_pre_id = courier_capture(b, fptr->writeconv_pre_ecopts); + uint32_t wc_ac_id = courier_capture(b, fptr->writeconv_asciicompat); + uint32_t timeout_id = courier_capture(b, fptr->timeout); fptr->self = Qnil; /* it points at the moved-from T_MOVED; attach rebuilds it */ fptr->pathv = Qnil; fptr->encs.ecopts = Qnil; @@ -2787,7 +2788,7 @@ move_capture(struct move_build *b, VALUE obj) fptr->write_lock = Qnil; fptr->wakeup_mutex = Qnil; fptr->tied_io_for_writing = 0; /* io.c tests it as a C boolean, so 0 rather than Qnil */ - b->c->nodes[id].kind = MOVE_KIND_IO; + b->c->nodes[id].kind = COURIER_KIND_IO; b->c->nodes[id].u.io.fptr = fptr; b->c->nodes[id].u.io.klass = RBASIC_CLASS(obj); b->c->nodes[id].u.io.pathv_id = pathv_id; @@ -2804,7 +2805,7 @@ move_capture(struct move_build *b, VALUE obj) if (b->copy && rb_backtrace_p(obj)) { int size; void *blob = rb_backtrace_blob_dump(obj, &size); - b->c->nodes[id].kind = MOVE_KIND_BACKTRACE; + b->c->nodes[id].kind = COURIER_KIND_BACKTRACE; b->c->nodes[id].u.bt.blob = blob; b->c->nodes[id].u.bt.size = size; break; @@ -2842,7 +2843,7 @@ move_preflight_hash_i(st_data_t key, st_data_t val, st_data_t arg) return ST_CONTINUE; } -/* A read-only pre-walk of move_capture's decision tree. Capture turns sources into +/* A read-only pre-walk of courier_capture's decision tree. Capture turns sources into * T_MOVED as it goes, so an unmovable object midway would leave the graph broken beyond * repair; every "can not move" error is raised here, before anything is mutated. */ static void @@ -2912,7 +2913,7 @@ move_preflight(VALUE obj, struct move_preflight_ctx *ctx) } /* The walk also sizes the courier: one node per distinct unshareable object, one ref - * per occurrence of a shareable one -- exactly what move_capture allocates, so the + * per occurrence of a shareable one -- exactly what courier_capture allocates, so the * arrays never have to grow while the graph is being captured. */ struct copy_support_ctx { st_table *seen; @@ -3006,8 +3007,8 @@ copy_courier_supported_p(VALUE obj, struct copy_support_ctx *ctx) /* Build a courier holding a copy of obj's graph, leaving the sources untouched. * Returns NULL when the graph has a type only the on-heap snapshot path handles. */ -struct rb_ractor_move_courier * -rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) +struct rb_ractor_courier * +rb_ractor_courier_build_copy(VALUE obj, struct rb_ractor_courier **slot) { struct copy_support_ctx scan = { st_init_numtable(), 0, 0, true }; { @@ -3016,9 +3017,9 @@ rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) if (!ok) return NULL; } - struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); - move_courier_reserve(c, scan.nodes, scan.refs); - struct move_build b = { c, st_init_numtable(), true }; + struct rb_ractor_courier *c = ZALLOC(struct rb_ractor_courier); + courier_reserve(c, scan.nodes, scan.refs); + struct courier_build b = { c, st_init_numtable(), true }; /* Publish it into the caller's basket before capturing anything: from here the * shareable payloads it collects are rooted by the basket's holder. */ @@ -3028,7 +3029,7 @@ rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) rb_execution_context_t *ec = GET_EC(); EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - c->root = move_capture(&b, obj); + c->root = courier_capture(&b, obj); } EC_POP_TAG(); st_free_table(b.seen); @@ -3037,10 +3038,10 @@ rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) return c; } -/* Build a move courier from obj and turn every captured source into a - * RactorMovedObject (move semantics). Returns the xmalloc'd courier. */ -struct rb_ractor_move_courier * -rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) +/* Build a courier from obj and turn every captured source into a RactorMovedObject + * (move semantics). Returns the xmalloc'd courier. */ +struct rb_ractor_courier * +rb_ractor_courier_build_move(VALUE obj, struct rb_ractor_courier **slot) { /* Two phases, preflight then commit, so an unmovable object is raised from the * read-only walk while the graph is still intact. */ @@ -3057,9 +3058,9 @@ rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) if (state != TAG_NONE) EC_JUMP_TAG(ec, state); } - struct rb_ractor_move_courier *c = ZALLOC(struct rb_ractor_move_courier); - move_courier_reserve(c, scan.nodes, scan.refs); - struct move_build b = { c, st_init_numtable(), false }; + struct rb_ractor_courier *c = ZALLOC(struct rb_ractor_courier); + courier_reserve(c, scan.nodes, scan.refs); + struct courier_build b = { c, st_init_numtable(), false }; /* Publish it into the caller's basket before the sources become T_MOVED: from here * the basket's holder roots what the courier carries, and partial nodes are @@ -3070,14 +3071,14 @@ rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) rb_execution_context_t *ec = GET_EC(); EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - c->root = move_capture(&b, obj); + c->root = courier_capture(&b, obj); } EC_POP_TAG(); st_free_table(b.seen); if (state != TAG_NONE) { - /* move_capture raised (an unmovable type, an interrupt). The courier belongs to - * the basket from the publish above, so leave it there and re-raise: the basket - * frees it, once, on the way out. */ + /* courier_capture raised (an unmovable type, an interrupt). The courier belongs + * to the basket from the publish above, so leave it there and re-raise: the + * basket frees it, once, on the way out. */ EC_JUMP_TAG(ec, state); } return c; @@ -3087,7 +3088,7 @@ rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot) * singleton class (classes are shareable; the reference is safe). A singleton's * attached object still points at the sender's source: re-attach it to the shell. */ static void -move_apply_moved_klass(VALUE shell, VALUE klass) +courier_apply_klass(VALUE shell, VALUE klass) { if (klass != RBASIC_CLASS(shell)) { RBASIC_SET_CLASS(shell, klass); @@ -3100,105 +3101,105 @@ move_apply_moved_klass(VALUE shell, VALUE klass) /* Rebuild the courier's graph in the current Ractor's objspace and return its root. * Two passes (allocate shells, then fill) break reference cycles. */ VALUE -rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) +rb_ractor_courier_materialize(struct rb_ractor_courier *c) { /* A hidden Array roots every shell, keeping them alive while the allocations that * build the rest of the graph (which can start this Ractor's GC) run. */ VALUE shells = rb_ary_hidden_new(c->count); for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; + struct courier_node *n = &c->nodes[i]; VALUE shell; switch (n->kind) { - case MOVE_KIND_REF: + case COURIER_KIND_REF: shell = n->u.ref; break; - case MOVE_KIND_STRING: + case COURIER_KIND_STRING: /* Hand the courier's buffer to the String instead of copying it again: the * bytes were already copied (or taken from the source) when the node was * built. */ shell = rb_str_new_owned(n->u.str.ptr, n->u.str.len, n->u.str.capa, n->u.str.encidx); n->u.str.ptr = NULL; /* consumed: the new String owns it now */ - move_apply_moved_klass(shell, n->u.str.klass); + courier_apply_klass(shell, n->u.str.klass); break; - case MOVE_KIND_ARRAY: + case COURIER_KIND_ARRAY: shell = rb_ary_new_capa(n->u.ary.len); - move_apply_moved_klass(shell, n->u.ary.klass); + courier_apply_klass(shell, n->u.ary.klass); break; - case MOVE_KIND_HASH: + case COURIER_KIND_HASH: shell = n->u.hash.compare_by_id ? rb_ident_hash_new() : rb_hash_new(); - move_apply_moved_klass(shell, n->u.hash.klass); + courier_apply_klass(shell, n->u.hash.klass); break; - case MOVE_KIND_OBJECT: + case COURIER_KIND_OBJECT: /* A singleton class cannot allocate, so make an instance of the real class * and re-attach it afterwards */ shell = rb_obj_alloc(rb_class_real(n->u.obj.klass)); - move_apply_moved_klass(shell, n->u.obj.klass); + courier_apply_klass(shell, n->u.obj.klass); break; - case MOVE_KIND_STRUCT: + case COURIER_KIND_STRUCT: shell = rb_obj_alloc(rb_class_real(n->u.strct.klass)); - move_apply_moved_klass(shell, n->u.strct.klass); + courier_apply_klass(shell, n->u.strct.klass); break; - case MOVE_KIND_MATCH: - shell = rb_match_move_alloc(rb_class_real(n->u.match.klass), n->u.match.num_regs); - move_apply_moved_klass(shell, n->u.match.klass); + case COURIER_KIND_MATCH: + shell = rb_match_blob_alloc(rb_class_real(n->u.match.klass), n->u.match.num_regs); + courier_apply_klass(shell, n->u.match.klass); break; - case MOVE_KIND_BACKTRACE: + case COURIER_KIND_BACKTRACE: shell = rb_backtrace_blob_load(n->u.bt.blob, n->u.bt.size); break; - case MOVE_KIND_IO: + case COURIER_KIND_IO: shell = rb_obj_alloc(rb_class_real(n->u.io.klass)); - move_apply_moved_klass(shell, n->u.io.klass); + courier_apply_klass(shell, n->u.io.klass); RFILE(shell)->fptr = n->u.io.fptr; n->u.io.fptr->self = shell; n->u.io.fptr = NULL; /* consumed: the new IO owns it now */ break; default: - rb_bug("rb_ractor_move_courier_materialize: bad node kind"); + rb_bug("rb_ractor_courier_materialize: bad node kind"); } rb_ary_push(shells, shell); } for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; + struct courier_node *n = &c->nodes[i]; VALUE shell = RARRAY_AREF(shells, i); switch (n->kind) { - case MOVE_KIND_ARRAY: { + case COURIER_KIND_ARRAY: { /* The length is known, so set it once and write the slots, rather than * pushing each element through the capacity check. */ const long len = n->u.ary.len; if (len > 0) { rb_ary_resize(shell, len); for (long j = 0; j < len; j++) { - RARRAY_ASET(shell, j, move_child(c, shells, n->u.ary.elems[j])); + RARRAY_ASET(shell, j, courier_child(c, shells, n->u.ary.elems[j])); } } break; } - case MOVE_KIND_HASH: + case COURIER_KIND_HASH: /* Entry insertion is deferred to a third pass: insertion calls the key's * #hash / #eql?, and a content-based #hash would collide on every key while * the graph is still empty, collapsing entries. */ break; - case MOVE_KIND_STRUCT: + case COURIER_KIND_STRUCT: for (long j = 0; j < n->u.strct.len; j++) { - RSTRUCT_SET(shell, (int)j, move_child(c, shells, n->u.strct.elems[j])); + RSTRUCT_SET(shell, (int)j, courier_child(c, shells, n->u.strct.elems[j])); } break; - case MOVE_KIND_MATCH: - rb_match_move_load(shell, move_child(c, shells, n->u.match.regexp_id), - move_child(c, shells, n->u.match.str_id), + case COURIER_KIND_MATCH: + rb_match_blob_load(shell, courier_child(c, shells, n->u.match.regexp_id), + courier_child(c, shells, n->u.match.str_id), n->u.match.num_regs, n->u.match.regs); break; - case MOVE_KIND_IO: { + case COURIER_KIND_IO: { /* Write the rebuilt VALUE members back into fptr (capture detached them). * write_lock and wakeup_mutex stay nil; io.c recreates them lazily. */ struct rb_io *fptr = RFILE(shell)->fptr; - RB_OBJ_WRITE(shell, &fptr->pathv, move_child(c, shells, n->u.io.pathv_id)); - RB_OBJ_WRITE(shell, &fptr->encs.ecopts, move_child(c, shells, n->u.io.ecopts_id)); - RB_OBJ_WRITE(shell, &fptr->writeconv_pre_ecopts, move_child(c, shells, n->u.io.wc_pre_ecopts_id)); - RB_OBJ_WRITE(shell, &fptr->writeconv_asciicompat, move_child(c, shells, n->u.io.wc_asciicompat_id)); - RB_OBJ_WRITE(shell, &fptr->timeout, move_child(c, shells, n->u.io.timeout_id)); + RB_OBJ_WRITE(shell, &fptr->pathv, courier_child(c, shells, n->u.io.pathv_id)); + RB_OBJ_WRITE(shell, &fptr->encs.ecopts, courier_child(c, shells, n->u.io.ecopts_id)); + RB_OBJ_WRITE(shell, &fptr->writeconv_pre_ecopts, courier_child(c, shells, n->u.io.wc_pre_ecopts_id)); + RB_OBJ_WRITE(shell, &fptr->writeconv_asciicompat, courier_child(c, shells, n->u.io.wc_asciicompat_id)); + RB_OBJ_WRITE(shell, &fptr->timeout, courier_child(c, shells, n->u.io.timeout_id)); break; } default: @@ -3206,7 +3207,7 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) } /* Restore instance and generic ivars (any non-REF node can have them) */ for (uint32_t j = 0; j < n->niv; j++) { - rb_ivar_set(shell, n->iv_ids[j], move_child(c, shells, n->iv_vals[j])); + rb_ivar_set(shell, n->iv_ids[j], courier_child(c, shells, n->iv_vals[j])); } } @@ -3214,15 +3215,15 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) * depth-first (children larger), so inserting in reverse settles nested hash keys * inside-out (a #hash cycling through itself is out of scope). */ for (uint32_t i = c->count; i > 0; i--) { - struct move_node *n = &c->nodes[i - 1]; - if (n->kind != MOVE_KIND_HASH) continue; + struct courier_node *n = &c->nodes[i - 1]; + if (n->kind != COURIER_KIND_HASH) continue; VALUE shell = RARRAY_AREF(shells, i - 1); for (long j = 0; j < n->u.hash.size; j++) { - rb_hash_aset(shell, move_child(c, shells, n->u.hash.kv[2 * j]), - move_child(c, shells, n->u.hash.kv[2 * j + 1])); + rb_hash_aset(shell, courier_child(c, shells, n->u.hash.kv[2 * j]), + courier_child(c, shells, n->u.hash.kv[2 * j + 1])); } /* Restore the default value and default proc (before freezing) */ - VALUE ifnone = move_child(c, shells, n->u.hash.ifnone_id); + VALUE ifnone = courier_child(c, shells, n->u.hash.ifnone_id); if (n->u.hash.proc_default) { rb_hash_set_default_proc(shell, ifnone); } @@ -3239,38 +3240,38 @@ rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c) } } - VALUE root = (c->count || c->refs_count) ? move_child(c, shells, c->root) : Qnil; + VALUE root = (c->count || c->refs_count) ? courier_child(c, shells, c->root) : Qnil; RB_GC_GUARD(shells); return root; } void -rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) +rb_ractor_courier_free(struct rb_ractor_courier *c) { for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; + struct courier_node *n = &c->nodes[i]; ruby_xfree(n->iv_ids); ruby_xfree(n->iv_vals); switch (n->kind) { - case MOVE_KIND_STRING: + case COURIER_KIND_STRING: ruby_xfree(n->u.str.ptr); break; - case MOVE_KIND_ARRAY: + case COURIER_KIND_ARRAY: ruby_xfree(n->u.ary.elems); break; - case MOVE_KIND_HASH: + case COURIER_KIND_HASH: ruby_xfree(n->u.hash.kv); break; - case MOVE_KIND_STRUCT: + case COURIER_KIND_STRUCT: ruby_xfree(n->u.strct.elems); break; - case MOVE_KIND_MATCH: - rb_match_move_free(n->u.match.regs); + case COURIER_KIND_MATCH: + rb_match_blob_free(n->u.match.regs); break; - case MOVE_KIND_BACKTRACE: + case COURIER_KIND_BACKTRACE: ruby_xfree(n->u.bt.blob); break; - case MOVE_KIND_IO: + case COURIER_KIND_IO: /* A delivered IO left fptr == NULL (the rebuilt IO owns it). An * undelivered one still owns the fd and its source is already a * RactorMovedObject nobody can close: close it here, not leak it. */ @@ -3292,39 +3293,39 @@ rb_ractor_move_courier_free(struct rb_ractor_move_courier *c) * classes of its objects. All of them are shareable, so marking cannot race, and the * global GC keeps them reachable through the courier. */ void -rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c) +rb_ractor_courier_mark(struct rb_ractor_courier *c) { if (!c) return; for (uint32_t i = 0; i < c->refs_count; i++) { rb_gc_mark(c->refs[i]); } for (uint32_t i = 0; i < c->count; i++) { - struct move_node *n = &c->nodes[i]; - if (n->kind == MOVE_KIND_REF) { + struct courier_node *n = &c->nodes[i]; + if (n->kind == COURIER_KIND_REF) { rb_gc_mark(n->u.ref); } - else if (n->kind == MOVE_KIND_OBJECT) { + else if (n->kind == COURIER_KIND_OBJECT) { rb_gc_mark(n->u.obj.klass); } - else if (n->kind == MOVE_KIND_STRUCT) { + else if (n->kind == COURIER_KIND_STRUCT) { rb_gc_mark(n->u.strct.klass); } - else if (n->kind == MOVE_KIND_MATCH) { + else if (n->kind == COURIER_KIND_MATCH) { rb_gc_mark(n->u.match.klass); } - else if (n->kind == MOVE_KIND_IO) { + else if (n->kind == COURIER_KIND_IO) { rb_gc_mark(n->u.io.klass); } - else if (n->kind == MOVE_KIND_STRING) { + else if (n->kind == COURIER_KIND_STRING) { rb_gc_mark(n->u.str.klass); } - else if (n->kind == MOVE_KIND_BACKTRACE) { + else if (n->kind == COURIER_KIND_BACKTRACE) { rb_backtrace_blob_mark(n->u.bt.blob, n->u.bt.size); } - else if (n->kind == MOVE_KIND_ARRAY) { + else if (n->kind == COURIER_KIND_ARRAY) { rb_gc_mark(n->u.ary.klass); } - else if (n->kind == MOVE_KIND_HASH) { + else if (n->kind == COURIER_KIND_HASH) { rb_gc_mark(n->u.hash.klass); } } diff --git a/ractor_core.h b/ractor_core.h index d5833d34e721d3..d8933c658b43e6 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -13,7 +13,7 @@ #define RUBY_TYPED_FROZEN_SHAREABLE_NO_REC RUBY_FL_FINALIZE /* An in-flight move payload, serialized off-heap (defined in ractor.c). */ -struct rb_ractor_move_courier; +struct rb_ractor_courier; struct rb_ractor_sync { // ractor lock diff --git a/ractor_sync.c b/ractor_sync.c index b28f6feba2a8c5..c700bfb2561d22 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -19,14 +19,14 @@ static struct ractor_basket *ractor_basket_new_ref(VALUE shareable); static void ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, struct ractor_basket *b, bool raise_on_error); static void ractor_add_port(rb_ractor_t *r, st_data_t id); -// The off-heap courier used for moves. It is defined in ractor.c. -struct rb_ractor_move_courier *rb_ractor_move_courier_build(VALUE obj, struct rb_ractor_move_courier **slot); -VALUE rb_ractor_move_courier_materialize(struct rb_ractor_move_courier *c); -void rb_ractor_move_courier_free(struct rb_ractor_move_courier *c); +// The off-heap courier a copy or a move payload travels in. Defined in ractor.c. +struct rb_ractor_courier *rb_ractor_courier_build_move(VALUE obj, struct rb_ractor_courier **slot); +VALUE rb_ractor_courier_materialize(struct rb_ractor_courier *c); +void rb_ractor_courier_free(struct rb_ractor_courier *c); static void ractor_off_queue_add(rb_ractor_t *cr, struct ractor_basket *b); static void ractor_off_queue_remove(struct ractor_basket *b); -void rb_ractor_move_courier_mark(struct rb_ractor_move_courier *c); -struct rb_ractor_move_courier *rb_ractor_copy_courier_build(VALUE obj, struct rb_ractor_move_courier **slot); +void rb_ractor_courier_mark(struct rb_ractor_courier *c); +struct rb_ractor_courier *rb_ractor_courier_build_copy(VALUE obj, struct rb_ractor_courier **slot); static void ractor_port_mark(void *ptr) @@ -233,10 +233,10 @@ struct ractor_basket { * Marshal byte String. The receiver rebuilds it with Marshal.load instead * of walking it natively. */ bool marshaled; - /* The off-heap (xmalloc) courier of a basket_type_move. A move basket does - * not use v. */ - struct rb_ractor_move_courier *move_courier; - /* The marshaled bytes of a copy payload, off-heap like a move courier. When + /* The off-heap (xmalloc) courier the payload graph was serialized into. + * Copy and move both use it; when set, v is unused. */ + struct rb_ractor_courier *courier; + /* The marshaled bytes of a copy payload, off-heap like the courier. When * set, v is unused: an in-flight payload that is not a GC object needs no * in-flight pin, so it never keeps a page of the sender's heap alive. */ char *mbuf; @@ -264,12 +264,12 @@ ractor_basket_none_p(const struct ractor_basket *b) static void ractor_basket_mark(const struct ractor_basket *b) { - if (b->p.move_courier != NULL) { + if (b->p.courier != NULL) { /* The payload became this Ractor's to root the moment the message was enqueued * here: the sender's own roots stop at the send. Before and after the queue the * basket is on its holder's off_queue_baskets instead, so a courier is rooted * from the moment it is allocated to the moment it is freed. */ - rb_ractor_move_courier_mark(b->p.move_courier); + rb_ractor_courier_mark(b->p.courier); } else if (b->p.mbuf == NULL) { /* Marshaled bytes are off-heap and hold nothing to mark. */ @@ -284,10 +284,10 @@ ractor_basket_free(struct ractor_basket *b) ruby_xfree(b->p.mbuf); b->p.mbuf = NULL; b->p.mlen = 0; - if (b->p.move_courier) { - /* A move courier that was never consumed (a queue being torn down, say). */ - rb_ractor_move_courier_free(b->p.move_courier); - b->p.move_courier = NULL; + if (b->p.courier) { + /* A courier that was never consumed (a queue being torn down, say). */ + rb_ractor_courier_free(b->p.courier); + b->p.courier = NULL; } SIZED_FREE(b); } @@ -305,7 +305,7 @@ ractor_basket_alloc(void) b->p.v = Qnil; b->p.exception = false; b->p.marshaled = false; - b->p.move_courier = NULL; + b->p.courier = NULL; b->p.mbuf = NULL; b->p.mlen = 0; ccan_list_node_init(&b->off_queue_node); @@ -1013,7 +1013,7 @@ ractor_marshal_dump_rescue(VALUE obj, VALUE errinfo) static VALUE ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type *ptype, bool *pmarshaled, - struct rb_ractor_move_courier **pcourier) + struct rb_ractor_courier **pcourier) { switch (*ptype) { case basket_type_ref: @@ -1031,7 +1031,7 @@ ractor_prepare_payload(rb_execution_context_t *ec, VALUE obj, enum ractor_basket * types; anything else is marshaled here, so its user hooks run on the * sender, and the dump travels as plain bytes. */ *ptype = basket_type_copy; - if (rb_ractor_copy_courier_build(obj, pcourier) != NULL) return Qundef; + if (rb_ractor_courier_build_copy(obj, pcourier) != NULL) return Qundef; *pmarshaled = true; return rb_rescue2(ractor_marshal_dump_body, obj, @@ -1064,10 +1064,10 @@ ractor_basket_new(rb_execution_context_t *ec, VALUE obj, enum ractor_basket_type * RactorMovedObject. While in flight there is no GC object left for the * sender's GC to mark, sweep or move. The build publishes the courier into * the basket as soon as it exists. */ - rb_ractor_move_courier_build(obj, &b->p.move_courier); + rb_ractor_courier_build_move(obj, &b->p.courier); } else { - v = ractor_prepare_payload(ec, obj, &type, &marshaled, &b->p.move_courier); + v = ractor_prepare_payload(ec, obj, &type, &marshaled, &b->p.courier); if (type == basket_type_copy && marshaled) { /* Take the dump off-heap: the sender's copy of it is ordinary garbage * from here, so nothing of its heap is held while the message waits. */ @@ -1103,7 +1103,7 @@ ractor_basket_value(struct ractor_basket *b) case basket_type_copy: { /* An off-heap copy courier rebuilds exactly like a move one; only the sources * differ (still alive here, already shells there). */ - if (b->p.move_courier != NULL) goto materialize_courier; + if (b->p.courier != NULL) goto materialize_courier; /* The payload is the marshaled bytes. Marshal.load allocates through this * Ractor's normal newobj and write-barrier paths, and can raise (load hooks and * autoload run user code, an async interrupt can arrive anywhere), so it runs @@ -1151,7 +1151,7 @@ ractor_basket_value(struct ractor_basket *b) * #hash runs user code, and an async interrupt can arrive). On a raise the * courier is still owned by the basket, whose teardown frees it. */ rb_execution_context_t *ec = rb_current_ec_noinline(); - struct rb_ractor_move_courier *courier = b->p.move_courier; + struct rb_ractor_courier *courier = b->p.courier; /* Keep the materialized graph on the machine stack (result): it is the only * root until it reaches the caller. courier_free below runs a long loop, and * only the malloc'd basket's p.v holding it would give a concurrent global GC a @@ -1160,16 +1160,16 @@ ractor_basket_value(struct ractor_basket *b) enum ruby_tag_type state; EC_PUSH_TAG(ec); if ((state = EC_EXEC_TAG()) == TAG_NONE) { - result = rb_ractor_move_courier_materialize(courier); + result = rb_ractor_courier_materialize(courier); } EC_POP_TAG(); if (state != TAG_NONE) { - /* An unconsumed courier stays in b->p.move_courier; basket_free frees it. */ + /* An unconsumed courier stays in b->p.courier; basket_free frees it. */ ractor_basket_free(b); EC_JUMP_TAG(ec, state); } - rb_ractor_move_courier_free(courier); - b->p.move_courier = NULL; + rb_ractor_courier_free(courier); + b->p.courier = NULL; b->p.v = result; RB_GC_GUARD(result); break; @@ -1563,7 +1563,7 @@ ractor_basket_new_ref(VALUE shareable) b->p.v = shareable; b->p.exception = false; b->p.marshaled = false; - b->p.move_courier = NULL; + b->p.courier = NULL; b->p.mbuf = NULL; b->p.mlen = 0; diff --git a/re.c b/re.c index 35d675f71aa117..81aa79f801cf5b 100644 --- a/re.c +++ b/re.c @@ -1084,7 +1084,7 @@ match_set_regs(VALUE match, int num_regs, const OnigPosition *beg, const OnigPos * registers are written out to an onig-independent blob so the original malloc'd area can be * freed, leaving an empty shell behind, and rebuilt from the blob on the receiving side. */ void * -rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source) +rb_match_blob_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs_out, bool release_source) { struct RMatch *rm = RMATCH(match); int n = rm->num_regs; @@ -1117,13 +1117,13 @@ rb_match_move_dump(VALUE match, VALUE *regexp_out, VALUE *str_out, int *num_regs } VALUE -rb_match_move_alloc(VALUE klass, int num_regs) +rb_match_blob_alloc(VALUE klass, int num_regs) { return match_alloc_n(klass, num_regs); } void -rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob_) +rb_match_blob_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const void *blob_) { const OnigPosition *blob = blob_; struct RMatch *rm = RMATCH(match); @@ -1142,7 +1142,7 @@ rb_match_move_load(VALUE match, VALUE regexp, VALUE str, int num_regs, const voi } void -rb_match_move_free(void *blob) +rb_match_blob_free(void *blob) { ruby_xfree(blob); } From 5be69f02462fa5ce2581b373a8f594032925b86b Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 01:57:46 +0000 Subject: [PATCH 09/14] Ractor: free the basket when the port turned out to be closed ractor_send_basket freed the basket only on the path that raises Ractor::ClosedError. With raise_on_error false the closed port leaves the basket neither enqueued nor freed, and nothing else holds a pointer to it, so it leaks. The one caller that passes false is ractor_send_exit_tokens, so a monitor port that is closed before the Ractor it watches exits leaks one basket per exit. Counting live baskets over 400 rounds of mon = Ractor::Port.new r = Ractor.new { Ractor.receive } r.monitor(mon) mon.close r.send(1) r.join gives 401 live before and 1 after; receiving the token instead of closing the port was already flat, which is what the closed path should match. Predates this branch -- the same code is in master. It matters more here because a basket now sits on its holder's off_queue_baskets while it is being built: a caller that combined ractor_basket_new with raise_on_error false would leave it on that list, marked in every collection, rather than only leaking the allocation. No caller does that today. Found by the Copilot review on ruby/ruby#18392. Co-Authored-By: Claude Opus 5 (1M context) --- ractor_sync.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ractor_sync.c b/ractor_sync.c index c700bfb2561d22..5bf15a0d4b9717 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -1543,8 +1543,11 @@ ractor_send_basket(rb_execution_context_t *ec, const struct ractor_port *rp, str else { RUBY_DEBUG_LOG("closed:%u@r%u", (unsigned int)ractor_port_id(rp), rb_ractor_id(rp->r)); + /* Nothing took the basket: it was not enqueued, so free it whether or not the + * caller wants the error raised. */ + ractor_basket_free(b); + if (raise_on_error) { - ractor_basket_free(b); rb_raise(rb_eRactorClosedError, "The port was already closed"); } } From ed0b427b4ba3c776c763d2673070e88372b2096a Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 02:24:30 +0000 Subject: [PATCH 10/14] Ractor: fill a courier node before it is counted courier_alloc_node bumped c->count and then wrote the placeholder fields, so for those few stores the mark walk over nodes[0, count) covered a slot that had not been written yet. courier_alloc_ref already had the other order and says why; make the two match. Not reachable as written: nothing between the bump and the stores allocates or reaches a safepoint, so no collection can start there, and the growth above it happens before the bump. The order is what keeps that true without having to check. Found by the Copilot review on ruby/ruby#18392. Co-Authored-By: Claude Opus 5 (1M context) --- ractor.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ractor.c b/ractor.c index f30c3dbe262e20..c449490981927a 100644 --- a/ractor.c +++ b/ractor.c @@ -2463,16 +2463,17 @@ static uint32_t courier_alloc_node(struct rb_ractor_courier *c) { if (RB_UNLIKELY(c->count == c->capa)) courier_grow_nodes(c); - uint32_t id = c->count++; - /* Initialize to a harmless REF/Qnil so the courier mark (a GC root while sending) - * is safe even mid-construction; a captured node overwrites it later. */ - c->nodes[id].kind = COURIER_KIND_REF; - c->nodes[id].frozen = false; - c->nodes[id].niv = 0; - c->nodes[id].iv_ids = NULL; - c->nodes[id].iv_vals = NULL; - c->nodes[id].u.ref = Qnil; - return id; + /* Fill the slot with a harmless REF/Qnil and bump the count only after, the way + * courier_alloc_ref does: the courier is a GC root while it is being built, and + * the mark walks nodes[0, count). A captured node overwrites this later. */ + struct courier_node *n = &c->nodes[c->count]; + n->kind = COURIER_KIND_REF; + n->frozen = false; + n->niv = 0; + n->iv_ids = NULL; + n->iv_vals = NULL; + n->u.ref = Qnil; + return c->count++; } /* Size the arrays from the preflight's count, so capture never grows them. A count From 93a5504fce886e1d7cef800210677e5cc17e64e6 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 02:42:52 +0000 Subject: [PATCH 11/14] GC: call the wrapper rb_gc_rest, the name the rest of the GC uses rb_gc_finish_in_flight_gc is a one-line wrapper around rb_gc_impl_gc_rest, and every other layer of the same call already says "gc rest": gc_rest in gc/default/default.c, rb_gc_impl_gc_rest across the modular GC boundary. The wrapper exists only because ractor.c cannot call rb_gc_impl_* itself, so it should not rename the operation on the way out. "in flight" also means a message in transit elsewhere in the Ractor code (rb_ractor_repin_in_flight, rb_ractor_mark_in_flight_for_single_objspace), which is a second sense for the word in the subsystem where both GC cycles and messages are the subject. The comment above the definition described the one caller; the caller already carries that, so drop it there and keep the reason where the call is. Co-Authored-By: Claude Opus 5 (1M context) --- gc.c | 5 +---- internal/gc.h | 2 +- ractor.c | 7 ++++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/gc.c b/gc.c index f3731acd5702f8..0328bd98405b91 100644 --- a/gc.c +++ b/gc.c @@ -4137,11 +4137,8 @@ rb_gc_vm_refresh_zombie_pages(void) vm->gc.zombie_total_pages = total; } -/* Incremental marking only runs single-objspace; vm_insert_ractor0 calls this just - * before a second Ractor becomes visible so any cycle in progress finishes; a settle - * cannot resume, nor inheritance extend, another objspace's partial mark. */ void -rb_gc_finish_in_flight_gc(void) +rb_gc_rest(void) { rb_gc_impl_gc_rest(rb_gc_get_objspace()); } diff --git a/internal/gc.h b/internal/gc.h index f79e39ae952133..5bb87ef1543907 100644 --- a/internal/gc.h +++ b/internal/gc.h @@ -312,7 +312,7 @@ void rb_gc_zombie_objspaces_atfork(void); void rb_gc_disable_holders_atfork(void); void rb_gc_atfork_global_locks(void); void rb_gc_stash_cleanup_objspace(void); -void rb_gc_finish_in_flight_gc(void); +void rb_gc_rest(void); bool rb_gc_during_global_gc_p(void); bool rb_gc_single_objspace_p(void); const char *rb_obj_info(VALUE obj); diff --git a/ractor.c b/ractor.c index c449490981927a..5bf49b0c59d986 100644 --- a/ractor.c +++ b/ractor.c @@ -516,10 +516,11 @@ vm_insert_ractor0(rb_vm_t *vm, rb_ractor_t *r, bool single_ractor_mode) RUBY_DEBUG_LOG("r:%u ractor.cnt:%u++", r->pub.id, vm->ractor.cnt); VM_ASSERT(single_ractor_mode || RB_VM_LOCKED_P()); - /* Just before the process goes multi-objspace. Incremental marking only runs in a - * single-objspace world, so finish any cycle in progress before the count changes. */ + /* Incremental marking only runs in a single-objspace world, and nothing later can + * finish another objspace's partial mark, so end any cycle in progress before a + * second Ractor becomes visible. */ if (vm->ractor.cnt == 1) { - rb_gc_finish_in_flight_gc(); + rb_gc_rest(); } ccan_list_add_tail(&vm->ractor.set, &r->vmlr_node); From 0541de372fea6a61e840bad3533e18090070cd89 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 02:56:34 +0000 Subject: [PATCH 12/14] Ractor: mark a terminated Ractor's join value through one function rb_ractor_mark_in_flight_for_single_objspace marked r->sync.legacy and nothing else once the pin machinery went away, which is what rb_ractor_mark_terminated_join_value next to it already does -- two functions over the same field, and the surviving one names it correctly. Call that one from the single-objspace branch and drop the other. It also pins, which the plain rb_gc_mark did not. sync.legacy is a C-struct slot with no compaction update hook anywhere, so pinning is the treatment the field needs; the comment on rb_ractor_mark_terminated_join_value says so. The name had stopped describing the function: "in flight" meant the sending basket and the pinned copy snapshot it used to mark, and both are gone. The word is also the GC-cycle sense elsewhere in this code, which is why the wrapper it sat next to is now rb_gc_rest. Only the ractor.c translation unit used it -- ractor.c includes ractor_sync.c -- so the ractor_core.h declaration goes too. Co-Authored-By: Claude Opus 5 (1M context) --- ractor.c | 6 +++--- ractor_core.h | 1 - ractor_sync.c | 9 --------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/ractor.c b/ractor.c index 5bf49b0c59d986..ad32d9976e01d1 100644 --- a/ractor.c +++ b/ractor.c @@ -306,11 +306,11 @@ ractor_mark(void *ptr) * both the set and zombie_objspaces (orphan-merged) this marker is its only cover. */ rb_gc_mark(r->sync.default_port_value); /* A single-objspace impl (mmtk) has no zombie_objspaces and no pin/shref bits, so - * the root scan cannot reach a terminated Ractor's legacy value, queue or in-flight - * payloads; and no shref rule forbids following them from the wrapper. */ + * the root scan cannot reach a terminated Ractor's queue, in-flight payloads or + * join value; and no shref rule forbids following them from the wrapper. */ if (!rb_gc_multi_objspace_p()) { ractor_mark_unshareable_parts(r); - rb_ractor_mark_in_flight_for_single_objspace(r); + rb_ractor_mark_terminated_join_value(r); } } diff --git a/ractor_core.h b/ractor_core.h index d8933c658b43e6..21787329ce817d 100644 --- a/ractor_core.h +++ b/ractor_core.h @@ -159,7 +159,6 @@ struct rb_ractor_struct { /* Mark the GC roots held in Ractor r's C structs (from the root scan in gc.c). */ void rb_ractor_mark_local_roots(rb_ractor_t *r); void rb_ractor_mark_terminated_join_value(rb_ractor_t *r); -void rb_ractor_mark_in_flight_for_single_objspace(rb_ractor_t *r); /* Move src's registered_marks to dst and leave src empty (on join or when an orphan * is absorbed). An absorb can run during a GC sweep, so the implementation uses raw diff --git a/ractor_sync.c b/ractor_sync.c index 5bf15a0d4b9717..74a3daec193559 100644 --- a/ractor_sync.c +++ b/ractor_sync.c @@ -825,15 +825,6 @@ ractor_sync_mark(rb_ractor_t *r) } } -/* A single-objspace impl (mmtk) has no pin or shref bits and no zombie_objspaces, so - * plain marking from the wrapper keeps these alive; the default GC covers the same set - * with its pins and its zombie scan. */ -void -rb_ractor_mark_in_flight_for_single_objspace(rb_ractor_t *r) -{ - rb_gc_mark(r->sync.legacy); -} - static int ractor_sync_free_ports_i(st_data_t _key, st_data_t val, st_data_t _args) { From 8d77e28dadf4e9010171a9d8007ac9e647441b9e Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Thu, 20 Aug 2026 04:26:55 +0000 Subject: [PATCH 13/14] Ractor: reach a terminated Ractor's join value through its wrapper The root scan marked r->sync.legacy for every zombie objspace, deliberately "without depending on wrapper reachability". That makes the value a GC root, and a Ractor whose return value can reach the Ractor object roots itself: Ractor.new { Ractor.current }.join Nothing else names either one, but the value is a root, the value reaches the wrapper, so the wrapper never dies, so ractor_free never runs, rb_gc_objspace_disown is never called, and the objspace stays in zombie_objspaces for the life of the process. The count never returning to zero also holds rb_gc_single_objspace_p() false forever, which is the gate on page_pool_reclaim, so the page pool stops returning arenas to the OS. A Ractor::Port, an array, an ivar -- any path from the value back to the wrapper does it; 10000 rounds of the line above leave 10001 live Ractors. The value is of use only to whoever can still call Ractor#value, and that means holding the wrapper, so mark it as the wrapper's child instead. A self-referential pair is then ordinary garbage, and the transitive case (one Ractor's value naming another) falls out of normal tracing rather than needing a pass that iterates to a fixpoint. Only during a global GC: it stops the world and marks every objspace together, which is what lets the shareable wrapper reach an unshareable value. A local GC must not, and never did -- the zombie scan this replaces also ran only in a global GC. 300 rounds of the line above 10001 -> 1 live Ractors 8000 rounds with a dropped message 2.3 GB -> 216 MB, and it now plateaus (287 MB at 20000, 288 MB at 40000) Verified with RUBY_DEBUG=1: bootstraptest/test_ractor.rb, the GC and Ractor unit tests, GC.verify_internal_consistency after holding 100 terminated Ractors across GC.compact (their values, including self-references, all still readable), and the same under GC.stress. Co-Authored-By: Claude Opus 5 (1M context) --- gc.c | 6 ++---- ractor.c | 7 +++++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/gc.c b/gc.c index 0328bd98405b91..f72d2c09af374c 100644 --- a/gc.c +++ b/gc.c @@ -3279,15 +3279,13 @@ rb_gc_mark_roots(void *objspace, const char **categoryp) } /* A Ractor that terminated (left vm->ractor.set) but whose struct is not freed * still owns rb_gc_register_mark_object pins. Keep them alive until - * ractor_free hands them to main; an orphan (owner == NULL) was moved above. */ + * ractor_free hands them to main; an orphan (owner == NULL) was moved above. + * The join value is not rooted here: ractor_mark marks it from the wrapper. */ for (size_t i = 0; i < vm->gc.zombie_objspaces_count; i++) { rb_ractor_t *owner = vm->gc.zombie_objspaces[i].owner; if (owner) { rb_gc_mark_vm_stack_values((long)owner->registered_marks_cnt, owner->registered_marks); - /* Keep a terminated Ractor's join value (read by Ractor#value) alive - * without depending on wrapper reachability. Threads are not walked. */ - rb_ractor_mark_terminated_join_value(owner); } } diff --git a/ractor.c b/ractor.c index ad32d9976e01d1..a26176619a15b2 100644 --- a/ractor.c +++ b/ractor.c @@ -312,6 +312,13 @@ ractor_mark(void *ptr) ractor_mark_unshareable_parts(r); rb_ractor_mark_terminated_join_value(r); } + else if (rb_gc_during_global_gc_p()) { + /* The join value is only of use to whoever can still call Ractor#value, which + * means holding this wrapper, so mark it as the wrapper's child rather than as a + * root. A global GC stops the world and marks every objspace together, which is + * what lets the shareable wrapper reach an unshareable value at all. */ + rb_ractor_mark_terminated_join_value(r); + } } /* Mark the GC roots reachable from Ractor r's C structs. A local GC cannot rely on the From 4012a96bf084d035bf582d58c7dafb31f4762ac6 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Thu, 20 Aug 2026 08:02:45 +0900 Subject: [PATCH 14/14] Shrink lvar_states in iseq lvar_states only has 3 states, but an enum uses 4 bytes. This commit shrinks lvar_states to use only 2 bits (0.25 bytes), which means we save 3.75 bytes for every local variable in an iseq. We can see that for this script: def foo a = 1 b = 1 c = 1 d = 1 end iseq = RubyVM::InstructionSequence.of(method(:foo)) puts ObjectSpace.memsize_of(iseq) It outputs 736 before this commit, and 721 now, which is 15 bytes saved over the 4 local variables. --- compile.c | 32 +++++++++++++------------------- iseq.c | 4 ++-- iseq.h | 20 ++++++++++++++++++++ vm.c | 2 +- vm_core.h | 12 +++++++----- zjit/src/cruby_bindings.inc.rs | 4 ---- 6 files changed, 43 insertions(+), 31 deletions(-) diff --git a/compile.c b/compile.c index 5cc888bcc39cf2..95f49b53a9ea27 100644 --- a/compile.c +++ b/compile.c @@ -1864,14 +1864,14 @@ update_lvar_state(const rb_iseq_t *iseq, int level, int idx) iseq = ISEQ_BODY(iseq)->parent_iseq; } - enum lvar_state *states = ISEQ_BODY(iseq)->lvar_states; + uint8_t *states = ISEQ_BODY(iseq)->lvar_states; int table_idx = ISEQ_BODY(iseq)->local_table_size - idx; - switch (states[table_idx]) { + switch (iseq_lvar_state_get(states, table_idx)) { case lvar_uninitialized: - states[table_idx] = lvar_initialized; + iseq_lvar_state_set(states, table_idx, lvar_initialized); break; case lvar_initialized: - states[table_idx] = lvar_reassigned; + iseq_lvar_state_set(states, table_idx, lvar_reassigned); break; case lvar_reassigned: /* nothing */ @@ -1885,13 +1885,13 @@ static int iseq_set_parameters_lvar_state(const rb_iseq_t *iseq) { for (unsigned int i=0; iparam.size; i++) { - ISEQ_BODY(iseq)->lvar_states[i] = lvar_initialized; + iseq_lvar_state_set(ISEQ_BODY(iseq)->lvar_states, i, lvar_initialized); } int lead_num = ISEQ_BODY(iseq)->param.lead_num; int opt_num = ISEQ_BODY(iseq)->param.opt_num; for (int i=0; ilvar_states[lead_num + i] = lvar_uninitialized; + iseq_lvar_state_set(ISEQ_BODY(iseq)->lvar_states, lead_num + i, lvar_uninitialized); } return COMPILE_OK; @@ -2257,13 +2257,7 @@ iseq_set_local_table(rb_iseq_t *iseq, const rb_ast_id_table_t *tbl, const NODE * MEMCPY(ids, tbl->ids + offset, ID, size); ISEQ_BODY(iseq)->local_table = ids; - enum lvar_state *states = ALLOC_N(enum lvar_state, size); - // fprintf(stderr, "iseq:%p states:%p size:%d\n", iseq, states, (int)size); - for (unsigned int i=0; ilocal_table[i])); - } - ISEQ_BODY(iseq)->lvar_states = states; + ISEQ_BODY(iseq)->lvar_states = ZALLOC_N(uint8_t, ISEQ_LVAR_STATES_BUFLEN(size)); } ISEQ_BODY(iseq)->local_table_size = size; @@ -12616,7 +12610,7 @@ typedef uint32_t ibf_offset_t; #define IBF_MAJOR_VERSION ISEQ_MAJOR_VERSION #ifdef RUBY_DEVEL -#define IBF_DEVEL_VERSION 6 +#define IBF_DEVEL_VERSION 7 #define IBF_MINOR_VERSION (ISEQ_MINOR_VERSION * 10000 + IBF_DEVEL_VERSION) #else #define IBF_MINOR_VERSION ISEQ_MINOR_VERSION @@ -13443,12 +13437,12 @@ static ibf_offset_t ibf_dump_lvar_states(struct ibf_dump *dump, const rb_iseq_t *iseq) { const struct rb_iseq_constant_body *const body = ISEQ_BODY(iseq); - const int size = body->local_table_size; - IBF_W_ALIGN(enum lvar_state); - return ibf_dump_write(dump, body->lvar_states, sizeof(enum lvar_state) * (body->lvar_states ? size : 0)); + const int size = ISEQ_LVAR_STATES_BUFLEN(body->local_table_size); + IBF_W_ALIGN(uint8_t); + return ibf_dump_write(dump, body->lvar_states, sizeof(uint8_t) * (body->lvar_states ? size : 0)); } -static enum lvar_state * +static uint8_t * ibf_load_lvar_states(const struct ibf_load *load, ibf_offset_t lvar_states_offset, int size, const ID *local_table) { if (local_table == rb_iseq_shared_exc_local_tbl || @@ -13456,7 +13450,7 @@ ibf_load_lvar_states(const struct ibf_load *load, ibf_offset_t lvar_states_offse return NULL; } else { - enum lvar_state *states = IBF_R(lvar_states_offset, enum lvar_state, size); + uint8_t *states = IBF_R(lvar_states_offset, uint8_t, ISEQ_LVAR_STATES_BUFLEN(size)); return states; } } diff --git a/iseq.c b/iseq.c index 5a12633dd78b1c..e1cc8a364bb6ae 100644 --- a/iseq.c +++ b/iseq.c @@ -231,7 +231,7 @@ rb_iseq_free(const rb_iseq_t *iseq) if (LIKELY(body->local_table != rb_iseq_shared_exc_local_tbl)) { SIZED_FREE_N(body->local_table, body->local_table_size); } - SIZED_FREE_N(body->lvar_states, body->local_table_size); + SIZED_FREE_N(body->lvar_states, ISEQ_LVAR_STATES_BUFLEN(body->local_table_size)); compile_data_free(ISEQ_COMPILE_DATA(iseq)); if (body->outer_variables) rb_id_table_free(body->outer_variables); @@ -544,7 +544,7 @@ rb_iseq_memsize(const rb_iseq_t *iseq) size += body->iseq_size * sizeof(VALUE); size += body->insns_info.size * (sizeof(struct iseq_insn_info_entry) + sizeof(unsigned int)); size += body->local_table_size * sizeof(ID); // body->local_table - if (body->lvar_states) size += body->local_table_size * sizeof(enum lvar_state); + if (body->lvar_states) size += ISEQ_LVAR_STATES_BUFLEN(body->local_table_size) * sizeof(uint8_t); size += ISEQ_MBITS_BUFLEN(body->iseq_size) * ISEQ_MBITS_SIZE; if (body->catch_table) { size += iseq_catch_table_bytes(body->catch_table->size); diff --git a/iseq.h b/iseq.h index c9bdfcb484759f..e641dd6aed1f5e 100644 --- a/iseq.h +++ b/iseq.h @@ -26,6 +26,26 @@ RUBY_EXTERN const int ruby_api_version[]; #define ISEQ_MBITS_SET_P(buf, i) ((buf[(i) / ISEQ_MBITS_BITLENGTH] >> ((i) % ISEQ_MBITS_BITLENGTH)) & 0x1) #define ISEQ_MBITS_BUFLEN(size) roomof(size, ISEQ_MBITS_BITLENGTH) +#define ISEQ_LVAR_STATE_BITS 2 +#define ISEQ_LVAR_STATES_PER_BYTE (CHAR_BIT / ISEQ_LVAR_STATE_BITS) +#define ISEQ_LVAR_STATES_BUFLEN(size) roomof(size, ISEQ_LVAR_STATES_PER_BYTE) +STATIC_ASSERT(lvar_state_fits_in_iseq_lvar_state_bits, lvar_reassigned < (1 << ISEQ_LVAR_STATE_BITS)); + +static inline enum lvar_state +iseq_lvar_state_get(const uint8_t *buf, unsigned int i) +{ + const unsigned int shift = (i % ISEQ_LVAR_STATES_PER_BYTE) * ISEQ_LVAR_STATE_BITS; + return (enum lvar_state)((buf[i / ISEQ_LVAR_STATES_PER_BYTE] >> shift) & ((1 << ISEQ_LVAR_STATE_BITS) - 1)); +} + +static inline void +iseq_lvar_state_set(uint8_t *buf, unsigned int i, enum lvar_state state) +{ + uint8_t *const byte = &buf[i / ISEQ_LVAR_STATES_PER_BYTE]; + const unsigned int shift = (i % ISEQ_LVAR_STATES_PER_BYTE) * ISEQ_LVAR_STATE_BITS; + *byte = (*byte & ~(((1 << ISEQ_LVAR_STATE_BITS) - 1) << shift)) | ((uint8_t)state << shift); +} + #ifndef USE_ISEQ_NODE_ID #define USE_ISEQ_NODE_ID 1 #endif diff --git a/vm.c b/vm.c index eed553494a38fc..749fe5cc1a760c 100644 --- a/vm.c +++ b/vm.c @@ -1495,7 +1495,7 @@ env_copy(const VALUE *src_ep, VALUE read_only_variables) for (unsigned int j=0; jlocal_table_size; j++) { if (id == body->local_table[j]) { // check reassignment - if (body->lvar_states[j] == lvar_reassigned) { + if (iseq_lvar_state_get(body->lvar_states, j) == lvar_reassigned) { VALUE name = rb_id2str(id); VALUE msg = rb_sprintf("cannot make a shareable Proc because " "the outer variable '%" PRIsVALUE "' may be reassigned.", name); diff --git a/vm_core.h b/vm_core.h index 4c851bccac3d7a..e5700b11140f41 100644 --- a/vm_core.h +++ b/vm_core.h @@ -410,6 +410,12 @@ enum rb_builtin_attr { typedef VALUE (*rb_jit_func_t)(struct rb_execution_context_struct *, struct rb_control_frame_struct *); typedef VALUE (*rb_zjit_func_t)(struct rb_execution_context_struct *, struct rb_control_frame_struct *, rb_jit_func_t); +enum lvar_state { + lvar_uninitialized, + lvar_initialized, + lvar_reassigned, +}; + struct rb_iseq_constant_body { enum rb_iseq_type type; @@ -507,11 +513,7 @@ struct rb_iseq_constant_body { const ID *local_table; /* must free */ - enum lvar_state { - lvar_uninitialized, - lvar_initialized, - lvar_reassigned, - } *lvar_states; + uint8_t *lvar_states; /* catch table */ struct iseq_catch_table *catch_table; diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 32b76962b8262c..08d4181f77e1f6 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -1267,10 +1267,6 @@ pub struct rb_iseq_constant_body_iseq_insn_info { pub size: ::std::os::raw::c_uint, pub succ_index_table: *mut succ_index_table, } -pub const lvar_uninitialized: rb_iseq_constant_body_lvar_state = 0; -pub const lvar_initialized: rb_iseq_constant_body_lvar_state = 1; -pub const lvar_reassigned: rb_iseq_constant_body_lvar_state = 2; -pub type rb_iseq_constant_body_lvar_state = u32; #[repr(C)] pub struct rb_iseq_constant_body__bindgen_ty_1 { pub flip_count: rb_snum_t,