From 05f1c9e244bdc7c03aed822740bc715e622c3e90 Mon Sep 17 00:00:00 2001 From: technomancer Date: Thu, 19 Nov 2026 19:49:29 -0800 Subject: [PATCH 1/4] idle-pause: a uniprocessor park ends on a time bound, not only on a line The IP30 PROM menu and fx run with IE and IM set but every HEART source disabled, so no Cause.IP line ever rises. Their loops look idle, and a uniprocessor park waited only for a line or a stop: after the warm-up the machine slept for good. A uniprocessor park now ends after 100 ms with no line; later parks last one 1 ms slice until an interrupt wakes one again. IRIX's clock tick always ends a park first, so a settled IRIX guest is unchanged. Counted under wake_slice. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + clanker/INDEX.md | 1 + .../idle-pause-uniprocessor-park-must-end.md | 29 ++++++++ debug.md | 2 +- src/cpu/idle_park.rs | 68 ++++++++++++++++--- 5 files changed, 92 insertions(+), 10 deletions(-) create mode 100644 clanker/perf/idle-pause-uniprocessor-park-must-end.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ae73d664..f2ce962c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -529,6 +529,8 @@ image) where the guest test suites showed they differed: ### Idle parking +- With `idle-pause`, a single-processor machine whose guest has no interrupt + source (the Octane PROM menu, fx) no longer parks for good after a minute. - With `idle-pause`, multiprocessors (Octane, Tezro, Origin, Onyx) no longer stall at boot; `IRIS_IDLE_PEER_US` and `IRIS_IDLE_SLICE_US` bound a park. - An idle multiprocessor under jitv2 wakes far less often, and busy kernel diff --git a/clanker/INDEX.md b/clanker/INDEX.md index bc13a810f..732c138a4 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -436,6 +436,7 @@ Host performance: idle parking, thread parking, GFIFO, benchmarks. | [hot-path-inlining-must-be-explicit.md](perf/hot-path-inlining-must-be-explicit.md) | rule | Hot-path inlining must be stated, not left to the inliner | | [idle-park-smp-run-queue-polling.md](perf/idle-park-smp-run-queue-polling.md) | rule | SMP idle parking: idle IRIX CPUs poll run queues without an IPI and stop their clocks; every park slice ends to poll, a CPU that ran a device/IPI handler or work kicks the others to poll (not on its own CP0 timer tick) so all-idle parks last 10 ms, only the newest batch's samples decide a re-park, a CPU parks only in a loop another idle CPU shares, and under jitv2 a probe run of interpreted batches finds the idle loop and learns its (PC, return address) exit signature | | [idle-pause-masked-interrupt-must-wake.md](perf/idle-pause-masked-interrupt-must-wake.md) | rule | idle-pause: a masked interrupt must end the park too | +| [idle-pause-uniprocessor-park-must-end.md](perf/idle-pause-uniprocessor-park-must-end.md) | rule | idle-pause: a uniprocessor park must end on a time bound, not only on a line | | [idle-pause-must-not-park-with-interrupts-masked.md](perf/idle-pause-must-not-park-with-interrupts-masked.md) | rule | The idle-park heuristic must not park when Status.IM is zero | | [idle-pause-wake-the-parked-cpu.md](perf/idle-pause-wake-the-parked-cpu.md) | rule | idle-pause: an interrupt must unpark the CPU thread, not wait for the slice | | [idle-pause-work.md](perf/idle-pause-work.md) | history | Idle-pause work — findings & handoff | diff --git a/clanker/perf/idle-pause-uniprocessor-park-must-end.md b/clanker/perf/idle-pause-uniprocessor-park-must-end.md new file mode 100644 index 000000000..f507e9a91 --- /dev/null +++ b/clanker/perf/idle-pause-uniprocessor-park-must-end.md @@ -0,0 +1,29 @@ +# idle-pause: a uniprocessor park must end on its own, not only on a line + +Status: rule (2026-11-19). + +**Symptom.** Octane (single R10000), V12, `idle-pause` build: a minute after +start (the warm-up) the PROM menu and fx stopped responding. `idle status` +showed parks=6, slept=601 s, wake_irq=0, every wake `wake_other` (the +monitor stopping the CPU). Status 0x34005081: IE set, IM = IP4|IP6. + +**Cause.** The PROM's menu loop (drawing the cursor into the VPro FIFO and +polling the console) is ~21 distinct kernel-mode PCs in one cluster with IE +and IM set, so the detector calls it idle. But the PROM enables no HEART +source, so no Cause.IP line ever rises. A uniprocessor `park` had no time +bound (the slice only re-checked `running`), so it slept for good. IE/IM +being set says a line *could* be taken, not that any source will send one. + +**Fix.** `park` on a uniprocessor ends after `UP_PARK_MAX_NS` (100 ms) with +no line; that sets `quiet`, and later parks last one `SLICE_NS` (1 ms) until +an interrupt wake clears it. IRIX's clock tick always wakes it first, so a +settled IRIX guest is unchanged; a guest with no interrupt source runs +about a thousand short batches a second (still low host CPU). Counted in +`wake_slice`. Test: `a_park_with_no_interrupt_source_ends_and_then_slices`. + +A poll loop the detector mistakes for idle still runs at one batch per +millisecond while it waits; if fx-style polled I/O turns out slow, back off +after a quiet park that is followed by leaving the cluster. + +See also `idle-pause-must-not-park-with-interrupts-masked.md` (IM = 0) and +`idle-pause-masked-interrupt-must-wake.md`. diff --git a/debug.md b/debug.md index 638b04f90..c33202a72 100644 --- a/debug.md +++ b/debug.md @@ -273,7 +273,7 @@ exception of that class is taken. `ex tlb on` stops on every TLB exception. | `debug >` | Print every executed instruction, or write them to a file **[DEV]** | | `trace start ` / `trace stop` / `trace status` | Record full per-instruction architectural state to a file **[DEV]** | | `idleprof ` | Sample the PC to find idle and spin loops (`idle-pause` builds); `stats` prints park counters | -| `idle ` | Turn idle parking on or off at runtime, or show park counters (`wake_peer`: parks a multiprocessor ended early because another CPU was busy; `wake_slice`: parks it ended after a slice with every CPU idle, to poll the run queue; `alone`: idle-looking windows not parked because no other CPU was in the same loop; `probes`: batches jitv2 ran interpreted to look inside a compiled loop; `known_parks`: parks decided from a learned compiled exit signature; `learned`: probe runs whose signature was learned; `wake_kick`: parks ended because another CPU asked this one to poll; `kicks`: such requests; then interrupt wakes by Cause.IP line and by CPU), the CPUs seen, idle and parked, and the detector's last window (`idle-pause` builds) | +| `idle ` | Turn idle parking on or off at runtime, or show park counters (`wake_peer`: parks a multiprocessor ended early because another CPU was busy; `wake_slice`: parks that ended on their time bound with no line pending — on a multiprocessor after a slice with every CPU idle, to poll the run queue; on a uniprocessor after 100 ms with no interrupt (a PROM or standalone tool polling with every board source disabled), after which it parks 1 ms at a time until a line wakes it again; `alone`: idle-looking windows not parked because no other CPU was in the same loop; `probes`: batches jitv2 ran interpreted to look inside a compiled loop; `known_parks`: parks decided from a learned compiled exit signature; `learned`: probe runs whose signature was learned; `wake_kick`: parks ended because another CPU asked this one to poll; `kicks`: such requests; then interrupt wakes by Cause.IP line and by CPU), the CPUs seen, idle and parked, and the detector's last window (`idle-pause` builds) | | `instrstats [report\|clear\|dump]` | Per-instruction decode/execute counters (`--features instr_stats`) **[DEV]** | The undo buffer tracks register changes and memory writes. It does not diff --git a/src/cpu/idle_park.rs b/src/cpu/idle_park.rs index 767960940..5bcd56005 100644 --- a/src/cpu/idle_park.rs +++ b/src/cpu/idle_park.rs @@ -55,8 +55,18 @@ const PROBE_BACKOFF: u32 = 256; /// interpreted for good. const PROBE_RUN: u32 = 16; /// The slice of a parked uniprocessor: how often it looks at `running` -/// while it waits for a line. +/// while it waits for a line, and the whole park once a park has ended with +/// no line at all (see [`UP_PARK_MAX_NS`]). const SLICE_NS: u64 = 1_000_000; +/// The longest park of a uniprocessor. A guest with a running clock wakes +/// on its tick long before this, so IRIX never reaches it. A guest with no +/// interrupt source (the IP30 PROM and fx poll the console and the +/// graphics board with IE set but every HEART source disabled) would +/// otherwise sleep for good on a loop that only looks idle. Once a park +/// ends here, the next ones last one [`SLICE_NS`] each until a line wakes +/// one again: about a thousand short batches a second, enough for a menu +/// to see a key and a poll loop to see its device. +const UP_PARK_MAX_NS: u64 = 100_000_000; /// The longest park of a multiprocessor's CPU while every CPU is idle. Work /// reaches an idle CPU without an interrupt, but only after some CPU has run /// an interrupt handler (a disk or network completion, an inter-processor @@ -310,6 +320,10 @@ pub struct IdleParkState { probe_sig_len: usize, /// The last park was decided on a learned signature. parked_known: bool, + /// The last park of this uniprocessor ended on its time bound with no + /// line pending: the guest has no interrupt source, so park in single + /// slices (see [`UP_PARK_MAX_NS`]). + quiet: bool, /// `hot.cycles` at the last update: a reset starts it again from zero, /// and the signatures of the old kernel go with it. last_cycles: u64, @@ -369,6 +383,7 @@ impl IdleParkState { probe_sig: [(0, 0); CLUSTER_MIN_DISTINCT], probe_sig_len: 0, parked_known: false, + quiet: false, last_cycles: 0, park_skip_until_cycle: 0, started: *PROCESS_START.get_or_init(Instant::now), @@ -889,6 +904,7 @@ impl IdleParkState { // can only do running. (Waiting for `ip & im` slept through the // timer for good in such a loop: an IRIX 6.5 hang.) if ip != 0 { + self.quiet = false; PARK_WAKE_IRQ.fetch_add(1, Ordering::Relaxed); PARK_WAKE_IRQ_LINE[((ip >> 8).trailing_zeros() & 7) as usize].fetch_add(1, Ordering::Relaxed); PARK_WAKE_IRQ_CPU[core.cpu_id as usize % MAX_CPUS].fetch_add(1, Ordering::Relaxed); @@ -916,7 +932,8 @@ impl IdleParkState { // CPUs' run queues, and the IP35 kernel stops an idle CPU's own // clock. That CPU kicks the others when it parks again // (`kick_peers`), so the slice is only a bound. Alone, sleep on - // until a line. + // until a line, but not for good: a guest with no interrupt + // source never sends one (`UP_PARK_MAX_NS`). let peer_running = core.smp && { let others = SEEN.load(Ordering::Relaxed) & !(1 << core.cpu_id); (PARKED.load(Ordering::SeqCst) | IDLE.load(Ordering::Relaxed)) & others != others @@ -925,15 +942,20 @@ impl IdleParkState { peer_slice_ns() << self.peer_backoff } else if core.smp { all_idle_slice_ns() - } else { + } else if self.quiet { SLICE_NS + } else { + UP_PARK_MAX_NS }; - // A multiprocessor's park ends when its slice is up. Every device - // interrupt unparks all parked CPUs (`wake`), so a wake that was - // for another CPU sleeps on for the rest of the slice. - let waited = if core.smp { start.elapsed().as_nanos() as u64 } else { 0 }; + // A park ends when its slice is up. Every device interrupt + // unparks all parked CPUs (`wake`), so a wake that was for + // another CPU sleeps on for the rest of the slice. + let waited = start.elapsed().as_nanos() as u64; if waited >= slice { - if peer_running { + if !core.smp { + self.quiet = true; + PARK_WAKE_SLICE.fetch_add(1, Ordering::Relaxed); + } else if peer_running { PARK_WAKE_PEER.fetch_add(1, Ordering::Relaxed); self.peer_backoff = (self.peer_backoff + 1).min(PEER_BACKOFF_MAX); } else { @@ -944,7 +966,9 @@ impl IdleParkState { let t0 = Instant::now(); // Still a bounded slice — `running` and the ci_clock threshold are // only polled — but an interrupt now ends it at once. - std::thread::park_timeout(Duration::from_nanos(slice - waited)); + // A uniprocessor looks at `running` once a `SLICE_NS`. + let nap = if core.smp { slice - waited } else { (slice - waited).min(SLICE_NS) }; + std::thread::park_timeout(Duration::from_nanos(nap)); let elapsed_ns = t0.elapsed().as_nanos() as u64; PARK_SLEEP_NS.fetch_add(elapsed_ns, Ordering::Relaxed); advance_parked(core, elapsed_ns); @@ -1665,6 +1689,32 @@ mod wake_tests { "park slept through a pending interrupt masked only by the spl"); } + /// A uniprocessor guest with no interrupt source (the IP30 PROM's menu + /// and fx, IE set but every HEART source off) must not park for good: + /// the park ends on its time bound, and the next ones last one slice. + /// An interrupt wake restores the long bound. + #[test] + fn a_park_with_no_interrupt_source_ends_and_then_slices() { + let _one = PARK_TEST_LOCK.lock(); + let mut core = MipsCore::default(); + core.cp0_status = STATUS_IE | (1 << (STATUS_IM_SHIFT + 4)) | (1 << (STATUS_IM_SHIFT + 6)); + let running = AtomicBool::new(true); + let mut st = IdleParkState::ready(); + let t0 = Instant::now(); + st.park(&mut core, &running); + let first = t0.elapsed(); + assert!(first >= Duration::from_nanos(UP_PARK_MAX_NS), "the first park ended early: {first:?}"); + assert!(first < Duration::from_secs(2), "the first park did not end on its bound: {first:?}"); + assert!(st.quiet); + let t1 = Instant::now(); + st.park(&mut core, &running); + let second = t1.elapsed(); + assert!(second < Duration::from_millis(50), "a quiet guest parks one slice at a time: {second:?}"); + core.hot.interrupts.store(CAUSE_IP7 as u64, Ordering::SeqCst); + st.park(&mut core, &running); + assert!(!st.quiet, "an interrupt wake ends quiet slicing"); + } + /// A park that wakes on a masked line was a false positive: back off so /// a device-poll or DELAY loop keeps running instead of sleeping a slice /// per batch. A park woken by an unmasked line leaves parking enabled. From ee5d415d6a85dfeaf177f019ad41fbbe9dcacf07 Mon Sep 17 00:00:00 2001 From: technomancer Date: Thu, 19 Nov 2026 19:49:32 -0800 Subject: [PATCH 2/4] jitv2: key code pages by bank and offset; one pool per machine (#779) A code page was keyed by its physical frame. Each node of an IP27 or IP35 machine has its own address space, so each node had its own page pool and compile queue, and a memory controller that moved a bank had to flush every CPU's compiled code: dozens of flushes while a PROM sizes memory. A page is now keyed by the memory it holds. Every device that holds runnable memory (a RAM bank, a PROM, a flash part) takes an id from one counter (CodeBank), and a bus answers BusDevice::code_page(addr) with the bank and the page within it, the bank's own write counter, and for RAM and PROM a pointer to the page in the bank's storage. The pool finds a page by that key. The direct-mapped frame table stays as a lock-free cache, one per address space, and each page records where each CPU of the pool last reached it. A page no bank names (MMIO) is keyed by its address within its address space, as every page was. So code follows its bank through a remap: the request every CPU takes after one (REQUEST_JIT_REMAP, was REQUEST_JIT_FLUSH) now only forgets the CPU's addresses. Addresses that reach the same bytes share a page, and a cached and an uncached fetch that decode to different bytes get a page each without a split flag in the page. On PROM memory sizing the Octane takes 94 such requests, the Tezro 16 and the Indy 12, with no flush. Every CPU of a machine now shares one pool, one compile queue and one body of code (share_pool, for IP27 and IP35); nodes with different processor modules keep a pool each. Compile workers read RAM and PROM pages in the bank's storage, by key; a flash page is read through the bus of a CPU that reached it, and the bus is asked for the key before and after the read. The Bridge did not forward the write counter of the flash behind it, so the Octane's PROM ran as code that never changes; it forwards the counter and the page question now. The IP27 node flash and the BaseIO flash had no counter and get one, bumped by every write cycle. j2 pcp shows a page's bank, offset and per-CPU frames; j2 status counts remaps. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 8 + HACKING.md | 49 +- clanker/INDEX.md | 1 + ...as-pages-must-share-generation-counters.md | 5 + .../pages-are-keyed-by-bank-and-offset.md | 72 +++ debug.md | 4 +- src/cpu/jitv2/comp.rs | 102 ++- src/cpu/jitv2/equiv_test.rs | 4 + src/cpu/jitv2/jitv2.rs | 599 ++++++++++++++---- src/cpu/jitv2/pcp_dump.rs | 34 +- src/cpu/jitv2_html_j2wp.rs | 13 +- src/cpu/mips_exec.rs | 217 ++++--- src/cpu/mips_exec/gdb.rs | 2 +- src/cpu/mips_exec/monitor.rs | 31 +- src/cpu/mips_exec_test.rs | 105 ++- src/cpu/smp.rs | 42 +- src/dev/baseio_flash.rs | 26 +- src/dev/bedrock/md.rs | 12 +- src/dev/bedrock_flash.rs | 11 +- src/dev/bridge.rs | 30 + src/dev/bridge_tests.rs | 25 + src/dev/heart_mem.rs | 4 + src/dev/hub/tests.rs | 4 + src/dev/mem.rs | 22 + src/dev/node_flash.rs | 41 ++ src/dev/prom.rs | 18 + src/dev/sflash.rs | 14 +- src/dev/widths.rs | 4 + src/ip27.rs | 14 +- src/ip30.rs | 5 + src/ip30_machine.rs | 5 +- src/ip32.rs | 2 + src/ip35.rs | 29 +- src/ip35/tests.rs | 23 +- src/physical.rs | 13 + src/ppmem/ppmem.rs | 20 + src/sn.rs | 4 + src/sn/tests.rs | 8 +- src/traits.rs | 115 ++++ src/window_map.rs | 8 + 40 files changed, 1400 insertions(+), 345 deletions(-) create mode 100644 clanker/jitv2/pages-are-keyed-by-bank-and-offset.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ce962c7..e3271f69c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -504,6 +504,14 @@ image) where the guest test suites showed they differed: ### jitv2 +- jitv2: compiled code is kept when a memory controller moves a bank (the + PROM's memory sizing on Indy, O2, Octane and the IP27/IP35 machines no + longer flushes it), and addresses that reach the same memory share one + compiled page. +- jitv2 on machines of several nodes (Origin 2000/Onyx2, Origin 3200/Onyx + 3000): every processor shares one compile pool and one body of compiled + code, where each node had its own. `j2 pcp` shows a page's bank and offset; + `j2 status` counts remaps. - The compile pool no longer polls: an idle worker parks until a request arrives, instead of waking every 200µs (which cost about 1.4% of a host core per worker), and a small host can now settle. All machines. diff --git a/HACKING.md b/HACKING.md index ec18598a5..ffa1e90f2 100644 --- a/HACKING.md +++ b/HACKING.md @@ -146,6 +146,7 @@ pub trait BusDevice: Send + Sync { fn describe_fault(&self, addr, size) -> Option // why a failed access failed, for the log fn mem_ptr(&self, addr: u64) -> Option<*const u64>; // direct backing store, for RAM fn gen_ptr(&self, addr: u64) -> *const AtomicU64; // jitv2 page generation counter + fn code_page(&self, addr: u64) -> Option; // jitv2: which bank's page this is (see section 8) } pub trait Device: Send + Sync { @@ -540,7 +541,9 @@ one `Physical`. `src/cpu/smp.rs` holds what they share: - **Getting the machine to yourself.** `World::stop_others` parks every other CPU and runs a closure. Call it holding no lock another CPU could be waiting on. Two callers take turns, and the one waiting counts as parked. Remapping - memory and flushing jitv2 go through it. + memory and flushing jitv2 go through it. A remap that moved memory then asks + every CPU (`World::layout_changed`) to forget the addresses of its JIT code + pages; no compiled code is dropped for it (section 8). The monitor reaches CPUs through `CpuMonitor` (`src/cpu/cpu_monitor.rs`), on every machine: it routes each command and the commands themselves live in @@ -1335,6 +1338,37 @@ code into host code with Cranelift. The concepts: the pool if a worker is asleep (a dropped push, on a full queue, wakes nobody), and the park's timeout is only a backstop for the periodic pending-publish sweep. +- **A page is keyed by the memory it holds, not by its address.** Every + device that holds runnable memory (a RAM bank, a PROM, a flash part) owns a + `CodeBank`, a number taken from one process-wide counter. A bus answers + `code_page(addr)` with the page's `CodeKey` (the bank and the page within + it), the bank's own generation counter for it, and, where the bank can hand + it out, a pointer to the page's words in the bank's storage. Buses forward + the question as they forward accesses. A bridge forwards it + to the device behind it (the Octane's PROM flash sits behind the Bridge's + flash chip select). Where no bank answers (MMIO, or a device that does not + say), the page is keyed by its address within its address + space and counted by `gen_ptr`, so it compiles as before but is shared + only among the CPUs of that space. The pool finds a `PhysicalCodePage` by + that key. So code follows its bank when a memory controller moves it + (a remap flushes nothing), two addresses for the same bytes share one page, + and two fetches of one address that decode to different bytes (cached and + uncached on a node board) get a page each. +- **Finding a page from an address.** Each address space (the CPUs that see + memory at the same addresses: every CPU of an Octane, the CPUs of one node) + has a direct-mapped frame table, `frame -> slot`, read lock-free on a page + switch. It only caches what the bus and the key index say: a miss asks the + bus under the pool lock, and a remap empties the table of every CPU that + takes the request (`Jitv2::forget_addresses`). Each page also records, per + CPU of the pool, the frame that CPU last reached it at; the monitor shows + them, and a compile uses one only for a page whose bank hands out no + storage (below). +- **Reading a page to compile it.** A compile worker reads RAM and PROM pages + in the bank's own storage, by the pointer the page was claimed with: no bus, + no address. A page of a device that gives no pointer (a flash part) is read + through the bus of a CPU that reached it, at that CPU's address, and the + worker asks the bus for the key there before and after the read, so an + address a remap moved other memory under is never compiled as this page. - **Invalidation by generation counters.** Every RAM page has a generation counter owned by the memory device (`gen_ptr`). Any write to the page, from the CPU or from DMA, bumps it; compiled code checks it and stops being @@ -1358,16 +1392,19 @@ code into host code with Cranelift. The concepts: an emitter end the compiled region or call the interpreter's handler. The equivalence tests, the cpu-tests suite run on both engines, `jitcheck` and the `jitv2_lockstep` feature compare the JIT against the interpreter. -- **Several CPUs share one pool.** Compiled code reaches CPU state only through - its core pointer, so it runs on either CPU. Dispatch reads pages lock-free, the - pfn map publishes a slot after its claim, and inline stores bump generations +- **Every CPU of a machine shares one pool.** Compiled code reaches CPU state + only through its core pointer, and a page is keyed by its memory, so one page + pool, one compile queue and one body of code serve every CPU, on whichever + node (`share_pool`; nodes with different processor modules get a pool each, + because code is specific to the module). Dispatch reads pages lock-free, a + frame table publishes a slot after its claim, and inline stores bump generations atomically. Compiled code is freed only by a full flush, and every flush parks all CPUs first (`jitv2_flush_from_here`, `run_leader_flush`). Compiled loads and stores are plain host accesses, so on a multiprocessor a guest `SYNC` compiles to a host fence on hosts that are not x86 (the interpreter's `SYNC` does the same). -- **Uncached fetches the board decodes apart.** A page is keyed by the - physical address of the fetch. The IP35 reset PROM answers an uncached fetch +- **Uncached fetches the board decodes apart.** A frame table is indexed by + the physical address of the fetch. The IP35 reset PROM answers an uncached fetch at 0x1fc00000 whatever RAM sits behind the cached view there, so a board whose banks reach that window presents the fetch at an address of its own (`uncached_fetch_alias`, which only the R16000 model takes) and the JIT sees diff --git a/clanker/INDEX.md b/clanker/INDEX.md index 732c138a4..860154f9d 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -62,6 +62,7 @@ The MIPS JIT (`src/cpu/jitv2/`): design, Cranelift codegen, delay slots, invalid | Note | Kind | Title | |---|---|---| | [alias-pages-must-share-generation-counters.md](jitv2/alias-pages-must-share-generation-counters.md) | rule | An aliased physical page must share its generation counter | +| [pages-are-keyed-by-bank-and-offset.md](jitv2/pages-are-keyed-by-bank-and-offset.md) | design | A code page is keyed by bank:offset (`CodeKey`), found through `code_page` on the bus; frame tables per address space; remaps flush nothing; one pool per machine | | [bc1-is-an-ordinary-branch.md](jitv2/bc1-is-an-ordinary-branch.md) | measure | BC1 is an ordinary PC-relative branch, and compiling it is worth ~4x | | [block-fragmentation-blocks-cse.md](jitv2/block-fragmentation-blocks-cse.md) | measure | Block fragmentation, not callouts, is what starves `opt_level=speed` | | [callout-arg0-is-core-ptr.md](jitv2/callout-arg0-is-core-ptr.md) | rule | JIT→Rust callouts: arg 0 is always the `MipsCore` pointer | diff --git a/clanker/jitv2/alias-pages-must-share-generation-counters.md b/clanker/jitv2/alias-pages-must-share-generation-counters.md index 0577f27aa..7a55615ff 100644 --- a/clanker/jitv2/alias-pages-must-share-generation-counters.md +++ b/clanker/jitv2/alias-pages-must-share-generation-counters.md @@ -1,5 +1,10 @@ # An aliased physical page must share its generation counter +Status: still true for the counters. Since 2026-11-19 an aliased page also +shares its code page: pages are keyed by the memory behind them +([pages-are-keyed-by-bank-and-offset.md](pages-are-keyed-by-bank-and-offset.md)), +and a page's counter comes from `code_page`, the bank's own. + Two independent bugs, one per memory path, both with the same effect: a page reachable at two physical addresses had **two different** JIT generation counters, so writes through one address never invalidated code compiled through diff --git a/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md b/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md new file mode 100644 index 000000000..c67b7587e --- /dev/null +++ b/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md @@ -0,0 +1,72 @@ +# A code page is keyed by bank:offset, not by physical address + +Status: current (2026-11-19, issue #779). + +## Rule + +`PhysicalCodePage.key` is a `CodeKey { bank, page }`: the device that holds the +bytes (`CodeBank`, an id from one atomic counter, taken by every RAM bank, PROM +and flash part when built) and the 4 KiB page within it. The bus answers +`BusDevice::code_page(addr)` with the key, the bank's own generation counter +and, for RAM and PROM, a pointer to the page's words in the bank's storage. +Call it with the start of the page. + +- Never take a page's counter from `gen_ptr` on a `Physical`: with ppmem that + is the *window's* counter for the address, which a remap points at another + bank. `code_page().gen` is the bank's own. +- A bus that forwards `gen_ptr` should forward `code_page`. Where the bus + answers `None` the page is keyed by address in its space + (`CodeKey::at_address`, counter from `gen_ptr`), which is what every page + was before: it compiles, but a remap does not follow it and other nodes do + not share it. That is a fallback for MMIO, not a place for real code: every + device code runs from sits behind `BusDevice`s, so forward the question. + The Bridge did not (the Octane's PROM flash is on its chip select 0, the + BaseIO flash likewise), nor did it forward `gen_ptr`, so on main that code + ran against a counter that never moved. An earlier draft returned "not + code" for unnamed pages and the Octane PROM ran interpreted: its 2 GB + memory clear took minutes. +- A test bus with no banks answers `CodeKey::flat_page(addr, gen)` (bank 0). + Two devices behind one bus must not both do that for the same address. +- The key assumes the bank offset of a page start is page-aligned. + +## What sits on top + +- `Jitv2::by_key`: key -> slot, pre-sized, touched only on a frame-table miss. +- `Jitv2::spaces`: one `PfnMap` (frame -> slot) per address space, the + lock-free page-switch path. A cache; `forget_addresses` empties it. +- `PhysicalCodePage::addr[cpu]`: frame (and uncached bit) each CPU of the pool + last reached the page at. Display, and the bus read below. May be stale + after a remap; nothing trusts it without re-asking the bus for the key. +- The executor keeps its own `cur_code_pfn` + `pcp_uncached` for the + same-page test; it never compares against the page. +- Compile read (`comp::snapshot_page`): bank storage when `page.words()` is + not null; else `CodeBuses::locate` (a CPU's bus + address, key checked + before and after the read). + +## Remaps + +`World::layout_changed` sets `REQUEST_JIT_REMAP` on every CPU; each, before +running guest code again, calls `jitv2_layout_changed`: clears its nanotlb and +`pcp`, its space's frame table and its `addr` slots. No flush, no world stop. +Measured on PROM memory sizing: Octane 2 CPUs 94 requests, Tezro 4 CPUs 16, +Indy 12, all with 0 mega-flushes (each was a full flush before). + +## One pool per machine + +`jitv2::share_pool` (IP27, IP35 family) puts every CPU on one `Jitv2`: CPU +numbers in the pool are dense (not the board's CPU ids), each node is a space. +Nodes with different `CpuModule`s get separate pools: `JitConsts`, the ISA +level and the D-cache geometry are per pool. + +## Open + +- An address-keyed page survives a remap with its code. Remaps move RAM + banks, which are always bank-keyed, so nothing known breaks; a controller + that swapped two unnamed devices would. +- Flash parts (`CfiFlash`, bedrock `Flash`) hand out no storage pointer (their + `Vec` lifetime and mode-dependent reads were not audited), so their + pages still read through the bus. +- `POOL_CPUS` is `smp::MAX_CPUS` (4). +- Seen once in five Tezro boots, on the bus-read build: all CPUs spinning in + the PROM near 0xc00000001fc33614 with 67 pages claimed. Not reproduced, not + yet compared with main. diff --git a/debug.md b/debug.md index c33202a72..a61b585c7 100644 --- a/debug.md +++ b/debug.md @@ -305,7 +305,7 @@ take effect after `j2 flush`. | Command | Effect | Default (release / `developer`) | |---|---|---| -| `j2 status` (alias `j2 stats`) | Arena use, compile counts, skipped/rejected recompiles, reject reasons | — | +| `j2 status` (alias `j2 stats`) | Arena use, compile counts, skipped/rejected recompiles, reject reasons, and `remaps`: times a CPU forgot its page addresses because memory moved (no code is dropped for one) | — | | `j2 dispatch [on\|off]` | Main switch for JIT dispatch; off = interpreter only | `on` | | `j2 inline [on\|off]` | Compile synchronously on the CPU thread instead of the compile pool; a page with code recompiles only when new entries make it hot, as the pool would (`[jitv2] compile = "sync"` at startup). Refused with several CPUs, and `on` is refused in `lightning` builds | `off` (`on` under lockstep) | | `j2 threads` | Compile-pool thread count and why (read-only; set with `[jitv2] threads` or `--jitv2-threads`) | one per guest CPU, at most 4, capped by the host's usable CPUs | @@ -324,7 +324,7 @@ take effect after `j2 flush`. | `j2 flush` | Drop all compiled code and reset the arena (stop the CPU first) | — | | `j2 clear ` / `j2 deny ` | Reset one physical code page / deny one entry point. The address has to be in the machine's memory; the error lists its ranges | — | | `j2 analyze ` | Run the analyzer's walk on live memory and print each word with its classification | — | -| `j2 pcp [vaddr]` | Physical code page state: entries, generation, compiled code. Takes a virtual address; on an R10000, xkphys (`0x9000000000000000` plus the physical address) reaches any page | — | +| `j2 pcp [vaddr]` | Code page state: its key (`bank`, `page` within the bank), the frame each CPU reaches it at, entries, generation, compiled code. Takes a virtual address; on an R10000, xkphys (`0x9000000000000000` plus the physical address) reaches any page | — | | `j2 dumppcp [vaddr] [path]` | Capture a page and its memory for the offline `jitv2-pcp-dump` | — | | `j2 corpus [dir]` | Dump every cached page as `.pcp` files for offline codegen measurement | `jitv2_corpus/` | | `j2 html [path]` | Render the physical code page visualiser, with one section per memory range of the machine | — | diff --git a/src/cpu/jitv2/comp.rs b/src/cpu/jitv2/comp.rs index b226836f8..c0309f2da 100644 --- a/src/cpu/jitv2/comp.rs +++ b/src/cpu/jitv2/comp.rs @@ -127,7 +127,15 @@ pub(crate) enum Snapshot { /// page under sustained concurrent SMC just keeps losing the race to /// itself, correctly punted to the next arrival's request rather than /// compiled against torn bytes. -pub(crate) fn snapshot_page(page: &PhysicalCodePage, bus: &dyn BusDevice, phys_base: u64, words: &mut [u32; ENTRIES_PER_PAGE]) -> Snapshot { +/// +/// A page whose bank hands out its storage is read there, by its key alone: +/// no bus, and no address a remap could have moved. `bus` and `phys_base` +/// serve a page whose bank does not. +pub(crate) fn snapshot_page(page: &PhysicalCodePage, bus: Option<(&dyn BusDevice, u64)>, words: &mut [u32; ENTRIES_PER_PAGE]) -> Snapshot { + let storage = page.words(); + if storage.is_null() && bus.is_none() { + return Snapshot::Unreadable; + } const MAX_SEQLOCK_RETRIES: u32 = 8; for _ in 0..MAX_SEQLOCK_RETRIES { let gen0 = page.current_gen(); @@ -142,12 +150,21 @@ pub(crate) fn snapshot_page(page: &PhysicalCodePage, bus: &dyn BusDevice, phys_b // a word before `gen0`, or see a bump before the bytes it follows, // and the snapshot pairs a generation with older bytes. std::sync::atomic::fence(std::sync::atomic::Ordering::Acquire); - for (i, w) in words.iter_mut().enumerate() { - let r = bus.read32(phys_base + i as u64 * 4); - if !r.is_ok() { - return Snapshot::Unreadable; + if !storage.is_null() { + // A guest store can land while this reads; the generation check + // below discards such a read, so the words are read as plain + // memory, volatile so none is assumed stable. + for (i, w) in words.iter_mut().enumerate() { + *w = unsafe { std::ptr::read_volatile(storage.add(i)) }; + } + } else if let Some((bus, phys_base)) = bus { + for (i, w) in words.iter_mut().enumerate() { + let r = bus.read32(phys_base + i as u64 * 4); + if !r.is_ok() { + return Snapshot::Unreadable; + } + *w = r.data; } - *w = r.data; } std::sync::atomic::fence(std::sync::atomic::Ordering::Acquire); let gen1 = page.current_gen(); @@ -169,14 +186,27 @@ pub(crate) fn snapshot_page(page: &PhysicalCodePage, bus: &dyn BusDevice, phys_b /// `finalize_batch_nonforced` + seal-queue publish). fn prepare_multi_entry_compile( req: &CompileRequest, - bus: &Arc, + buses: &crate::cpu::jitv2::jitv2::CodeBuses, analyzer: &mut Analyzer, cache_fp: Option<&crate::cpu::jitv2::pcache::Fingerprint>, #[cfg(feature = "developer")] stats: &crate::cpu::jitv2::JitStats, ) -> PrepareOutcome { let page = unsafe { &*req.page }; - let bus = page.uncached.as_ref().unwrap_or(bus); - let phys_base = page.pfn as u64 * PAGE_SIZE as u64; + // The page is keyed by the memory behind it, and read in that memory's + // own storage when its bank hands that out (RAM, a PROM). Else its bytes + // are read where some CPU reaches the memory, through that CPU's bus; + // if none does right now (a remap since the request), the next arrival + // asks again. + let located = if page.words().is_null() { + let Some(at) = buses.locate(page, req.cpu as usize) else { + page.mark_prepare_bounced(); + return PrepareOutcome::Done(false); + }; + Some(at) + } else { + None + }; + let via_bus = located.as_ref().map(|(bus, at)| (&**bus, *at)); // Dirty-cache gate. The seqlock snapshot below reads RAM off the bus; the // guest CPU sees RAM overlaid with its own dirty L1-D/L2 lines, and a @@ -205,7 +235,14 @@ fn prepare_multi_entry_compile( // §13.3 step 2: seqlock-snapshot the page bytes. let mut words = [0u32; ENTRIES_PER_PAGE]; - let gen_snap = match snapshot_page(page, &**bus, phys_base, &mut words) { + let gen_snap = match snapshot_page(page, via_bus, &mut words) { + // A remap during a read through the bus: the address was the page's + // before it (`locate`), and is not now, so the words may be other + // memory's. + Snapshot::Taken(_) if via_bus.is_some_and(|(bus, at)| !crate::cpu::jitv2::jitv2::page_is_at(page, bus, at)) => { + page.mark_prepare_bounced(); + return PrepareOutcome::Done(false); + } Snapshot::Taken(gen) => { // The lines are gone: lift the held publish's denials (below). if page.entry_gen() >= crate::cpu::jitv2::jitv2::GEN_HELD { @@ -365,7 +402,7 @@ fn prepare_multi_entry_compile( // churn-avoidance check ignore writes to it. let mut used = [0u64; crate::cpu::jitv2::BITMAP_WORDS]; let instr_count = { - let instrs = analyzer.walk_multi_entry(&words, &candidates, phys_base as u32, max_instrs_per_compile()); + let instrs = analyzer.walk_multi_entry(&words, &candidates, 0, max_instrs_per_compile()); let mut n = 0usize; for instr in crate::cpu::jitv2::analyzer::instrs_linear(instrs) { let w = instr.word as usize; @@ -449,7 +486,7 @@ fn prepare_multi_entry_compile( for &o in analyzer.covered() { entries[o as usize >> 6] |= 1u64 << (o % 64); } - crate::cpu::jitv2::hashstats::record(page.pfn, &words, &used, &entries, req.compiled_for_fr1, instr_count, had_snapshot, + crate::cpu::jitv2::hashstats::record(page.pfn(), &words, &used, &entries, req.compiled_for_fr1, instr_count, had_snapshot, page.last_skip_reject.load(std::sync::atomic::Ordering::Relaxed)); } @@ -471,7 +508,7 @@ fn prepare_multi_entry_compile( /// a real codegen decline, or a transient bus-read failure). pub fn handle_request( req: &CompileRequest, - bus: &Arc, + bus: &crate::cpu::jitv2::jitv2::CodeBuses, analyzer: &mut Analyzer, codegen: &mut Codegen, #[cfg(feature = "developer")] stats: &crate::cpu::jitv2::JitStats, @@ -664,7 +701,7 @@ pub type PendingCount = usize; /// same reason. pub fn handle_request_deferred( req: &CompileRequest, - bus: &Arc, + bus: &crate::cpu::jitv2::jitv2::CodeBuses, analyzer: &mut Analyzer, codegen: &mut Codegen, pending: &mut PendingCount, @@ -932,11 +969,11 @@ mod tests { #[cfg(feature = "developer")] { let stats = crate::cpu::jitv2::JitStats::default(); - handle_request(req, bus, analyzer, codegen, &stats) + handle_request(req, &crate::cpu::jitv2::jitv2::CodeBuses::single(bus.clone()), analyzer, codegen, &stats) } #[cfg(not(feature = "developer"))] { - handle_request(req, bus, analyzer, codegen) + handle_request(req, &crate::cpu::jitv2::jitv2::CodeBuses::single(bus.clone()), analyzer, codegen) } } @@ -957,6 +994,7 @@ mod tests { fn read64(&self, _addr: u64) -> BusRead64 { BusRead64::err() } fn write64(&self, _addr: u64, _val: u64) -> u32 { BUS_ERR } fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { std::ptr::null() } + fn code_page(&self, addr: u64) -> Option { Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) } } /// Every word decodes as the JIT region-boundary sentinel — a genuine @@ -979,6 +1017,7 @@ mod tests { fn read64(&self, _addr: u64) -> BusRead64 { BusRead64::err() } fn write64(&self, _addr: u64, _val: u64) -> u32 { BUS_ERR } fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { std::ptr::null() } + fn code_page(&self, addr: u64) -> Option { Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) } } struct ErrDevice; @@ -992,6 +1031,7 @@ mod tests { fn read64(&self, _addr: u64) -> BusRead64 { BusRead64::err() } fn write64(&self, _addr: u64, _val: u64) -> u32 { BUS_ERR } fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { std::ptr::null() } + fn code_page(&self, addr: u64) -> Option { Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) } } #[test] @@ -1001,7 +1041,7 @@ mod tests { let counter = AtomicU64::new(0); let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(4); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); @@ -1021,7 +1061,7 @@ mod tests { let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); let compile = |page: &mut PhysicalCodePage| { page.mark_requested(4); - let req = CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; handle_request_for_test(&req, &bus, &mut Analyzer::new(), &mut Codegen::new()); }; compile(&mut page); @@ -1051,7 +1091,7 @@ mod tests { let mut codegen = Codegen::new(); page.mark_requested(4); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; handle_request_for_test(&req, &bus, &mut analyzer, &mut codegen); assert!(page.is_runnable(4), "offset 4 must compile and publish on its own"); let func_after_first_compile = page.func(); @@ -1074,7 +1114,7 @@ mod tests { let counter = AtomicU64::new(0); let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(0); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); @@ -1099,7 +1139,7 @@ mod tests { let bus: Arc = Arc::new(ErrDevice); let mut page = PhysicalCodePage::new(0, std::ptr::null()); page.mark_requested(0); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); @@ -1133,7 +1173,7 @@ mod tests { let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(4); assert!(page.try_schedule_page()); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); handle_request_for_test(&req, &bus, &mut analyzer, &mut codegen); @@ -1149,7 +1189,7 @@ mod tests { let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(0); assert!(page.try_schedule_page()); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); handle_request_for_test(&req, &bus, &mut analyzer, &mut codegen); @@ -1163,7 +1203,7 @@ mod tests { let mut page = PhysicalCodePage::new(0, std::ptr::null()); page.mark_requested(0); assert!(page.try_schedule_page()); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); handle_request_for_test(&req, &bus, &mut analyzer, &mut codegen); @@ -1179,11 +1219,11 @@ mod tests { #[cfg(feature = "developer")] { let stats = crate::cpu::jitv2::JitStats::default(); - handle_request_deferred(req, bus, analyzer, codegen, pending, &stats) + handle_request_deferred(req, &crate::cpu::jitv2::jitv2::CodeBuses::single(bus.clone()), analyzer, codegen, pending, &stats) } #[cfg(not(feature = "developer"))] { - handle_request_deferred(req, bus, analyzer, codegen, pending) + handle_request_deferred(req, &crate::cpu::jitv2::jitv2::CodeBuses::single(bus.clone()), analyzer, codegen, pending) } } @@ -1203,7 +1243,7 @@ mod tests { let counter = AtomicU64::new(0); let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(4); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); @@ -1224,7 +1264,7 @@ mod tests { let counter = AtomicU64::new(0); let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(0); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); @@ -1243,7 +1283,7 @@ mod tests { let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); page.mark_requested(4); assert!(page.try_schedule_page()); - let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = Analyzer::new(); let mut codegen = Codegen::new(); @@ -1277,7 +1317,7 @@ mod tests { for i in 0..N { let before = pages.iter().filter(|p| p.is_runnable(4)).count(); pages[i].mark_requested(4); - let req = CompileRequest { page: &mut pages[i] as *mut PhysicalCodePage, compiled_for_fr1: true }; + let req = CompileRequest { page: &mut pages[i] as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }; let mut pending: PendingCount = 0; handle_request_deferred_for_test(&req, &bus, &mut analyzer, &mut codegen, &mut pending); let own_published = pages[i].is_runnable(4); @@ -1336,7 +1376,7 @@ mod tests { let mut words = [0u32; ENTRIES_PER_PAGE]; let mut taken = 0u64; while std::time::Instant::now() < deadline && !stop.load(Relaxed) { - if let Snapshot::Taken(g) = snapshot_page(&page, &*bus, 0, &mut words) { + if let Snapshot::Taken(g) = snapshot_page(&page, Some((&*bus, 0)), &mut words) { taken += 1; if (words[0] as u64) < g { stop.store(true, Relaxed); diff --git a/src/cpu/jitv2/equiv_test.rs b/src/cpu/jitv2/equiv_test.rs index 81e57af47..28c42af60 100644 --- a/src/cpu/jitv2/equiv_test.rs +++ b/src/cpu/jitv2/equiv_test.rs @@ -227,6 +227,10 @@ mod tests { let counter = gens.entry(page).or_insert_with(|| Box::new(AtomicU64::new(0))); counter.as_ref() as *const AtomicU64 } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) + } } fn make_r(op: u32, rs: u32, rt: u32, rd: u32, sa: u32, funct: u32) -> u32 { diff --git a/src/cpu/jitv2/jitv2.rs b/src/cpu/jitv2/jitv2.rs index 14c27701f..ef69a7f99 100644 --- a/src/cpu/jitv2/jitv2.rs +++ b/src/cpu/jitv2/jitv2.rs @@ -29,7 +29,7 @@ use parking_lot::Mutex; use crate::cpu::mips_core::MipsCore; use crate::cpu::mips_exec::ExecStatus; use crate::cpu::jitv2::pool_barrier::PoolBarrier; -use crate::traits::{BusDevice, Device}; +use crate::traits::{BusDevice, CodeKey, Device}; /// A compile request pushed from the mips exec thread to the compile pool /// (§6.4). Carries the page pointer rather than a @@ -55,10 +55,90 @@ pub struct CompileRequest { /// (same value `exec_decoded` used to run the interpreter fallback for /// this same arrival). pub compiled_for_fr1: bool, + /// The CPU that asked, by its number in the pool: the page's bytes are + /// read at that CPU's address for it, through that CPU's bus + /// ([`CodeBuses::locate`]). + pub cpu: u8, } unsafe impl Send for CompileRequest {} +/// Most CPUs one pool serves: a page keeps an address per CPU. +pub const POOL_CPUS: usize = crate::cpu::smp::MAX_CPUS; + +/// "This CPU has not reached the page", in [`PhysicalCodePage`]'s `addr`. +const NO_ADDR: u32 = u32::MAX; + +/// The two buses a CPU fetches through. +#[derive(Clone)] +pub struct CpuBuses { + /// The CPU's address space in the pool. + pub space: usize, + /// Decodes a cached fetch. + pub cached: Arc, + /// Decodes an uncached fetch (the CPU's own bus). + pub uncached: Arc, +} + +/// The buses of every CPU that shares a pool, by its number in the pool. A +/// page is keyed by the memory behind it, which each CPU may reach at its own +/// address, so a compile reads the bytes at one CPU's address through that +/// CPU's bus. +#[derive(Default)] +pub struct CodeBuses(parking_lot::RwLock<[Option; POOL_CPUS]>); + +impl CodeBuses { + /// One CPU, number 0, that fetches everything through `bus`. + pub fn single(bus: Arc) -> Self { + let buses = Self::default(); + buses.set(0, 0, bus.clone(), bus); + buses + } + + pub fn set(&self, cpu: usize, space: usize, cached: Arc, uncached: Arc) { + self.0.write()[cpu] = Some(CpuBuses { space, cached, uncached }); + } + + /// `bus` for CPU 0 if nothing named its buses (a pool started on its own). + fn set_default(&self, bus: Arc) { + let mut g = self.0.write(); + if g[0].is_none() { + g[0] = Some(CpuBuses { space: 0, cached: bus.clone(), uncached: bus }); + } + } + + /// Let go of every bus (the machine is going away). + pub fn clear(&self) { + *self.0.write() = Default::default(); + } + + /// A bus and the address on it where `page`'s bytes are right now: + /// `first`'s if it has one, else any CPU's. An address a remap has since + /// moved other memory under no longer decodes to the page's key and is + /// passed over. + pub fn locate(&self, page: &PhysicalCodePage, first: usize) -> Option<(Arc, u64)> { + let g = self.0.read(); + std::iter::once(first).chain(0..POOL_CPUS).find_map(|cpu| { + let (pfn, uncached) = page.addr_of(cpu)?; + let buses = g.get(cpu)?.as_ref()?; + let bus = if uncached { &buses.uncached } else { &buses.cached }; + let at = pfn as u64 * PAGE_SIZE as u64; + // A page keyed by its address is its own space's alone. + let mine = !page.key.is_address() || page.key == CodeKey::at_address(buses.space, pfn, uncached); + (mine && page_is_at(page, &**bus, at)).then(|| (bus.clone(), at)) + }) + } +} + +/// Whether `bus` still decodes `at` to `page`: to its bank's page, or, for a +/// page no bank names, to no bank still. +pub fn page_is_at(page: &PhysicalCodePage, bus: &dyn BusDevice, at: u64) -> bool { + match bus.code_page(at) { + Some(code) => code.key == page.key, + None => page.key.is_address(), + } +} + // ============================================================================ // Tunables & initial settings. Single source of truth: every knob that shapes // JIT v2's steady-state memory footprint and flush/queue behavior lives here. @@ -91,8 +171,10 @@ pub fn page_key(pfn: Pfn, uncached: bool, split: bool) -> u32 { if split { pfn << 1 | uncached as u32 } else { pfn } } -/// Direct-mapped `pfn -> PageSlot` lookup, replacing the `HashMap` both page pools used to carry. +/// Direct-mapped `pfn -> PageSlot` lookup for one address space: which pool +/// slot holds the page a CPU of that space reaches at a frame. A cache of the +/// answer the bus ([`BusDevice::code_page`]) and the pool's key index give, +/// so a page switch to a frame already reached takes two loads and no lock. /// /// One `u32` per physical frame up to the end of the machine's memory /// ([`frames_for`]), two on an IP27 or IP35 machine ([`page_key`]). Allocated once, at its @@ -465,7 +547,7 @@ pub type Pfn = u32; /// `reset_to_unclaimed`) — deliberately NOT `0`: pfn 0 is a real, legitimate /// physical page (low RAM), so zeroing `pfn` on reset let any stale reference /// to a freed slot (a dangling raw pointer held past a `mega_flush` eviction, -/// or a `pfn_to_slot` entry that wasn't removed before the reset) silently +/// or a frame-table entry that wasn't removed before the reset) silently /// read back as "page 0's real data" instead of something obviously wrong — /// undetectable by inspection (`j2 pcp` on a stale pointer would show a /// plausible-looking pfn/gen/entry_gen for the wrong page, not a crash or an @@ -653,6 +735,9 @@ impl RejectReason { #[derive(Default)] pub struct JitStats { + /// Times a CPU forgot its addresses because a memory controller moved + /// memory (`Jitv2::forget_addresses`). No code is dropped for one. + pub remaps: AtomicU64, /// Successful `compile_region` calls that got published /// (`comp::handle_request`'s `Some(jit_fn)` arm). pub compiles: AtomicU64, @@ -797,7 +882,21 @@ static NEVER_COMPILABLE_GEN: AtomicU64 = AtomicU64::new(0); /// has ever been a JIT compilation target; the executor holds a pointer to /// the page it is currently executing out of. pub struct PhysicalCodePage { - pub pfn: Pfn, + /// The memory this page is: its bank and the page within it + /// ([`CodeKey`]), the same for every CPU and through every remap. + /// [`CodeKey::NONE`] while the slot is unclaimed. + pub key: CodeKey, + /// Where each CPU that shares the pool last reached this page, by the + /// CPU's number in the pool: `frame << 1 | uncached`, or `NO_ADDR`. A + /// compile reads the bytes there through that CPU's bus + /// ([`CodeBuses::locate`]). A remap can leave one stale; nothing trusts it + /// without asking the bus for the key there again. + addr: [AtomicU32; POOL_CPUS], + /// The page's 1024 words in its bank's own storage + /// ([`crate::traits::CodePage::words`]), where a compile reads them, or + /// null for a bank that hands none out: then the compile reads through a + /// CPU's bus at `addr`. + words: *const u32, /// Free list pointer: singly-linked free list threaded through /// `Jitv2::free_head`. `NO_SLOT` when claimed or at tail of free list. next: u32, @@ -809,8 +908,6 @@ pub struct PhysicalCodePage { /// returns null) gets pointed at the shared [`NEVER_COMPILABLE_GEN`] /// fallback instead, by `claim()`. gen: *const AtomicU64, - /// The bus a page fetched uncached is read through; `None` for a cached page. - pub uncached: Option>, /// Offsets some dispatch has asked for an entry point at (§13.2) — a /// superset of `compiled` in general (a bit can be requested long before /// any compile picks it up, and stays requested across however many @@ -1350,12 +1447,17 @@ impl PhysicalCodePage { /// slot, or when the real device has no gen tracking) and normalized to /// `NEVER_COMPILABLE_GEN` — `self.gen` itself is never null, so nothing /// downstream ever needs to check. + /// + /// A `pfn` other than [`UNCLAIMED_PFN`] makes a page on its own, outside + /// a pool (tests): frame `pfn` of a flat bus, reached there by CPU 0. pub fn new(pfn: Pfn, gen: *const AtomicU64) -> Self { + let claimed = pfn != UNCLAIMED_PFN; Self { - pfn, + key: if claimed { CodeKey::flat(pfn as u64 * PAGE_SIZE as u64) } else { CodeKey::NONE }, + addr: std::array::from_fn(|cpu| AtomicU32::new(if claimed && cpu == 0 { pfn << 1 } else { NO_ADDR })), + words: std::ptr::null(), next: NO_SLOT, gen: if gen.is_null() { &NEVER_COMPILABLE_GEN } else { gen }, - uncached: None, requested: new_bitmap(), denied: new_bitmap_all_set(), compiled: new_bitmap(), @@ -1446,16 +1548,57 @@ impl PhysicalCodePage { /// invalidation and on an FR-guard bail (see `Self::fr1`'s doc comment). #[inline] pub fn is_claimed(&self) -> bool { - self.pfn != UNCLAIMED_PFN + self.key != CodeKey::NONE + } + + /// Where CPU `cpu` of the pool reaches this page: its frame, and whether + /// through an uncached fetch. `None` if it has not, or not since a remap. + #[inline] + pub fn addr_of(&self, cpu: usize) -> Option<(Pfn, bool)> { + match self.addr.get(cpu)?.load(Ordering::Relaxed) { + NO_ADDR => None, + a => Some((a >> 1, a & 1 != 0)), + } } - pub fn claim(&mut self, pfn: Pfn, gen: *const AtomicU64, fr1: bool) { - debug_assert!(std::ptr::eq(self.gen, &NEVER_COMPILABLE_GEN) && self.pfn == UNCLAIMED_PFN, - "claim() called on a slot that wasn't clean (pfn={:#x}) — every path that reuses a slot must reset it first (see reset_to_unclaimed)", - self.pfn); + /// CPU `cpu` reached this page at frame `pfn`. + #[inline] + pub fn set_addr(&self, cpu: usize, pfn: Pfn, uncached: bool) { + self.addr[cpu].store(pfn << 1 | uncached as u32, Ordering::Relaxed); + } + + /// CPU `cpu`'s address for this page no longer holds (a remap). + #[inline] + pub fn forget_addr(&self, cpu: usize) { + self.addr[cpu].store(NO_ADDR, Ordering::Relaxed); + } + + /// A frame some CPU reaches this page at, for display: the lowest-numbered + /// CPU's that has one, else [`UNCLAIMED_PFN`]. CPUs of different nodes may + /// each have their own. + pub fn pfn(&self) -> Pfn { + (0..POOL_CPUS).find_map(|cpu| self.addr_of(cpu)).map_or(UNCLAIMED_PFN, |(pfn, _)| pfn) + } + + /// The page's words in its bank's storage, or null: see the field. + #[inline] + pub fn words(&self) -> *const u32 { + self.words + } + + /// [`Self::claim`] for the page a bus answered with. + pub fn claim_code(&mut self, code: crate::traits::CodePage, fr1: bool) { + self.claim(code.key, code.gen, fr1); + self.words = code.words; + } + + pub fn claim(&mut self, key: CodeKey, gen: *const AtomicU64, fr1: bool) { + debug_assert!(std::ptr::eq(self.gen, &NEVER_COMPILABLE_GEN) && !self.is_claimed(), + "claim() called on a slot that wasn't clean (key={:?}) — every path that reuses a slot must reset it first (see reset_to_unclaimed)", + self.key); debug_assert!(self.func.load(Ordering::Relaxed).is_null(), "claim() called on a slot with a still-published function — mega_flush must reset_to_unclaimed before this slot can be reused"); - self.pfn = pfn; + self.key = key; self.gen = if gen.is_null() { &NEVER_COMPILABLE_GEN } else { gen }; self.fr1.store(fr1, Ordering::Relaxed); // A freshly-claimed slot has no bail outstanding: `fr1` was just set @@ -1528,14 +1671,17 @@ impl PhysicalCodePage { /// `Jitv2::mega_flush`'s in-place counterpart to what used to be /// `Vec::clear()` dropping every `PhysicalCodePage` outright. `pfn` is /// poisoned to [`UNCLAIMED_PFN`], not `0` (see that constant's own doc - /// comment) — a dangling raw pointer or leftover `pfn_to_slot` entry that + /// comment) — a dangling raw pointer or leftover frame-table entry that /// reads this slot after it's been evicted and reused reads back an /// obviously-bogus pfn instead of silently aliasing page 0's real data. pub fn reset_to_unclaimed(&mut self) { self.reset_entries_and_bitmaps(); - self.pfn = UNCLAIMED_PFN; + self.key = CodeKey::NONE; + self.words = std::ptr::null(); + for a in &self.addr { + a.store(NO_ADDR, Ordering::Relaxed); + } self.gen = &NEVER_COMPILABLE_GEN; - self.uncached = None; } /// Reset `requested`/`denied`/`compiled` bitmaps, the compiled `func` @@ -2446,7 +2592,10 @@ const _: () = assert!(NO_SLOT == PFN_MAP_EMPTY); /// across a flush, threaded on an MRU list; that was removed — preserved /// entry points could be stale, and keeping them was at best neutral.) /// -/// Lookup from `pfn` to pool slot goes through `pfn_to_slot` — the design +/// A page is keyed by the memory it holds ([`CodeKey`]: bank and page within +/// it), found through `by_key`. Lookup from a CPU's `pfn` to pool slot goes +/// through its address space's frame table (`spaces`), which caches what the +/// bus and `by_key` say — the design /// doc's dense pfn-indexed alternative (§2.4), now built: page-switch lookup /// *did* show up hot in profiling (~4.5% of a real IRIX boot, with the /// HashMap's own `RawTable` grow-and-rehash allocation directly above it), @@ -2475,14 +2624,15 @@ pub struct Jitv2 { /// list up front); `page_for` pops from the head on a fresh claim, /// `mega_flush` resets the pool and rebuilds the free list. free_head: u32, - /// pfn -> index into `pages`. Consulted only on a page switch (fetch - /// lands on a different PFN than the currently-tracked one) — not on - /// every fetch. Direct-mapped, not hashed — see [`PfnMap`]. - pfn_to_slot: PfnMap, - /// Frames of memory the pool covers. - frames: usize, - /// Whether an uncached fetch of a frame gets its own page ([`page_key`]). - split_uncached: bool, + /// The pool's index: which slot holds the page for a key. Consulted when + /// a CPU reaches a frame its space's table has no slot for, which is once + /// per frame per space between flushes and remaps; sized for the whole + /// pool up front, so it never grows or rehashes. + by_key: std::collections::HashMap, + /// One frame table per address space: see [`Space`]. A machine whose CPUs + /// all see memory at the same addresses has one; a machine of several + /// nodes has one per node. + spaces: Vec, /// Pool capacity, fixed at construction (== `pages.len()`). Claiming past /// this (i.e. with the free list exhausted) triggers `mega_flush` (the /// "ran out of PCPs" resource-exhaustion trigger). @@ -2540,6 +2690,51 @@ pub struct CodeSizeBucket { pub max_bytes: u32, } +/// One address space's view of the pool: `frame -> slot` for the frames its +/// CPUs have reached, the lock-free path a page switch takes. It only caches +/// what the bus and `by_key` say, so a remap empties it and nothing else. +struct Space { + /// Frames of memory the table covers. Zero until a CPU sets it. + frames: usize, + /// Whether an uncached fetch of a frame has its own entry ([`page_key`]): + /// on a machine where it can decode to other memory than a cached one. + split: bool, + /// Empty until the first claim in this space. + map: PfnMap, +} + +impl Default for Space { + fn default() -> Self { + Space { frames: 0, split: false, map: PfnMap(Box::new([])) } + } +} + +impl Space { + fn entries(&self) -> usize { + if self.split { 2 * self.frames } else { self.frames } + } +} + +/// Where a CPU fetched from: what [`Jitv2::claim_page`] finds a page for. +#[derive(Clone, Copy, Debug)] +pub struct PageAt { + /// The CPU's number in the pool. + pub cpu: usize, + /// Its address space. + pub space: usize, + pub pfn: Pfn, + /// Fetched uncached, on a machine that keeps that apart. + pub uncached: bool, +} + +/// What [`Jitv2::claim_page`] found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Claim { + Page(PageSlot), + /// No slot is free: flush and ask again. + Full, +} + /// The workers read `pages` through raw pointers, and the fields drop in /// declaration order, so `pages` would be freed ahead of `compile_queue`'s /// own drop. Stop and join the workers first. @@ -2573,7 +2768,9 @@ impl Jitv2 { let _ = w.join(); } } - jit.lock().compile_queue.drain_pending_queue(); + let mut g = jit.lock(); + g.compile_queue.drain_pending_queue(); + g.buses().clear(); } /// Allocate the full page pool up front, `capacity` slots, every one @@ -2594,11 +2791,32 @@ impl Jitv2 { Self::with_frames(capacity, frames_for(memory) as usize, false) } - /// Keep an uncached page of a frame apart from its cached page. Before any - /// page is claimed. + /// The buses of the CPUs that share this pool: see [`CodeBuses`]. + pub fn buses(&self) -> &Arc { + &self.compile_queue.buses + } + + /// Describe address space `space`: the frames of memory its table covers, + /// and whether an uncached fetch of a frame is kept apart from a cached + /// one. Each CPU says so for its own space, before the space holds a page. + pub fn set_space(&mut self, space: usize, frames: usize, split: bool) { + if self.spaces.len() <= space { + self.spaces.resize_with(space + 1, Space::default); + } + let sp = &mut self.spaces[space]; + if sp.frames == frames && sp.split == split { + return; + } + assert!(sp.map.0.is_empty(), "an address space is laid out before it holds a page"); + sp.frames = frames; + sp.split = split; + } + + /// Keep an uncached page of a frame apart from its cached page, in space 0. + /// Before any page is claimed. pub fn split_uncached_code(&mut self) { - assert!(self.pfn_to_slot.0.is_empty(), "a page table splits before it holds a page"); - self.split_uncached = true; + let frames = self.spaces[0].frames; + self.set_space(0, frames, true); } fn with_frames(capacity: usize, frames: usize, split: bool) -> Self { @@ -2613,9 +2831,8 @@ impl Jitv2 { Self { pages, free_head: if capacity > 0 { 0 } else { NO_SLOT }, - pfn_to_slot: PfnMap(Box::new([])), - frames, - split_uncached: split, + by_key: std::collections::HashMap::with_capacity(capacity), + spaces: vec![Space { frames, split, map: PfnMap(Box::new([])) }], capacity, compile_queue: CompileQueue::new(), codegen: Mutex::new(Some(crate::cpu::jitv2::codegen::Codegen::new())), @@ -2623,69 +2840,98 @@ impl Jitv2 { } } - /// Release `slot` back to the free list. - fn free_page(&mut self, slot: PageSlot) { - let page = &self.pages[slot as usize]; - self.pfn_to_slot.remove(page_key(page.pfn, page.uncached.is_some(), self.split_uncached)); - self.pages[slot as usize].reset_to_unclaimed(); - self.pages[slot as usize].next = self.free_head; - self.free_head = slot; - } - - /// Look up the pool slot for `pfn`, claiming the next unclaimed slot - /// in place (`PhysicalCodePage::claim`, `gen_ptr(phys_addr)` on the bus) - /// if this is the first arrival at this page. Returns `None` if the pool - /// is exhausted — the caller (mips exec thread) is responsible for - /// running `mega_flush` and retrying. - pub fn page_for(&mut self, pfn: Pfn, phys_addr: u64, bus: &dyn BusDevice, fr1: bool) -> Option { - self.claim_page(pfn, phys_addr, bus, None, fr1) - } - - /// [`Self::page_for`], for a page fetched cached or, with `uncached` the - /// processor's own bus, uncached. - pub fn claim_page(&mut self, pfn: Pfn, phys_addr: u64, bus: &dyn BusDevice, uncached: Option<&Arc>, fr1: bool) -> Option { - let uncached = uncached.filter(|_| self.split_uncached); - let key = page_key(pfn, uncached.is_some(), self.split_uncached); - let bus = uncached.map_or(bus, |b| b.as_ref()); - if self.pfn_to_slot.0.is_empty() { - self.pfn_to_slot = PfnMap::new(self.table_entries()); - } - if let Some(slot) = self.pfn_to_slot.get(key) { - debug_assert_eq!(self.pages[slot as usize].pfn, pfn, - "pfn_to_slot[{:#x}] -> slot {} whose own pfn is {:#x} — the map and the slot it points at have desynced (a slot was reused/evicted without this map entry being updated to match)", - pfn, slot, self.pages[slot as usize].pfn); - return Some(slot); - } - if self.free_head == NO_SLOT { - return None; + /// The pool slot for frame `pfn` of a flat bus, as CPU 0 in space 0 + /// reaches it cached: [`Self::claim_page`] for a pool with one CPU. + /// `None` if the pool is exhausted. + pub fn page_for(&mut self, pfn: Pfn, _phys_addr: u64, bus: &dyn BusDevice, fr1: bool) -> Option { + self.page_at(PageAt { cpu: 0, space: 0, pfn, uncached: false }, bus, fr1) + } + + /// [`Self::claim_page`] for a caller with nothing to do about a full pool + /// (the monitor): the slot, or `None`. + pub fn page_at(&mut self, at: PageAt, bus: &dyn BusDevice, fr1: bool) -> Option { + match self.claim_page(at, bus, fr1) { + Claim::Page(slot) => Some(slot), + Claim::Full => None, } - let slot = self.free_head; - self.free_head = self.pages[slot as usize].next; - let gen = bus.gen_ptr(phys_addr); - self.pages[slot as usize].claim(pfn, gen, fr1); - self.pages[slot as usize].uncached = uncached.cloned(); - self.pages[slot as usize].next = NO_SLOT; - self.pfn_to_slot.insert(key, slot); - Some(slot) } - fn table_entries(&self) -> usize { - if self.split_uncached { 2 * self.frames } else { self.frames } + /// The pool slot for the page a CPU fetched from at `at`, through `bus` + /// (the bus that decodes that fetch: its cached one, or its own for an + /// uncached fetch on a machine that keeps those apart). The bus names the + /// memory there ([`BusDevice::code_page`]); the page for that memory is + /// claimed if no CPU has reached it yet, at whatever address. Where no + /// bank answers, the page is keyed by the address, within this space + /// ([`CodeKey::at_address`]), and counted as the bus counts it. Records + /// where this CPU reaches it and enters it in the space's frame table. + /// + /// [`Claim::Full`] if the pool is exhausted — the caller (mips exec + /// thread) is responsible for running `mega_flush` and retrying. + pub fn claim_page(&mut self, at: PageAt, bus: &dyn BusDevice, fr1: bool) -> Claim { + let space = &mut self.spaces[at.space]; + let uncached = at.uncached && space.split; + let entry = page_key(at.pfn, uncached, space.split); + if space.map.0.is_empty() { + space.map = PfnMap::new(space.entries()); + } + if let Some(slot) = space.map.get(entry) { + self.pages[slot as usize].set_addr(at.cpu, at.pfn, uncached); + return Claim::Page(slot); + } + let addr = at.pfn as u64 * PAGE_SIZE as u64; + let code = bus.code_page(addr).unwrap_or_else(|| crate::traits::CodePage { + key: CodeKey::at_address(at.space, at.pfn, uncached), + gen: bus.gen_ptr(addr), + words: std::ptr::null(), + }); + let slot = match self.by_key.get(&code.key) { + Some(&slot) => slot, + None => { + if self.free_head == NO_SLOT { + return Claim::Full; + } + let slot = self.free_head; + self.free_head = self.pages[slot as usize].next; + self.pages[slot as usize].claim_code(code, fr1); + self.pages[slot as usize].next = NO_SLOT; + self.by_key.insert(code.key, slot); + slot + } + }; + debug_assert_eq!(self.pages[slot as usize].key, code.key, "the key index and slot {slot} disagree"); + self.pages[slot as usize].set_addr(at.cpu, at.pfn, uncached); + self.spaces[at.space].map.insert(entry, slot); + Claim::Page(slot) + } + + /// Memory moved under the addresses of `space`: forget which frame holds + /// which page there, and where CPU `cpu` reached each page. The pages and + /// their code stay: they are keyed by the memory, not by its address, so + /// the next fetch finds them again wherever the memory went. Each CPU of + /// the space calls this for itself before it runs guest code again. + pub fn forget_addresses(&mut self, space: usize, cpu: usize) { + self.spaces[space].map.clear(); + for page in self.pages.iter().filter(|p| p.is_claimed()) { + page.forget_addr(cpu); + } + self.stats.remaps.fetch_add(1, Ordering::Relaxed); } #[cfg(test)] pub(crate) fn table_bytes(&self) -> usize { - self.table_entries() * size_of::() + self.spaces.iter().map(Space::entries).sum::() * size_of::() } - /// Pointers for lock-free `pfn -> *mut PhysicalCodePage` fast lookup. - pub fn fast_lookup_ptrs(&mut self) -> (*const AtomicU32, *mut PhysicalCodePage) { - (self.pfn_map_ptr(), self.pages.as_mut_ptr()) + /// Pointers for lock-free `frame -> *mut PhysicalCodePage` fast lookup in + /// `space`. + pub fn fast_lookup_ptrs(&mut self, space: usize) -> (*const AtomicU32, *mut PhysicalCodePage) { + (self.pfn_map_ptr(space), self.pages.as_mut_ptr()) } - /// The page table pointer, null before the first claim. - pub fn pfn_map_ptr(&self) -> *const AtomicU32 { - if self.pfn_to_slot.0.is_empty() { std::ptr::null() } else { self.pfn_to_slot.as_ptr() } + /// `space`'s frame table pointer, null before its first claim. + pub fn pfn_map_ptr(&self, space: usize) -> *const AtomicU32 { + let map = &self.spaces[space].map; + if map.0.is_empty() { std::ptr::null() } else { map.as_ptr() } } /// Raw pointer to the page at `slot`. Valid for the process's entire @@ -2788,9 +3034,8 @@ impl Jitv2 { /// Every claimed page in the pool, exposed as a borrowing iterator /// rather than folded into an aggregate, for callers (`j2 html`) that /// need the raw per-page detail instead of a summary statistic. Not - /// necessarily a contiguous `pages[..n]` prefix — `free_page` can return - /// an individual slot mid-run, so claimed slots may be scattered anywhere - /// in the array — hence the full scan filtered on + /// necessarily a contiguous `pages[..n]` prefix, hence the full scan + /// filtered on /// [`PhysicalCodePage::is_claimed`]. `pages` itself stays private (index /// stability, see the field's own doc comment, is an invariant only this /// module should rely on). @@ -2805,7 +3050,7 @@ impl Jitv2 { /// /// Every slot gets [`PhysicalCodePage::reset_to_unclaimed`] (which clears /// `pfn` back to `UNCLAIMED_PFN`, so `pages_used()` reads 0 afterwards), - /// `pfn_to_slot` is cleared, and the free list is relinked across the + /// every frame table and `by_key` are cleared, and the free list is relinked across the /// entire array from slot 0. Slot *indices* stay stable — the array is /// reset in place and never reallocates (see the `pages` field's own doc /// comment) — but a slot's *contents* do not survive, so any surviving @@ -2832,7 +3077,10 @@ impl Jitv2 { /// compile-queue lifecycle independently of `MipsCpu::stop()`/`start()`). fn mega_flush(&mut self) { crate::cpu::jitv2::hashstats::FLUSH_EPOCH.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - self.pfn_to_slot.clear(); + for space in &mut self.spaces { + space.map.clear(); + } + self.by_key.clear(); let cap = self.pages.len(); for (i, page) in self.pages.iter_mut().enumerate() { page.reset_to_unclaimed(); @@ -3007,6 +3255,58 @@ pub fn attach_pool( let _ = (mode, bus); } +/// What [`share_pool`] needs of one node: its processors, the module they +/// are, and the bus they read cached code through. +pub struct PoolNode<'a> { + pub cpus: &'a [Arc], + pub module: &'a crate::cpu::parts::CpuModule, + pub bus: Arc, +} + +/// Give a machine of several nodes its JIT: one page pool, one compile queue +/// and one body of compiled code for every processor, whichever node it is +/// on. A page is keyed by the memory it was compiled from, so the node whose +/// processor reaches that memory at another address runs the same code. Each +/// node is an address space of the pool, with its own frame table. +/// +/// Compiled code is specific to the processor module (its instruction set, +/// its cache geometry), so nodes with different modules get a pool each. +pub fn share_pool( + nodes: &[PoolNode], + pool_size: (usize, String), + mode: crate::config::JitCompile, + world: &Arc, +) -> Result<(), String> { + let mut done = vec![false; nodes.len()]; + for first in 0..nodes.len() { + if done[first] { + continue; + } + let members: Vec = (first..nodes.len()).filter(|&n| !done[n] && nodes[n].module == nodes[first].module).collect(); + members.iter().for_each(|&n| done[n] = true); + let group: Vec<&PoolNode> = members.iter().map(|&n| &nodes[n]).collect(); + let Some(lead) = group.iter().find_map(|n| n.cpus.first()) else { + continue; + }; + let several = group.iter().map(|n| n.cpus.len()).sum::() > 1; + if several { + if cfg!(feature = "jitv2_lockstep") { + return Err("several processors cannot run with jitv2_lockstep".into()); + } + let jit = lead.jitv2(); + let mut number = 0; + for (space, node) in group.iter().enumerate() { + for cpu in node.cpus { + cpu.share_jitv2(jit.clone(), world.clone(), number, space); + number += 1; + } + } + } + attach_pool(lead, pool_size.clone(), mode, group[0].bus.clone(), several.then(|| world.clone())); + } + Ok(()) +} + pub struct CompileQueue { /// The ISA level every worker's `Analyzer` walks at — the `CpuModel`'s /// `MIPS4` const, set by `MipsExecutor::new` through [`set_isa`] before @@ -3044,6 +3344,9 @@ pub struct CompileQueue { /// a request holds raw pointers into its own JIT's pages and its own /// machine's memory, so another machine's worker must never take it. hot: Arc, + /// The buses the workers read page bytes through, one pair per CPU that + /// shares the pool: see [`CodeBuses`]. + pub buses: Arc, /// The pool's wake-up channel, shared by both lanes — the same workers /// drain both. See [`CompileWake`]. wake: Arc, @@ -3123,6 +3426,7 @@ impl CompileQueue { jit_consts: Mutex::new(crate::cpu::jitv2::codegen::JitConsts::default()), queue: Arc::new(CompileLane::new(COMPILE_QUEUE_CAPACITY, wake.clone())), hot: Arc::new(CompileLane::new(HOT_QUEUE_CAPACITY, wake.clone())), + buses: Arc::new(CodeBuses::default()), wake, threads: Vec::new(), cpu: Mutex::new(None), @@ -3400,6 +3704,7 @@ impl CompileQueue { if !self.threads.is_empty() { return; } + self.buses.set_default(bus); // Captured before the spawn loop: each worker builds its own // `Analyzer` and needs the level by value, not through `&self`. let worker_mips4 = self.mips4; @@ -3421,7 +3726,7 @@ impl CompileQueue { codegen.jit_consts = *self.jit_consts.lock(); let queue = self.queue.clone(); let hot = self.hot.clone(); - let bus = bus.clone(); + let bus = self.buses.clone(); let cpu = self.cpu.lock().clone(); let world = self.world.lock().clone(); let jitv2 = self.jitv2.lock().clone(); @@ -3561,7 +3866,7 @@ impl CompileQueue { fn worker_loop( queue: Arc, hot: Arc, - bus: Arc, + bus: Arc, mut codegen: crate::cpu::jitv2::codegen::Codegen, cpu: Option>, world: Option>, @@ -4000,7 +4305,7 @@ mod tests { /// `claim` debug-asserts the slot is clean, so it must start unclaimed. fn claimed_page(fr1: bool) -> Box { let mut page = Box::new(PhysicalCodePage::new(UNCLAIMED_PFN, std::ptr::null())); - page.claim(0x1234, std::ptr::null(), fr1); + page.claim(CodeKey::flat(0x1234 << 12), std::ptr::null(), fr1); page } @@ -4341,7 +4646,7 @@ mod tests { let mut jit = Jitv2::new(sizes.len()); for (i, sz) in sizes.iter().enumerate() { - jit.pages[i].claim(i as Pfn + 1, &counter as *const AtomicU64, false); + jit.pages[i].claim(CodeKey::flat((i as u64 + 1) << 12), &counter as *const AtomicU64, false); let mut bits = [0u64; BITMAP_WORDS]; bits[0] |= 1u64; assert!(jit.pages[i].publish(&bits, 0x1000 as *const (), 0, 1, *sz, false)); @@ -5206,6 +5511,7 @@ mod tests { fn read64(&self, _addr: u64) -> BusRead64 { BusRead64::err() } fn write64(&self, _addr: u64, _val: u64) -> u32 { crate::traits::BUS_ERR } fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { &self.0 as *const AtomicU64 } + fn code_page(&self, addr: u64) -> Option { Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) } } #[test] @@ -5269,24 +5575,94 @@ mod tests { if split { jit.split_uncached_code(); } - assert!(jit.fast_lookup_ptrs().0.is_null()); + assert!(jit.fast_lookup_ptrs(0).0.is_null()); jit.page_for(1, PAGE_SIZE as u64, &dev, false).unwrap(); assert_eq!(jit.table_bytes(), (64 << 20) / PAGE_SIZE as usize * 4 * entries); - assert_eq!(size_of_val(&*jit.pfn_to_slot.0), jit.table_bytes()); + assert_eq!(size_of_val(&*jit.spaces[0].map.0), jit.table_bytes()); } } + /// One RAM bank answering `base` bytes up a node's bus. + struct NodeBus { + ram: Arc, + base: u64, + } + impl BusDevice for NodeBus { + fn read32(&self, a: u64) -> BusRead32 { + a.checked_sub(self.base).map_or_else(BusRead32::err, |a| self.ram.read32(a)) + } + fn gen_ptr(&self, a: u64) -> *const AtomicU64 { + a.checked_sub(self.base).map_or(std::ptr::null(), |a| self.ram.gen_ptr(a)) + } + fn code_page(&self, a: u64) -> Option { + self.ram.code_page(a.checked_sub(self.base)?) + } + } + + /// Two nodes that reach one bank at different addresses share its page, + /// each through its own frame table, and a compile reads it through + /// whichever CPU's address still decodes to it. + #[test] + fn two_address_spaces_share_the_page_of_one_bank() { + let ram = Arc::new(crate::dev::mem::Memory::new(1)); + let near: Arc = Arc::new(NodeBus { ram: ram.clone(), base: 0 }); + let far: Arc = Arc::new(NodeBus { ram: ram.clone(), base: 0x40 * PAGE_SIZE as u64 }); + let mut jit = Jitv2::new(4); + jit.set_space(1, PFN_MAP_ENTRIES, false); + jit.buses().set(0, 0, near.clone(), near.clone()); + jit.buses().set(1, 1, far.clone(), far.clone()); + let a = jit.claim_page(PageAt { cpu: 0, space: 0, pfn: 3, uncached: false }, &*near, false); + let b = jit.claim_page(PageAt { cpu: 1, space: 1, pfn: 0x43, uncached: false }, &*far, false); + let Claim::Page(slot) = a else { panic!("{a:?}") }; + assert_eq!(b, Claim::Page(slot), "the same bank page from the other node is the same slot"); + // The other node's frame 3 is not this memory, nor any bank: a page + // of its own, keyed by that node's address. + let Claim::Page(hole) = jit.claim_page(PageAt { cpu: 1, space: 1, pfn: 3, uncached: false }, &*far, false) else { panic!() }; + assert_ne!(hole, slot); + assert_eq!(unsafe { &*jit.page_ptr(hole) }.key, CodeKey::at_address(1, 3, false)); + let page = unsafe { &*jit.page_ptr(slot) }; + assert_eq!(page.key, ram.code_page(3 * PAGE_SIZE as u64).unwrap().key); + assert_eq!((page.addr_of(0), page.addr_of(1)), (Some((3, false)), Some((0x43, false)))); + assert!(std::ptr::eq(page.gen, ram.gen_ptr(3 * PAGE_SIZE as u64)), "counted by the bank's own counter"); + + let (_, at) = jit.buses().locate(page, 0).unwrap(); + assert_eq!(at, 3 * PAGE_SIZE as u64); + // CPU 0's bus stops decoding that address to the bank: its address + // for the page is stale, and the page is read through CPU 1. + jit.buses().set(0, 0, far.clone(), far.clone()); + let (_, at) = jit.buses().locate(page, 0).unwrap(); + assert_eq!(at, 0x43 * PAGE_SIZE as u64); + + // A compile needs neither: RAM hands out its storage, so the page is + // read by its key alone, whatever the buses decode. + ram.write32(3 * PAGE_SIZE as u64 + 8, 0x1234_5678); + let mut words = [0u32; crate::cpu::jitv2::ENTRIES_PER_PAGE]; + assert!(!page.words().is_null()); + assert!(matches!(crate::cpu::jitv2::comp::snapshot_page(page, None, &mut words), crate::cpu::jitv2::comp::Snapshot::Taken(_))); + assert_eq!(words[2], 0x1234_5678); + + // A remap in space 0 forgets its frames and CPU 0's addresses only. + jit.forget_addresses(0, 0); + assert_eq!((page.addr_of(0), page.addr_of(1)), (None, Some((0x43, false)))); + assert_eq!(jit.pages_used(), 2, "and drops no page"); + assert_eq!(jit.claim_page(PageAt { cpu: 0, space: 0, pfn: 0x43, uncached: false }, &*far, false), Claim::Page(slot)); + jit.mega_flush(); + assert_eq!(jit.pages_used(), 0); + assert!(matches!(jit.claim_page(PageAt { cpu: 1, space: 1, pfn: 0x43, uncached: false }, &*far, false), Claim::Page(_))); + } + #[test] fn fast_lookup_ptrs_lookup_hit() { let dev = FakeDevice(AtomicU64::new(0)); let mut jit = Jitv2::new(4); let slot = jit.page_for(7, (7 * PAGE_SIZE) as u64, &dev, false).unwrap(); - let (pfn_map_ptr, pages_base) = jit.fast_lookup_ptrs(); + let (pfn_map_ptr, pages_base) = jit.fast_lookup_ptrs(0); unsafe { let mapped_slot = (*pfn_map_ptr.add(page_key(7, false, false) as usize)).load(Ordering::Acquire); assert_eq!(mapped_slot, slot); let page = &*pages_base.add(mapped_slot as usize); - assert_eq!(page.pfn, 7); + assert_eq!(page.pfn(), 7); + assert_eq!(page.key, CodeKey::flat(7 << 12)); } } @@ -5350,7 +5726,7 @@ mod tests { let stats = JitStats::default(); for i in 0..8u16 { let _ = i; - assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); } // stop() joins the worker; must return promptly even with requests in flight. q.stop(); @@ -5461,6 +5837,7 @@ mod tests { fn read64(&self, _addr: u64) -> BusRead64 { BusRead64::err() } fn write64(&self, _addr: u64, _val: u64) -> u32 { crate::traits::BUS_ERR } fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { &self.0 as *const AtomicU64 } + fn code_page(&self, addr: u64) -> Option { Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) } } #[test] @@ -5507,7 +5884,7 @@ mod tests { // dequeue time. page.mark_requested(0); let stats = JitStats::default(); - assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); // 30s, not the original 5s: this test passes in ~0.1s in isolation, // but the compile-worker thread genuinely needs real CPU time to get @@ -5562,7 +5939,7 @@ mod tests { for page in pages.iter_mut() { // §13.2: same as the queue-drain-fallback test above. page.mark_requested(0); - assert!(q.send(CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); } // Poll for the *first* page specifically, well before the full @@ -5640,7 +6017,7 @@ mod tests { // offset must be marked BEFORE sending, or handle_request_deferred // finds zero candidates and silently no-ops. page.mark_requested(0); - assert!(q.send(CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); } // catch_unwind for the same reason as the other threaded tests in @@ -5671,7 +6048,7 @@ mod tests { let mut q = CompileQueue::new(); let mut page = PhysicalCodePage::new(0, std::ptr::null()); let stats = JitStats::default(); - assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); q.start(dev, std::sync::Arc::new(JitStats::default())); q.stop(); } @@ -5686,7 +6063,7 @@ mod tests { let stats = JitStats::default(); let mut accepted = 0; for i in 0..(COMPILE_QUEUE_CAPACITY + 10) { - if q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats) { + if q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats) { accepted += 1; } } @@ -5707,7 +6084,7 @@ mod tests { let mut page = PhysicalCodePage::new(0, std::ptr::null()); assert!(page.try_schedule_page()); let stats = JitStats::default(); - assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); assert!(page.is_scheduled(), "sanity: the send above left the page marked in-flight"); q.drain_pending_queue(); @@ -5724,7 +6101,7 @@ mod tests { assert!(page.try_schedule_page()); let mut ours = CompileQueue::new(); let mut theirs = CompileQueue::new(); - push_hot_request(&ours.hot_handle(), CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true }); + push_hot_request(&ours.hot_handle(), CompileRequest { page: &mut page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }); theirs.drain_pending_queue(); assert!(page.is_scheduled(), "another pool's flush discarded this pool's hot request"); assert_eq!(theirs.hot_handle().len(), 0); @@ -5798,14 +6175,14 @@ mod tests { } assert_eq!(wake.sleepers.load(Ordering::SeqCst), 1, "the worker never parked"); let stats = JitStats::default(); - let req = CompileRequest { page: std::ptr::null_mut(), compiled_for_fr1: false }; + let req = CompileRequest { page: std::ptr::null_mut(), compiled_for_fr1: false, cpu: 0 }; assert!(push_compile_request(&queue, req, &stats)); let waited = worker.join().expect("the worker thread panicked"); assert!(waited < std::time::Duration::from_secs(30), "the sleeper waited out its backstop: {waited:?}"); assert_eq!(wake.sleepers.load(Ordering::SeqCst), 0); // A dropped push (the lane holds one request already) must not wake // anyone, and must not fail for want of a sleeper. - let req = CompileRequest { page: std::ptr::null_mut(), compiled_for_fr1: false }; + let req = CompileRequest { page: std::ptr::null_mut(), compiled_for_fr1: false, cpu: 0 }; assert!(!push_compile_request(&queue, req, &stats)); } @@ -5823,7 +6200,7 @@ mod tests { let mut page = Box::new(PhysicalCodePage::new(0, std::ptr::null())); assert!(page.try_schedule_page()); let stats = JitStats::default(); - assert!(q.send(CompileRequest { page: &mut *page as *mut PhysicalCodePage, compiled_for_fr1: true }, &stats)); + assert!(q.send(CompileRequest { page: &mut *page as *mut PhysicalCodePage, compiled_for_fr1: true, cpu: 0 }, &stats)); // The worker pops the request and drops it (the page has no code to // compile), which clears the in-flight flag. Waiting for that proves // the request was taken, not left in the queue. diff --git a/src/cpu/jitv2/pcp_dump.rs b/src/cpu/jitv2/pcp_dump.rs index 2fd460f0f..6b06adf4c 100644 --- a/src/cpu/jitv2/pcp_dump.rs +++ b/src/cpu/jitv2/pcp_dump.rs @@ -133,20 +133,32 @@ impl PcpDump { /// a live SMC race are an acceptable, clearly-labeled-by-context /// possibility for a debugging dump, not a correctness-critical path). /// Returns `Err` with the failing physical address if any word read fails. - pub fn capture(page: &PhysicalCodePage, bus: &dyn BusDevice) -> Result { - let bus = page.uncached.as_deref().unwrap_or(bus); - let phys_base = page.pfn as u64 * PAGE_SIZE as u64; + /// + /// The bytes are read in the bank's own storage when it hands that out, + /// else where a CPU of the pool reaches the page (`CodeBuses::locate`); + /// `Err(u64::MAX)` if none does right now. + pub fn capture(page: &PhysicalCodePage, buses: &crate::cpu::jitv2::jitv2::CodeBuses) -> Result { + // The bank's own storage when it hands that out, else a CPU's bus. let mut words = [0u32; ENTRIES_PER_PAGE]; - for (i, w) in words.iter_mut().enumerate() { - let addr = phys_base + i as u64 * 4; - let r = bus.read32(addr); - if !r.is_ok() { - return Err(addr); + let pfn = if !page.words().is_null() { + for (i, w) in words.iter_mut().enumerate() { + *w = unsafe { std::ptr::read_volatile(page.words().add(i)) }; } - *w = r.data; - } + page.pfn() + } else { + let (bus, phys_base) = buses.locate(page, 0).ok_or(u64::MAX)?; + for (i, w) in words.iter_mut().enumerate() { + let addr = phys_base + i as u64 * 4; + let r = bus.read32(addr); + if !r.is_ok() { + return Err(addr); + } + *w = r.data; + } + (phys_base / PAGE_SIZE as u64) as u32 + }; Ok(Self { - pfn: page.pfn, + pfn, current_gen: page.current_gen(), entry_gen: page.dump_entry_gen(), fr1: page.dump_fr1(), diff --git a/src/cpu/jitv2_html_j2wp.rs b/src/cpu/jitv2_html_j2wp.rs index 7922326cd..1e46ac858 100644 --- a/src/cpu/jitv2_html_j2wp.rs +++ b/src/cpu/jitv2_html_j2wp.rs @@ -477,13 +477,20 @@ pub fn write_jitv2_html( // (Jitv2::code_bytes_used doc comment) — fine for an on-demand dump, // not a hot path. for page in jit.claimed_pages() { - let phys_addr = page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; + // Where the lowest-numbered CPU that reaches the page has it. + let located = jit.buses().locate(page, 0); + let phys_addr = located.as_ref().map_or(page.pfn() as u64 * crate::cpu::jitv2::PAGE_SIZE as u64, |(_, at)| *at); + let bus = located.as_ref().map_or(bus, |(b, _)| b); let window = memory.find(phys_addr).map_or(OTHER_REGION, |r| r.name); let page_gen = page.current_gen(); let mut words = [0u32; crate::cpu::jitv2::ENTRIES_PER_PAGE]; for (i, w) in words.iter_mut().enumerate() { - *w = bus.read32(phys_addr + i as u64 * 4).data; + *w = if page.words().is_null() { + bus.read32(phys_addr + i as u64 * 4).data + } else { + unsafe { std::ptr::read_volatile(page.words().add(i)) } + }; } // §13: one function/gen/instr_count/code_size/call_count/ @@ -547,7 +554,7 @@ pub fn write_jitv2_html( let guest_code_size = (rows.len() as u32) * 4; pages.push(jitv2_html::PageDump { - pfn: page.pfn, + pfn: (phys_addr / crate::cpu::jitv2::PAGE_SIZE as u64) as u32, phys_addr, window, gen: page_gen, diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 75bf30967..b3d92fc0b 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -1321,6 +1321,18 @@ pub struct MipsExecutor { /// Whether the JIT keys a code page by uncached fetch too (`set_cached_bus`). #[cfg(feature = "jitv2")] jitv2_split: bool, + /// This CPU's number in the JIT pool it runs on: its slot in each code + /// page's addresses and in the pool's buses. 0 with a pool of its own. + #[cfg(feature = "jitv2")] + jitv2_cpu: usize, + /// The pool's frame table this CPU looks pages up in: one per set of CPUs + /// that see memory at the same addresses (a node). 0 with a pool of its own. + #[cfg(feature = "jitv2")] + jitv2_space: usize, + /// Whether the tracked code page was fetched uncached (`jitv2_split`): + /// with `core.cur_code_pfn`, where this CPU reached `pcp`. + #[cfg(feature = "jitv2")] + pcp_uncached: bool, /// Refused posted stores so far, so `post_refused_store` can log sparsely. refused_stores: u64, /// `jitcheck`'s hardware-read fixup: `validate_jit_determinism`'s two @@ -1828,7 +1840,7 @@ unsafe extern "C" fn jit_fetch_verify(ctx: *mut core::ffi:: // address while `expected` is right, which looks like a coherent // "shift" but is really two unrelated regions. `tracked_pfn` vs // `phys_page` is what tells those apart. - let pcp_pfn: i64 = if exec.pcp.is_null() { -1 } else { unsafe { (*exec.pcp).pfn as i64 } }; + let pcp_pfn: i64 = if exec.pcp.is_null() { -1 } else { exec.core.cur_code_pfn as i64 }; eprintln!( "\n=== STALE COMPILED CODE at {:#018x}{} phys={:#010x} ===\n compiled-in: {:08x} {}\n now in memory: {:08x} {}\n live pc={:#018x} va_page={:#x} phys_page={:#x} tracked_pfn={}\n", va, sym, tr.phys, @@ -2681,9 +2693,11 @@ impl MipsExecutor { #[cfg(feature = "jitv2")] if !self.jitv2_split && !Arc::ptr_eq(&bus, &self.sysad) { self.jitv2_split = true; - self.jitv2.lock().split_uncached_code(); + self.jitv2.lock().set_space(self.jitv2_space, self.jitv2_frames as usize, true); } self.cached_bus = bus; + #[cfg(feature = "jitv2")] + self.jitv2.lock().buses().set(self.jitv2_cpu, self.jitv2_space, self.cached_bus.clone(), self.sysad.clone()); } /// Create a new executor from a config and a bus (sysad) and a TLB. @@ -2968,6 +2982,12 @@ impl MipsExecutor { jitv2_frames: crate::cpu::jitv2::jitv2::frames_for(&cfg.memory), #[cfg(feature = "jitv2")] jitv2_split: false, + #[cfg(feature = "jitv2")] + jitv2_cpu: 0, + #[cfg(feature = "jitv2")] + jitv2_space: 0, + #[cfg(feature = "jitv2")] + pcp_uncached: false, refused_stores: 0, #[cfg(feature = "developer")] hw_read_fixup_replay: None, @@ -3534,13 +3554,14 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// structural rather than a convention six call sites have to remember. #[cfg(feature = "jitv2")] #[inline] - fn set_pcp(&mut self, page: *mut crate::cpu::jitv2::PhysicalCodePage) { + fn set_pcp(&mut self, page: *mut crate::cpu::jitv2::PhysicalCodePage, pfn: u32, uncached: bool) { self.pcp = page; - self.core.cur_code_pfn = if page.is_null() { - u32::MAX - } else { - unsafe { (*page).pfn } - }; + self.core.cur_code_pfn = if page.is_null() { u32::MAX } else { pfn }; + self.pcp_uncached = uncached; + // A compile for this CPU's request reads the page here. + if let Some(page) = unsafe { page.as_ref() } { + page.set_addr(self.jitv2_cpu, pfn, uncached); + } } /// Drop the currently tracked code page — the `null` counterpart to @@ -3549,7 +3570,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[cfg(feature = "jitv2")] #[inline] fn clear_pcp(&mut self) { - self.set_pcp(std::ptr::null_mut()); + self.set_pcp(std::ptr::null_mut(), u32::MAX, false); } /// Flush the page pool and compiled code from this CPU: every other CPU @@ -3595,9 +3616,14 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } #[cfg(feature = "jitv2")] - fn jitv2_share(&mut self, jit: Arc>, world: Arc) { + fn jitv2_share(&mut self, jit: Arc>, world: Arc, cpu: usize, space: usize) { + assert!(cpu < crate::cpu::jitv2::jitv2::POOL_CPUS, "a JIT pool serves at most {} CPUs", crate::cpu::jitv2::jitv2::POOL_CPUS); + self.clear_pcp(); + self.jitv2_cpu = cpu; + self.jitv2_space = space; { - let j = jit.lock(); + let mut j = jit.lock(); + j.set_space(space, self.jitv2_frames as usize, self.jitv2_split); self.jitv2_compile_queue_handle = j.compile_queue.queue_handle(); self.jitv2_hot_queue_handle = j.compile_queue.hot_handle(); self.jitv2_stats = j.stats.clone(); @@ -3611,11 +3637,25 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", /// `page_for` fast path (`jitv2_pfn_map`/`jitv2_pages_base`). #[cfg(feature = "jitv2")] pub fn jitv2_bind_fast_lookup(&mut self) { - let (map, pages) = self.jitv2.lock().fast_lookup_ptrs(); + let mut j = self.jitv2.lock(); + j.buses().set(self.jitv2_cpu, self.jitv2_space, self.cached_bus.clone(), self.sysad.clone()); + let (map, pages) = j.fast_lookup_ptrs(self.jitv2_space); self.jitv2_pfn_map = map; self.jitv2_pages_base = pages; } + /// A memory controller moved memory under this CPU's addresses. Compiled + /// code is keyed by the memory it was compiled from, not by its address, + /// so none is dropped: the CPU forgets which frame holds which page and + /// finds each again, by asking the bus, at its next fetch from it. + #[cfg(feature = "jitv2")] + pub(crate) fn jitv2_layout_changed(&mut self) { + // Translations hold addresses the move may have changed too. + self.nanotlb_invalidate(); + self.clear_pcp(); + self.jitv2.lock().forget_addresses(self.jitv2_space, self.jitv2_cpu); + } + /// Lock-free `pfn -> *mut PhysicalCodePage`, or `None` on a miss (which /// the caller must then resolve under the lock, since claiming mutates /// the free list). @@ -3663,7 +3703,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.clear_pcp(); return; }; - let same_page = !self.pcp.is_null() && unsafe { (*self.pcp).pfn == pfn && (!SPLIT || (*self.pcp).uncached.is_some() == uncached) }; + // Where this CPU reached the tracked page, kept beside the pointer: + // the page itself is keyed by its memory and may sit at another + // address for another CPU. + let same_page = !self.pcp.is_null() && self.core.cur_code_pfn == pfn && (!SPLIT || self.pcp_uncached == uncached); if same_page { return; } @@ -3702,84 +3745,59 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", self.core.jit_trigger = true; } - // Lock-free hit path: an already-claimed page needs nothing but the - // `pfn -> slot -> page` indirection, which is two loads from arrays that - // never move; this skips `jitv2.lock()`. + // Lock-free hit path: a frame this address space has already reached + // needs nothing but the `frame -> slot -> page` indirection, which is + // two loads from arrays that never move; this skips `jitv2.lock()` + // and the bus. if let Some(page) = self.jitv2_lookup_page_fast(crate::cpu::jitv2::jitv2::page_key(pfn, uncached, SPLIT)) { - self.pcp = page; - self.core.cur_code_pfn = pfn; - debug_assert_eq!(unsafe { (*self.pcp).pfn }, pfn, - "jitv2_track_pcp fast path: pfn_map[{:#x}] pointed at a slot whose own pfn is {:#x}", - pfn, unsafe { (*self.pcp).pfn }); + self.set_pcp(page, pfn, uncached); return; } - let page_base = pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; // §13: live FR mode at first-arrival time, pinned into the page for // its whole lifetime (PhysicalCodePage::fr1's own doc comment) — - // ignored by page_for on a lookup hit (an already-claimed page keeps - // whatever it was pinned to). + // ignored by `claim_page` on a lookup hit (an already-claimed page + // keeps whatever it was pinned to). let fr1 = (self.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; - - let (mut jit, lookup) = { - crate::pf_scope!(crate::perfetto::events::JIT_CPU_TRACK); - let mut jit = self.jitv2.lock(); - let lookup = jit.claim_page(pfn, page_base, self.cached_bus.as_ref(), uncached.then_some(&self.sysad), fr1); - // The first claim allocates the table, possibly after this executor bound. - if self.jitv2_pfn_map.is_null() { - self.jitv2_pfn_map = jit.pfn_map_ptr(); - } - (jit, lookup) + let at = crate::cpu::jitv2::jitv2::PageAt { cpu: self.jitv2_cpu, space: self.jitv2_space, pfn, uncached }; + let claim = self.jitv2_claim(at, fr1); + let claim = if claim == crate::cpu::jitv2::jitv2::Claim::Full { + // Pool exhausted: `flush_from_cpu_thread` resets to initial + // state and retries — always leaves room for at least one + // fresh page. Called from the CPU thread itself (this + // function), which is "as good as stopped" for its own + // purposes — no need to pause it — so this only pauses the + // *compile* queue internally, not the CPU. + self.jitv2_flush_from_here(true); + // `self.pcp` points into the pool the flush just cleared. The + // nanotlb's translations are untouched (memory did not move). + self.clear_pcp(); + self.jitv2_claim(at, fr1) + } else { + claim }; - match lookup { - Some(slot) => { - // Assigned directly rather than through `set_pcp`: the - // `jit` guard is still live here, so a `&mut self` call - // would conflict. `pfn` is the value `set_pcp` would have - // read back out of the slot anyway — this keeps the mirror - // exact without the borrow. - self.pcp = jit.page_ptr(slot); - self.core.cur_code_pfn = pfn; - // `pfn` here is the real physical frame number (`phys_addr` - // is post-translation, from nanotlb_translate's successful - // result — never a virtual page) — this must match the slot - // `page_for` just handed back, or self.pcp is now tracking - // the wrong physical page for this fetch (the exact failure - // shape `j2 pcp` can otherwise only reveal after the fact). - debug_assert_eq!(unsafe { (*self.pcp).pfn }, pfn, - "jitv2_track_pcp: page_for({:#x}) returned a slot whose own pfn is {:#x}", - pfn, unsafe { (*self.pcp).pfn }); - } - None => { - // Pool exhausted: `flush_from_cpu_thread` resets to initial - // state and retries — always leaves room for at least one - // fresh page. Called from the CPU thread itself (this - // function), which is "as good as stopped" for its own - // purposes — no need to pause it — so this only pauses the - // *compile* queue internally, not the CPU. - drop(jit); // re-locks internally below; nothing needs it held across that call - self.jitv2_flush_from_here(true); - // Only `self.pcp` is actually stale here — it's a raw - // pointer into the `Vec` mega_flush just - // cleared. The nanotlb's own translation slots (Fetch/Read/ - // Write) are untouched by mega_flush (physical memory didn't - // move, only the JIT's compiled-code bookkeeping did), so - // there's no reason to blow those away too — just null the - // dangling pointer directly and re-derive it below for the - // exact page this fetch already landed on. - self.clear_pcp(); - let mut jit = self.jitv2.lock(); - let slot = jit.claim_page(pfn, page_base, self.cached_bus.as_ref(), uncached.then_some(&self.sysad), fr1) - .expect("flush must leave room for at least one page"); - // Direct assignment for the same borrow reason as the - // hit path above; `pfn` is what `set_pcp` would mirror. - self.pcp = jit.page_ptr(slot); - self.core.cur_code_pfn = pfn; - debug_assert_eq!(unsafe { (*self.pcp).pfn }, pfn, - "jitv2_track_pcp (post-flush retry): page_for({:#x}) returned a slot whose own pfn is {:#x}", - pfn, unsafe { (*self.pcp).pfn }); + match claim { + crate::cpu::jitv2::jitv2::Claim::Page(slot) => { + let page = self.jitv2.lock().page_ptr(slot); + self.set_pcp(page, pfn, uncached); } + crate::cpu::jitv2::jitv2::Claim::Full => unreachable!("a flush leaves room for at least one page"), + } + } + + /// Ask the pool for the page at `at`, through the bus that decodes the fetch. + #[cfg(feature = "jitv2")] + #[cold] + fn jitv2_claim(&mut self, at: crate::cpu::jitv2::jitv2::PageAt, fr1: bool) -> crate::cpu::jitv2::jitv2::Claim { + crate::pf_scope!(crate::perfetto::events::JIT_CPU_TRACK); + let bus = if at.uncached { &self.sysad } else { &self.cached_bus }; + let mut jit = self.jitv2.lock(); + let claim = jit.claim_page(at, bus.as_ref(), fr1); + // The first claim allocates the table, possibly after this executor bound. + if self.jitv2_pfn_map.is_null() { + self.jitv2_pfn_map = jit.pfn_map_ptr(self.jitv2_space); } + claim } /// Re-derive `translate_fn` from the current CP0 Status register. @@ -4270,6 +4288,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let req = crate::cpu::jitv2::CompileRequest { page: self.pcp, compiled_for_fr1, + cpu: self.jitv2_cpu as u8, }; if inline_compile { // Sync compiles a page with no code, or one this arrival makes hot, @@ -9513,7 +9532,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let over_threshold = self.jitv2.lock().codegen.lock().as_ref() .is_some_and(|c| c.packing_stats().1 > crate::cpu::jitv2::CODEGEN_ARENA_FLUSH_THRESHOLD_BYTES); if over_threshold { - let (phys_page_base, uncached) = (page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64, page.uncached.is_some()); + let (phys_page_base, uncached) = (self.core.cur_code_pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64, self.pcp_uncached); unsafe { self.jitv2.lock().flush_from_cpu_thread(self.cached_bus.clone()); } self.clear_pcp(); self.jitv2_track(phys_page_base, uncached); @@ -9533,13 +9552,14 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // Stamp the L1-D geometry the inline load/store path needs. No // helper build here — see the note in Jitv2::worker_loop. codegen.dc_geometry = self.cache.jit_dc_geometry(); + let buses = self.jitv2.lock().buses().clone(); #[cfg(feature = "developer")] { let stats = self.jitv2.lock().stats.clone(); - ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.cached_bus, &mut self.jitv2_inline_analyzer, codegen, &stats); + ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &buses, &mut self.jitv2_inline_analyzer, codegen, &stats); } #[cfg(not(feature = "developer"))] - { ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.cached_bus, &mut self.jitv2_inline_analyzer, codegen); } + { ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &buses, &mut self.jitv2_inline_analyzer, codegen); } } *self.jitv2.lock().codegen.lock() = codegen; { @@ -9549,7 +9569,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", } if ran_out_of_memory { let page = unsafe { &mut *self.pcp }; - let (phys_page_base, uncached) = (page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64, page.uncached.is_some()); + let (phys_page_base, uncached) = (self.core.cur_code_pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64, self.pcp_uncached); unsafe { self.jitv2.lock().flush_from_cpu_thread(self.cached_bus.clone()); } self.clear_pcp(); self.jitv2_track(phys_page_base, uncached); @@ -10891,13 +10911,12 @@ impl MipsCpu { Ok(()) } - /// A flush a remap posted while the CPU was not running. It shares the park + /// A remap request posted while the CPU was not running. It shares the park /// bit, so a stepper must take it or `step_int` retries for ever. - pub(crate) fn take_flush_request(&self, exec: &mut MipsExecutor) { + pub(crate) fn take_remap_request(&self, exec: &mut MipsExecutor) { #[cfg(feature = "jitv2")] - if self.park.release(crate::cpu::smp::REQUEST_JIT_FLUSH) { - exec.jitv2_flush_from_here(false); - exec.nanotlb_invalidate(); + if self.park.release(crate::cpu::smp::REQUEST_JIT_REMAP) { + exec.jitv2_layout_changed(); } #[cfg(not(feature = "jitv2"))] let _ = exec; @@ -10913,7 +10932,7 @@ impl MipsCpu { /// reach identical state after the same number of steps. pub fn step_n_inline(&self, n: u64) -> Result { let mut exec = self.try_lock_executor()?; - self.take_flush_request(&mut exec); + self.take_remap_request(&mut exec); let mut executed = 0u64; for _ in 0..n { let _status = exec.step_int(); @@ -11359,11 +11378,10 @@ impl Device for MipsCpu>, world: Arc); + /// Run on `jit`, a pool other CPUs run on too, as its CPU number `cpu`, + /// looking pages up in its address space `space` (the same for CPUs that + /// see memory at the same addresses, as the CPUs of one node do). + fn share_jitv2(&self, jit: Arc>, world: Arc, cpu: usize, space: usize); fn monitor_execute(&self, cmd: &str, args: &[&str], writer: Box) -> Result<(), String>; fn cycles_ptr(&self) -> crate::cpu::mips_core::CyclesPtr; fn interrupts_ptr(&self) -> *const AtomicU64; @@ -11644,9 +11665,9 @@ impl CpuDevice for MipsCp fn monitor_commands(&self) -> Vec<(String, String)> { MipsCpu::monitor_commands(self) } fn park_slot(&self) -> Arc { self.park.clone() } #[cfg(feature = "jitv2")] - fn share_jitv2(&self, jit: Arc>, world: Arc) { + fn share_jitv2(&self, jit: Arc>, world: Arc, cpu: usize, space: usize) { let mut exec = self.executor.lock(); - exec.jitv2_share(jit, world); + exec.jitv2_share(jit, world, cpu, space); } fn monitor_execute(&self, cmd: &str, args: &[&str], writer: Box) -> Result<(), String> { MipsCpu::monitor_execute(self, cmd, args, writer) diff --git a/src/cpu/mips_exec/gdb.rs b/src/cpu/mips_exec/gdb.rs index ab15cc93d..3be546030 100644 --- a/src/cpu/mips_exec/gdb.rs +++ b/src/cpu/mips_exec/gdb.rs @@ -61,7 +61,7 @@ impl CpuDebug use crate::cpu::mips_exec::{EXEC_BREAKPOINT, EXEC_RETRY}; self.cpu.stop(); // ensure no thread is running let mut exec = self.cpu.executor.lock(); - self.cpu.take_flush_request(&mut exec); + self.cpu.take_remap_request(&mut exec); exec.last_bp_hit = None; // EXEC_RETRY means the instruction did not retire and PC is unmoved, so diff --git a/src/cpu/mips_exec/monitor.rs b/src/cpu/mips_exec/monitor.rs index 9f12f237e..4f698a12a 100644 --- a/src/cpu/mips_exec/monitor.rs +++ b/src/cpu/mips_exec/monitor.rs @@ -263,7 +263,7 @@ pub(super) fn is_call_instruction(instr: u32) -> bool { impl MipsCpu { pub fn run_debug_loop(&self, mut count: Option, wait: bool, mut writer: Box) { self.halt(); // Ensure stopped before running - self.take_flush_request(&mut self.executor.lock()); + self.take_remap_request(&mut self.executor.lock()); self.running.store(true, Ordering::SeqCst); @@ -2589,15 +2589,16 @@ impl MipsCpu { let bus = exec.cached_bus.clone(); let mut jit = exec.jitv2.lock(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; - let slot = jit.page_for(pfn, page_base, bus.as_ref(), fr1); - match slot { - Some(slot) => jit.page_ptr(slot), - None => return Err("page pool full".to_string()), + let _ = page_base; + let at = crate::cpu::jitv2::jitv2::PageAt { cpu: exec.jitv2_cpu, space: exec.jitv2_space, pfn, uncached: false }; + match jit.claim_page(at, bus.as_ref(), fr1) { + crate::cpu::jitv2::jitv2::Claim::Page(slot) => jit.page_ptr(slot), + crate::cpu::jitv2::jitv2::Claim::Full => return Err("page pool full".to_string()), } }; let page = unsafe { &*page_ptr }; - let bus = exec.cached_bus.clone(); - let dump = crate::cpu::jitv2::pcp_dump::PcpDump::capture(page, bus.as_ref()) + let buses = exec.jitv2.lock().buses().clone(); + let dump = crate::cpu::jitv2::pcp_dump::PcpDump::capture(page, &buses) .map_err(|st| format!("capture failed: bus status {:#x}", st))?; // A path argument is whichever arg is not the address. let path = actual_args.iter().skip(1) @@ -2640,7 +2641,7 @@ impl MipsCpu { // now (device unmapped under it, say) is skipped // rather than failing the whole dump — one bad // page shouldn't cost the other few thousand. - match crate::cpu::jitv2::pcp_dump::PcpDump::capture(page, bus.as_ref()) { + match crate::cpu::jitv2::pcp_dump::PcpDump::capture(page, jit.buses()) { Ok(d) => v.push(d), Err(_) => unreadable += 1, } @@ -2732,7 +2733,7 @@ impl MipsCpu { let bus = exec.cached_bus.clone(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let mut jit = exec.jitv2.lock(); - match jit.page_for(pfn, page_base, bus.as_ref(), fr1) { + match jit.page_at(crate::cpu::jitv2::jitv2::PageAt { cpu: exec.jitv2_cpu, space: exec.jitv2_space, pfn, uncached: false }, bus.as_ref(), fr1) { Some(slot) => jit.page_ptr(slot), None => { writeln!(writer, "No PhysicalCodePage tracked (self.pcp is null) and the page pool is full (can't even allocate a fresh lookup entry).").unwrap(); @@ -2759,8 +2760,11 @@ impl MipsCpu { // block_include_count (overlap between entries is // structurally impossible under this design, so that // diagnostic no longer applies). + writeln!(writer, "bank={} page={:#x} ({})", page.key.bank, page.key.page, + (0..crate::cpu::jitv2::jitv2::POOL_CPUS).filter_map(|c| page.addr_of(c).map(|(pfn, unc)| format!("cpu{c} at pfn {pfn:#x}{}", if unc { " uncached" } else { "" }))) + .collect::>().join(", ")).unwrap(); writeln!(writer, "pfn={:#010x} gen={} entry_gen={} scheduled(in-flight)={} compiles_since_flush={} rejected_compiles={}", - page.pfn, page.current_gen(), page.entry_gen(), page.is_scheduled(), page.compiles_since_flush(), page.rejected_compiles()).unwrap(); + page.pfn(), page.current_gen(), page.entry_gen(), page.is_scheduled(), page.compiles_since_flush(), page.rejected_compiles()).unwrap(); // §13: one function per page, so FR mode is a single // whole-page value — pinned at page-claim time from the // live STATUS_FR (PhysicalCodePage::fr1) and every @@ -2894,7 +2898,7 @@ impl MipsCpu { let bus = exec.cached_bus.clone(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let mut jit = exec.jitv2.lock(); - match jit.page_for(pfn, page_base, bus.as_ref(), fr1) { + match jit.page_at(crate::cpu::jitv2::jitv2::PageAt { cpu: exec.jitv2_cpu, space: exec.jitv2_space, pfn, uncached: false }, bus.as_ref(), fr1) { Some(slot) => { let page_ptr = jit.page_ptr(slot); unsafe { (*page_ptr).reset_compiled_state(); } @@ -2942,7 +2946,7 @@ impl MipsCpu { let bus = exec.cached_bus.clone(); let fr1 = (exec.core.cp0_status & crate::cpu::mips_core::STATUS_FR) != 0; let mut jit = exec.jitv2.lock(); - match jit.page_for(pfn, page_base, bus.as_ref(), fr1) { + match jit.page_at(crate::cpu::jitv2::jitv2::PageAt { cpu: exec.jitv2_cpu, space: exec.jitv2_space, pfn, uncached: false }, bus.as_ref(), fr1) { Some(slot) => { let page_ptr = jit.page_ptr(slot); unsafe { (*page_ptr).denylist(offset); (*page_ptr).kill(offset); } @@ -3003,7 +3007,7 @@ impl MipsCpu { // signal, not a crash risk. let page = unsafe { &*page_ptr }; writeln!(writer, " page: pfn={:#010x} phys_base={:#010x}", - page.pfn, page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64).unwrap(); + page.pfn(), page.pfn() as u64 * crate::cpu::jitv2::PAGE_SIZE as u64).unwrap(); } }; match jit.compile_queue.seal_queue_entries() { @@ -3047,6 +3051,7 @@ impl MipsCpu { "stats" | "status" => { let jit = exec.jitv2.lock(); writeln!(writer, "pages: {} / {} used", jit.pages_used(), jit.capacity()).unwrap(); + writeln!(writer, "remaps: {} (a CPU forgot its page addresses after memory moved; no code dropped)", jit.stats.remaps.load(Ordering::Relaxed)).unwrap(); // Churn avoidance: "skipped" is compile work that did not // happen because the page's generation moved without // any word the last compile decoded changing; diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 524ff3f90..9a3871167 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -180,6 +180,10 @@ mod tests { fn gen_ptr(&self, _addr: u64) -> *const std::sync::atomic::AtomicU64 { std::ptr::null() } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + Some(crate::traits::CodeKey::flat_page(addr, std::ptr::null())) + } } /// Disable every jitv2 dispatch path on a freshly-constructed executor — @@ -5871,6 +5875,9 @@ mod tests { let counter = gens.entry(page).or_insert_with(|| Box::new(std::sync::atomic::AtomicU64::new(0))); counter.as_ref() as *const std::sync::atomic::AtomicU64 } + fn code_page(&self, addr: u64) -> Option { + Some(crate::traits::CodeKey::flat_page(addr, self.gen_ptr(addr))) + } } #[test] @@ -7402,10 +7409,98 @@ mod tests { assert!(out.contains("[hits 1]"), "{}", out); } - /// A page compiled before a remap is not dispatched over the new layout. + /// Two RAM banks of 1 MiB at 0 and 1 MiB, which a store to nothing swaps: + /// a memory controller moving banks. + #[cfg(feature = "jitv2")] + struct SwapBus { + banks: [crate::dev::mem::Memory; 2], + swapped: std::sync::atomic::AtomicBool, + } + + #[cfg(feature = "jitv2")] + impl SwapBus { + fn at(&self, addr: u64) -> (&crate::dev::mem::Memory, u64) { + let swapped = self.swapped.load(std::sync::atomic::Ordering::Relaxed); + let bank = ((addr >> 20) & 1) as usize ^ swapped as usize; + (&self.banks[bank], addr & 0xF_FFFF) + } + } + + #[cfg(feature = "jitv2")] + impl BusDevice for SwapBus { + fn read8(&self, a: u64) -> BusRead8 { let (d, a) = self.at(a); d.read8(a) } + fn write8(&self, a: u64, v: u8) -> u32 { let (d, a) = self.at(a); d.write8(a, v) } + fn read16(&self, a: u64) -> BusRead16 { let (d, a) = self.at(a); d.read16(a) } + fn write16(&self, a: u64, v: u16) -> u32 { let (d, a) = self.at(a); d.write16(a, v) } + fn read32(&self, a: u64) -> BusRead32 { let (d, a) = self.at(a); d.read32(a) } + fn write32(&self, a: u64, v: u32) -> u32 { let (d, a) = self.at(a); d.write32(a, v) } + fn read64(&self, a: u64) -> BusRead64 { let (d, a) = self.at(a); d.read64(a) } + fn write64(&self, a: u64, v: u64) -> u32 { let (d, a) = self.at(a); d.write64(a, v) } + fn gen_ptr(&self, a: u64) -> *const std::sync::atomic::AtomicU64 { let (d, a) = self.at(a); d.gen_ptr(a) } + fn code_page(&self, a: u64) -> Option { let (d, a) = self.at(a); d.code_page(a) } + } + + /// Compiled code is keyed by the bank it was compiled from: when a remap + /// puts another bank at its address, the address runs that bank's page, + /// and the code is found again, not recompiled, where its bank went. + #[cfg(all(feature = "jitv2", not(feature = "lightning")))] + #[test] + fn compiled_code_follows_its_bank_through_a_remap() { + use std::sync::atomic::Ordering; + let bus = Arc::new(SwapBus { banks: [crate::dev::mem::Memory::new(1), crate::dev::mem::Memory::new(1)], swapped: Default::default() }); + let mut exec: MipsExecutor = + MipsExecutor::new(bus.clone() as Arc, PassthroughTlb::default(), &MipsCpuConfig::indy()); + exec.jitv2_inline_compile = true; + #[cfg(feature = "fetchverify")] + { exec.core.fetch_verify_on = false; } + exec.install_jit_hooks(); + // Each bank adds its own number to r2 four times and returns. + for (bank, add) in [(0u64, 1u16), (1, 2)] { + for i in 0..4 { + bus.write32((bank << 20) + 0x1000 + 4 * i, make_i(OP_ADDIU, 2, 2, add)); + } + bus.write32((bank << 20) + 0x1010, make_r(OP_SPECIAL, 31, 0, 0, 0, FUNCT_JR)); + } + exec.core.gpr[31] = 0xffff_ffff_8000_f000; + let arrive = |exec: &mut MipsExecutor, pc: u64| { + exec.core.pc = pc; + exec.core.in_delay_slot = false; + exec.core.jit_trigger = true; + exec.step_jit(); + exec.jitv2_current_page() + }; + let low = arrive(&mut exec, 0xffff_ffff_8000_1000); + assert!(!low.is_null()); + let low = unsafe { &*low }; + let key0 = bus.banks[0].code_page(0x1000).unwrap().key; + let key1 = bus.banks[1].code_page(0x1000).unwrap().key; + assert_eq!(low.key, key0, "the page is keyed by its bank, not its address"); + assert!(low.is_runnable(0)); + let compiles = low.compiles_since_flush(); + + // The controller swaps the banks, and tells the CPU as a machine does. + bus.swapped.store(true, Ordering::Relaxed); + exec.jitv2_layout_changed(); + assert!(low.is_runnable(0), "a remap drops no code"); + assert_eq!(low.addr_of(0), None, "the CPU no longer knows where the page is"); + + let other = arrive(&mut exec, 0xffff_ffff_8000_1000); + assert!(!std::ptr::eq(other, low), "the address now holds the other bank's page"); + assert_eq!(unsafe { &*other }.key, key1); + + let moved = arrive(&mut exec, 0xffff_ffff_8010_1000); + assert!(std::ptr::eq(moved, low), "the first bank's page is found at its new address"); + assert_eq!(low.addr_of(0), Some((0x101, false))); + assert_eq!(low.compiles_since_flush(), compiles, "and runs the code it already had"); + let jit = exec.jitv2.clone(); + assert_eq!(jit.lock().pages_used(), 2); + } + + /// A remap request posted while the CPU is stopped is taken by a step, + /// and drops no compiled code. #[cfg(feature = "jitv2")] #[test] - fn a_compiled_page_is_not_dispatched_after_a_remap() { + fn a_remap_request_keeps_compiled_code() { use crate::cpu::mips_exec::CpuDevice; let mem = Arc::new(MockMemory::new()); let bus: Arc = mem.clone(); @@ -7434,10 +7529,12 @@ mod tests { } crate::traits::Device::stop(&*cpu); let before = page(); - assert!(unsafe { (*before).is_runnable(0) }); + let remaps = jit.lock().stats.remaps.load(std::sync::atomic::Ordering::Relaxed); world.layout_changed(); cpu.step_n_inline(0).unwrap(); - assert!(!unsafe { (*before).is_runnable(0) }, "the page compiled before the remap is still runnable"); + assert_eq!(jit.lock().stats.remaps.load(std::sync::atomic::Ordering::Relaxed), remaps + 1, "the step took the request"); + assert!(unsafe { (*before).is_runnable(0) }, "the page keeps its code through a remap"); + assert_eq!(jit.lock().pages_used(), 1); } /// 64 entries on one page (four adds and a return each), sync compile. diff --git a/src/cpu/smp.rs b/src/cpu/smp.rs index 364f1acb6..b1826d045 100644 --- a/src/cpu/smp.rs +++ b/src/cpu/smp.rs @@ -9,8 +9,9 @@ //! until the boundary. Holding is cold; the running CPU pays one load per //! batch. //! -//! [`REQUEST_JIT_FLUSH`] rides the same bit but is not a hold: the CPU empties its -//! JIT pool at the boundary and runs on ([`World::layout_changed`]). +//! [`REQUEST_JIT_REMAP`] rides the same bit but is not a hold: at the boundary +//! the CPU forgets where its JIT's code pages are and runs on +//! ([`World::layout_changed`]). use std::cell::Cell; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; @@ -36,8 +37,10 @@ pub const HOLD_WORLD: u32 = 1 << 1; /// The board holds this processor at a firmware rendezvous until the ones /// it must follow catch up (the IP35 PROM's presence publication). pub const HOLD_RENDEZVOUS: u32 = 1 << 2; -/// Not a hold: the CPU empties its JIT pool at its next batch boundary. -pub const REQUEST_JIT_FLUSH: u32 = 1 << 3; +/// Not a hold: memory moved, so at its next batch boundary the CPU forgets +/// the addresses it reached its JIT's code pages at. The pages keep their +/// code, which is keyed by the memory it came from. +pub const REQUEST_JIT_REMAP: u32 = 1 << 3; const NOT_A_CPU: u8 = u8::MAX; @@ -117,8 +120,8 @@ impl ParkSlot { if s.holds == 0 { self.word().fetch_and(!HOST_PARK_BIT, Ordering::SeqCst); } - // A parked CPU does not wait on a flush request; wake it now, not at its next timeout. - if s.holds & !REQUEST_JIT_FLUSH == 0 { + // A parked CPU does not wait on a remap request; wake it now, not at its next timeout. + if s.holds & !REQUEST_JIT_REMAP == 0 { self.changed.notify_all(); } posted @@ -149,12 +152,12 @@ impl ParkSlot { } /// The CPU thread, with its executor unlocked: wait out the holds, or - /// until the CPU is stopped. A flush request is not waited on. + /// until the CPU is stopped. A remap request is not waited on. pub fn park(&self, running: &AtomicBool) { let mut s = self.state.lock(); s.parked = true; self.changed.notify_all(); - while s.holds & !REQUEST_JIT_FLUSH != 0 && running.load(Ordering::Relaxed) { + while s.holds & !REQUEST_JIT_REMAP != 0 && running.load(Ordering::Relaxed) { self.changed.wait_for(&mut s, Duration::from_millis(1)); } s.parked = false; @@ -210,11 +213,12 @@ impl World { self.leader.try_lock().map(|turn| self.hold_others(turn)) } - /// Memory moved under an address: every CPU, the caller's included, empties - /// its JIT pool before it runs guest code again. + /// Memory moved under an address: every CPU, the caller's included, + /// forgets the addresses of its JIT's code pages before it runs guest code + /// again, and finds each page again through the bus. pub fn layout_changed(&self) { #[cfg(feature = "jitv2")] - self.slots().iter().for_each(|s| s.hold(REQUEST_JIT_FLUSH)); + self.slots().iter().for_each(|s| s.hold(REQUEST_JIT_REMAP)); } fn slots(&self) -> &[Arc] { @@ -351,9 +355,9 @@ mod tests { assert!(world.try_lead().is_some()); } - /// A flush request sets the park bit but does not hold a parked CPU. + /// A remap request sets the park bit but does not hold a parked CPU. #[test] - fn a_layout_change_asks_every_cpu_to_flush_without_holding_it() { + fn a_layout_change_asks_every_cpu_without_holding_it() { let (s, word) = slot(0); let world = World::new(); world.set_slots(vec![s.clone()]); @@ -362,18 +366,18 @@ mod tests { assert_ne!(word.load(Ordering::SeqCst) & HOST_PARK_BIT, 0); } s.park(&AtomicBool::new(true)); - assert_eq!(s.release(REQUEST_JIT_FLUSH), cfg!(feature = "jitv2")); - assert!(!s.release(REQUEST_JIT_FLUSH)); + assert_eq!(s.release(REQUEST_JIT_REMAP), cfg!(feature = "jitv2")); + assert!(!s.release(REQUEST_JIT_REMAP)); assert_eq!(word.load(Ordering::SeqCst) & HOST_PARK_BIT, 0); } - /// The flush leaves another hold's park bit set. + /// The request leaves another hold's park bit set. #[test] - fn releasing_the_flush_keeps_another_hold() { + fn releasing_the_remap_request_keeps_another_hold() { let (s, word) = slot(0); - s.hold(REQUEST_JIT_FLUSH); + s.hold(REQUEST_JIT_REMAP); s.hold(HOLD_HEART); - assert!(s.release(REQUEST_JIT_FLUSH)); + assert!(s.release(REQUEST_JIT_REMAP)); assert_ne!(word.load(Ordering::SeqCst) & HOST_PARK_BIT, 0); } diff --git a/src/dev/baseio_flash.rs b/src/dev/baseio_flash.rs index 3dd612a02..54173c54e 100644 --- a/src/dev/baseio_flash.rs +++ b/src/dev/baseio_flash.rs @@ -35,6 +35,10 @@ const GEOMETRY: Geometry = Geometry { pub struct BaseioFlash { image: Box<[u8]>, core: Mutex, + /// JIT v2: the part's identity in a page key, and its write counter: + /// a command cycle changes what a read returns. + #[cfg(feature = "jitv2")] + code: (crate::traits::CodeBank, std::sync::atomic::AtomicU64), } impl BaseioFlash { @@ -45,7 +49,12 @@ impl BaseioFlash { } let mut image = vec![0xff; SIZE].into_boxed_slice(); image[..file.len()].copy_from_slice(file); - Ok(Self { image, core: Mutex::new(AmdCore::new(GEOMETRY)) }) + Ok(Self { + image, + core: Mutex::new(AmdCore::new(GEOMETRY)), + #[cfg(feature = "jitv2")] + code: Default::default(), + }) } } @@ -64,11 +73,24 @@ impl BusDevice for BaseioFlash { fn write16(&self, addr: u64, val: u16) -> u32 { // The engine logs a cycle it does not take, including the program and // erase this part refuses. - match self.core.lock().write16(addr as usize, val) { + let cycle = self.core.lock().write16(addr as usize, val); + #[cfg(feature = "jitv2")] + self.code.1.fetch_add(1, std::sync::atomic::Ordering::Release); + match cycle { Ok(_) => BUS_OK, Err(()) => BUS_ERR, } } + + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, _addr: u64) -> *const std::sync::atomic::AtomicU64 { + &self.code.1 + } + + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + ((addr as usize) < self.image.len()).then(|| self.code.0.page(addr, &self.code.1)) + } } #[cfg(test)] diff --git a/src/dev/bedrock/md.rs b/src/dev/bedrock/md.rs index 5fc8f862a..1702ab2f0 100644 --- a/src/dev/bedrock/md.rs +++ b/src/dev/bedrock/md.rs @@ -611,27 +611,27 @@ mod tests { #[cfg(feature = "jitv2")] #[test] fn a_store_that_moves_banks_asks_for_a_flush() { - use crate::cpu::smp::{ParkSlot, World, REQUEST_JIT_FLUSH}; + use crate::cpu::smp::{ParkSlot, World, REQUEST_JIT_REMAP}; let word = AtomicU64::new(0); let processor = Arc::new(ParkSlot::new(0, &word)); let world = Arc::new(World::new()); world.set_slots(vec![processor.clone()]); let md = TestMd::with_world(&FUEL_BANKS, Arc::new(|| 0), world).unwrap(); md.md_bus().write64(reg::MEMORY_CONFIG::OFFSET, 0x33); - assert!(processor.release(REQUEST_JIT_FLUSH), "mapping the banks asks for a flush"); + assert!(processor.release(REQUEST_JIT_REMAP), "mapping the banks asks for a flush"); md.md_bus().write64(reg::MEMORY_CONFIG::OFFSET, 0x33 | (1 << 32)); - assert!(processor.release(REQUEST_JIT_FLUSH), "the swap asks for a flush"); + assert!(processor.release(REQUEST_JIT_REMAP), "the swap asks for a flush"); md.md_bus().write64(reg::MEMORY_CONFIG::OFFSET, 0x33 | (1 << 32)); - assert!(!processor.release(REQUEST_JIT_FLUSH), "a write that moves nothing does not"); + assert!(!processor.release(REQUEST_JIT_REMAP), "a write that moves nothing does not"); let counter = || unsafe { (*md.physical().banks()[0].gen_ptr(0)).load(Ordering::Relaxed) }; md.md_bus().write64(reg::MEMORY_CONFIG::OFFSET, 0x13); md.md_bus().write64(0x20038, 0x80); let before = counter(); md.md_bus().write64(reg::SELF_TEST_CONTROL::OFFSET, 1); assert!(counter() > before, "the zero fill invalidates the bank's code pages"); - let _ = processor.release(REQUEST_JIT_FLUSH); + let _ = processor.release(REQUEST_JIT_REMAP); md.0.power_on(); - assert!(processor.release(REQUEST_JIT_FLUSH), "a reset that moves the banks asks for a flush"); + assert!(processor.release(REQUEST_JIT_REMAP), "a reset that moves the banks asks for a flush"); } /// Directory and ECC state is keyed per bank, so banks of different sizes never share an entry. diff --git a/src/dev/bedrock_flash.rs b/src/dev/bedrock_flash.rs index 613cdd2e6..efb0dec01 100644 --- a/src/dev/bedrock_flash.rs +++ b/src/dev/bedrock_flash.rs @@ -239,6 +239,9 @@ pub struct Flash { /// state and take the writer lock, as do commands, resets and the erase clock. st: RwLock, gen: AtomicU64, + /// JIT v2: the flash's identity in a page key. + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank, /// The host time the model clock last stood at: [`Self::advance_to`] /// advances it from there. clock: AtomicU64, @@ -282,6 +285,8 @@ impl Flash { Ok(Self { st: RwLock::new(State::new(data)), gen: AtomicU64::new(0), + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank::new(), clock: AtomicU64::new(0), machine_clock: OnceLock::new(), }) @@ -314,7 +319,7 @@ impl Flash { backing.write_range(0, &data)?; let mut st = State::new(data); st.backing = Some(backing); - Ok(Self { st: RwLock::new(st), gen: AtomicU64::new(0), clock: AtomicU64::new(0), machine_clock: OnceLock::new() }) + Ok(Self { st: RwLock::new(st), gen: AtomicU64::new(0), #[cfg(feature = "jitv2")] code_bank: crate::traits::CodeBank::new(), clock: AtomicU64::new(0), machine_clock: OnceLock::new() }) } pub fn contents(&self) -> Vec { @@ -548,6 +553,10 @@ impl BusDevice for FlashView { fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { &self.flash.gen } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + self.offset(addr, 1).map(|off| self.flash.code_bank.page(off as u64, &self.flash.gen)) + } } #[cfg(test)] diff --git a/src/dev/bridge.rs b/src/dev/bridge.rs index 4d540ecf8..be0e94237 100644 --- a/src/dev/bridge.rs +++ b/src/dev/bridge.rs @@ -2084,6 +2084,16 @@ impl XioWindow { } impl BusDevice for XioWindow { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, addr: u64) -> *const std::sync::atomic::AtomicU64 { + let off = addr & self.mask; + if off < WIDGET_SMALL_SIZE as u64 { self.bridge.gen_ptr(off) } else { std::ptr::null() } + } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + let off = addr & self.mask; + if off < WIDGET_SMALL_SIZE as u64 { self.bridge.code_page(off) } else { None } + } /// A configuration master time-out reached through the window's first /// 16 MB is explained as through the small window. fn describe_fault(&self, addr: u64, size: usize) -> Option { @@ -2116,7 +2126,27 @@ impl BusDevice for XioWindow { } } +impl Bridge { + /// The flash part a widget address reaches and the offset in it, for the + /// questions a bus forwards about memory code runs from. The PROM runs + /// from the part on chip select 0, at the reset vector. + #[cfg(feature = "jitv2")] + fn flash_at(&self, off: u32) -> Option<(Arc, u64)> { + let woff = off.checked_sub(FLASH_BASE).filter(|_| off <= 0xff_ffff)?; + self.flash_part(woff).map(|(dev, at)| (dev, at as u64)) + } +} + impl BusDevice for Bridge { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, addr: u64) -> *const std::sync::atomic::AtomicU64 { + self.flash_at(addr as u32 & WINDOW_MASK).map_or(std::ptr::null(), |(dev, at)| dev.gen_ptr(at)) + } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + self.flash_at(addr as u32 & WINDOW_MASK).and_then(|(dev, at)| dev.code_page(at)) + } + fn describe_fault(&self, addr: u64, size: usize) -> Option { let off = addr as u32 & WINDOW_MASK; if (CFG0_BASE..CFG0_BASE + CFG0_SIZE).contains(&off) { diff --git a/src/dev/bridge_tests.rs b/src/dev/bridge_tests.rs index f451a27e5..90f44b2d1 100644 --- a/src/dev/bridge_tests.rs +++ b/src/dev/bridge_tests.rs @@ -1598,6 +1598,31 @@ fn flash_sits_where_the_reset_vector_is() { assert_eq!(0x1000_0000 + ((WIDGET_BRIDGE as u32) << 24) + FLASH_BASE, 0x1fc0_0000); } +/// The PROM runs from the flash behind the Bridge: the Bridge names the part +/// and the page for the JIT, and hands on the part's counter, by chip select. +#[cfg(feature = "jitv2")] +#[test] +fn the_bridge_names_its_flash_pages_for_the_jit() { + use crate::dev::sflash::CfiFlash; + let r = Rig::new(); + let parts = [Arc::new(CfiFlash::new(&image(), 0x10_0000)), Arc::new(CfiFlash::new(&image(), 0x10_0000))]; + r.bridge.attach_flash(0, parts[0].clone(), 0x10_0000); + r.bridge.attach_flash(1, parts[1].clone(), 0x10_0000); + let at = |off: u32| r.bridge.code_page((FLASH_BASE + off) as u64).expect("a flash page"); + let page = at(0x3000); + assert_eq!(page.key, parts[0].code_page(0x3000).unwrap().key); + assert_eq!(page.gen, parts[0].gen_ptr(0x3000)); + assert_eq!(r.bridge.gen_ptr((FLASH_BASE + 0x3000) as u64), page.gen); + // The part repeats above the address lines it decodes: the same page. + assert_eq!(at(0x10_3000).key, page.key); + // The second chip select is the other part. + assert_eq!(at(FLASH_SELECT_SIZE + 0x3000).key, parts[1].code_page(0x3000).unwrap().key); + assert_ne!(at(FLASH_SELECT_SIZE + 0x3000).key, page.key); + // Registers are not code. + assert!(r.bridge.code_page(0x100).is_none()); + assert!(r.bridge.gen_ptr(0x100).is_null()); +} + #[test] fn dma_above_4_gib_master_aborts_instead_of_wrapping() { let r = Rig::new(); diff --git a/src/dev/heart_mem.rs b/src/dev/heart_mem.rs index f7d7e995c..8a1c9d096 100644 --- a/src/dev/heart_mem.rs +++ b/src/dev/heart_mem.rs @@ -367,6 +367,10 @@ impl BusDevice for DmaView { fn gen_ptr(&self, addr: u64) -> *const AtomicU64 { self.at(addr).map_or(std::ptr::null(), |(d, a)| d.gen_ptr(a)) } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + self.at(addr).and_then(|(d, a)| d.code_page(a)) + } } // The memory's contents for a snapshot. Where the banks answer is HEART's diff --git a/src/dev/hub/tests.rs b/src/dev/hub/tests.rs index e70b2f2f3..db39e6e67 100644 --- a/src/dev/hub/tests.rs +++ b/src/dev/hub/tests.rs @@ -34,6 +34,10 @@ impl BusDevice for Ram { fn gen_ptr(&self, _a: u64) -> *const std::sync::atomic::AtomicU64 { &self.1 } + #[cfg(feature = "jitv2")] + fn code_page(&self, a: u64) -> Option { + Some(crate::traits::CodeKey::flat_page(a, &self.1)) + } crate::dev::widths::access_widths!(); } diff --git a/src/dev/mem.rs b/src/dev/mem.rs index 49b94e0db..81bfcf65e 100644 --- a/src/dev/mem.rs +++ b/src/dev/mem.rs @@ -34,6 +34,9 @@ pub struct Memory { /// a page is valid iff its recorded generation still matches this counter (§4.1, §6.5). #[cfg(feature = "jitv2")] gen: Vec, + /// JIT v2: this bank's identity in a page key. + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank, } // Safety: Memory is safe to share across threads because access is controlled @@ -55,6 +58,8 @@ impl Memory { addr_mask: (size_bytes - 1) as u64, #[cfg(feature = "jitv2")] gen: (0..size_bytes.div_ceil(JITV2_PAGE_SIZE)).map(|_| AtomicU64::new(0)).collect(), + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank::new(), } } @@ -334,6 +339,18 @@ impl BusDevice for Memory { fn gen_ptr(&self, addr: u64) -> *const AtomicU64 { Memory::gen_ptr(self, addr) } + + #[cfg(feature = "jitv2")] + #[inline] + fn code_page(&self, addr: u64) -> Option { + let offset = addr & self.addr_mask & !(JITV2_PAGE_SIZE as u64 - 1); + let mut page = self.code_bank.page(offset, Memory::gen_ptr(self, addr)); + // The buffer is allocated once and never resized. + if offset as usize + JITV2_PAGE_SIZE <= self.size_words * 4 { + page.words = unsafe { self.data().as_ptr().add(offset as usize / 4) }; + } + Some(page) + } } // Implement BusDevice for Arc to allow using Arc-wrapped memory directly @@ -386,6 +403,11 @@ impl BusDevice for Arc { fn gen_ptr(&self, addr: u64) -> *const AtomicU64 { (**self).gen_ptr(addr) } + + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + (**self).code_page(addr) + } } /// Black Hole Region diff --git a/src/dev/node_flash.rs b/src/dev/node_flash.rs index 29f2bdcf8..85cdd1f51 100644 --- a/src/dev/node_flash.rs +++ b/src/dev/node_flash.rs @@ -208,6 +208,10 @@ impl State { pub struct NodeFlash { st: Mutex, + /// JIT v2: the part's identity in a page key, and its write counter: + /// every write cycle bumps it, since a command changes what a read returns. + #[cfg(feature = "jitv2")] + code: (crate::traits::CodeBank, std::sync::atomic::AtomicU64), } impl NodeFlash { @@ -217,6 +221,8 @@ impl NodeFlash { } Ok(Arc::new(Self { st: Mutex::new(State::new(image, None)), + #[cfg(feature = "jitv2")] + code: Default::default(), })) } @@ -227,6 +233,8 @@ impl NodeFlash { let backing = FlashBacking::open(path, &mut data)?; Ok(Arc::new(Self { st: Mutex::new(State::new(data, Some(backing))), + #[cfg(feature = "jitv2")] + code: Default::default(), })) } @@ -282,7 +290,29 @@ macro_rules! chip_read { } }; } +impl NodeFlash { + /// A write cycle reached the part: compiled code read from it is stale. + fn wrote(&self) { + #[cfg(feature = "jitv2")] + self.code.1.fetch_add(1, std::sync::atomic::Ordering::Release); + } + + /// The page holding byte `offset` of the part, for the JIT. + #[cfg(feature = "jitv2")] + fn code_page(&self, offset: usize) -> Option { + (offset < SIZE).then(|| self.code.0.page(offset as u64, &self.code.1)) + } +} + impl BusDevice for Chip { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, _a: u64) -> *const std::sync::atomic::AtomicU64 { + &self.0.code.1 + } + #[cfg(feature = "jitv2")] + fn code_page(&self, a: u64) -> Option { + self.0.code_page(usize::try_from(a).ok()?) + } chip_read!(read8, u8, BusRead8); chip_read!(read16, u16, BusRead16); chip_read!(read32, u32, BusRead32); @@ -291,6 +321,7 @@ impl BusDevice for Chip { match usize::try_from(a) { Ok(a) if a < SIZE => { self.0.st.lock().write(a, v); + self.0.wrote(); BUS_OK } _ => BUS_ERR, @@ -313,12 +344,22 @@ macro_rules! window_read { }; } impl BusDevice for Window { + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, _a: u64) -> *const std::sync::atomic::AtomicU64 { + &self.0.code.1 + } + // Reads repeat the image, so the page is the image's, wherever it repeats. + #[cfg(feature = "jitv2")] + fn code_page(&self, a: u64) -> Option { + self.0.code_page(a as usize % SIZE) + } window_read!(read8, u8, BusRead8); window_read!(read16, u16, BusRead16); window_read!(read32, u32, BusRead32); window_read!(read64, u64, BusRead64); fn write64(&self, a: u64, v: u64) -> u32 { self.0.st.lock().write((a as usize / 8) % SIZE, v as u8); + self.0.wrote(); BUS_OK } fn write8(&self, a: u64, v: u8) -> u32 { diff --git a/src/dev/prom.rs b/src/dev/prom.rs index 103cb64a6..f30fb2f33 100644 --- a/src/dev/prom.rs +++ b/src/dev/prom.rs @@ -19,6 +19,9 @@ struct PromInner { /// check against it therefore never fails due to ROM-side invalidation. #[cfg(feature = "jitv2")] gen: AtomicU64, + /// JIT v2: the PROM's identity in a page key. + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank, } pub struct PromPort { @@ -54,6 +57,8 @@ impl Prom { running: AtomicBool::new(false), #[cfg(feature = "jitv2")] gen: AtomicU64::new(0), + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank::new(), }), } } @@ -179,6 +184,19 @@ impl BusDevice for PromPort { fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { &self.inner.gen as *const AtomicU64 } + + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + (PROM_BASE..PROM_BASE + PROM_SIZE).contains(&addr).then(|| { + let offset = (addr - PROM_BASE) & !0xFFF; + let mut page = self.inner.code_bank.page(offset, &self.inner.gen); + // The image is immutable and lives as long as the device. + if offset as usize / 4 + 1024 <= self.inner.data.len() { + page.words = unsafe { self.inner.data.as_ptr().add(offset as usize / 4) }; + } + page + }) + } } #[cfg(test)] diff --git a/src/dev/sflash.rs b/src/dev/sflash.rs index 9d99c75a0..2b7c49975 100644 --- a/src/dev/sflash.rs +++ b/src/dev/sflash.rs @@ -89,6 +89,9 @@ pub struct CfiFlash { /// jitv2's code-generation counter for the whole part, bumped by every /// write cycle: a command changes what a read returns. gen: AtomicU64, + /// JIT v2: the part's identity in a page key. + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank, } /// The name this part had when it was only a PROM store. @@ -117,7 +120,11 @@ impl CfiFlash { assert!(image.len() <= size, "image must be within the part's size"); let mut data = vec![0xff; size]; data[..image.len()].copy_from_slice(image); - Self { st: Mutex::new(State { data, mode: Mode::Array, status: SR_READY, backing: None }), gen: AtomicU64::new(0) } + Self { st: Mutex::new(State { data, mode: Mode::Array, status: SR_READY, backing: None }), + gen: AtomicU64::new(0), + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank::new(), + } } /// A reset: back to read-array mode with the status clear. The array @@ -251,6 +258,11 @@ impl BusDevice for CfiFlash { fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { &self.gen } + + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + (addr < self.st.lock().data.len() as u64).then(|| self.code_bank.page(addr, &self.gen)) + } } // The part for a snapshot. Its command state is device state; the array's diff --git a/src/dev/widths.rs b/src/dev/widths.rs index 867f09438..84462c3b7 100644 --- a/src/dev/widths.rs +++ b/src/dev/widths.rs @@ -112,6 +112,10 @@ macro_rules! access_widths { fn gen_ptr(&self, a: u64) -> *const std::sync::atomic::AtomicU64 { self.route(a).map_or(std::ptr::null(), |(d, at)| d.gen_ptr(at)) } + #[cfg(feature = "jitv2")] + fn code_page(&self, a: u64) -> Option<$crate::traits::CodePage> { + self.route(a).and_then(|(d, at)| d.code_page(at)) + } }; } pub(crate) use access_widths; diff --git a/src/ip27.rs b/src/ip27.rs index aeff95875..914d4d3e2 100644 --- a/src/ip27.rs +++ b/src/ip27.rs @@ -326,6 +326,18 @@ pub fn machine( cpus.push(cpu); } world.set_slots(cpus.iter().map(|c| c.park_slot()).collect()); + // One JIT for the machine: both nodes' processors compile through one + // queue into one page pool, each node an address space of it. + #[cfg(feature = "jitv2")] + { + let pool: Vec<_> = nodes + .iter() + .zip(&specs) + .zip(cpus.chunks(1)) + .map(|((node, spec), cpus)| crate::cpu::jitv2::jitv2::PoolNode { cpus, module: spec.module, bus: node.chip.decode.physical().clone() }) + .collect(); + crate::cpu::jitv2::jitv2::share_pool(&pool, cfg.jitv2_pool_size(), cfg.jitv2.compile, &world)?; + } eprintln!( "iris: {}, {} node{} of {} MiB, {} x {}", cfg.machine.profile.product_name().unwrap_or_default(), @@ -364,8 +376,6 @@ fn build_cpu( Arc::new(MipsCpu::new(*exec)) as Arc }); physical.connect_cpu(&*cpu, false)?; - #[cfg(feature = "jitv2")] - crate::cpu::jitv2::jitv2::attach_pool(&cpu, cfg.jitv2_pool_size(), cfg.jitv2.compile, physical.clone(), None); Ok(cpu) } diff --git a/src/ip30.rs b/src/ip30.rs index b5675f536..4fa4452b7 100644 --- a/src/ip30.rs +++ b/src/ip30.rs @@ -535,6 +535,11 @@ impl BusDevice for CachedFlashLines { fn gen_ptr(&self, addr: u64) -> *const AtomicU64 { self.bus.gen_ptr(addr) } + + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + self.bus.code_page(addr) + } } /// Cross-device tests: the chips wired as the machine wires them, no CPU. diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 0427b027f..5c1f299cc 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -571,8 +571,9 @@ impl OctaneMachine { #[cfg(feature = "jitv2_lockstep")] return Err("[machine] cpus = 2 cannot run with jitv2_lockstep".into()); let jit = cpu.jitv2(); - for c in &cpus { - c.share_jitv2(jit.clone(), world.clone()); + // Every CPU sees memory at the same addresses: one address space. + for (n, c) in cpus.iter().enumerate() { + c.share_jitv2(jit.clone(), world.clone(), n, 0); } } status.set_cpu_cycles(cpu.cycles_ptr()); diff --git a/src/ip32.rs b/src/ip32.rs index 04c227eb7..f1f527fa0 100644 --- a/src/ip32.rs +++ b/src/ip32.rs @@ -852,6 +852,8 @@ impl crate::traits::Device for Ip32Bus { impl BusDevice for Ip32Bus { #[cfg(feature = "jitv2")] fn gen_ptr(&self, addr: u64) -> *const std::sync::atomic::AtomicU64 { self.phys.gen_ptr(addr) } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { self.phys.code_page(addr) } fn read8(&self, addr: u64) -> BusRead8 { self.phys.read8(addr) } fn read16(&self, addr: u64) -> BusRead16 { self.phys.read16(addr) } fn read32(&self, addr: u64) -> BusRead32 { self.phys.read32(addr) } diff --git a/src/ip35.rs b/src/ip35.rs index 0f870fadd..92d75fb1b 100644 --- a/src/ip35.rs +++ b/src/ip35.rs @@ -291,26 +291,17 @@ pub(crate) fn machine( connect_cpus(&node.chip, cpus)?; node.chip.set_timer_manager(&timers); } - // One JIT for the processors of a node, which share its memory, its - // pool parking them with the rest; one of its own for a lone one. + // One JIT for the machine: every processor of every node compiles through + // one queue into one page pool, each node an address space of it. #[cfg(feature = "jitv2")] - for (node, cpus) in nodes.iter().zip(&node_cpus) { - let several = cpus.len() > 1; - if several { - if cfg!(feature = "jitv2_lockstep") { - return Err(format!("{name}: a node of several processors cannot run with jitv2_lockstep")); - } - for c in cpus { - c.share_jitv2(cpus[0].jitv2(), world.clone()); - } - } - crate::cpu::jitv2::jitv2::attach_pool( - &cpus[0], - cfg.jitv2_pool_size(), - cfg.jitv2.compile, - node.chip.decode.physical().clone(), - several.then(|| world.clone()), - ); + { + let pool: Vec<_> = nodes + .iter() + .zip(&specs) + .zip(&node_cpus) + .map(|((node, spec), cpus)| crate::cpu::jitv2::jitv2::PoolNode { cpus, module: spec.module, bus: node.chip.decode.physical().clone() }) + .collect(); + crate::cpu::jitv2::jitv2::share_pool(&pool, cfg.jitv2_pool_size(), cfg.jitv2.compile, &world).map_err(|e| format!("{name}: {e}"))?; } if let Some(hook) = &io.cpus { hook(&cpus); diff --git a/src/ip35/tests.rs b/src/ip35/tests.rs index 0003d4bed..7bf20c7ce 100644 --- a/src/ip35/tests.rs +++ b/src/ip35/tests.rs @@ -483,7 +483,7 @@ mod code_pages { use crate::cpu::jitv2::jitv2::CompileRequest; use crate::cpu::mips_isa::{JIT_REGION_BOUNDARY_SENTINEL, OP_ADDIU}; - struct Words(u32, AtomicU64); + struct Words(u32, AtomicU64, crate::traits::CodeBank); impl BusDevice for Words { fn read32(&self, _: u64) -> BusRead32 { BusRead32::ok(self.0) @@ -494,6 +494,9 @@ mod code_pages { fn gen_ptr(&self, _: u64) -> *const AtomicU64 { &self.1 } + fn code_page(&self, _: u64) -> Option { + Some(self.2.page(0, &self.1)) + } } const RESET: u64 = 0x1fc0_0000; @@ -504,8 +507,8 @@ mod code_pages { /// (a region boundary, which compiles to nothing) in the cached one. fn rig() -> (Arc, Arc, Box) { use crate::cpu::mips_exec::{MipsCpuConfig, MipsExecutor}; - let rom = Arc::new(Words(ADDIU_R1, AtomicU64::new(0))); - let ram = Arc::new(Words(JIT_REGION_BOUNDARY_SENTINEL, AtomicU64::new(0))); + let rom = Arc::new(Words(ADDIU_R1, AtomicU64::new(0), Default::default())); + let ram = Arc::new(Words(JIT_REGION_BOUNDARY_SENTINEL, AtomicU64::new(0), Default::default())); let physical = Physical::new(Vec::new()); physical.set_default(Arc::new(crate::sn::NoMemory(crate::devlog::LogModule::Bedrock)), "absent"); map_device(&physical, RESET, 0x1000, ram.clone(), "RAM over the reset window"); @@ -534,13 +537,14 @@ mod code_pages { fn compile(cpu: &Cpu, page: &mut crate::cpu::jitv2::PhysicalCodePage) { page.mark_requested(0); - let req = CompileRequest { page: page as *mut _, compiled_for_fr1: true }; + let req = CompileRequest { page: page as *mut _, compiled_for_fr1: true, cpu: 0 }; let mut analyzer = crate::cpu::jitv2::analyzer::Analyzer::new(); let mut codegen = crate::cpu::jitv2::codegen::Codegen::new(); + let buses = cpu.jitv2.lock().buses().clone(); #[cfg(feature = "developer")] - crate::cpu::jitv2::comp::handle_request(&req, cpu.cached_bus(), &mut analyzer, &mut codegen, &crate::cpu::jitv2::JitStats::default()); + crate::cpu::jitv2::comp::handle_request(&req, &buses, &mut analyzer, &mut codegen, &crate::cpu::jitv2::JitStats::default()); #[cfg(not(feature = "developer"))] - crate::cpu::jitv2::comp::handle_request(&req, cpu.cached_bus(), &mut analyzer, &mut codegen); + crate::cpu::jitv2::comp::handle_request(&req, &buses, &mut analyzer, &mut codegen); } #[test] @@ -550,7 +554,8 @@ mod code_pages { // The reset vector in KSEG1, then the same frame in KSEG0. let prom = page_at(&mut cpu, 0xffff_ffff_bfc0_0000); let cached = page_at(&mut cpu, 0xffff_ffff_9fc0_0000); - assert_eq!((prom.pfn, cached.pfn), (0x1fc00, 0x1fc00)); + assert_eq!((prom.addr_of(0), cached.addr_of(0)), (Some((0x1fc00, true)), Some((0x1fc00, false)))); + assert_ne!(prom.key, cached.key, "the frame decodes to the PROM uncached and to RAM cached"); assert_ne!(prom as *mut _, cached as *mut _, "the two fetches key two pages"); compile(&cpu, prom); compile(&cpu, cached); @@ -587,12 +592,12 @@ mod code_pages { } cpu.jitv2_flush_from_here(false); let page = page_at(&mut cpu, 0xffff_ffff_bfc0_0000); - assert!(page.uncached.is_some()); + assert_eq!(page.addr_of(0), Some((0x1fc00, true))); page.mark_requested(0); let jit = cpu.jitv2.clone(); let g = jit.lock(); let stats = g.stats.clone(); - assert!(g.compile_queue.send(CompileRequest { page: page as *mut _, compiled_for_fr1: true }, &stats)); + assert!(g.compile_queue.send(CompileRequest { page: page as *mut _, compiled_for_fr1: true, cpu: 0 }, &stats)); drop(g); assert!(wait_runnable(page), "the restarted pool must compile the PROM's page"); jit.lock().compile_queue.stop(); diff --git a/src/physical.rs b/src/physical.rs index 22f210fa4..8a984d551 100644 --- a/src/physical.rs +++ b/src/physical.rs @@ -141,6 +141,10 @@ impl BusDevice for AliasBus { fn gen_ptr(&self, addr: u64) -> *const std::sync::atomic::AtomicU64 { unsafe { (*self.target).gen_ptr(self.target_addr(addr)) } } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + unsafe { (*self.target).code_page(self.target_addr(addr)) } + } fn cas32(&self, addr: u64, old: u32, new: u32) -> Option { unsafe { (*self.target).cas32(self.target_addr(addr), old, new) } } @@ -190,6 +194,8 @@ impl BusDevice for BankView { #[inline] fn write_block(&self, a: u64, buf: &[u64]) -> u32 { self.bank().write_block(self.at(a), buf) } #[cfg(feature = "jitv2")] fn gen_ptr(&self, a: u64) -> *const std::sync::atomic::AtomicU64 { self.bank().gen_ptr(self.at(a)) } + #[cfg(feature = "jitv2")] + fn code_page(&self, a: u64) -> Option { self.bank().code_page(self.at(a)) } } /// RAM bank `bank` (an index into the `Physical`'s banks) answering @@ -1142,6 +1148,13 @@ impl BusDevice for Physical { unsafe { (*device_ptr).gen_ptr(addr) } } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { + let addr = addr & PHYS_MASK; + let device_ptr = self.dev_at(addr); + unsafe { (*device_ptr).code_page(addr) } + } + /// Direct pointer into ppmem's window for a directly-mapped physical /// address, so the cache's line-fill fast path (`mips_cache_v2.rs`) reads /// RAM without a device dispatch. diff --git a/src/ppmem/ppmem.rs b/src/ppmem/ppmem.rs index 03cd79e46..f14d70fb5 100644 --- a/src/ppmem/ppmem.rs +++ b/src/ppmem/ppmem.rs @@ -214,6 +214,9 @@ pub struct PpMemory { gen_base: *mut AtomicU64, #[cfg(feature = "jitv2")] _gen_own: AddrSpace, + /// JIT v2: this bank's identity in a page key. + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank, } // Safety: same contract as `Memory` — access is serialized through the bus. @@ -273,6 +276,8 @@ impl PpMemory { gen_base, #[cfg(feature = "jitv2")] _gen_own: gen_own, + #[cfg(feature = "jitv2")] + code_bank: crate::traits::CodeBank::new(), }) } @@ -613,6 +618,19 @@ impl BusDevice for PpMemory { fn gen_ptr(&self, addr: u64) -> *const AtomicU64 { PpMemory::gen_ptr(self, addr) } + + #[cfg(feature = "jitv2")] + #[inline] + fn code_page(&self, addr: u64) -> Option { + let offset = self.off(addr) & !0xFFF; + let mut page = self.code_bank.page(offset as u64, PpMemory::gen_ptr(self, addr)); + // The bank's own mapping, where `read32` reads: it does not move when + // a window maps the bank elsewhere. + if offset + 4096 <= self.size_bytes { + page.words = unsafe { (self.base as *const u32).add(offset >> 2) }; + } + Some(page) + } } // Mirrors `impl BusDevice for Arc` so Arc-wrapped banks work directly. @@ -648,6 +666,8 @@ impl BusDevice for Arc { #[cfg(feature = "jitv2")] #[inline] fn gen_ptr(&self, addr: u64) -> *const AtomicU64 { (**self).gen_ptr(addr) } + #[cfg(feature = "jitv2")] + fn code_page(&self, addr: u64) -> Option { (**self).code_page(addr) } } // --------------------------------------------------------------------------- diff --git a/src/sn.rs b/src/sn.rs index c0989e2e9..9b54bc872 100644 --- a/src/sn.rs +++ b/src/sn.rs @@ -505,6 +505,10 @@ impl BusDevice for NodeQualifiedView { fn gen_ptr(&self, a: u64) -> *const std::sync::atomic::AtomicU64 { self.target(a).map_or(std::ptr::null(), |(d, at)| d.gen_ptr(at)) } + #[cfg(feature = "jitv2")] + fn code_page(&self, a: u64) -> Option { + self.target(a).and_then(|(d, at)| d.code_page(at)) + } access_widths!(); } diff --git a/src/sn/tests.rs b/src/sn/tests.rs index edc2bb9d2..e5ebddc43 100644 --- a/src/sn/tests.rs +++ b/src/sn/tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::dev::{bedrock, hub}; -use crate::cpu::smp::{set_current_cpu, ParkSlot, HOST_PARK_BIT, REQUEST_JIT_FLUSH}; +use crate::cpu::smp::{set_current_cpu, ParkSlot, HOST_PARK_BIT, REQUEST_JIT_REMAP}; use crate::physical::RamBank; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -293,7 +293,7 @@ impl ParkProbe { while running.load(Ordering::Relaxed) { if word.load(Ordering::Relaxed) & HOST_PARK_BIT == 0 { std::thread::yield_now(); - } else if slot.release(REQUEST_JIT_FLUSH) { + } else if slot.release(REQUEST_JIT_REMAP) { flushes.fetch_add(1, Ordering::SeqCst); } else { slot.park(&running); @@ -307,13 +307,13 @@ impl ParkProbe { s.spawn(|| { set_current_cpu(0); store(); - if self.slots[0].release(REQUEST_JIT_FLUSH) { + if self.slots[0].release(REQUEST_JIT_REMAP) { self.flushes.fetch_add(1, Ordering::SeqCst); } }); }); // Processor 1 takes a flush posted to it before it stops. - while self.slots[1].holds() & REQUEST_JIT_FLUSH != 0 { + while self.slots[1].holds() & REQUEST_JIT_REMAP != 0 { std::thread::yield_now(); } running.store(false, Ordering::Relaxed); diff --git a/src/traits.rs b/src/traits.rs index 05ac413a3..391f721eb 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -55,6 +55,109 @@ pub enum Signal { Interrupt(u32, bool), } +/// What 4 KiB of runnable memory is, whatever address a processor reaches it +/// at: the device that holds the bytes ([`CodeBank`]) and the page within it. +/// The JIT keys compiled code by it, so the code follows the bytes when a +/// memory controller moves a bank, and two processors that see the same bytes +/// at different addresses share one compiled page. +#[cfg(feature = "jitv2")] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct CodeKey { + pub bank: u32, + /// Page number within the bank: byte offset / 4096. + pub page: u32, +} + +#[cfg(feature = "jitv2")] +impl CodeKey { + /// The key of no page. + pub const NONE: CodeKey = CodeKey { bank: u32::MAX, page: u32::MAX }; + + /// The first of the bank numbers kept for pages no bank names + /// ([`Self::at_address`]); no [`CodeBank`] counts that far. + const ADDRESS_BANKS: u32 = u32::MAX - 0x100; + + /// The key of a page that no bank on the bus names (a device behind a + /// bridge that does not forward the question): its address, in address + /// space `space` of the JIT pool. Private to that space, since only its + /// CPUs are known to see the same bytes there. + pub const fn at_address(space: usize, frame: u32, uncached: bool) -> CodeKey { + CodeKey { bank: Self::ADDRESS_BANKS + space as u32, page: frame << 1 | uncached as u32 } + } + + /// Whether this is an [`Self::at_address`] key. + pub const fn is_address(self) -> bool { + self.bank >= Self::ADDRESS_BANKS && self.bank != u32::MAX + } + + /// A bus that is one flat memory (a test bus, a tool) names a page by its + /// address, in bank 0, which no [`CodeBank`] is ever given. + pub const fn flat(addr: u64) -> CodeKey { + CodeKey { bank: 0, page: (addr >> 12) as u32 } + } + + /// [`Self::flat`] as a bus answers it, counted by `gen`. + pub const fn flat_page(addr: u64, gen: *const std::sync::atomic::AtomicU64) -> CodePage { + CodePage { key: Self::flat(addr), gen, words: std::ptr::null() } + } +} + +/// What a bus answers for a page of runnable memory: its key, and the bank's +/// own write counter for the page. The counter belongs to the bank, not to an +/// address: it stays the page's counter wherever a memory controller puts the +/// bank (a ppmem window's counter for an address does not). +#[cfg(feature = "jitv2")] +#[derive(Clone, Copy, Debug)] +pub struct CodePage { + pub key: CodeKey, + pub gen: *const std::sync::atomic::AtomicU64, + /// The page's 1024 words in the bank's own storage, for a bank that can + /// hand them out: the JIT's compile workers read the page there, with no + /// bus and no address. Like `gen` it belongs to the bank, stays put when + /// the bank moves on the bus, and is valid while the device lives. Null + /// for a device that cannot (a flash whose reads depend on its mode, a + /// test bus); such a page is read through the bus at a CPU's address. + pub words: *const u32, +} + +/// The identity of one device that holds runnable memory: a RAM bank, a PROM, +/// a flash part. Each takes the next number from one counter when it is built, +/// so no two in the process share one. +#[cfg(feature = "jitv2")] +#[derive(Debug)] +pub struct CodeBank(u32); + +#[cfg(feature = "jitv2")] +impl CodeBank { + pub fn new() -> Self { + static NEXT: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(1); + CodeBank(NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)) + } + + pub fn id(&self) -> u32 { + self.0 + } + + /// The key of the page holding byte `offset` of this bank. + #[inline] + pub fn key(&self, offset: u64) -> CodeKey { + CodeKey { bank: self.0, page: (offset >> 12) as u32 } + } + + /// The page holding byte `offset` of this bank, counted by `gen`. + #[inline] + pub fn page(&self, offset: u64, gen: *const std::sync::atomic::AtomicU64) -> CodePage { + CodePage { key: self.key(offset), gen, words: std::ptr::null() } + } +} + +#[cfg(feature = "jitv2")] +impl Default for CodeBank { + fn default() -> Self { + Self::new() + } +} + /// Unified bus device interface supporting 8/16/32/64-bit accesses. /// /// Read methods return `BusRead{N}` with `status == BUS_OK` on success. @@ -250,6 +353,18 @@ pub trait BusDevice: Send + Sync { fn gen_ptr(&self, _addr: u64) -> *const std::sync::atomic::AtomicU64 { std::ptr::null() } + + /// JIT v2: which bank holds the 4 KiB page at `addr` and where in it + /// (the page's [`CodeKey`]), with the bank's counter for it, or `None` + /// where no bank answers (MMIO, a hole, a device that does not say). The + /// JIT keys a page fetched from there by its address and counts it with + /// [`Self::gen_ptr`], as it would a ROM. The answer changes when a memory controller + /// moves a bank; the bytes and the counter a key names do not. A bus that + /// forwards accesses forwards this too. `addr` is the start of the page. + #[cfg(feature = "jitv2")] + fn code_page(&self, _addr: u64) -> Option { + None + } } pub trait FifoDevice: Send + Sync { diff --git a/src/window_map.rs b/src/window_map.rs index 35f68d668..839a3e3e5 100644 --- a/src/window_map.rs +++ b/src/window_map.rs @@ -77,6 +77,10 @@ impl BusDevice for Window { fn gen_ptr(&self, address: u64) -> *const AtomicU64 { self.local(address, 1).map(|local| self.device.gen_ptr(local)).unwrap_or(std::ptr::null()) } + #[cfg(feature = "jitv2")] + fn code_page(&self, address: u64) -> Option { + self.local(address, 1).and_then(|local| self.device.code_page(local)) + } fn write64_masked(&self, a: u64, val: u64, mask: u64) -> u32 { if a & 7 != 0 { return BUS_ERR; @@ -169,6 +173,10 @@ impl BusDevice for SparseWindowMap { fn gen_ptr(&self, address: u64) -> *const AtomicU64 { self.target(address).map(|window| window.gen_ptr(address)).unwrap_or(std::ptr::null()) } + #[cfg(feature = "jitv2")] + fn code_page(&self, address: u64) -> Option { + self.target(address).and_then(|window| window.code_page(address)) + } fn write64_masked(&self, address: u64, value: u64, mask: u64) -> u32 { self.target(address).map(|window| window.write64_masked(address, value, mask)) .unwrap_or_else(|| Unclaimed.write64_masked(address, value, mask)) From 9021f070d2f5cc8c83427f103d34896ec788340b Mon Sep 17 00:00:00 2001 From: technomancer Date: Thu, 19 Nov 2026 19:49:34 -0800 Subject: [PATCH 3/4] ip35: memory in the boot window is the zero holder's for both nodes On a two-node Bedrock machine the node that does not hold number 0 kept its own RAM mapped at node-field-0 addresses 0x1fc00000-0x1fffffff, the window where its boot flash sits before memory is up. The two CPUs then saw different pages at the same physical address: IRIX processes on the second node read zero pages and died with memory faults, and bulk I/O ended in a kernel panic. NodeDecode::map_banks now maps the zero holder's banks there once the node has memory behind the window; flash stays local only where it has none. Verified on Origin 3200 and Onyx 3000 with two C-bricks, both CPUs and jitv2 on: clean boot, raw disk reads and /usr/lib checksum sweeps match the host. j2 status gains a "shared pages" line: pages reached by more than one CPU, and how many of them at different addresses. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 3 + clanker/INDEX.md | 1 + ...-boot-window-memory-is-the-zero-holders.md | 71 +++++++++++++++++++ .../pages-are-keyed-by-bank-and-offset.md | 4 ++ debug.md | 2 +- src/cpu/jitv2/jitv2.rs | 17 +++++ src/cpu/mips_exec/monitor.rs | 2 + src/sn.rs | 21 ++++-- src/sn/tests.rs | 23 ++++++ 9 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 clanker/devices/ip35-boot-window-memory-is-the-zero-holders.md diff --git a/CHANGELOG.md b/CHANGELOG.md index e3271f69c..24c5759ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -181,6 +181,9 @@ hashes are given where a change is easiest to understand by reading the commit. boots on both from a Fibre Channel disk (`[fibre_channel.disk.N]`). - The I-brick has a USB keyboard and mouse, driven by the `ps2` monitor commands. +- Origin 3200 and Onyx 3000 with two C-bricks of 512 MiB or more each: IRIX + no longer corrupts memory (processes dying with memory faults and bus + errors, kernel panics and hangs under disk load). ### Onyx 3000 diff --git a/clanker/INDEX.md b/clanker/INDEX.md index 860154f9d..cb6e76d06 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -114,6 +114,7 @@ Non-graphics devices: MC/MEMCFG and RAM banks, HPC3, VDMA, EEPROMs and NVRAM, PS | [refreshing-the-generated-register-files.md](devices/refreshing-the-generated-register-files.md) | rule | The Hub and Bedrock register files are byte-for-byte copies of the chips repository's generated files; how to refresh them and what the contract test then catches | | [ip35-board-eeprom-cpu-config.md](devices/ip35-board-eeprom-cpu-config.md) | rule | The IP35 PROM checks its flash CPU record against the node board EEPROM's format-3 extended data (`CpuContext`); the field layout and how it was read out of the PROM | | [a-dropped-machine-must-free-its-cpus.md](devices/a-dropped-machine-must-free-its-cpus.md) | rule | Never `forget` a CPU `Arc` after handing out a weak one: CPU 0's batch hook holds the board, so the Fuel, Tezro and Origin 3200 kept every disk and the flash open after drop (failed on Windows) | +| [ip35-boot-window-memory-is-the-zero-holders.md](devices/ip35-boot-window-memory-is-the-zero-holders.md) | rule | Two-node IP35: memory behind the boot window at node field zero is the zero holder's for every node; keeping it on the reader gave node 1 its own RAM for 4 MiB of node 0 and corrupted IRIX (found by bisecting FC, JIT, graphics, second CPU, then diffing a piped disk region) | | [ip35-rendezvous-hold-must-stop-the-cpu-at-once.md](devices/ip35-rendezvous-hold-must-stop-the-cpu-at-once.md) | rule | A held CPU must retire nothing more; running out its batch let a Tezro PROM's PI1 pass a barrier alone and start the copied PROM early (measured on a fork's rendezvous) | | [draw-marks-tell-the-display-what-shows.md](devices/draw-marks-tell-the-display-what-shows.md) | rule | IMPACT and VPro compose only when drawing landed in a buffer the screen shows: a draw marks its store, planes and box after its stores (`display::Marks`); a path that cannot must say "anything"; `IRIS_DISPLAY_VERIFY=1` checks the verdicts; measurements | | [pci-layer-design.md](devices/pci-layer-design.md) | design | The PCI layer: headers change only through the bus, byte orders are per-window values, `pci/` imports no card or SCSI; the stale-run and delivery-order hazards, the SeqCst handshake, and what the first attempt got wrong | diff --git a/clanker/devices/ip35-boot-window-memory-is-the-zero-holders.md b/clanker/devices/ip35-boot-window-memory-is-the-zero-holders.md new file mode 100644 index 000000000..6e7c9578f --- /dev/null +++ b/clanker/devices/ip35-boot-window-memory-is-the-zero-holders.md @@ -0,0 +1,71 @@ +# IP35: memory in the boot window belongs to the zero holder, on every node + +Status: fixed 2026-11-19. + +## Symptom + +A two-node IP35 machine (Onyx 3000, Origin 3200 with `c_bricks = 2`) with +512 MiB or more per node booted IRIX and then fell apart: `/etc/rc2[55]` +died with `Memory fault(coredump)`, later processes faulted at random +(`Bus error`, shell scripts printing nonsense), raw disk reads returned +different data each time, and under load the kernel panicked (`KERNEL FAULT, +PC: 0x0`) or hung. Same with jitv2 dispatch off; same without the V-brick. +One node was clean, and so were two nodes with the second brick's CPU +disabled in the PROM. + +The corruption itself, captured by piping a known disk region through two +processes and comparing with the image: whole 16 KiB pages (the IRIX page +size) of zeros where data belonged. + +## Cause + +`NodeDecode::map_banks` kept "the boot window at node field zero" on the +reader for a renumbered Bedrock node. Where the node has no memory behind +the window that shows its own boot flash, which the PROM needs. But it kept +the node's own *banks* there too: with 512 MiB or more, physical +0x1fc0_0000..0x2000_0000 at node field zero was the second node's own RAM +for its processors, and the zero holder's RAM for the first node's +processors, for DMA and for the block transfer engine. IRIX hands those +four megabytes out as ordinary pages of node 0. + +On the hardware only *uncached* references to 0x1fc0_0000 are the boot +vector, and they reach the local flash through the hub-special space. The +PROM's copy in memory is at a low address, mapped at virtual +0xc000_0000_1fc0_0000 through the TLB. A cached reference to physical +0x1fc0_0000 is plain memory by node number. + +## Rule + +At node field zero a renumbered node keeps its alias (the first bytes its +chip says) and, on the Bedrock, its boot *flash* where it has no memory +behind the window. Memory is never the reader's own there: it is the zero +holder's. Every path to a physical address has to agree: each node's +`Physical` and ppmem window (what its processors see), and memory by number +(`NodeDecode` as a bus: DMA, block transfers). + +## How it was found + +The order matters more than the result; each step halved the search: + +1. One node, same Fibre Channel disk: byte-exact against the host + (`dd ... | cksum` in the guest and on the image). Not the ISP2200. +2. Two nodes, `IRIS_JIT_DISPATCH=off`: still fails. Not the JIT. +3. Two nodes, no graphics: still fails. Not the V-brick. +4. Two nodes, second CPU disabled (PROM menu 5, `disable -m 001c07 -cpu a`, + `reset`): clean with all memory in use. Not node-1 memory or DMA. +5. Pipe a raw region to the NFS share and `cmp` with the image: 16 KiB + pages of zeros. A page seen as two different pages. +6. Compare what each CPU reads at the same physical address from the + monitor (`cpu N mem 0xa8...`), and CPU 1's view of a node-0 address + against its view of its own node at the same offset. Sample every + 64 KiB; a coarse stride over the whole of memory missed the 4 MiB. + +Traps on the way: `0x9000...` is the hub-special space, where a per-node +alias is legitimate (uncached memory is `0x9600...`); breakpoints do nothing +in a `lightning` build; the Tezro image has no network on these machines +until `ifconfig ef0` by hand. + +Still open: a node with less than 512 MiB keeps its flash in the window +even when the zero holder has memory there, so its processors cannot reach +those pages of node 0. The PROM path that needs the flash there has not +been separated from the kernel's need for the memory. diff --git a/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md b/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md index c67b7587e..52c5543b8 100644 --- a/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md +++ b/clanker/jitv2/pages-are-keyed-by-bank-and-offset.md @@ -67,6 +67,10 @@ level and the D-cache geometry are per pool. `Vec` lifetime and mode-dependent reads were not audited), so their pages still read through the bus. - `POOL_CPUS` is `smp::MAX_CPUS` (4). +- `j2 status` prints `shared pages`: pages more than one CPU reached, and + how many at different addresses. IRIX replicates kernel text per node, so + each node's CPUs run their own copy (different banks, no sharing there); + what two nodes share is user and library code. - Seen once in five Tezro boots, on the bus-read build: all CPUs spinning in the PROM near 0xc00000001fc33614 with 67 pages claimed. Not reproduced, not yet compared with main. diff --git a/debug.md b/debug.md index a61b585c7..17b04e94a 100644 --- a/debug.md +++ b/debug.md @@ -305,7 +305,7 @@ take effect after `j2 flush`. | Command | Effect | Default (release / `developer`) | |---|---|---| -| `j2 status` (alias `j2 stats`) | Arena use, compile counts, skipped/rejected recompiles, reject reasons, and `remaps`: times a CPU forgot its page addresses because memory moved (no code is dropped for one) | — | +| `j2 status` (alias `j2 stats`) | Arena use, compile counts, skipped/rejected recompiles, reject reasons, `remaps` (times a CPU forgot its page addresses because memory moved; no code is dropped for one) and `shared pages` (pages more than one CPU has reached, and how many of those at different addresses) | — | | `j2 dispatch [on\|off]` | Main switch for JIT dispatch; off = interpreter only | `on` | | `j2 inline [on\|off]` | Compile synchronously on the CPU thread instead of the compile pool; a page with code recompiles only when new entries make it hot, as the pool would (`[jitv2] compile = "sync"` at startup). Refused with several CPUs, and `on` is refused in `lightning` builds | `off` (`on` under lockstep) | | `j2 threads` | Compile-pool thread count and why (read-only; set with `[jitv2] threads` or `--jitv2-threads`) | one per guest CPU, at most 4, capped by the host's usable CPUs | diff --git a/src/cpu/jitv2/jitv2.rs b/src/cpu/jitv2/jitv2.rs index ef69a7f99..76e3e23f5 100644 --- a/src/cpu/jitv2/jitv2.rs +++ b/src/cpu/jitv2/jitv2.rs @@ -2972,6 +2972,23 @@ impl Jitv2 { (skipped, rejected) } + /// Pages more than one CPU of the pool has reached, and how many of those + /// some CPU reached at another frame than the rest (another node's address + /// for the same memory): the sharing a key by memory makes possible. + pub fn shared_pages(&self) -> (usize, usize) { + let (mut shared, mut apart) = (0, 0); + for page in self.pages.iter().filter(|p| p.is_claimed()) { + let mut frames = (0..POOL_CPUS).filter_map(|cpu| page.addr_of(cpu)).map(|(pfn, _)| pfn); + let Some(first) = frames.next() else { continue }; + let rest: Vec = frames.collect(); + if !rest.is_empty() { + shared += 1; + apart += usize::from(rest.iter().any(|&pfn| pfn != first)); + } + } + (shared, apart) + } + /// Pool capacity, as passed to `new()`. #[inline] pub fn capacity(&self) -> usize { diff --git a/src/cpu/mips_exec/monitor.rs b/src/cpu/mips_exec/monitor.rs index 4f698a12a..248331bb4 100644 --- a/src/cpu/mips_exec/monitor.rs +++ b/src/cpu/mips_exec/monitor.rs @@ -3052,6 +3052,8 @@ impl MipsCpu { let jit = exec.jitv2.lock(); writeln!(writer, "pages: {} / {} used", jit.pages_used(), jit.capacity()).unwrap(); writeln!(writer, "remaps: {} (a CPU forgot its page addresses after memory moved; no code dropped)", jit.stats.remaps.load(Ordering::Relaxed)).unwrap(); + let (shared, apart) = jit.shared_pages(); + writeln!(writer, "shared pages: {shared} reached by more than one CPU, {apart} of them at different addresses").unwrap(); // Churn avoidance: "skipped" is compile work that did not // happen because the page's generation moved without // any word the last compile decoded changing; diff --git a/src/sn.rs b/src/sn.rs index 9b54bc872..e87ad695d 100644 --- a/src/sn.rs +++ b/src/sn.rs @@ -145,8 +145,10 @@ pub struct Node { memory: Arc, /// The boot flash, mapped at node field zero wherever no bank covers its window. boot: Option>, - /// Whether a cached reference to the boot window at node field zero stays on the - /// reader once the node is renumbered: the Bedrock keeps it, the Hub does not. + /// Whether the boot flash stays in its window at node field zero for the node's own + /// processors once the node is renumbered, where the node has no memory behind the + /// window: the Bedrock keeps it, the Hub does not. Memory in the window is never the + /// reader's own: it is the zero holder's, as everywhere above the alias. boot_local: bool, spaces: [OnceLock>; 2], /// Where the remap shows the boot flash in the node's table, `at` and `len`; @@ -392,14 +394,25 @@ impl NodeDecode { match far { Some(f) if n != 0 => { // What stays on the reader at node field zero: the alias, and - // on the Bedrock the boot window. The rest is the zero holder's. + // on the Bedrock the boot flash in its window. The rest is the + // zero holder's. That includes the window itself wherever the + // reader has memory of its own behind it and so shows no flash: + // a cached reference there is ordinary memory by node number, + // the same bytes for every processor, DMA master and block + // transfer. Kept on the reader, a node with 512 MiB or more + // read and wrote its own memory for four megabytes the kernel + // hands out as the zero holder's pages. let local = boot.clone().filter(|_| me.boot_local).unwrap_or(u64::MAX..u64::MAX); own.extend(shifted(&banks, 0, 0..alias)); - own.extend(shifted(&banks, 0, local.clone())); if f.number() == 0 { let far_banks = far_banks.as_deref().unwrap_or_default(); across.extend(shifted(far_banks, 0, alias..local.start.max(alias))); + for mine in shifted(&banks, 0, local.clone()) { + across.extend(shifted(far_banks, 0, mine.base.max(alias)..mine.end)); + } across.extend(shifted(far_banks, 0, local.end.max(alias)..u64::MAX)); + } else { + own.extend(shifted(&banks, 0, local.clone())); } } _ => own.extend(shifted(&banks, 0, 0..u64::MAX)), diff --git a/src/sn/tests.rs b/src/sn/tests.rs index e5ebddc43..ff724b054 100644 --- a/src/sn/tests.rs +++ b/src/sn/tests.rs @@ -221,6 +221,29 @@ fn a_boot_local_node_keeps_its_flash_at_node_field_zero() { release(&nodes); } +/// With memory of its own behind the boot window a renumbered IP35 node shows no flash there, +/// and the window is the zero holder's memory like the rest of node field zero: IRIX hands those +/// pages out, and every processor, DMA master and block transfer has to reach the same bytes. +#[test] +fn memory_in_the_boot_window_is_the_zero_holders_for_both_nodes() { + let nodes = pair([1024, 1024], Some(0x5a), bedrock::NODE_SHIFT); + renumber(&nodes[0], 1); + let at = RESET_WINDOW + 0x2000; + // The zero holder stores; the renumbered node reads the same memory, and stores back. + assert_eq!(nodes[1].physical().write64(at, 0x1122_3344_5566_7788), BUS_OK); + assert_eq!(nodes[0].physical().read64(at).data, 0x1122_3344_5566_7788, "the zero holder's memory, not its own"); + assert_eq!(nodes[0].physical().write64(at + 8, 0xa5a5), BUS_OK); + assert_eq!(nodes[1].physical().read64(at + 8).data, 0xa5a5); + // Memory by number, as DMA and the block transfer engine reach it, agrees. + let dma: &dyn BusDevice = &*nodes[0]; + assert_eq!(dma.read64(at).data, 0x1122_3344_5566_7788); + // The renumbered node's own memory behind the window is at its number. + let own = (1u64 << bedrock::NODE_SHIFT) + at; + assert_ne!(nodes[0].physical().read64(own).data, 0x1122_3344_5566_7788); + assert_eq!(nodes[1].physical().read64(own).data, nodes[0].physical().read64(own).data); + release(&nodes); +} + /// A remap publishes and maps with every other processor parked. #[test] fn a_remap_publishes_with_the_other_processor_parked() { From 1f72e2387d183a6cab5308873e85ea3c8184ae7a Mon Sep 17 00:00:00 2001 From: technomancer Date: Thu, 19 Nov 2026 19:49:39 -0800 Subject: [PATCH 4/4] monitor: the CPUs of one machine share one symbol table Each CPU had its own table. `loadsym` goes to the selected CPU and the breakpoint commands to every CPU, so on a machine with more than one CPU `b ` failed with "unknown symbol" from the CPUs that had no map, right after `sym ` resolved it. CpuMonitor now hands the first CPU's table to the others. Co-Authored-By: Claude Fable 5.2 --- CHANGELOG.md | 3 +++ debug.md | 2 ++ src/cpu/cpu_monitor.rs | 7 +++++++ src/cpu/mips_exec.rs | 9 +++++++++ src/cpu/mips_exec_test.rs | 29 +++++++++++++++++++++++++++++ 5 files changed, 50 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24c5759ba..3b2c1060f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -281,6 +281,9 @@ hashes are given where a change is easiest to understand by reading the commit. `monitor_history` file left by an earlier nightly is no longer read. - Tab at the terminal console completes monitor command names, also after `cpu` and `cpu `. +- Machines with more than one CPU: `b ` and other breakpoint commands + accept names from a map loaded with `loadsym` (they failed with "unknown + symbol"). - The terminal is left usable however iris ends (monitor `quit`, a signal, the guest powering off, the window closing, a panic), not only after Ctrl-C. - The monitor keeps a client that sends non-UTF-8 bytes, refuses a line over diff --git a/debug.md b/debug.md index 17b04e94a..4e328dc33 100644 --- a/debug.md +++ b/debug.md @@ -164,6 +164,8 @@ or Onyx system with several) adds a choice of which CPU a command means. Without a number, `start`, `stop` and the breakpoint commands (`bp`, `b`, `bd`, `be`, `bb`) apply to every CPU, `status` answers for each, and everything else goes to the selected CPU. A number always means that CPU alone, even for `stop`. +The CPUs share one symbol table, so a map loaded with `loadsym` names +breakpoints and addresses on all of them. **Debugging a running guest on two CPUs is not supported.** The commands run, but there is no way to step one CPU that keeps the guest healthy. If the other diff --git a/src/cpu/cpu_monitor.rs b/src/cpu/cpu_monitor.rs index e150c0c08..5e12f2325 100644 --- a/src/cpu/cpu_monitor.rs +++ b/src/cpu/cpu_monitor.rs @@ -113,6 +113,13 @@ pub struct CpuMonitor { impl CpuMonitor { pub fn new(cpus: Vec>) -> Self { assert!(!cpus.is_empty(), "a machine has at least one CPU"); + // One symbol table for the machine. `loadsym` goes to the selected + // CPU and a breakpoint to every CPU; with a table each, the others + // refused the name. + let symbols = cpus[0].symbol_table(); + for cpu in &cpus[1..] { + cpu.set_symbol_table(symbols.clone()); + } Self { cpus, selected: AtomicUsize::new(0) } } diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index b3d92fc0b..8c624246c 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -11590,6 +11590,13 @@ pub trait CpuDevice: Device + Resettable + Saveable + Send + Sync { /// see memory at the same addresses, as the CPUs of one node do). fn share_jitv2(&self, jit: Arc>, world: Arc, cpu: usize, space: usize); fn monitor_execute(&self, cmd: &str, args: &[&str], writer: Box) -> Result<(), String>; + /// The symbol table this CPU's monitor commands and breakpoints resolve + /// names in. + fn symbol_table(&self) -> Arc>; + /// Resolve names in `table` from now on: the CPUs of one machine share + /// one, so `loadsym` on any of them serves a breakpoint set on all. + /// Before the CPU first runs. + fn set_symbol_table(&self, table: Arc>); fn cycles_ptr(&self) -> crate::cpu::mips_core::CyclesPtr; fn interrupts_ptr(&self) -> *const AtomicU64; /// ppmem: the executor's inline `ppmem_bitmap` field, for @@ -11672,6 +11679,8 @@ impl CpuDevice for MipsCp fn monitor_execute(&self, cmd: &str, args: &[&str], writer: Box) -> Result<(), String> { MipsCpu::monitor_execute(self, cmd, args, writer) } + fn symbol_table(&self) -> Arc> { self.executor.lock().symbols.clone() } + fn set_symbol_table(&self, table: Arc>) { self.executor.lock().symbols = table; } fn cycles_ptr(&self) -> crate::cpu::mips_core::CyclesPtr { MipsCpu::cycles_ptr(self) } fn interrupts_ptr(&self) -> *const AtomicU64 { MipsCpu::interrupts_ptr(self) } fn ppmem_bitmap_ptr(&self) -> *mut u64 { MipsCpu::ppmem_bitmap_ptr(self) } diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 9a3871167..51c4e5804 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -7363,6 +7363,35 @@ mod tests { crate::cpu::mips_exec::MipsCpu::new(exec) } + /// `loadsym` goes to the selected CPU and a breakpoint to every CPU. With + /// a symbol table per CPU the unselected ones refused the name and the + /// whole command failed with "unknown symbol". + #[test] + fn a_symbol_loaded_once_names_a_breakpoint_on_every_cpu() { + use crate::traits::Device; + let map = std::env::temp_dir().join(format!("iris-symtest-{}.map", std::process::id())); + std::fs::write(&map, "ffffffff80001000 T sendsig\n").unwrap(); + let cpus: Vec> = vec![Arc::new(nop_cpu()), Arc::new(nop_cpu())]; + let monitor = crate::cpu::cpu_monitor::CpuMonitor::new(cpus); + let run = |line: &str| { + let words: Vec<&str> = line.split_whitespace().collect(); + let cap = Capture::default(); + let r = monitor.execute_command(words[0], &words[1..], Box::new(cap.clone())); + let out = String::from_utf8_lossy(&cap.0.lock()).into_owned(); + (r, out) + }; + let (r, _) = run(&format!("loadsym {}", map.display())); + std::fs::remove_file(&map).ok(); + assert!(r.is_ok(), "{:?}", r); + let (r, out) = run("b sendsig if a1 == 11"); + assert!(r.is_ok(), "{:?}", r); + assert!(out.contains("sendsig") && out.contains("if a1 == 11"), "{}", out); + // The unselected CPU has it too, by name. + let (r, out) = run("cpu 1 bl"); + assert!(r.is_ok(), "{:?}", r); + assert!(out.contains("sendsig"), "{}", out); + } + // `lightning` compiles breakpoint checks out of the run loop, so nothing // would ever stop the CPU at the breakpoint. #[cfg(not(feature = "lightning"))]