From e39201f7cc595d83247e335d32a3ca21bdaf7e78 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 20 Nov 2026 13:22:34 -0700 Subject: [PATCH] mips: fold the IP7 ticket module back into mips_core A timer callback delivers IP7 in two steps on two words: it claims the armed ticket, then sets IP7 in the pending word. A Compare write issues a new ticket and clears IP7. With only "armed" and "consumed" in the ticket word, a callback that had claimed the old ticket could set IP7 after the write had cleared it, and the guest saw a timer interrupt for a deadline it had just replaced. 8b4b6ba41 closed that with a third state, held from the claim until IP7 is set, which a Compare write waits out. That fix put the word in a module of its own, behind closures, so a loom model could drive it. The model could not use the real pending word, which is a std atomic that compiled code reads, so it checked the ticket word against a copy of the two closures and of the Compare write's order. The module, its wrappers and the model were about 170 lines for a protocol that is one extra state. Keep the state and drop the rest. claim_ip7 and arm_ip7_sequence hold the protocol directly, as they did before, and the Compare write takes its ticket at the top again. The new test runs the real claim_ip7 on one thread against real Compare writes on another and checks IP7 after both finish. With the third state removed it failed 60 runs of 60 on two or more CPUs, and it passed 250 of 250 with it. Pinned to one CPU it passed 20 of 20 without the state, so it only catches a regression on a multiprocessor host. --- TESTING.md | 4 +- src/cpu/ip7_ticket.rs | 175 ---------------------------------------- src/cpu/mips_core.rs | 181 ++++++++++++++++++++++++++++-------------- src/cpu/mod.rs | 1 - 4 files changed, 123 insertions(+), 238 deletions(-) delete mode 100644 src/cpu/ip7_ticket.rs diff --git a/TESTING.md b/TESTING.md index 6e7a61764..edc312b0f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -637,8 +637,8 @@ trees. The lock-free and condition-variable protocols have [loom](https://docs.rs/loom) models in `loom_tests` modules beside the code they check: the GFIFO ring (`src/gfifo.rs`), the interrupt guard `IrqMutex` (`src/irq.rs`), PCI INTx -routing (`src/pci/intx.rs`), the IP7 timer ticket (`src/cpu/ip7_ticket.rs`) -and the jitv2 compile pool's flush barrier (`src/cpu/jitv2/pool_barrier.rs`). +routing (`src/pci/intx.rs`) and the jitv2 compile pool's flush barrier +(`src/cpu/jitv2/pool_barrier.rs`). They are built only under `--cfg loom`, where `src/sync.rs` swaps `parking_lot`'s `Mutex` and `Condvar` for loom's, so the model drives the code the device runs rather than a copy of it. Run them all with: diff --git a/src/cpu/ip7_ticket.rs b/src/cpu/ip7_ticket.rs deleted file mode 100644 index 4319a78b8..000000000 --- a/src/cpu/ip7_ticket.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! The ticket word that makes compare-timer (IP7) delivery exactly-once per -//! armed Compare deadline. -//! -//! Every Compare write arms a fresh ticket. Whoever delivers the deadline -//! (the CPU thread's own Count check or the hptimer callback) claims the -//! ticket first, then asserts IP7. A Compare write is also the guest's -//! acknowledgement of IP7, so it must not be possible for a delivery that -//! claimed the *old* ticket to assert IP7 after the acknowledgement has -//! cleared it. Claiming and asserting are two steps on two words, so the word -//! has a third state, `CLAIMING`, held from the claim until the assertion is -//! done, and `arm` waits it out. The wait is only as long as the claimer's -//! `deliver` closure, which is a few atomic operations and an unpark. -//! -//! The per-instruction Count path never touches this word: it runs only when -//! Count crosses Compare or Compare is written. -//! -//! `loom_tests` checks every interleaving of one delivery against one -//! acknowledgement. Run them with -//! `RUSTFLAGS="--cfg loom" cargo test --release --lib ip7_ticket::loom_tests`. - -#[cfg(loom)] -use loom::{sync::atomic::{AtomicU64, Ordering::SeqCst}, thread::yield_now}; -#[cfg(not(loom))] -use std::{sync::atomic::{AtomicU64, Ordering::SeqCst}, thread::yield_now}; - -/// The armed deadline has been delivered. Never a valid ticket. -pub const CONSUMED: u64 = 0; - -/// A claimer has won the ticket and is asserting IP7. Never a valid ticket. -pub const CLAIMING: u64 = u64::MAX; - -/// True for the two values that are states, not tickets. -#[inline] -pub fn is_sentinel(ticket: u64) -> bool { - ticket == CONSUMED || ticket == CLAIMING -} - -/// One CPU's IP7 ticket word, shared between its CPU thread and the hptimer -/// callback of the deadline it armed (`MipsCore::ip7_seq_shared`). -pub struct Ip7Ticket(AtomicU64); - -impl Ip7Ticket { - /// A word with nothing armed (`CONSUMED`). - pub fn new() -> Self { - Ip7Ticket(AtomicU64::new(CONSUMED)) - } - - /// The word: the armed ticket, or `CONSUMED`, or `CLAIMING` mid-delivery. - pub fn load(&self) -> u64 { - self.0.load(SeqCst) - } - - /// Claim the right to deliver `ticket`, and run `deliver` (which asserts - /// IP7) before anyone can arm a new one. Returns false, running nothing, - /// if the ticket is no longer the armed one. `deliver` must not panic or - /// block: a Compare write waits for it to return. - #[inline] - pub fn claim(&self, ticket: u64, deliver: impl FnOnce()) -> bool { - if is_sentinel(ticket) { - return false; - } - if self.0.compare_exchange(ticket, CLAIMING, SeqCst, SeqCst).is_err() { - return false; - } - deliver(); - self.0.store(CONSUMED, SeqCst); - true - } - - /// Replace the armed ticket with `ticket`, after any delivery in flight - /// has asserted its IP7. The caller clears IP7 next: nothing that claimed - /// an older ticket can assert it after this returns. - pub fn arm(&self, ticket: u64) { - debug_assert!(!is_sentinel(ticket)); - loop { - let cur = self.0.load(SeqCst); - if cur != CLAIMING && self.0.compare_exchange(cur, ticket, SeqCst, SeqCst).is_ok() { - return; - } - yield_now(); - } - } - - /// A Compare write's acknowledgement: arm `ticket`, then run `clear`, - /// which removes the pending IP7. The order is the point. Arming first - /// means a delivery of the old deadline either lost its claim, or has - /// finished asserting IP7 for `clear` to remove. - #[inline] - pub fn acknowledge(&self, ticket: u64, clear: impl FnOnce()) { - self.arm(ticket); - clear(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_ticket_is_claimed_once_and_only_while_armed() { - let t = Ip7Ticket::new(); - t.arm(1); - assert!(!t.claim(2, || panic!("stale ticket delivered"))); - let mut n = 0; - assert!(t.claim(1, || n += 1)); - assert!(!t.claim(1, || n += 1)); - assert_eq!((n, t.load()), (1, CONSUMED)); - t.arm(2); - assert!(t.claim(2, || n += 1)); - assert!(!t.claim(CONSUMED, || n += 1)); - assert!(!t.claim(CLAIMING, || n += 1)); - assert_eq!(n, 2); - } -} - -#[cfg(loom)] -mod loom_tests { - use super::*; - use loom::sync::Arc; - use loom::thread; - - const IP7: u64 = 1 << 15; - const OTHER: u64 = 1 << 10; - - /// The timer callback delivers deadline 1 while the CPU thread's Compare - /// write arms deadline 2 and clears IP7. `post` runs on the CPU thread - /// after the acknowledgement. - fn run(post: bool) -> u64 { - let t = Arc::new(Ip7Ticket::new()); - let irq = Arc::new(AtomicU64::new(OTHER)); - t.arm(1); - - let cb = { - let (t, irq) = (t.clone(), irq.clone()); - thread::spawn(move || { - t.claim(1, || { - irq.fetch_or(IP7, SeqCst); - }); - }) - }; - - // The Compare write, through the function `MipsCore::write_cp0` calls. - t.acknowledge(2, || { - irq.fetch_and(!IP7, SeqCst); - }); - if post { - // A deadline the write found already missed: the CPU thread - // delivers it itself. - assert!(t.claim(2, || { - irq.fetch_or(IP7, SeqCst); - })); - } - cb.join().unwrap(); - irq.load(SeqCst) - } - - /// After the guest's acknowledgement, a delivery of the superseded - /// deadline must not leave IP7 set. - #[test] - fn an_acknowledged_deadline_is_not_delivered_afterwards() { - loom::model(|| { - let irq = run(false); - assert_eq!(irq, OTHER, "IP7 asserted after the Compare write acknowledged it"); - }); - } - - /// The new deadline's own delivery survives the old one's. - #[test] - fn the_new_deadline_is_still_delivered() { - loom::model(|| { - let irq = run(true); - assert_eq!(irq, OTHER | IP7, "the new deadline's IP7 was lost"); - }); - } -} diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 838bf84c1..713d89b1c 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use crate::cpu::ip7_ticket::{self, Ip7Ticket}; // CP0 Status Register bit definitions pub const STATUS_IE: u32 = 1 << 0; // Interrupt Enable @@ -95,32 +94,47 @@ pub const NS_PER_GUEST_CYCLE: u64 = 10; /// over Compare, or when a Compare write is classified as an already-missed /// deadline) and the hptimer callback (when its one-shot expires). The /// sequence word holds the ticket of the currently-armed deadline; the winner -/// asserts IP7 and leaves the word consumed, and every later claim for the -/// same ticket -- including an in-flight timer callback whose arm the -/// guest has already superseded -- fails and does nothing. A Compare write -/// waits for a winner that is still asserting IP7 (see `Ip7Ticket`), so the -/// assertion can never land after the acknowledgement that supersedes it. +/// asserts IP7 and leaves the word `IP7_SEQ_CONSUMED`, and every later claim +/// for the same ticket -- including an in-flight timer callback whose arm the +/// guest has already superseded -- fails the compare-exchange and does +/// nothing. /// /// Returns true if this caller delivered the interrupt. /// /// Level-triggered semantics are preserved: the word stays consumed until the -/// next Compare write re-arms it (it takes `next_ip7_ticket()` and calls -/// `acknowledge()`), which is exactly the guest's acknowledgement. +/// next Compare write re-arms it via `MipsCore::arm_ip7_sequence`, which is +/// exactly the guest's acknowledgement. #[inline] -fn claim_ip7(seq: &Ip7Ticket, ticket: u64, irq: &AtomicU64, fasttick: &AtomicU64) -> bool { - seq.claim(ticket, || { +fn claim_ip7(seq: &AtomicU64, ticket: u64, irq: &AtomicU64, fasttick: &AtomicU64) -> bool { + if ticket == IP7_SEQ_CONSUMED { + return false; + } + // The claim and the IP7 it asserts are two words. The word reads + // IP7_SEQ_CLAIMING between them, and `arm_ip7_sequence` waits that out. + if seq + .compare_exchange(ticket, IP7_SEQ_CLAIMING, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { irq.fetch_or(CAUSE_IP7 as u64, Ordering::SeqCst); + seq.store(IP7_SEQ_CONSUMED, Ordering::SeqCst); crate::pf_level!(crate::perfetto::events::IP7_LEVEL, crate::perfetto::events::IP7_COMPARE, true); #[cfg(feature = "idle-pause")] crate::cpu::idle_park::wake(); fasttick.fetch_add(1, Ordering::Relaxed); - }) + true + } else { + false + } } /// Sentinel meaning "the armed deadline has already been delivered". Never a -/// valid ticket: `next_ip7_ticket` only ever issues values from a monotonic -/// counter starting at 1, skipping both sentinels (`ip7_ticket::is_sentinel`). -const IP7_SEQ_CONSUMED: u64 = ip7_ticket::CONSUMED; +/// valid ticket: `arm_ip7_sequence` only ever issues values from a monotonic +/// counter starting at 1. +const IP7_SEQ_CONSUMED: u64 = 0; + +/// Sentinel meaning "a deliverer has claimed the ticket and has not asserted +/// IP7 yet". Never a valid ticket either. +const IP7_SEQ_CLAIMING: u64 = u64::MAX; /// Raw pointer to a `MipsCore`'s `hot.interrupts` word, captured by the armed /// compare-timer callback so the hptimer thread can raise IP7. Same @@ -981,9 +995,8 @@ pub struct MipsCore { /// emulator approximation for host scheduling, not processor behaviour. pub count_write_samples: bool, /// IP7 delivery sequence, CPU-thread copy. Incremented by - /// `next_ip7_ticket()` on every Compare write (which then calls - /// `acknowledge()`) and on every re-arm; the value it becomes is the - /// ticket that the *next* IP7 delivery must present. + /// `arm_ip7_sequence` on every Compare write / re-arm; the value it + /// becomes is the ticket that the *next* IP7 delivery must present. /// /// Only the CPU thread writes this; the timer thread never touches it. pub(crate) ip7_seq: u64, @@ -991,7 +1004,7 @@ pub struct MipsCore { /// ticket of the currently-armed deadline. Exactly one deliverer -- /// whichever of the CPU thread (crossing detection / late-deadline /// classification) or the timer callback gets there first -- wins the - /// claim that consumes it (`Ip7Ticket::claim`); the loser does nothing. + /// compare-exchange that consumes it; the loser does nothing. /// /// This is what makes IP7 delivery exactly-once per armed deadline. The /// hptimer's `remove()` cannot retract a callback the timer thread has @@ -999,11 +1012,9 @@ pub struct MipsCore { /// armed for a superseded Compare can still execute after the guest has /// acked -- re-raising the interrupt it just cleared, which is what /// breaks Linux's second `c0_compare_int_usable` verification window. - /// A Compare write arms a new ticket before it clears IP7, waiting out a - /// claimer that is still asserting it (`Ip7Ticket::acknowledge`), which - /// invalidates any such in-flight callback by construction rather than - /// by timing. - pub(crate) ip7_seq_shared: Arc, + /// Bumping the sequence at the top of the Compare write invalidates any + /// such in-flight callback by construction rather than by timing. + pub(crate) ip7_seq_shared: Arc, /// developer_ip7: Compare writes classified as a pure acknowledgement. #[cfg(feature = "developer_ip7")] pub ip7_acks: u64, @@ -1477,7 +1488,7 @@ impl MipsCore { count_last_guest_read: 0, count_write_samples: false, ip7_seq: 0, - ip7_seq_shared: Arc::new(Ip7Ticket::new()), + ip7_seq_shared: Arc::new(AtomicU64::new(0)), #[cfg(feature = "developer_ip7")] ip7_acks: 0, #[cfg(feature = "developer_ip7")] @@ -1880,30 +1891,30 @@ impl MipsCore { self.count_hz_atomic.store(hz, Ordering::Relaxed); } - /// Issue and publish a fresh IP7 delivery ticket, invalidating any - /// previously armed one, on the CPU thread. Waits for a delivery that - /// already claimed the old ticket to finish asserting IP7. Used where no - /// acknowledgement goes with it (`restore_compare_deadline`); a Compare - /// write takes `next_ip7_ticket` and arms it through - /// `Ip7Ticket::acknowledge`, which clears IP7 after the wait so a timer - /// callback racing the write cannot re-assert the interrupt the guest is - /// acknowledging. + /// Issue a fresh IP7 delivery ticket, invalidating any previously armed + /// one. Called from the CPU thread at the top of a Compare write, before + /// IP7 is cleared, so that a timer callback racing us cannot re-assert + /// the interrupt the guest is in the middle of acknowledging. /// /// Returns the new ticket, which the caller hands to whoever may deliver /// this deadline (the armed one-shot, and the CPU thread's own crossing /// and late-deadline checks). fn arm_ip7_sequence(&mut self) -> u64 { - let ticket = self.next_ip7_ticket(); - self.ip7_seq_shared.arm(ticket); - ticket - } - - /// The next delivery ticket, not yet published to the timer callback. - fn next_ip7_ticket(&mut self) -> u64 { self.ip7_seq = self.ip7_seq.wrapping_add(1); - if ip7_ticket::is_sentinel(self.ip7_seq) { + if self.ip7_seq == IP7_SEQ_CLAIMING { self.ip7_seq = 1; // never issue a sentinel } + // A callback that already claimed the old ticket sets IP7 next. Wait + // for it, so the caller's clear lands after that and not before. + loop { + let cur = self.ip7_seq_shared.load(Ordering::SeqCst); + if cur != IP7_SEQ_CLAIMING + && self.ip7_seq_shared.compare_exchange(cur, self.ip7_seq, Ordering::SeqCst, Ordering::SeqCst).is_ok() + { + break; + } + std::hint::spin_loop(); + } self.ip7_seq } @@ -2152,27 +2163,24 @@ impl MipsCore { // schedule_compare_timer arm/classify against the true // current Count/Compare relationship, then arm the // interrupt. + // Issue a new delivery ticket before anything else. Any + // one-shot armed for the previous Compare -- including one the + // timer thread has already dequeued and is about to run + // outside its lock, which `remove()` cannot retract -- now + // holds a stale ticket and can no longer assert IP7 over the + // acknowledgement we are about to perform. + let ticket = self.arm_ip7_sequence(); let count_before = self.materialize_count(false); self.cp0_compare = value as u32 as u64; // Writing Compare always acknowledges the pending timer - // interrupt: publish a new delivery ticket, then clear IP7 in - // both Cause and the shared pending word (the step() preamble - // mirrors the pending word into Cause, so leaving the atomic - // bit set would immediately re-raise it). Any one-shot armed - // for the previous Compare -- including one the timer thread - // has already dequeued and is about to run outside its lock, - // which `remove()` cannot retract -- now holds a stale ticket - // and cannot assert IP7 over this acknowledgement, and one - // already asserting it is waited for, so the clear removes it. - // Count was materialized above without delivering a crossing, - // so nothing between here and the clear claims a ticket. - let ticket = self.next_ip7_ticket(); - self.ip7_seq_shared.acknowledge(ticket, || { - self.cp0_cause &= !CAUSE_IP7; - self.hot.interrupts.fetch_and(!(CAUSE_IP7 as u64), Ordering::SeqCst); - crate::pf_level!(crate::perfetto::events::IP7_LEVEL, crate::perfetto::events::IP7_COMPARE, false); - }); + // interrupt first: clear IP7 in both Cause and the shared + // pending word (the step() preamble mirrors the pending word + // into Cause, so leaving the atomic bit set would immediately + // re-raise it). + self.cp0_cause &= !CAUSE_IP7; + self.hot.interrupts.fetch_and(!(CAUSE_IP7 as u64), Ordering::SeqCst); + crate::pf_level!(crate::perfetto::events::IP7_LEVEL, crate::perfetto::events::IP7_COMPARE, false); // Now classify what the guest actually asked for. Count here // is virtual and materialized in lumps from the wall clock, @@ -2911,7 +2919,7 @@ mod ip7_ticket_tests { let before = core.count_peek(); core.write_cp0(11, (before.wrapping_add(33_000_000)) as u64); - let shared = core.ip7_seq_shared.load(); + let shared = core.ip7_seq_shared.load(Ordering::SeqCst); assert_ne!( shared, IP7_SEQ_CONSUMED, "the Compare write consumed its own ticket; the one-shot it just \ @@ -2934,7 +2942,7 @@ mod ip7_ticket_tests { core.cp0_count = 0x1000; core.cp0_compare = 0x2000; core.restore_compare_deadline(); - let shared = core.ip7_seq_shared.load(); + let shared = core.ip7_seq_shared.load(Ordering::SeqCst); assert_ne!(shared, IP7_SEQ_CONSUMED, "the restored deadline was armed with the delivered sentinel"); assert_eq!(shared, core.ip7_seq, "the armed one-shot's ticket must be the live sequence"); assert!(claim_ip7(&core.ip7_seq_shared, core.ip7_seq, &core.hot.interrupts, &core.fasttick_count), @@ -2958,7 +2966,7 @@ mod ip7_ticket_tests { core.restore_compare_deadline(); assert!(ip7_pending(&core), "IP7 was pending in the snapshot but not after the restore"); // Delivered once: the restored deadline must not fire it again. - assert_eq!(core.ip7_seq_shared.load(), IP7_SEQ_CONSUMED); + assert_eq!(core.ip7_seq_shared.load(Ordering::SeqCst), IP7_SEQ_CONSUMED); } /// The other direction, for a restore over a running core: an IP7 the @@ -2971,6 +2979,59 @@ mod ip7_ticket_tests { assert!(!ip7_pending(&core)); } + /// A timer claim races the Compare write that supersedes its deadline. Once + /// both are done IP7 is clear: a claim that won set it before the clear. + #[test] + fn a_claim_racing_a_compare_write_does_not_leave_ip7_set() { + const STOP: u64 = u64::MAX; + let mut core = MipsCore::default(); + let seq = core.ip7_seq_shared.clone(); + let fasttick = core.fasttick_count.clone(); + let irq = TimerIrqPtr(&core.hot.interrupts); + // The ticket the claimer may claim, and the last one it tried. + let (offered, tried) = (&AtomicU64::new(0), &AtomicU64::new(0)); + let mut stale = None; + std::thread::scope(|s| { + s.spawn(move || { + let irq = irq; + loop { + let ticket = offered.load(Ordering::SeqCst); + if ticket == STOP { + break; + } + if ticket != tried.load(Ordering::SeqCst) { + // SAFETY: the core outlives the scope, as it does the armed timer. + claim_ip7(&seq, ticket, unsafe { &*irq.0 }, &fasttick); + tried.store(ticket, Ordering::SeqCst); + } + std::thread::yield_now(); + } + }); + for i in 0..200_000u32 { + let ahead = core.count_peek().wrapping_add(1 << 30) as u64; + core.write_cp0(11, ahead); + let ticket = core.ip7_seq; + offered.store(ticket, Ordering::SeqCst); + // Move the write across the claim from one round to the next. + for _ in 0..i % 64 { + std::hint::spin_loop(); + } + core.write_cp0(11, ahead); + let start = std::time::Instant::now(); + while tried.load(Ordering::SeqCst) != ticket { + assert!(start.elapsed().as_secs() < 10, "the claimer thread stopped answering"); + std::thread::yield_now(); + } + if ip7_pending(&core) { + stale = Some(i); + break; + } + } + offered.store(STOP, Ordering::SeqCst); + }); + assert_eq!(stale, None, "IP7 was asserted after the Compare write that acknowledged it"); + } + /// The timer sets IP7 in `hot.interrupts`, and Cause only sees it at the /// next step: a snapshot in between must still record it. #[test] diff --git a/src/cpu/mod.rs b/src/cpu/mod.rs index f3f5e474b..650781d24 100644 --- a/src/cpu/mod.rs +++ b/src/cpu/mod.rs @@ -5,7 +5,6 @@ pub mod mips_isa; pub mod mips_dis; pub mod mips_core; -pub mod ip7_ticket; pub mod mips_tlb; pub mod mips_cache_v2; pub mod mips_cache_shadow;