diff --git a/TESTING.md b/TESTING.md index 6e7a61764..edc312b0f 100644 --- a/TESTING.md +++ b/TESTING.md @@ -637,8 +637,8 @@ trees. The lock-free and condition-variable protocols have [loom](https://docs.rs/loom) models in `loom_tests` modules beside the code they check: the GFIFO ring (`src/gfifo.rs`), the interrupt guard `IrqMutex` (`src/irq.rs`), PCI INTx -routing (`src/pci/intx.rs`), the IP7 timer ticket (`src/cpu/ip7_ticket.rs`) -and the jitv2 compile pool's flush barrier (`src/cpu/jitv2/pool_barrier.rs`). +routing (`src/pci/intx.rs`) and the jitv2 compile pool's flush barrier +(`src/cpu/jitv2/pool_barrier.rs`). They are built only under `--cfg loom`, where `src/sync.rs` swaps `parking_lot`'s `Mutex` and `Condvar` for loom's, so the model drives the code the device runs rather than a copy of it. Run them all with: diff --git a/src/cpu/ip7_ticket.rs b/src/cpu/ip7_ticket.rs deleted file mode 100644 index 4319a78b8..000000000 --- a/src/cpu/ip7_ticket.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! The ticket word that makes compare-timer (IP7) delivery exactly-once per -//! armed Compare deadline. -//! -//! Every Compare write arms a fresh ticket. Whoever delivers the deadline -//! (the CPU thread's own Count check or the hptimer callback) claims the -//! ticket first, then asserts IP7. A Compare write is also the guest's -//! acknowledgement of IP7, so it must not be possible for a delivery that -//! claimed the *old* ticket to assert IP7 after the acknowledgement has -//! cleared it. Claiming and asserting are two steps on two words, so the word -//! has a third state, `CLAIMING`, held from the claim until the assertion is -//! done, and `arm` waits it out. The wait is only as long as the claimer's -//! `deliver` closure, which is a few atomic operations and an unpark. -//! -//! The per-instruction Count path never touches this word: it runs only when -//! Count crosses Compare or Compare is written. -//! -//! `loom_tests` checks every interleaving of one delivery against one -//! acknowledgement. Run them with -//! `RUSTFLAGS="--cfg loom" cargo test --release --lib ip7_ticket::loom_tests`. - -#[cfg(loom)] -use loom::{sync::atomic::{AtomicU64, Ordering::SeqCst}, thread::yield_now}; -#[cfg(not(loom))] -use std::{sync::atomic::{AtomicU64, Ordering::SeqCst}, thread::yield_now}; - -/// The armed deadline has been delivered. Never a valid ticket. -pub const CONSUMED: u64 = 0; - -/// A claimer has won the ticket and is asserting IP7. Never a valid ticket. -pub const CLAIMING: u64 = u64::MAX; - -/// True for the two values that are states, not tickets. -#[inline] -pub fn is_sentinel(ticket: u64) -> bool { - ticket == CONSUMED || ticket == CLAIMING -} - -/// One CPU's IP7 ticket word, shared between its CPU thread and the hptimer -/// callback of the deadline it armed (`MipsCore::ip7_seq_shared`). -pub struct Ip7Ticket(AtomicU64); - -impl Ip7Ticket { - /// A word with nothing armed (`CONSUMED`). - pub fn new() -> Self { - Ip7Ticket(AtomicU64::new(CONSUMED)) - } - - /// The word: the armed ticket, or `CONSUMED`, or `CLAIMING` mid-delivery. - pub fn load(&self) -> u64 { - self.0.load(SeqCst) - } - - /// Claim the right to deliver `ticket`, and run `deliver` (which asserts - /// IP7) before anyone can arm a new one. Returns false, running nothing, - /// if the ticket is no longer the armed one. `deliver` must not panic or - /// block: a Compare write waits for it to return. - #[inline] - pub fn claim(&self, ticket: u64, deliver: impl FnOnce()) -> bool { - if is_sentinel(ticket) { - return false; - } - if self.0.compare_exchange(ticket, CLAIMING, SeqCst, SeqCst).is_err() { - return false; - } - deliver(); - self.0.store(CONSUMED, SeqCst); - true - } - - /// Replace the armed ticket with `ticket`, after any delivery in flight - /// has asserted its IP7. The caller clears IP7 next: nothing that claimed - /// an older ticket can assert it after this returns. - pub fn arm(&self, ticket: u64) { - debug_assert!(!is_sentinel(ticket)); - loop { - let cur = self.0.load(SeqCst); - if cur != CLAIMING && self.0.compare_exchange(cur, ticket, SeqCst, SeqCst).is_ok() { - return; - } - yield_now(); - } - } - - /// A Compare write's acknowledgement: arm `ticket`, then run `clear`, - /// which removes the pending IP7. The order is the point. Arming first - /// means a delivery of the old deadline either lost its claim, or has - /// finished asserting IP7 for `clear` to remove. - #[inline] - pub fn acknowledge(&self, ticket: u64, clear: impl FnOnce()) { - self.arm(ticket); - clear(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_ticket_is_claimed_once_and_only_while_armed() { - let t = Ip7Ticket::new(); - t.arm(1); - assert!(!t.claim(2, || panic!("stale ticket delivered"))); - let mut n = 0; - assert!(t.claim(1, || n += 1)); - assert!(!t.claim(1, || n += 1)); - assert_eq!((n, t.load()), (1, CONSUMED)); - t.arm(2); - assert!(t.claim(2, || n += 1)); - assert!(!t.claim(CONSUMED, || n += 1)); - assert!(!t.claim(CLAIMING, || n += 1)); - assert_eq!(n, 2); - } -} - -#[cfg(loom)] -mod loom_tests { - use super::*; - use loom::sync::Arc; - use loom::thread; - - const IP7: u64 = 1 << 15; - const OTHER: u64 = 1 << 10; - - /// The timer callback delivers deadline 1 while the CPU thread's Compare - /// write arms deadline 2 and clears IP7. `post` runs on the CPU thread - /// after the acknowledgement. - fn run(post: bool) -> u64 { - let t = Arc::new(Ip7Ticket::new()); - let irq = Arc::new(AtomicU64::new(OTHER)); - t.arm(1); - - let cb = { - let (t, irq) = (t.clone(), irq.clone()); - thread::spawn(move || { - t.claim(1, || { - irq.fetch_or(IP7, SeqCst); - }); - }) - }; - - // The Compare write, through the function `MipsCore::write_cp0` calls. - t.acknowledge(2, || { - irq.fetch_and(!IP7, SeqCst); - }); - if post { - // A deadline the write found already missed: the CPU thread - // delivers it itself. - assert!(t.claim(2, || { - irq.fetch_or(IP7, SeqCst); - })); - } - cb.join().unwrap(); - irq.load(SeqCst) - } - - /// After the guest's acknowledgement, a delivery of the superseded - /// deadline must not leave IP7 set. - #[test] - fn an_acknowledged_deadline_is_not_delivered_afterwards() { - loom::model(|| { - let irq = run(false); - assert_eq!(irq, OTHER, "IP7 asserted after the Compare write acknowledged it"); - }); - } - - /// The new deadline's own delivery survives the old one's. - #[test] - fn the_new_deadline_is_still_delivered() { - loom::model(|| { - let irq = run(true); - assert_eq!(irq, OTHER | IP7, "the new deadline's IP7 was lost"); - }); - } -} diff --git a/src/cpu/mips_core.rs b/src/cpu/mips_core.rs index 838bf84c1..713d89b1c 100644 --- a/src/cpu/mips_core.rs +++ b/src/cpu/mips_core.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use crate::cpu::ip7_ticket::{self, Ip7Ticket}; // CP0 Status Register bit definitions pub const STATUS_IE: u32 = 1 << 0; // Interrupt Enable @@ -95,32 +94,47 @@ pub const NS_PER_GUEST_CYCLE: u64 = 10; /// over Compare, or when a Compare write is classified as an already-missed /// deadline) and the hptimer callback (when its one-shot expires). The /// sequence word holds the ticket of the currently-armed deadline; the winner -/// asserts IP7 and leaves the word consumed, and every later claim for the -/// same ticket -- including an in-flight timer callback whose arm the -/// guest has already superseded -- fails and does nothing. A Compare write -/// waits for a winner that is still asserting IP7 (see `Ip7Ticket`), so the -/// assertion can never land after the acknowledgement that supersedes it. +/// asserts IP7 and leaves the word `IP7_SEQ_CONSUMED`, and every later claim +/// for the same ticket -- including an in-flight timer callback whose arm the +/// guest has already superseded -- fails the compare-exchange and does +/// nothing. /// /// Returns true if this caller delivered the interrupt. /// /// Level-triggered semantics are preserved: the word stays consumed until the -/// next Compare write re-arms it (it takes `next_ip7_ticket()` and calls -/// `acknowledge()`), which is exactly the guest's acknowledgement. +/// next Compare write re-arms it via `MipsCore::arm_ip7_sequence`, which is +/// exactly the guest's acknowledgement. #[inline] -fn claim_ip7(seq: &Ip7Ticket, ticket: u64, irq: &AtomicU64, fasttick: &AtomicU64) -> bool { - seq.claim(ticket, || { +fn claim_ip7(seq: &AtomicU64, ticket: u64, irq: &AtomicU64, fasttick: &AtomicU64) -> bool { + if ticket == IP7_SEQ_CONSUMED { + return false; + } + // The claim and the IP7 it asserts are two words. The word reads + // IP7_SEQ_CLAIMING between them, and `arm_ip7_sequence` waits that out. + if seq + .compare_exchange(ticket, IP7_SEQ_CLAIMING, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { irq.fetch_or(CAUSE_IP7 as u64, Ordering::SeqCst); + seq.store(IP7_SEQ_CONSUMED, Ordering::SeqCst); crate::pf_level!(crate::perfetto::events::IP7_LEVEL, crate::perfetto::events::IP7_COMPARE, true); #[cfg(feature = "idle-pause")] crate::cpu::idle_park::wake(); fasttick.fetch_add(1, Ordering::Relaxed); - }) + true + } else { + false + } } /// Sentinel meaning "the armed deadline has already been delivered". Never a -/// valid ticket: `next_ip7_ticket` only ever issues values from a monotonic -/// counter starting at 1, skipping both sentinels (`ip7_ticket::is_sentinel`). -const IP7_SEQ_CONSUMED: u64 = ip7_ticket::CONSUMED; +/// valid ticket: `arm_ip7_sequence` only ever issues values from a monotonic +/// counter starting at 1. +const IP7_SEQ_CONSUMED: u64 = 0; + +/// Sentinel meaning "a deliverer has claimed the ticket and has not asserted +/// IP7 yet". Never a valid ticket either. +const IP7_SEQ_CLAIMING: u64 = u64::MAX; /// Raw pointer to a `MipsCore`'s `hot.interrupts` word, captured by the armed /// compare-timer callback so the hptimer thread can raise IP7. Same @@ -981,9 +995,8 @@ pub struct MipsCore { /// emulator approximation for host scheduling, not processor behaviour. pub count_write_samples: bool, /// IP7 delivery sequence, CPU-thread copy. Incremented by - /// `next_ip7_ticket()` on every Compare write (which then calls - /// `acknowledge()`) and on every re-arm; the value it becomes is the - /// ticket that the *next* IP7 delivery must present. + /// `arm_ip7_sequence` on every Compare write / re-arm; the value it + /// becomes is the ticket that the *next* IP7 delivery must present. /// /// Only the CPU thread writes this; the timer thread never touches it. pub(crate) ip7_seq: u64, @@ -991,7 +1004,7 @@ pub struct MipsCore { /// ticket of the currently-armed deadline. Exactly one deliverer -- /// whichever of the CPU thread (crossing detection / late-deadline /// classification) or the timer callback gets there first -- wins the - /// claim that consumes it (`Ip7Ticket::claim`); the loser does nothing. + /// compare-exchange that consumes it; the loser does nothing. /// /// This is what makes IP7 delivery exactly-once per armed deadline. The /// hptimer's `remove()` cannot retract a callback the timer thread has @@ -999,11 +1012,9 @@ pub struct MipsCore { /// armed for a superseded Compare can still execute after the guest has /// acked -- re-raising the interrupt it just cleared, which is what /// breaks Linux's second `c0_compare_int_usable` verification window. - /// A Compare write arms a new ticket before it clears IP7, waiting out a - /// claimer that is still asserting it (`Ip7Ticket::acknowledge`), which - /// invalidates any such in-flight callback by construction rather than - /// by timing. - pub(crate) ip7_seq_shared: Arc, + /// Bumping the sequence at the top of the Compare write invalidates any + /// such in-flight callback by construction rather than by timing. + pub(crate) ip7_seq_shared: Arc, /// developer_ip7: Compare writes classified as a pure acknowledgement. #[cfg(feature = "developer_ip7")] pub ip7_acks: u64, @@ -1477,7 +1488,7 @@ impl MipsCore { count_last_guest_read: 0, count_write_samples: false, ip7_seq: 0, - ip7_seq_shared: Arc::new(Ip7Ticket::new()), + ip7_seq_shared: Arc::new(AtomicU64::new(0)), #[cfg(feature = "developer_ip7")] ip7_acks: 0, #[cfg(feature = "developer_ip7")] @@ -1880,30 +1891,30 @@ impl MipsCore { self.count_hz_atomic.store(hz, Ordering::Relaxed); } - /// Issue and publish a fresh IP7 delivery ticket, invalidating any - /// previously armed one, on the CPU thread. Waits for a delivery that - /// already claimed the old ticket to finish asserting IP7. Used where no - /// acknowledgement goes with it (`restore_compare_deadline`); a Compare - /// write takes `next_ip7_ticket` and arms it through - /// `Ip7Ticket::acknowledge`, which clears IP7 after the wait so a timer - /// callback racing the write cannot re-assert the interrupt the guest is - /// acknowledging. + /// Issue a fresh IP7 delivery ticket, invalidating any previously armed + /// one. Called from the CPU thread at the top of a Compare write, before + /// IP7 is cleared, so that a timer callback racing us cannot re-assert + /// the interrupt the guest is in the middle of acknowledging. /// /// Returns the new ticket, which the caller hands to whoever may deliver /// this deadline (the armed one-shot, and the CPU thread's own crossing /// and late-deadline checks). fn arm_ip7_sequence(&mut self) -> u64 { - let ticket = self.next_ip7_ticket(); - self.ip7_seq_shared.arm(ticket); - ticket - } - - /// The next delivery ticket, not yet published to the timer callback. - fn next_ip7_ticket(&mut self) -> u64 { self.ip7_seq = self.ip7_seq.wrapping_add(1); - if ip7_ticket::is_sentinel(self.ip7_seq) { + if self.ip7_seq == IP7_SEQ_CLAIMING { self.ip7_seq = 1; // never issue a sentinel } + // A callback that already claimed the old ticket sets IP7 next. Wait + // for it, so the caller's clear lands after that and not before. + loop { + let cur = self.ip7_seq_shared.load(Ordering::SeqCst); + if cur != IP7_SEQ_CLAIMING + && self.ip7_seq_shared.compare_exchange(cur, self.ip7_seq, Ordering::SeqCst, Ordering::SeqCst).is_ok() + { + break; + } + std::hint::spin_loop(); + } self.ip7_seq } @@ -2152,27 +2163,24 @@ impl MipsCore { // schedule_compare_timer arm/classify against the true // current Count/Compare relationship, then arm the // interrupt. + // Issue a new delivery ticket before anything else. Any + // one-shot armed for the previous Compare -- including one the + // timer thread has already dequeued and is about to run + // outside its lock, which `remove()` cannot retract -- now + // holds a stale ticket and can no longer assert IP7 over the + // acknowledgement we are about to perform. + let ticket = self.arm_ip7_sequence(); let count_before = self.materialize_count(false); self.cp0_compare = value as u32 as u64; // Writing Compare always acknowledges the pending timer - // interrupt: publish a new delivery ticket, then clear IP7 in - // both Cause and the shared pending word (the step() preamble - // mirrors the pending word into Cause, so leaving the atomic - // bit set would immediately re-raise it). Any one-shot armed - // for the previous Compare -- including one the timer thread - // has already dequeued and is about to run outside its lock, - // which `remove()` cannot retract -- now holds a stale ticket - // and cannot assert IP7 over this acknowledgement, and one - // already asserting it is waited for, so the clear removes it. - // Count was materialized above without delivering a crossing, - // so nothing between here and the clear claims a ticket. - let ticket = self.next_ip7_ticket(); - self.ip7_seq_shared.acknowledge(ticket, || { - self.cp0_cause &= !CAUSE_IP7; - self.hot.interrupts.fetch_and(!(CAUSE_IP7 as u64), Ordering::SeqCst); - crate::pf_level!(crate::perfetto::events::IP7_LEVEL, crate::perfetto::events::IP7_COMPARE, false); - }); + // interrupt first: clear IP7 in both Cause and the shared + // pending word (the step() preamble mirrors the pending word + // into Cause, so leaving the atomic bit set would immediately + // re-raise it). + self.cp0_cause &= !CAUSE_IP7; + self.hot.interrupts.fetch_and(!(CAUSE_IP7 as u64), Ordering::SeqCst); + crate::pf_level!(crate::perfetto::events::IP7_LEVEL, crate::perfetto::events::IP7_COMPARE, false); // Now classify what the guest actually asked for. Count here // is virtual and materialized in lumps from the wall clock, @@ -2911,7 +2919,7 @@ mod ip7_ticket_tests { let before = core.count_peek(); core.write_cp0(11, (before.wrapping_add(33_000_000)) as u64); - let shared = core.ip7_seq_shared.load(); + let shared = core.ip7_seq_shared.load(Ordering::SeqCst); assert_ne!( shared, IP7_SEQ_CONSUMED, "the Compare write consumed its own ticket; the one-shot it just \ @@ -2934,7 +2942,7 @@ mod ip7_ticket_tests { core.cp0_count = 0x1000; core.cp0_compare = 0x2000; core.restore_compare_deadline(); - let shared = core.ip7_seq_shared.load(); + let shared = core.ip7_seq_shared.load(Ordering::SeqCst); assert_ne!(shared, IP7_SEQ_CONSUMED, "the restored deadline was armed with the delivered sentinel"); assert_eq!(shared, core.ip7_seq, "the armed one-shot's ticket must be the live sequence"); assert!(claim_ip7(&core.ip7_seq_shared, core.ip7_seq, &core.hot.interrupts, &core.fasttick_count), @@ -2958,7 +2966,7 @@ mod ip7_ticket_tests { core.restore_compare_deadline(); assert!(ip7_pending(&core), "IP7 was pending in the snapshot but not after the restore"); // Delivered once: the restored deadline must not fire it again. - assert_eq!(core.ip7_seq_shared.load(), IP7_SEQ_CONSUMED); + assert_eq!(core.ip7_seq_shared.load(Ordering::SeqCst), IP7_SEQ_CONSUMED); } /// The other direction, for a restore over a running core: an IP7 the @@ -2971,6 +2979,59 @@ mod ip7_ticket_tests { assert!(!ip7_pending(&core)); } + /// A timer claim races the Compare write that supersedes its deadline. Once + /// both are done IP7 is clear: a claim that won set it before the clear. + #[test] + fn a_claim_racing_a_compare_write_does_not_leave_ip7_set() { + const STOP: u64 = u64::MAX; + let mut core = MipsCore::default(); + let seq = core.ip7_seq_shared.clone(); + let fasttick = core.fasttick_count.clone(); + let irq = TimerIrqPtr(&core.hot.interrupts); + // The ticket the claimer may claim, and the last one it tried. + let (offered, tried) = (&AtomicU64::new(0), &AtomicU64::new(0)); + let mut stale = None; + std::thread::scope(|s| { + s.spawn(move || { + let irq = irq; + loop { + let ticket = offered.load(Ordering::SeqCst); + if ticket == STOP { + break; + } + if ticket != tried.load(Ordering::SeqCst) { + // SAFETY: the core outlives the scope, as it does the armed timer. + claim_ip7(&seq, ticket, unsafe { &*irq.0 }, &fasttick); + tried.store(ticket, Ordering::SeqCst); + } + std::thread::yield_now(); + } + }); + for i in 0..200_000u32 { + let ahead = core.count_peek().wrapping_add(1 << 30) as u64; + core.write_cp0(11, ahead); + let ticket = core.ip7_seq; + offered.store(ticket, Ordering::SeqCst); + // Move the write across the claim from one round to the next. + for _ in 0..i % 64 { + std::hint::spin_loop(); + } + core.write_cp0(11, ahead); + let start = std::time::Instant::now(); + while tried.load(Ordering::SeqCst) != ticket { + assert!(start.elapsed().as_secs() < 10, "the claimer thread stopped answering"); + std::thread::yield_now(); + } + if ip7_pending(&core) { + stale = Some(i); + break; + } + } + offered.store(STOP, Ordering::SeqCst); + }); + assert_eq!(stale, None, "IP7 was asserted after the Compare write that acknowledged it"); + } + /// The timer sets IP7 in `hot.interrupts`, and Cause only sees it at the /// next step: a snapshot in between must still record it. #[test] diff --git a/src/cpu/mod.rs b/src/cpu/mod.rs index f3f5e474b..650781d24 100644 --- a/src/cpu/mod.rs +++ b/src/cpu/mod.rs @@ -5,7 +5,6 @@ pub mod mips_isa; pub mod mips_dis; pub mod mips_core; -pub mod ip7_ticket; pub mod mips_tlb; pub mod mips_cache_v2; pub mod mips_cache_shadow;