From cf788d3655c3bedfc84a3273641abd080128a861 Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Tue, 13 Oct 2026 21:33:16 -0400 Subject: [PATCH 01/12] fix(event): resolve inter-CPU event ID collisions --- src/event.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/event.rs b/src/event.rs index ba94a616..bc872339 100644 --- a/src/event.rs +++ b/src/event.rs @@ -37,9 +37,9 @@ pub const IPI_EVENT_WAKEUP: usize = 0; pub const IPI_EVENT_SHUTDOWN: usize = 1; pub const IPI_EVENT_VIRTIO_INJECT_IRQ: usize = 2; pub const IPI_EVENT_WAKEUP_VIRTIO_DEVICE: usize = 3; -pub const IPI_EVENT_VIRTIO_PCI_CONFIG: usize = 7; -pub const IPI_EVENT_VIRTIO_PCI_DATA: usize = 8; -pub const IPI_EVENT_VIRTIO_PCI_DONE: usize = 9; +pub const IPI_EVENT_VIRTIO_PCI_CONFIG: usize = 10; +pub const IPI_EVENT_VIRTIO_PCI_DATA: usize = 11; +pub const IPI_EVENT_VIRTIO_PCI_DONE: usize = 12; #[percpu::def_percpu] static PERCPU_EVENTS: Mutex> = Mutex::new(VecDeque::new()); From 7873c1bd25c86f57ef298ba8693eff1bc8058860 Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Tue, 13 Oct 2026 21:33:52 -0400 Subject: [PATCH 02/12] refactor(arch): add architecture-private vCPU state --- src/arch/aarch64/mod.rs | 2 ++ src/arch/aarch64/vcpu.rs | 26 ++++++++++++++++++++++++++ src/arch/loongarch64/mod.rs | 2 ++ src/arch/loongarch64/vcpu.rs | 10 ++++++++++ src/arch/riscv64/mod.rs | 2 ++ src/arch/riscv64/vcpu.rs | 10 ++++++++++ src/arch/x86_64/mod.rs | 2 ++ src/arch/x86_64/vcpu.rs | 10 ++++++++++ 8 files changed, 64 insertions(+) create mode 100644 src/arch/aarch64/vcpu.rs create mode 100644 src/arch/loongarch64/vcpu.rs create mode 100644 src/arch/riscv64/vcpu.rs create mode 100644 src/arch/x86_64/vcpu.rs diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index f3ab85ad..ed0d8106 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -27,9 +27,11 @@ pub mod s2pt; pub mod sysreg; pub mod time; pub mod trap; +pub mod vcpu; pub mod zone; pub use s2pt::stage2_mode_detect; pub use s2pt::Stage2PageTable; +pub use vcpu::ArchVcpu; #[cfg(aarch64_uefi_support)] pub mod uefi; diff --git a/src/arch/aarch64/vcpu.rs b/src/arch/aarch64/vcpu.rs new file mode 100644 index 00000000..60b20c8d --- /dev/null +++ b/src/arch/aarch64/vcpu.rs @@ -0,0 +1,26 @@ +#[cfg(irq_gicv2)] +use core::sync::atomic::{AtomicU8, Ordering}; + +/// Architecture-private vCPU state, separate from the host pCPU's `ArchCpu`. +pub struct ArchVcpu { + #[cfg(irq_gicv2)] + pub sgi_source: [AtomicU8; 16], +} + +impl ArchVcpu { + pub const fn new() -> Self { + #[cfg(irq_gicv2)] + const ZERO_SOURCE: AtomicU8 = AtomicU8::new(0); + Self { + #[cfg(irq_gicv2)] + sgi_source: [ZERO_SOURCE; 16], + } + } + + pub fn clear_irq_state(&self) { + #[cfg(irq_gicv2)] + for source in &self.sgi_source { + source.store(0, Ordering::Release); + } + } +} diff --git a/src/arch/loongarch64/mod.rs b/src/arch/loongarch64/mod.rs index 26f045ad..28347182 100644 --- a/src/arch/loongarch64/mod.rs +++ b/src/arch/loongarch64/mod.rs @@ -29,7 +29,9 @@ pub mod s1pt; pub mod s2pt; pub mod time; pub mod trap; +pub mod vcpu; pub mod zone; +pub use vcpu::ArchVcpu; pub use s1pt::Stage1PageTable; pub use s2pt::stage2_mode_detect; diff --git a/src/arch/loongarch64/vcpu.rs b/src/arch/loongarch64/vcpu.rs new file mode 100644 index 00000000..351b847c --- /dev/null +++ b/src/arch/loongarch64/vcpu.rs @@ -0,0 +1,10 @@ +/// Architecture-private vCPU state placeholder. +pub struct ArchVcpu; + +impl ArchVcpu { + pub const fn new() -> Self { + Self + } + + pub fn clear_irq_state(&self) {} +} diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs index 6288400b..fe7a3615 100644 --- a/src/arch/riscv64/mod.rs +++ b/src/arch/riscv64/mod.rs @@ -31,7 +31,9 @@ pub mod s2pt; pub mod sbi; pub mod time; pub mod trap; +pub mod vcpu; pub mod zone; +pub use vcpu::ArchVcpu; // pub use s1pt::Stage1PageTable; pub use s2pt::stage2_mode_detect; diff --git a/src/arch/riscv64/vcpu.rs b/src/arch/riscv64/vcpu.rs new file mode 100644 index 00000000..351b847c --- /dev/null +++ b/src/arch/riscv64/vcpu.rs @@ -0,0 +1,10 @@ +/// Architecture-private vCPU state placeholder. +pub struct ArchVcpu; + +impl ArchVcpu { + pub const fn new() -> Self { + Self + } + + pub fn clear_irq_state(&self) {} +} diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 6f775e21..fc24aa6c 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -36,9 +36,11 @@ pub mod s1pt; pub mod s2pt; pub mod time; pub mod trap; +pub mod vcpu; pub mod vmcs; pub mod vmx; pub mod zone; +pub use vcpu::ArchVcpu; pub use s1pt::Stage1PageTable; pub use s2pt::stage2_mode_detect; diff --git a/src/arch/x86_64/vcpu.rs b/src/arch/x86_64/vcpu.rs new file mode 100644 index 00000000..351b847c --- /dev/null +++ b/src/arch/x86_64/vcpu.rs @@ -0,0 +1,10 @@ +/// Architecture-private vCPU state placeholder. +pub struct ArchVcpu; + +impl ArchVcpu { + pub const fn new() -> Self { + Self + } + + pub fn clear_irq_state(&self) {} +} From ff2f37255422d7f1e4cdaffe553749ab25bca33b Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Tue, 13 Oct 2026 21:35:05 -0400 Subject: [PATCH 03/12] refactor(cpu): add guest IRQ state to pinned vCPUs --- src/arch/aarch64/trap.rs | 2 + src/cpu_data.rs | 157 ++++++++++++++++++++++++++++++++++++++- src/hypercall/mod.rs | 1 + 3 files changed, 158 insertions(+), 2 deletions(-) diff --git a/src/arch/aarch64/trap.rs b/src/arch/aarch64/trap.rs index 26eab6d9..5fe37f13 100644 --- a/src/arch/aarch64/trap.rs +++ b/src/arch/aarch64/trap.rs @@ -476,10 +476,12 @@ fn handle_psci_smc( for cpu_id in zone.read().cpu_set().iter_except(this_cpu_data().id) { let target_cpu = get_cpu_data(cpu_id); let _lock = target_cpu.ctrl_lock.lock(); + target_cpu.vcpu.clear_irq_state(); target_cpu.zone = None; send_event(cpu_id, SGI_IPI_ID as _, IPI_EVENT_SHUTDOWN); } + this_cpu_data().vcpu.clear_irq_state(); this_cpu_data().zone = None; drop(zone); remove_zone(zone_id); diff --git a/src/cpu_data.rs b/src/cpu_data.rs index fc151919..b78795a9 100644 --- a/src/cpu_data.rs +++ b/src/cpu_data.rs @@ -14,12 +14,140 @@ // Authors: // use alloc::sync::Arc; +use core::sync::atomic::AtomicU64; use spin::Mutex; -use crate::arch::cpu::{store_cpu_pointer_to_reg, this_cpu_id, ArchCpu}; +use crate::arch::{ + cpu::{store_cpu_pointer_to_reg, this_cpu_id, ArchCpu}, + ArchVcpu, +}; use crate::consts::{INVALID_ADDRESS, PER_CPU_ARRAY_PTR, PER_CPU_SIZE}; use crate::memory::addr::VirtAddr; use crate::zone::Zone; + +const GUEST_IRQ_WORD_BITS: usize = u64::BITS as usize; +const GUEST_IRQ_WORDS: usize = (crate::config::CONFIG_MAX_INTERRUPTS + 63) / 64; + +/// Pending software IRQ state owned by one guest CPU. +/// +/// Bits are guest INTIDs directly. Architecture-specific LR metadata remains +/// in `ArchVcpu`; this state is the common producer/consumer contract. +pub struct GuestCpuIrqState { + pub pending: [AtomicU64; GUEST_IRQ_WORDS], +} + +impl GuestCpuIrqState { + pub const fn new() -> Self { + const ZERO_WORD: AtomicU64 = AtomicU64::new(0); + Self { + pending: [ZERO_WORD; GUEST_IRQ_WORDS], + } + } + + pub fn set(&self, irq_id: usize) -> bool { + if irq_id >= crate::config::CONFIG_MAX_INTERRUPTS { + return false; + } + self.pending[irq_id / GUEST_IRQ_WORD_BITS].fetch_or( + 1u64 << (irq_id % GUEST_IRQ_WORD_BITS), + core::sync::atomic::Ordering::Release, + ); + true + } + + pub fn clear(&self, irq_id: usize) -> bool { + if irq_id >= crate::config::CONFIG_MAX_INTERRUPTS { + return false; + } + self.pending[irq_id / GUEST_IRQ_WORD_BITS].fetch_and( + !(1u64 << (irq_id % GUEST_IRQ_WORD_BITS)), + core::sync::atomic::Ordering::AcqRel, + ); + true + } + + pub fn clear_all(&self) { + for word in &self.pending { + word.store(0, core::sync::atomic::Ordering::Release); + } + } + + pub fn pending_mask(&self, first_irq: usize, count: usize) -> u64 { + if count == 0 || count > GUEST_IRQ_WORD_BITS { + return 0; + } + let word = first_irq / GUEST_IRQ_WORD_BITS; + let offset = first_irq % GUEST_IRQ_WORD_BITS; + let mut value = self + .pending + .get(word) + .map(|pending| pending.load(core::sync::atomic::Ordering::Acquire) >> offset) + .unwrap_or(0); + if offset != 0 { + value |= self + .pending + .get(word + 1) + .map(|pending| { + pending.load(core::sync::atomic::Ordering::Acquire) + << (GUEST_IRQ_WORD_BITS - offset) + }) + .unwrap_or(0); + } + if count == GUEST_IRQ_WORD_BITS { + value + } else { + value & ((1u64 << count) - 1) + } + } + + /// Atomically claim the lowest pending INTID in one bitmap word. + pub fn claim_pending_intid(&self, word_index: usize) -> Option { + let word = self.pending.get(word_index)?; + loop { + let old = word.load(core::sync::atomic::Ordering::Acquire); + if old == 0 { + return None; + } + let bit = old.trailing_zeros() as usize; + let mask = 1u64 << bit; + if word + .compare_exchange( + old, + old & !mask, + core::sync::atomic::Ordering::AcqRel, + core::sync::atomic::Ordering::Acquire, + ) + .is_ok() + { + return Some(word_index * GUEST_IRQ_WORD_BITS + bit); + } + } + } + + pub fn has_pending(&self) -> bool { + self.pending + .iter() + .any(|word| word.load(core::sync::atomic::Ordering::Acquire) != 0) + } + + /// Claim pending INTIDs and let the controller adapter install each one. + /// + /// The adapter owns INTID classes and LR encoding. Returning `false` means + /// the adapter could not install the claimed INTID, so this helper puts it + /// back and reports that maintenance must remain armed. + pub fn flush_pending_with(&self, mut install: impl FnMut(usize) -> bool) -> bool { + for word_index in 0..self.pending.len() { + while let Some(irq_id) = self.claim_pending_intid(word_index) { + if install(irq_id) { + continue; + } + self.set(irq_id); + return true; + } + } + false + } +} use crate::ENTERED_CPUS; use core::fmt::{Debug, Formatter, Result}; use core::sync::atomic::{AtomicU8, Ordering}; @@ -100,12 +228,37 @@ impl Debug for VcpuStateCell { } } +/// vCPU state temporarily colocated with its pinned host pCPU. +/// +/// This object, including its architecture-private `arch` member, is the unit +/// that can later move out of `PerCpu` when fixed 1:N vCPU scheduling returns. +pub struct Vcpu { + pub irq: GuestCpuIrqState, + pub arch: ArchVcpu, +} + +impl Vcpu { + pub const fn new() -> Self { + Self { + irq: GuestCpuIrqState::new(), + arch: ArchVcpu::new(), + } + } + + pub fn clear_irq_state(&self) { + self.irq.clear_all(); + self.arch.clear_irq_state(); + } +} + #[repr(C)] pub struct PerCpu { pub id: usize, pub cpu_on_entry: usize, pub dtb_ipa: usize, pub vcpu_state: VcpuStateCell, + /// Current pCPU:vCPU=1:1 temporary colocation; move out for fixed 1:N. + pub vcpu: Vcpu, pub arch_cpu: ArchCpu, pub zone: Option>, pub ctrl_lock: Mutex<()>, @@ -124,6 +277,7 @@ impl PerCpu { cpu_on_entry: INVALID_ADDRESS, dtb_ipa: INVALID_ADDRESS, vcpu_state: VcpuStateCell::new(VcpuState::Stopped), + vcpu: Vcpu::new(), arch_cpu, zone: None, ctrl_lock: Mutex::new(()), @@ -179,7 +333,6 @@ pub fn this_cpu_data<'a>() -> &'a mut PerCpu { pub fn this_zone() -> Arc { this_cpu_data().zone.clone().unwrap() } - /// Enter blocked state and wait until another CPU resumes it. #[allow(unused)] pub fn vcpu_suspend() { diff --git a/src/hypercall/mod.rs b/src/hypercall/mod.rs index 36587d3d..5c8979ee 100644 --- a/src/hypercall/mod.rs +++ b/src/hypercall/mod.rs @@ -363,6 +363,7 @@ impl<'a> HyperCall<'a> { zone_w.cpu_set().iter().for_each(|cpu_id| { let _lock = get_cpu_data(cpu_id).ctrl_lock.lock(); + get_cpu_data(cpu_id).vcpu.clear_irq_state(); get_cpu_data(cpu_id).zone = None; }); From 6d411fced1b792f5e042d72518138b19d634f4ce Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Tue, 13 Oct 2026 21:36:28 -0400 Subject: [PATCH 04/12] refactor(aarch64): post virtual SGIs through vCPU state Route GICv2 and GICv3 guest SGIs through per-vCPU pending state, host-doorbell dispatch, and local LR flush. --- src/arch/aarch64/cpu.rs | 1 + src/arch/aarch64/ipi.rs | 105 +++++++++++++++++ src/arch/aarch64/trap.rs | 10 +- src/arch/loongarch64/trap.rs | 18 +-- src/arch/riscv64/trap.rs | 6 +- src/arch/x86_64/ipi.rs | 8 +- src/device/irqchip/gicv2/gic.rs | 186 +++++++++++++++++++++++++++++-- src/device/irqchip/gicv2/gich.rs | 1 + src/device/irqchip/gicv2/vgic.rs | 171 ++++++++++++++++++++++++++-- src/device/irqchip/gicv3/mod.rs | 137 ++++++++++++++++++++--- src/device/irqchip/gicv3/vgic.rs | 29 +++++ src/device/irqchip/mod.rs | 75 +++++++++++++ src/event.rs | 13 +++ 13 files changed, 700 insertions(+), 60 deletions(-) diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 8c3a1c81..6b1ad01a 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -205,6 +205,7 @@ impl ArchCpu { self.cpuid, this_cpu_data().cpu_on_entry ); + crate::device::irqchip::flush_current_guest_irqs(); unsafe { vmreturn(self.guest_reg() as *mut _ as usize); } diff --git a/src/arch/aarch64/ipi.rs b/src/arch/aarch64/ipi.rs index 1904e61a..57fb1441 100644 --- a/src/arch/aarch64/ipi.rs +++ b/src/arch/aarch64/ipi.rs @@ -16,6 +16,63 @@ use crate::device::irqchip::gic_send_event; +#[cfg(irq_gicv2)] +use crate::{ + arch::cpu::this_cpu_id, + cpu_data::this_zone, + device::irqchip::gicv2::gicd::{GICD_SGIR_ROUTING_SHIFT, GICD_SGIR_TARGET_LIST_FILTER_SHIFT}, +}; + +#[cfg(irq_gicv3)] +use crate::{ + arch::cpu::{cpuid_to_mpidr_affinity, this_cpu_id}, + cpu_data::this_zone, +}; + +#[cfg(irq_gicv3)] +const ICC_SGI1R_TARGET_LIST_MASK: u64 = 0xffff; + +#[cfg(irq_gicv3)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Sgi1r { + target_list: u16, + aff1: u8, + intid: usize, + aff2: u8, + irm: bool, + range_selector: u8, + aff3: u8, +} + +#[cfg(irq_gicv3)] +impl Sgi1r { + fn decode(value: u64) -> Self { + Self { + target_list: (value & ICC_SGI1R_TARGET_LIST_MASK) as u16, + aff1: ((value >> 16) & 0xff) as u8, + intid: ((value >> 24) & 0xf) as usize, + aff2: ((value >> 32) & 0xff) as u8, + irm: value & (1 << 40) != 0, + range_selector: ((value >> 44) & 0xf) as u8, + aff3: ((value >> 48) & 0xff) as u8, + } + } + + fn targets(self, cpu_id: usize, source_cpu: usize) -> bool { + if self.irm { + return cpu_id != source_cpu; + } + let (aff3, aff2, aff1, aff0) = cpuid_to_mpidr_affinity(cpu_id as u64); + let target_aff0 = self.range_selector as u64 * 16; + aff3 == self.aff3 as u64 + && aff2 == self.aff2 as u64 + && aff1 == self.aff1 as u64 + && aff0 >= target_aff0 + && aff0 < target_aff0 + 16 + && self.target_list & (1 << (aff0 - target_aff0)) != 0 + } +} + pub fn arch_send_event(cpu_id: u64, sgi_num: u64) { gic_send_event(cpu_id, sgi_num); } @@ -24,6 +81,54 @@ pub fn arch_notify_event(cpu_id: u64, sgi_num: u64, _event_id: usize, _queue_was arch_send_event(cpu_id, sgi_num); } +#[cfg(irq_gicv3)] +pub fn handle_guest_sgi1r(value: u64) { + let sgi = Sgi1r::decode(value); + let source_cpu = this_cpu_id(); + for target_cpu in this_zone().cpu_set().iter() { + if sgi.targets(target_cpu, source_cpu) { + if !crate::device::irqchip::post_guest_irq(target_cpu, sgi.intid, None) { + error!( + "failed to post guest SGI {} to CPU{}", + sgi.intid, target_cpu + ); + } + } + } +} + +#[cfg(irq_gicv2)] +pub fn handle_guest_sgi1r(value: u64) { + let target_list = (value & 0xffff) as usize; + let routing_mode = if value & (1 << 40) != 0 { 1 } else { 0 }; + let irq_id = ((value >> 24) & 0xf) as usize; + handle_guest_sgir( + irq_id as u32 + | ((target_list as u32) << GICD_SGIR_TARGET_LIST_FILTER_SHIFT) + | ((routing_mode as u32) << GICD_SGIR_ROUTING_SHIFT), + ); +} + +#[cfg(irq_gicv2)] +pub fn handle_guest_sgir(value: u32) { + let source_cpu = this_cpu_id(); + let irq_id = (value & 0xf) as usize; + let target_list = ((value >> GICD_SGIR_TARGET_LIST_FILTER_SHIFT) & 0xff) as usize; + let routing_mode = (value >> GICD_SGIR_ROUTING_SHIFT) & 0x3; + for target_cpu in this_zone().cpu_set().iter() { + let selected = match routing_mode { + 0 => target_list & (1 << target_cpu) != 0, + 1 => target_cpu != source_cpu, + 2 => target_cpu == source_cpu, + _ => false, + }; + if selected && !crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(source_cpu)) + { + warn!("failed to post guest SGI {} to CPU{}", irq_id, target_cpu); + } + } +} + pub fn arch_check_events(event: Option) { match event { _ => { diff --git a/src/arch/aarch64/trap.rs b/src/arch/aarch64/trap.rs index 5fe37f13..bc2b3b29 100644 --- a/src/arch/aarch64/trap.rs +++ b/src/arch/aarch64/trap.rs @@ -21,10 +21,7 @@ use super::cpu::GeneralRegisters; use crate::arch::sysreg::smc_call; use crate::zone::zone_error; use crate::{ - arch::{ - cpu::mpidr_to_cpuid, - sysreg::{read_sysreg, write_sysreg}, - }, + arch::{cpu::mpidr_to_cpuid, sysreg::read_sysreg}, cpu_data::{get_cpu_data, this_cpu_data, this_zone, VcpuState}, device::irqchip::gic_handle_irq, event::{send_event, IPI_EVENT_SHUTDOWN, IPI_EVENT_WAKEUP}, @@ -122,6 +119,7 @@ pub fn arch_handle_exit(regs: &mut GeneralRegisters) -> ! { ExceptionType::EXIT_REASON_EL2_IRQ => irqchip_handle_irq2(), _ => arch_dump_exit(regs.exit_reason), } + crate::device::irqchip::flush_current_guest_irqs(); unsafe { vmreturn(regs as *const _ as usize) } } @@ -344,8 +342,8 @@ fn handle_sysreg(regs: &mut GeneralRegisters) { if !this_cpu_data().vcpu_state.is_running() { trace!("skip send sgi {:#x?}", sgi_id); } else { - trace!("send sgi {:#x?}", sgi_id); - write_sysreg!(icc_sgi1r_el1, val); + trace!("post guest sgi {:#x?}", sgi_id); + crate::arch::ipi::handle_guest_sgi1r(val); } arch_skip_instruction(regs); //skip sgi write } else { diff --git a/src/arch/loongarch64/trap.rs b/src/arch/loongarch64/trap.rs index 5e0591ab..25112cbc 100644 --- a/src/arch/loongarch64/trap.rs +++ b/src/arch/loongarch64/trap.rs @@ -23,7 +23,7 @@ use crate::consts::MAX_CPU_NUM; use crate::cpu_data::this_cpu_data; use crate::device::irqchip::inject_irq; use crate::device::irqchip::ls7a2000::chip::*; -use crate::event::{dump_events, handle_next_event, has_pending_events}; +use crate::event::{dispatch_host_events, dump_events, handle_next_event, has_pending_events}; use crate::hypercall::{SGI_IPI_ID, *}; use crate::memory::{addr, mmio_handle_access, MMIOAccess}; use crate::zone::Zone; @@ -1302,15 +1302,17 @@ fn handle_interrupt(is: usize) { } if ipi_status & event_doorbell != 0 { - // Clear before each fetch. If the fetch observes an empty queue while a - // producer enqueues concurrently, its doorbell remains pending and - // re-fires. Clearing after the fetch could lose that coalesced wakeup. + // Clear the observed host doorbell before checking logical state. + // A concurrent producer is either drained by the dispatcher or + // leaves/re-arms a later notification. clear_ipi_bits(event_doorbell); - if has_pending_events(cpu_id) { - while handle_next_event() { - clear_ipi_bits(event_doorbell); + dispatch_host_events(|| { + if has_pending_events(cpu_id) { + while handle_next_event() { + clear_ipi_bits(event_doorbell); + } } - } + }); } let handled_mask = event_doorbell | virtual_ipi_doorbell; diff --git a/src/arch/riscv64/trap.rs b/src/arch/riscv64/trap.rs index 4c8de184..c87bd02b 100644 --- a/src/arch/riscv64/trap.rs +++ b/src/arch/riscv64/trap.rs @@ -17,7 +17,7 @@ use super::cpu::ArchCpu; use crate::arch::sbi::sbi_vs_handler; #[cfg(plic)] use crate::device::irqchip::plic::{inject_irq, plic_get_hwirq}; -use crate::event::check_events; +use crate::event::{check_events, dispatch_host_events}; use crate::memory::GuestPhysAddr; use crate::memory::{mmio_handle_access, MMIOAccess}; use core::arch::{asm, global_asm}; @@ -407,12 +407,10 @@ pub fn handle_timer_interrupt(_current_cpu: &mut ArchCpu) { /// Handle supervisor software interrupt. pub fn handle_software_interrupt(_current_cpu: &mut ArchCpu) { - while check_events() { - // Get next event to handle, it is handled in check_events function. - } unsafe { riscv::register::sip::clear_ssoft(); } + dispatch_host_events(|| while check_events() {}); } /// Handle supervisor external interrupt. diff --git a/src/arch/x86_64/ipi.rs b/src/arch/x86_64/ipi.rs index 67a1588d..d99da9ca 100644 --- a/src/arch/x86_64/ipi.rs +++ b/src/arch/x86_64/ipi.rs @@ -149,13 +149,7 @@ pub fn arch_notify_event(dest: u64, vector: u64, _event_id: usize, _queue_was_em } pub fn handle_virt_ipi() { - // this may never return! - loop { - let ret = event::check_events(); - if !ret { - break; - } - } + event::dispatch_host_events(|| while event::check_events() {}); } pub fn arch_check_events(event: Option) { diff --git a/src/device/irqchip/gicv2/gic.rs b/src/device/irqchip/gicv2/gic.rs index d5e2b6a0..48e4784f 100644 --- a/src/device/irqchip/gicv2/gic.rs +++ b/src/device/irqchip/gicv2/gic.rs @@ -14,13 +14,14 @@ // Authors: // Hangqi Ren <2572131118@qq.com> use crate::arch::cpu::this_cpu_id; +use crate::cpu_data::this_cpu_data; use crate::device::irqchip::gicv2::gicc::GICC; use crate::device::irqchip::gicv2::gicd::GICV2_SGIS_NUM; use crate::device::irqchip::gicv2::gich::{ - GICH, GICV2_GICH_HCR_UIE, GICV2_GICH_LR_CPUID_SHIFT, GICV2_GICH_LR_HW, - GICV2_GICH_LR_PENDING_STATE, GICV2_GICH_LR_PHYSID_SHIFT, + GICH, GICV2_GICH_HCR_UIE, GICV2_GICH_LR_ACTIVE_STATE, GICV2_GICH_LR_CPUID_SHIFT, + GICV2_GICH_LR_HW, GICV2_GICH_LR_PENDING_STATE, GICV2_GICH_LR_PHYSID_SHIFT, }; -use crate::event::check_events; +use crate::event::{check_events, dispatch_host_events}; use crate::hypercall::SGI_IPI_ID; /// This file defines and implements the functional functions of physical gicv2. /// author: ForeverYolo @@ -36,17 +37,13 @@ pub const MAINTENACE_INTERRUPT: u64 = 25; pub fn gicv2_handle_irq() { if let Some(irq_id) = get_pending_irq() { - if irq_id < 8 { + if irq_id < GICV2_SGIS_NUM { deactivate_irq(irq_id); - let mut ipi_handled = false; if irq_id == SGI_IPI_ID as _ { - ipi_handled = check_events(); + dispatch_host_events(|| while check_events() {}); + } else { + warn!("unexpected physical SGI {}", irq_id); } - if !ipi_handled { - inject_irq(irq_id, true); - } - } else if irq_id < GICV2_SGIS_NUM { - deactivate_irq(irq_id); } else if irq_id == MAINTENACE_INTERRUPT as usize { handle_maintenace_interrupt(); } else { @@ -88,6 +85,7 @@ pub fn change_underflow_maintenance(is_enable: bool) { fn handle_maintenace_interrupt() { info!("handle_maintenace_interrupt"); + flush_current_guest_irqs(); while let Some((irq_id, is_sgi)) = PENDING_VIRQS.get().unwrap().fetch_irq() { let is_injected: bool = inject_irq(irq_id, is_sgi); if is_injected { @@ -99,7 +97,164 @@ fn handle_maintenace_interrupt() { return; } } - change_underflow_maintenance(false); + change_underflow_maintenance(has_pending_maintenance_work()); +} + +fn has_pending_maintenance_work() -> bool { + this_cpu_data().vcpu.irq.has_pending() || PENDING_VIRQS.get().unwrap().has_pending() +} + +const VINTID_MASK: usize = 0x3ff; + +fn try_install_software_sgi(irq_id: usize, source_cpu: usize) -> bool { + let gich = GICH.get().unwrap(); + let elrsr = (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64; + let lr_num = gich.get_lrs_num() as usize; + let mut free_lr = None; + for index in 0..lr_num { + if elrsr & (1u64 << index) != 0 { + free_lr.get_or_insert(index); + continue; + } + let lr = gich.get_lr(index) as usize; + let lr_source = (lr >> GICV2_GICH_LR_CPUID_SHIFT) & 0x7; + if lr & VINTID_MASK != irq_id || lr_source != source_cpu { + continue; + } + if lr & GICV2_GICH_LR_PENDING_STATE != 0 { + return true; + } + gich.set_lr( + index, + (lr | GICV2_GICH_LR_PENDING_STATE | (source_cpu << GICV2_GICH_LR_CPUID_SHIFT)) as u32, + ); + return true; + } + let Some(index) = free_lr else { + return false; + }; + gich.set_lr( + index, + (irq_id | GICV2_GICH_LR_PENDING_STATE | (source_cpu << GICV2_GICH_LR_CPUID_SHIFT)) as u32, + ); + true +} + +fn install_pending_guest_irq(cpu: &crate::cpu_data::PerCpu, irq_id: usize) -> bool { + if irq_id >= GICV2_SGIS_NUM { + return false; + } + let source_mask = cpu.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire); + let source_cpu = source_mask.trailing_zeros() as usize; + if source_cpu >= 8 { + return false; + } + cpu.vcpu.arch.sgi_source[irq_id] + .fetch_and(!(1u8 << source_cpu), core::sync::atomic::Ordering::AcqRel); + if !try_install_software_sgi(irq_id, source_cpu) { + cpu.vcpu.arch.sgi_source[irq_id] + .fetch_or(1u8 << source_cpu, core::sync::atomic::Ordering::AcqRel); + return false; + } + if cpu.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } + true +} + +pub fn flush_current_guest_irqs() { + let cpu = this_cpu_data(); + if cpu.zone.is_none() || cpu.vcpu_state.is_stopped() { + return; + } + let state = &cpu.vcpu.irq; + crate::device::irqchip::flush_guest_irq_state( + state, + |irq_id| install_pending_guest_irq(cpu, irq_id), + || change_underflow_maintenance(true), + ); +} + +pub fn clear_guest_sgi_pending(target_cpu: usize, irq_id: usize) -> bool { + clear_guest_sgi_pending_sources(target_cpu, irq_id, u8::MAX) +} + +pub fn clear_guest_sgi_pending_sources(target_cpu: usize, irq_id: usize, source_mask: u8) -> bool { + if irq_id >= GICV2_SGIS_NUM { + return false; + } + let target = crate::cpu_data::get_cpu_data(target_cpu); + let control = target.ctrl_lock.lock(); + target.vcpu.arch.sgi_source[irq_id] + .fetch_and(!source_mask, core::sync::atomic::Ordering::AcqRel); + let pending_cleared = + target.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire) == 0 + && target.vcpu.irq.clear(irq_id); + drop(control); + if target_cpu == this_cpu_id() { + clear_lr_pending_sgi(irq_id, source_mask); + } + pending_cleared +} + +fn clear_lr_pending_sgi(irq_id: usize, source_mask: u8) { + let gich = GICH.get().unwrap(); + let elrsr = (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64; + for index in 0..gich.get_lrs_num() as usize { + if elrsr & (1u64 << index) != 0 { + continue; + } + let lr = gich.get_lr(index) as usize; + let source = (lr >> GICV2_GICH_LR_CPUID_SHIFT) & 0x7; + if lr & VINTID_MASK != irq_id + || lr & GICV2_GICH_LR_PENDING_STATE == 0 + || source_mask & (1 << source) == 0 + { + continue; + } + if lr & GICV2_GICH_LR_ACTIVE_STATE != 0 { + gich.set_lr(index, (lr & !GICV2_GICH_LR_PENDING_STATE) as u32); + } else { + gich.set_lr(index, 0); + } + } +} + +pub(crate) struct PendingSgiLrSummary { + pub pending_mask: u32, + pub source_masks: [u8; GICV2_SGIS_NUM], +} + +pub(crate) fn read_pending_sgi_lr_summary() -> PendingSgiLrSummary { + let gich = GICH.get().unwrap(); + let elrsr = (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64; + let mut summary = PendingSgiLrSummary { + pending_mask: 0, + source_masks: [0; GICV2_SGIS_NUM], + }; + for index in 0..gich.get_lrs_num() as usize { + if elrsr & (1u64 << index) != 0 { + continue; + } + let lr = gich.get_lr(index) as usize; + let irq_id = lr & VINTID_MASK; + if irq_id < GICV2_SGIS_NUM && lr & GICV2_GICH_LR_PENDING_STATE != 0 { + summary.pending_mask |= 1 << irq_id; + summary.source_masks[irq_id] |= 1 << ((lr >> GICV2_GICH_LR_CPUID_SHIFT) & 0x7); + } + } + summary +} + +pub fn read_lr_pending_sgi_source_mask(irq_id: usize) -> u8 { + if irq_id >= GICV2_SGIS_NUM { + return 0; + } + read_pending_sgi_lr_summary().source_masks[irq_id] +} + +pub fn read_lr_pending_sgi_mask() -> u32 { + read_pending_sgi_lr_summary().pending_mask } pub fn inject_irq(irq_id: usize, is_sgi: bool) -> bool { @@ -186,4 +341,11 @@ impl PendingIrqs { _ => None, } } + + fn has_pending(&self) -> bool { + self.inner + .get(this_cpu_id()) + .map(|pending_irqs| !pending_irqs.lock().is_empty()) + .unwrap_or(false) + } } diff --git a/src/device/irqchip/gicv2/gich.rs b/src/device/irqchip/gicv2/gich.rs index 982ff7ad..3180df39 100644 --- a/src/device/irqchip/gicv2/gich.rs +++ b/src/device/irqchip/gicv2/gich.rs @@ -30,6 +30,7 @@ pub const GICV2_GICH_VMCR_VMGRP1EN: u32 = 0x1 << 1; pub const GICV2_GICH_VMCR_VMGRP0EN: u32 = 0x1; pub const GICV2_GICH_LR_GRP1: u32 = 0x1 << 30; pub const GICV2_GICH_LR_PENDING_STATE: usize = 0x1 << 28; +pub const GICV2_GICH_LR_ACTIVE_STATE: usize = 0x1 << 29; pub const GICV2_GICH_LR_HW: usize = 0x1 << 31; pub const GICV2_GICH_HCR_REG_OFFSET: usize = 0x0000; pub const GICV2_GICH_VTR_REG_OFFSET: usize = 0x0004; diff --git a/src/device/irqchip/gicv2/vgic.rs b/src/device/irqchip/gicv2/vgic.rs index 5016c330..640b4219 100644 --- a/src/device/irqchip/gicv2/vgic.rs +++ b/src/device/irqchip/gicv2/vgic.rs @@ -13,17 +13,19 @@ // // Authors: // Hangqi Ren <2572131118@qq.com> +use crate::arch::cpu::this_cpu_id; use crate::arch::zone::{GicConfig, Gicv2Config, HvArchZoneConfig}; use crate::config::{BitmapWord, CONFIG_INTERRUPTS_BITMAP_BITS_PER_WORD}; -use crate::cpu_data::this_zone; +use crate::cpu_data::{get_cpu_data, this_zone}; use crate::device::irqchip::gicv2::gicd::{ - get_max_int_num, GICD, GICD_CTRL_REG_OFFSET, GICD_ICACTIVER_REG_OFFSET, - GICD_ICENABLER_REG_OFFSET, GICD_ICFGR_REG_OFFSET, GICD_ICPENDR_REG_OFFSET, - GICD_IDENTIFICATION_NUM, GICD_IDENTIFICATION_OFFSET, GICD_IGROUPR_REG_OFFSET, - GICD_IIDR_REG_OFFSET, GICD_IPRIORITYR_REG_OFFSET, GICD_ISACTIVER_REG_OFFSET, - GICD_ISENABLER_REG_OFFSET, GICD_ISPENDR_REG_OFFSET, GICD_ITARGETSR_REG_OFFSET, GICD_LOCK, - GICD_SGIR_REG_OFFSET, GICD_SGIR_ROUTING_SHIFT, GICD_SGIR_TARGET_LIST_FILTER_SHIFT, - GICD_TYPER_REG_OFFSET, GICV2_CONFIG_REGS_NUM, GICV2_INT_REGS_NUM, GICV2_PRIO_REGS_NUM, + get_max_int_num, GICD, GICD_CPENDSGIR_REG_OFFSET, GICD_CTRL_REG_OFFSET, + GICD_ICACTIVER_REG_OFFSET, GICD_ICENABLER_REG_OFFSET, GICD_ICFGR_REG_OFFSET, + GICD_ICPENDR_REG_OFFSET, GICD_IDENTIFICATION_NUM, GICD_IDENTIFICATION_OFFSET, + GICD_IGROUPR_REG_OFFSET, GICD_IIDR_REG_OFFSET, GICD_IPRIORITYR_REG_OFFSET, + GICD_ISACTIVER_REG_OFFSET, GICD_ISENABLER_REG_OFFSET, GICD_ISPENDR_REG_OFFSET, + GICD_ITARGETSR_REG_OFFSET, GICD_LOCK, GICD_SGIR_REG_OFFSET, GICD_SGIR_ROUTING_SHIFT, + GICD_SGIR_TARGET_LIST_FILTER_SHIFT, GICD_SPENDSGIR_REG_OFFSET, GICD_TYPER_REG_OFFSET, + GICV2_CONFIG_REGS_NUM, GICV2_INT_REGS_NUM, GICV2_PRIO_REGS_NUM, GICV2_SGI_REGS_NUM, GICV2_TARGET_REGS_NUM, }; use crate::device::irqchip::gicv2::GICV2; @@ -272,6 +274,133 @@ pub fn set_sgi_irq(irq_id: usize, target_list: usize, routing_mode: usize) { GICD.get().unwrap().set_sgir(val as u32); } +fn read_guest_sgi_source_mask(target_cpu: usize, irq_id: usize, lr_source_mask: u8) -> u8 { + let target = get_cpu_data(target_cpu); + let software = if target.vcpu.irq.pending_mask(irq_id, 1) != 0 { + target.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire) + } else { + 0 + }; + software | lr_source_mask +} + +fn handle_gicd_sgi_pending_access(mmio: &mut MMIOAccess, is_clear: bool) -> HvResult { + let target_cpu = this_cpu_id(); + let word_offset = mmio.address & 0x3; + let shift = word_offset * 8; + let width_mask = if mmio.size == 4 { + u32::MAX + } else { + ((1u32 << (mmio.size * 8)) - 1) << shift + }; + if mmio.is_write { + let value = ((mmio.value as u32) << shift) & width_mask; + for byte in 0..4 { + if width_mask & (0xff << (byte * 8)) == 0 { + continue; + } + let source_mask = ((value >> (byte * 8)) & 0xff) as u8; + let base = if is_clear { + GICD_CPENDSGIR_REG_OFFSET + } else { + GICD_SPENDSGIR_REG_OFFSET + }; + let irq_id = ((mmio.address & !3) - base) / 4 * 4 + byte; + if irq_id >= 16 { + continue; + } + if is_clear { + crate::device::irqchip::gicv2::gic::clear_guest_sgi_pending_sources( + target_cpu, + irq_id, + source_mask, + ); + } else { + for source in 0..8 { + if source_mask & (1 << source) != 0 { + crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(source)); + } + } + } + } + return Ok(()); + } + + let register_base = if is_clear { + GICD_CPENDSGIR_REG_OFFSET + } else { + GICD_SPENDSGIR_REG_OFFSET + }; + let word_index = ((mmio.address & !3) - register_base) / 4; + let lr_summary = crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary(); + let mut value = 0u32; + for byte in 0..4 { + let irq_id = word_index * 4 + byte; + value |= (read_guest_sgi_source_mask(target_cpu, irq_id, lr_summary.source_masks[irq_id]) + as u32) + << (byte * 8); + } + mmio.value = ((value & width_mask) >> shift) as usize; + Ok(()) +} + +fn handle_gicd_spendsgir_access(mmio: &mut MMIOAccess) -> HvResult { + handle_gicd_sgi_pending_access(mmio, false) +} + +fn handle_gicd_cpendsgir_access(mmio: &mut MMIOAccess) -> HvResult { + handle_gicd_sgi_pending_access(mmio, true) +} + +fn handle_gicd_intid_pending_access( + mmio: &mut MMIOAccess, + is_clear: bool, + gicd_base: usize, +) -> HvResult { + let target_cpu = this_cpu_id(); + let shift = (mmio.address & 0x3) * 8; + let width_mask = if mmio.size == 4 { + u32::MAX + } else { + ((1u32 << (mmio.size * 8)) - 1) << shift + }; + if mmio.is_write { + let value = ((mmio.value as u32) << shift) & width_mask; + for irq_id in 0..16 { + if value & (1 << irq_id) == 0 { + continue; + } + if is_clear { + crate::device::irqchip::gicv2::gic::clear_guest_sgi_pending(target_cpu, irq_id); + } else { + crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(target_cpu)); + } + } + if mmio.size == 4 && shift == 0 { + mmio.value = (value & !0xffff) as usize; + mmio_perform_access(gicd_base, mmio); + } else if shift >= 16 { + mmio_perform_access(gicd_base, mmio); + } + return Ok(()); + } + mmio_perform_access(gicd_base, mmio); + let physical_value = ((mmio.value as u32) << shift) & width_mask; + let lr_pending = crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary(); + let mut value = get_cpu_data(target_cpu).vcpu.irq.pending_mask(0, 16) as u32; + value |= lr_pending.pending_mask; + mmio.value = ((((physical_value & !0xffff) | value) & width_mask) >> shift) as usize; + Ok(()) +} + +fn handle_gicd_ispendr_access(mmio: &mut MMIOAccess, gicd_base: usize) -> HvResult { + handle_gicd_intid_pending_access(mmio, false, gicd_base) +} + +fn handle_gicd_icpendr_access(mmio: &mut MMIOAccess, gicd_base: usize) -> HvResult { + handle_gicd_intid_pending_access(mmio, true, gicd_base) +} + // Handle GIC Distributor register accesses. pub fn vgicv2_dist_handler(mmio: &mut MMIOAccess, _arg: usize) -> HvResult { let gicd_base = GICV2.get().unwrap().gicd_base; @@ -282,9 +411,33 @@ pub fn vgicv2_dist_handler(mmio: &mut MMIOAccess, _arg: usize) -> HvResult { if !mmio.is_write { return Ok(()); } - mmio_perform_access(gicd_base, mmio); + crate::arch::ipi::handle_guest_sgir(mmio.value as u32); Ok(()) } + reg if reg_range(GICD_ISPENDR_REG_OFFSET, 1, GICV2_REG_WIDTH).contains(®) => { + handle_gicd_ispendr_access(mmio, gicd_base) + } + reg if reg_range(GICD_ICPENDR_REG_OFFSET, 1, GICV2_REG_WIDTH).contains(®) => { + handle_gicd_icpendr_access(mmio, gicd_base) + } + reg if reg_range( + GICD_CPENDSGIR_REG_OFFSET, + GICV2_SGI_REGS_NUM, + GICV2_REG_WIDTH, + ) + .contains(®) => + { + handle_gicd_cpendsgir_access(mmio) + } + reg if reg_range( + GICD_SPENDSGIR_REG_OFFSET, + GICV2_SGI_REGS_NUM, + GICV2_REG_WIDTH, + ) + .contains(®) => + { + handle_gicd_spendsgir_access(mmio) + } reg if reg_range( GICD_ITARGETSR_REG_OFFSET, GICV2_TARGET_REGS_NUM, diff --git a/src/device/irqchip/gicv3/mod.rs b/src/device/irqchip/gicv3/mod.rs index 4767e678..97e41f47 100644 --- a/src/device/irqchip/gicv3/mod.rs +++ b/src/device/irqchip/gicv3/mod.rs @@ -38,9 +38,10 @@ use crate::arch::cpu::this_cpu_id; use crate::arch::zone::GicConfig; use crate::config::root_zone_config; use crate::consts::{self, MAX_CPU_NUM}; +use crate::cpu_data::this_cpu_data; use crate::device::irqchip::gicv3::gits::gits_reset; -use crate::event::check_events; +use crate::event::{check_events, dispatch_host_events}; use crate::hypercall::SGI_IPI_ID; use crate::zone::Zone; @@ -96,20 +97,15 @@ const TIMER_INTERRUPT_PRINT_INTERVAL: u64 = 50; pub fn gicv3_handle_irq_el1() { while let Some(irq_id) = pending_irq() { - if irq_id < 8 { - trace!("sgi get {}, try to handle...", irq_id); + if irq_id < 16 { deactivate_irq(irq_id); - let mut ipi_handled = false; - if irq_id == SGI_IPI_ID as _ { - ipi_handled = check_events(); - } - if !ipi_handled { - trace!("sgi get {}, inject", irq_id); - inject_irq(irq_id, false); + if irq_id == SGI_IPI_ID as usize { + // SGI 7 is the reserved physical host doorbell. Guest SGIs, + // including guest SGI 7, are posted through software state. + dispatch_host_events(|| while check_events() {}); + } else { + warn!("unexpected physical SGI {}", irq_id); } - } else if irq_id < 16 { - warn!("skip sgi {}", irq_id); - deactivate_irq(irq_id); } else { #[cfg(all(dwc_pcie, dwc_msi))] let mut is_dwc_msi_irq = false; @@ -162,6 +158,107 @@ pub fn gicv3_handle_irq_el1() { trace!("handle done") } +const LR_VINTID_MASK: u64 = (1 << 32) - 1; +const LR_PENDING: u64 = 1 << 62; +const LR_ACTIVE: u64 = 1 << 63; +const LR_STATE_MASK: u64 = LR_PENDING | LR_ACTIVE; + +pub fn clear_guest_sgi(target_cpu: usize, irq_id: usize) -> bool { + if irq_id >= 16 { + return false; + } + let target = crate::cpu_data::get_cpu_data(target_cpu); + let control = target.ctrl_lock.lock(); + if !target.vcpu.irq.clear(irq_id) { + return false; + } + drop(control); + if target_cpu == this_cpu_id() { + clear_lr_pending_irq(irq_id); + } + true +} + +fn try_install_software_sgi(irq_id: usize) -> bool { + let elsr = read_sysreg!(ich_elrsr_el2); + let lr_num = (read_sysreg!(ich_vtr_el2) as usize & 0xf) + 1; + let mut free_lr = None; + for index in 0..lr_num { + if (1 << index) & elsr != 0 { + free_lr.get_or_insert(index); + continue; + } + let lr = read_lr(index); + if lr & LR_VINTID_MASK != irq_id as u64 { + continue; + } + let state = lr & LR_STATE_MASK; + if state == 0 || state == LR_ACTIVE { + write_lr(index, lr | LR_PENDING); + } + return true; + } + let Some(index) = free_lr else { + return false; + }; + write_lr(index, irq_id as u64 | (1 << 60) | LR_PENDING); + true +} + +pub(crate) fn clear_lr_pending_irq(irq_id: usize) { + let elsr = read_sysreg!(ich_elrsr_el2); + let lr_num = (read_sysreg!(ich_vtr_el2) as usize & 0xf) + 1; + for index in 0..lr_num { + if (1 << index) & elsr != 0 { + continue; + } + let lr = read_lr(index); + if lr & LR_VINTID_MASK != irq_id as u64 || lr & LR_PENDING == 0 { + continue; + } + if lr & LR_ACTIVE != 0 { + write_lr(index, lr & !LR_PENDING); + } else { + write_lr(index, 0); + } + } +} + +pub(crate) fn lr_pending_sgi_mask() -> u32 { + let elsr = read_sysreg!(ich_elrsr_el2); + let lr_num = (read_sysreg!(ich_vtr_el2) as usize & 0xf) + 1; + let mut pending = 0; + for index in 0..lr_num { + if (1 << index) & elsr != 0 { + continue; + } + let lr = read_lr(index); + let irq_id = (lr & LR_VINTID_MASK) as usize; + if irq_id < 16 && lr & LR_PENDING != 0 { + pending |= 1 << irq_id; + } + } + pending +} + +fn install_pending_guest_irq(irq_id: usize) -> bool { + if irq_id >= 16 { + return false; + } + try_install_software_sgi(irq_id) +} + +pub fn flush_current_guest_irqs() { + let cpu = this_cpu_data(); + if cpu.zone.is_none() || cpu.vcpu_state.is_stopped() { + return; + } + let state = &cpu.vcpu.irq; + crate::device::irqchip::flush_guest_irq_state(state, install_pending_guest_irq, || { + enable_maintenace_interrupt(true) + }); +} + fn pending_irq() -> Option { let iar = read_sysreg!(icc_iar1_el1) as usize; if iar == 0x3ff { @@ -268,6 +365,13 @@ impl PendingIrqs { _ => None, } } + + fn has_pending(&self) -> bool { + self.inner + .get(this_cpu_id()) + .map(|pending_irqs| !pending_irqs.lock().is_empty()) + .unwrap_or(false) + } } // Enable or disable an underflow maintenace interrupt. @@ -285,6 +389,7 @@ fn enable_maintenace_interrupt(is_enable: bool) { fn handle_maintenace_interrupt() { trace!("handle_maintenace_interrupt"); + flush_current_guest_irqs(); let pending_irqs = PENDING_VIRQS.get().unwrap(); while let Some((irq_id, is_hardware)) = pending_irqs.fetch_irq() { let is_injected: bool = inject_irq(irq_id, is_hardware); @@ -297,7 +402,11 @@ fn handle_maintenace_interrupt() { return; } } - enable_maintenace_interrupt(false); + enable_maintenace_interrupt(has_pending_maintenance_work()); +} + +fn has_pending_maintenance_work() -> bool { + this_cpu_data().vcpu.irq.has_pending() || PENDING_VIRQS.get().unwrap().has_pending() } /// Inject virtual interrupt to vCPU, return whether it not needs to add pending queue. diff --git a/src/device/irqchip/gicv3/vgic.rs b/src/device/irqchip/gicv3/vgic.rs index b20cf2b7..e877196b 100644 --- a/src/device/irqchip/gicv3/vgic.rs +++ b/src/device/irqchip/gicv3/vgic.rs @@ -271,6 +271,35 @@ pub fn vgicv3_redist_handler(mmio: &mut MMIOAccess, cpu: usize) -> HvResult { mmio.value &= !(1 << MAINTENACE_INTERRUPT); mmio.value &= !(1 << SGI_IPI_ID); } + + let is_sgi_set_pending = reg == GICR_SGI_BASE + GICR_ISPENDR; + let is_sgi_clear_pending = reg == GICR_SGI_BASE + GICR_ICPENDR; + if is_sgi_set_pending || is_sgi_clear_pending { + if mmio.is_write { + let mask = mmio.value as u32 & 0xffff; + for irq_id in 0..16 { + if mask & (1 << irq_id) == 0 { + continue; + } + if is_sgi_set_pending { + crate::device::irqchip::post_guest_irq(cpu, irq_id, None); + } else { + super::clear_guest_sgi(cpu, irq_id); + } + } + // Preserve physical PPI writes in bits 16..31 only. + mmio.value &= !0xffff; + mmio_perform_access(gicr_base, mmio); + } else { + mmio_perform_access(gicr_base, mmio); + let mut value = get_cpu_data(cpu).vcpu.irq.pending_mask(0, 16) as u32; + if cpu == crate::arch::cpu::this_cpu_id() { + value |= super::lr_pending_sgi_mask(); + } + mmio.value = (mmio.value & !0xffff) | value as usize; + } + return HvResult::Ok(()); + } // ignore access to foreign redistributors mmio_perform_access(gicr_base, mmio); } else { diff --git a/src/device/irqchip/mod.rs b/src/device/irqchip/mod.rs index baa70774..f3b86c8d 100644 --- a/src/device/irqchip/mod.rs +++ b/src/device/irqchip/mod.rs @@ -15,6 +15,8 @@ // use crate::arch::zone::HvArchZoneConfig; use crate::config::HvZoneConfig; +use crate::cpu_data::{get_cpu_data, this_cpu_data, GuestCpuIrqState}; +use crate::hypercall::SGI_IPI_ID; use crate::zone::Zone; #[cfg(target_arch = "loongarch64")] @@ -27,6 +29,79 @@ pub fn handle_virtio_irq_event() { crate::device::virtio_trampoline::handle_virtio_irq(); } +/// Synchronize posted IRQ state owned by the current vCPU. +/// +#[inline] +pub fn flush_current_guest_irqs() { + #[cfg(all(irq_gicv2, target_arch = "aarch64"))] + gicv2::gic::flush_current_guest_irqs(); + #[cfg(all(irq_gicv3, target_arch = "aarch64"))] + gicv3::flush_current_guest_irqs(); +} + +/// Run the controller-independent pending claim/retry flow. +pub fn flush_guest_irq_state( + state: &GuestCpuIrqState, + install: impl FnMut(usize) -> bool, + rearm_maintenance: impl FnOnce(), +) { + if state.flush_pending_with(install) { + rearm_maintenance(); + } +} + +/// Publish a guest software IRQ in the target guest CPU's common pending +/// bitmap and kick its fixed home pCPU. `source_cpu` is used only by GICv2 to +/// reconstruct the SGI source field in the virtual LR. +pub fn post_guest_irq(target_cpu: usize, irq_id: usize, source_cpu: Option) -> bool { + if irq_id >= crate::config::CONFIG_MAX_INTERRUPTS { + return false; + } + let target = get_cpu_data(target_cpu); + #[cfg(not(irq_gicv2))] + let _ = source_cpu; + let control = target.ctrl_lock.lock(); + if target.zone.is_none() || target.vcpu_state.is_stopped() { + return false; + } + #[cfg(irq_gicv2)] + if irq_id < 16 { + let Some(source_cpu) = source_cpu.filter(|source_cpu| *source_cpu < 8) else { + return false; + }; + let source_bit = 1u8 << source_cpu; + let previous_sources = target.vcpu.arch.sgi_source[irq_id] + .fetch_or(source_bit, core::sync::atomic::Ordering::AcqRel); + // A repeated SGI from the same source coalesces with the occurrence + // already represented by the source bit, including while it is being + // claimed by the target CPU. + if previous_sources & source_bit != 0 { + return true; + } + } + if !target.vcpu.irq.set(irq_id) { + return false; + } + drop(control); + kick_vcpu(target_cpu); + true +} + +/// Ring the host doorbell for the pCPU to which the target vCPU is pinned. +/// +/// Current mapping is pCPU:vCPU=1:1. A local target relies on the common +/// exit/entry flush hook and does not receive a self IPI. A future scheduler +/// can replace `target_pcpu_id` with a resolved `VcpuTarget`. +pub fn kick_vcpu(target_pcpu_id: usize) { + let target = get_cpu_data(target_pcpu_id); + if target.zone.is_none() || target.vcpu_state.is_stopped() { + return; + } + if target_pcpu_id != this_cpu_data().id { + crate::arch::ipi::arch_send_event(target_pcpu_id as u64, SGI_IPI_ID); + } +} + #[cfg(all(irq_gicv2, target_arch = "aarch64"))] pub mod gicv2; #[cfg(all(irq_gicv2, target_arch = "aarch64"))] diff --git a/src/event.rs b/src/event.rs index bc872339..4a8fdaa1 100644 --- a/src/event.rs +++ b/src/event.rs @@ -173,6 +173,19 @@ pub fn check_events() -> bool { handle_event(fetch_event(this_cpu_data().id)) } +/// Run the migration-time IRQ flush hooks around an architecture's existing +/// legacy event dispatch behavior. +/// +/// New posted IRQ state and the legacy FIFO intentionally coexist during the +/// migration. Each architecture retains its existing legacy FIFO drain +/// behavior through `drain_legacy_events`; this layer adds no batching or +/// scheduling policy. The IRQ flush is controller-specific. +pub fn dispatch_host_events(drain_legacy_events: impl FnOnce() -> T) -> T { + crate::device::irqchip::flush_current_guest_irqs(); + let result = drain_legacy_events(); + result +} + /// Handle one queued event, returning whether an event was present. pub fn handle_next_event() -> bool { let event = fetch_event(this_cpu_data().id); From ec58d629c349360a6b99e132db541a1cb4f99fb6 Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Tue, 13 Oct 2026 21:52:47 -0400 Subject: [PATCH 05/12] chore: add license headers to vCPU state files --- src/arch/aarch64/vcpu.rs | 15 +++++++++++++++ src/arch/loongarch64/vcpu.rs | 15 +++++++++++++++ src/arch/riscv64/vcpu.rs | 15 +++++++++++++++ src/arch/x86_64/vcpu.rs | 15 +++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/src/arch/aarch64/vcpu.rs b/src/arch/aarch64/vcpu.rs index 60b20c8d..e27a8ce8 100644 --- a/src/arch/aarch64/vcpu.rs +++ b/src/arch/aarch64/vcpu.rs @@ -1,3 +1,18 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// #[cfg(irq_gicv2)] use core::sync::atomic::{AtomicU8, Ordering}; diff --git a/src/arch/loongarch64/vcpu.rs b/src/arch/loongarch64/vcpu.rs index 351b847c..4c8e838d 100644 --- a/src/arch/loongarch64/vcpu.rs +++ b/src/arch/loongarch64/vcpu.rs @@ -1,3 +1,18 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// /// Architecture-private vCPU state placeholder. pub struct ArchVcpu; diff --git a/src/arch/riscv64/vcpu.rs b/src/arch/riscv64/vcpu.rs index 351b847c..4c8e838d 100644 --- a/src/arch/riscv64/vcpu.rs +++ b/src/arch/riscv64/vcpu.rs @@ -1,3 +1,18 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// /// Architecture-private vCPU state placeholder. pub struct ArchVcpu; diff --git a/src/arch/x86_64/vcpu.rs b/src/arch/x86_64/vcpu.rs index 351b847c..4c8e838d 100644 --- a/src/arch/x86_64/vcpu.rs +++ b/src/arch/x86_64/vcpu.rs @@ -1,3 +1,18 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// /// Architecture-private vCPU state placeholder. pub struct ArchVcpu; From ad0e09323ca8ed5f6dfb42973f1a683969fb7ada Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Sat, 17 Oct 2026 11:01:14 -0400 Subject: [PATCH 06/12] refactor(cpu): move lifecycle state into Vcpu --- src/arch/aarch64/cpu.rs | 4 ++-- src/arch/aarch64/trap.rs | 8 ++++---- src/arch/loongarch64/cpu.rs | 4 ++-- src/arch/loongarch64/zone.rs | 4 ++-- src/arch/riscv64/cpu.rs | 4 ++-- src/arch/riscv64/sbi.rs | 4 ++-- src/arch/x86_64/cpu.rs | 12 ++++++------ src/arch/x86_64/s2pt.rs | 2 +- src/arch/x86_64/trap.rs | 2 +- src/cpu_data.rs | 15 ++++++++------- src/device/irqchip/gicv2/gic.rs | 2 +- src/device/irqchip/gicv3/mod.rs | 2 +- src/device/irqchip/mod.rs | 4 ++-- src/hypercall/mod.rs | 6 +++--- 14 files changed, 37 insertions(+), 36 deletions(-) diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 6b1ad01a..f2d31924 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -199,7 +199,7 @@ impl ArchCpu { // Return to AArch32 Supervisor (SVC) mode, disable IRQ, FIQ, ABT SPSR_EL2.set(0x1D3); } - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); info!( "cpu {} started at {:#x?}", self.cpuid, @@ -216,7 +216,7 @@ impl ArchCpu { assert!(this_cpu_id() == self.cpuid); let cpu_data = this_cpu_data(); let _lock = cpu_data.ctrl_lock.lock(); - cpu_data.vcpu_state.store(VcpuState::Stopped); + cpu_data.vcpu.state.store(VcpuState::Stopped); drop(_lock); // reset current cpu -> pc = 0x0 (wfi) diff --git a/src/arch/aarch64/trap.rs b/src/arch/aarch64/trap.rs index bc2b3b29..5a26a3fc 100644 --- a/src/arch/aarch64/trap.rs +++ b/src/arch/aarch64/trap.rs @@ -339,7 +339,7 @@ fn handle_sysreg(regs: &mut GeneralRegisters) { let val = if rt == 31 { 0 } else { regs.usr[rt] }; trace!("esr_el2 rt{}: {:#x?}", rt, val); let sgi_id: u64 = (val & (0xf << 24)) >> 24; - if !this_cpu_data().vcpu_state.is_running() { + if !this_cpu_data().vcpu.state.is_running() { trace!("skip send sgi {:#x?}", sgi_id); } else { trace!("post guest sgi {:#x?}", sgi_id); @@ -430,9 +430,9 @@ fn psci_emulate_cpu_on(regs: &mut GeneralRegisters) -> u64 { let target_data = get_cpu_data(cpu as _); let _lock = target_data.ctrl_lock.lock(); - if target_data.vcpu_state.is_stopped() { + if target_data.vcpu.state.is_stopped() { target_data.cpu_on_entry = regs.usr[2] as _; - target_data.vcpu_state.store(VcpuState::Ready); + target_data.vcpu.state.store(VcpuState::Ready); send_event(cpu as _, SGI_IPI_ID as _, IPI_EVENT_WAKEUP); } else { error!("psci: cpu {} already on", cpu); @@ -461,7 +461,7 @@ fn handle_psci_smc( todo!(); } PsciFnId::PSCI_AFFINITY_INFO_32 | PsciFnId::PSCI_AFFINITY_INFO_64 => { - !get_cpu_data(arg0 as _).vcpu_state.is_online() as _ + !get_cpu_data(arg0 as _).vcpu.state.is_online() as _ } PsciFnId::PSCI_MIG_INFO_TYPE => PSCI_TOS_NOT_PRESENT_MP, PsciFnId::PSCI_FEATURES => psci_emulate_features_info(regs.usr[1]), diff --git a/src/arch/loongarch64/cpu.rs b/src/arch/loongarch64/cpu.rs index 7281572a..5bf779b7 100644 --- a/src/arch/loongarch64/cpu.rs +++ b/src/arch/loongarch64/cpu.rs @@ -62,7 +62,7 @@ impl ArchCpu { pub fn run(&mut self) -> ! { assert!(this_cpu_id() == self.get_cpuid()); this_cpu_data().activate_gpm(); - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); if !self.init { self.init(this_cpu_data().cpu_on_entry, this_cpu_data().id, 0); self.init = true; @@ -131,7 +131,7 @@ impl ArchCpu { ); } info!("loongarch64: ArchCpu::idle: cpuid={}", self.get_cpuid()); - this_cpu_data().vcpu_state.store(VcpuState::Stopped); + this_cpu_data().vcpu.state.store(VcpuState::Stopped); // enable ipi on ecfg ecfg_ipi_enable(); // The trap vector is installed with interrupts disabled. Enable them only diff --git a/src/arch/loongarch64/zone.rs b/src/arch/loongarch64/zone.rs index e692f672..569263b6 100644 --- a/src/arch/loongarch64/zone.rs +++ b/src/arch/loongarch64/zone.rs @@ -686,13 +686,13 @@ fn virtual_ipi_send(target_guest_cpu: usize, ipi_bits: u32, blocking: bool) -> H let entry = VIRTUAL_IPI_MAILBOX[target_cpu * 4].load(Ordering::Acquire) as usize; let target_data = get_cpu_data(target_cpu); let _lock = target_data.ctrl_lock.lock(); - if target_data.vcpu_state.is_stopped() { + if target_data.vcpu.state.is_stopped() { debug!( "loongarch64: guest boots vCPU{}(pCPU{}) through IPI, entry={:#x}", target_guest_cpu, target_cpu, entry ); target_data.cpu_on_entry = entry; - target_data.vcpu_state.store(VcpuState::Ready); + target_data.vcpu.state.store(VcpuState::Ready); send_event(target_cpu, SGI_IPI_ID as usize, IPI_EVENT_WAKEUP); } else { warn!( diff --git a/src/arch/riscv64/cpu.rs b/src/arch/riscv64/cpu.rs index 9e595e5f..8b1c666b 100644 --- a/src/arch/riscv64/cpu.rs +++ b/src/arch/riscv64/cpu.rs @@ -200,7 +200,7 @@ impl ArchCpu { } assert!(this_cpu_id() == self.cpuid); - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); self.init_interrupt(); // reset all registers related @@ -223,7 +223,7 @@ impl ArchCpu { fn vcpu_arch_entry() -> !; } assert!(this_cpu_id() == self.cpuid); - this_cpu_data().vcpu_state.store(VcpuState::Stopped); + this_cpu_data().vcpu.state.store(VcpuState::Stopped); PARKING_MEMORY_SET.call_once(|| { let parking_code: [u8; 8] = [0x73, 0x00, 0x50, 0x10, 0x6F, 0xF0, 0xDF, 0xFF]; // 1: wfi; b 1b diff --git a/src/arch/riscv64/sbi.rs b/src/arch/riscv64/sbi.rs index 1c95e12b..6d3dde89 100644 --- a/src/arch/riscv64/sbi.rs +++ b/src/arch/riscv64/sbi.rs @@ -347,12 +347,12 @@ pub fn sbi_hsm_start_handler(current_cpu: &mut ArchCpu) -> SbiRet { ); let target_cpu = get_cpu_data(cpuid); let _lock = target_cpu.ctrl_lock.lock(); - if !target_cpu.vcpu_state.is_stopped() { + if !target_cpu.vcpu.state.is_stopped() { sbi_ret.error = RET_ERR_FAILED; } else { target_cpu.cpu_on_entry = start_addr; target_cpu.dtb_ipa = opaque; - target_cpu.vcpu_state.store(VcpuState::Ready); + target_cpu.vcpu.state.store(VcpuState::Ready); send_event(cpuid, 0, IPI_EVENT_WAKEUP); } drop(_lock); diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 5741dce7..8ab75c21 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -226,7 +226,7 @@ impl ArchCpu { assert!(this_cpu_id() == self.cpuid); - this_cpu_data().vcpu_state.store(VcpuState::Stopped); + this_cpu_data().vcpu.state.store(VcpuState::Stopped); self.activate_vmx().unwrap(); // info!("idle! cpuid: {:x}", self.cpuid); @@ -268,7 +268,7 @@ impl ArchCpu { } pub fn run(&mut self) { - if this_cpu_data().vcpu_state.is_running() { + if this_cpu_data().vcpu.state.is_running() { // x86 wake up cpu will send ipi twice, but we only want once return; } @@ -280,7 +280,7 @@ impl ArchCpu { // info!("run! cpuid: {:x}", self.cpuid); - per_cpu.vcpu_state.store(VcpuState::Running); + per_cpu.vcpu.state.store(VcpuState::Running); self.activate_vmx().unwrap(); if !per_cpu.boot_cpu { @@ -475,7 +475,7 @@ impl ArchCpu { // pass-through exceptions, set I/O bitmap and MSR bitmaps VmcsControl32::EXCEPTION_BITMAP.write(0)?; - if this_cpu_data().vcpu_state.is_running() { + if this_cpu_data().vcpu.state.is_running() { let pio_bitmap = get_pio_bitmap(this_zone_id()); VmcsControl64::IO_BITMAP_A_ADDR.write(pio_bitmap.a.start_paddr() as _)?; VmcsControl64::IO_BITMAP_B_ADDR.write(pio_bitmap.b.start_paddr() as _)?; @@ -542,7 +542,7 @@ impl ArchCpu { VmcsGuest64::IA32_EFER.write(0)?; // for AP start up, set CS_BASE to entry address, and RIP to 0. - if this_cpu_data().vcpu_state.is_running() && !this_cpu_data().boot_cpu { + if this_cpu_data().vcpu.state.is_running() && !this_cpu_data().boot_cpu { VmcsGuestNW::RIP.write(0)?; VmcsGuestNW::CS_BASE.write(entry)?; } @@ -589,7 +589,7 @@ impl ArchCpu { fn vmexit_handler(&mut self) { crate::arch::trap::handle_vmexit(self).unwrap(); - if this_cpu_data().vcpu_state.is_running() { + if this_cpu_data().vcpu.state.is_running() { check_pending_vectors(self.cpuid); } } diff --git a/src/arch/x86_64/s2pt.rs b/src/arch/x86_64/s2pt.rs index 7d62106c..7204620c 100644 --- a/src/arch/x86_64/s2pt.rs +++ b/src/arch/x86_64/s2pt.rs @@ -262,7 +262,7 @@ impl PagingInstr for S2PTInstr { unsafe { invs2pt(InvS2PTType::SingleContext, s2ptp) }; // if this cpu is boot cpu and it is running - if this_cpu_data().vcpu_state.is_running() && this_cpu_data().boot_cpu { + if this_cpu_data().vcpu.state.is_running() && this_cpu_data().boot_cpu { #[cfg(intel_vtd)] iommu::fill_dma_translation_tables(this_zone_id(), root_paddr); } diff --git a/src/arch/x86_64/trap.rs b/src/arch/x86_64/trap.rs index 7c7fccd4..2a193b06 100644 --- a/src/arch/x86_64/trap.rs +++ b/src/arch/x86_64/trap.rs @@ -112,7 +112,7 @@ fn handle_irq(vector: u8) { IdtVector::I8042_KEYBOARD_VECTOR if !is_timer => {} IdtVector::APIC_SPURIOUS_VECTOR | IdtVector::APIC_ERROR_VECTOR if !is_timer => {} _ => { - if vector >= 0x20 && this_cpu_data().vcpu_state.is_running() { + if vector >= 0x20 && this_cpu_data().vcpu.state.is_running() { inject_vector(this_cpu_id(), vector, None, false); } } diff --git a/src/cpu_data.rs b/src/cpu_data.rs index b78795a9..04f49054 100644 --- a/src/cpu_data.rs +++ b/src/cpu_data.rs @@ -233,6 +233,7 @@ impl Debug for VcpuStateCell { /// This object, including its architecture-private `arch` member, is the unit /// that can later move out of `PerCpu` when fixed 1:N vCPU scheduling returns. pub struct Vcpu { + pub state: VcpuStateCell, pub irq: GuestCpuIrqState, pub arch: ArchVcpu, } @@ -240,6 +241,7 @@ pub struct Vcpu { impl Vcpu { pub const fn new() -> Self { Self { + state: VcpuStateCell::new(VcpuState::Stopped), irq: GuestCpuIrqState::new(), arch: ArchVcpu::new(), } @@ -256,7 +258,6 @@ pub struct PerCpu { pub id: usize, pub cpu_on_entry: usize, pub dtb_ipa: usize, - pub vcpu_state: VcpuStateCell, /// Current pCPU:vCPU=1:1 temporary colocation; move out for fixed 1:N. pub vcpu: Vcpu, pub arch_cpu: ArchCpu, @@ -276,7 +277,6 @@ impl PerCpu { id: arch_cpu.cpuid, cpu_on_entry: INVALID_ADDRESS, dtb_ipa: INVALID_ADDRESS, - vcpu_state: VcpuStateCell::new(VcpuState::Stopped), vcpu: Vcpu::new(), arch_cpu, zone: None, @@ -337,16 +337,16 @@ pub fn this_zone() -> Arc { #[allow(unused)] pub fn vcpu_suspend() { info!("cpu {} suspending...", this_cpu_data().id); - this_cpu_data().vcpu_state.store(VcpuState::Blocked); + this_cpu_data().vcpu.state.store(VcpuState::Blocked); loop { // TODO: use wfi to optimize the loop - if !this_cpu_data().vcpu_state.is_blocked() { + if !this_cpu_data().vcpu.state.is_blocked() { break; } core::hint::spin_loop(); } // Remote sets `Ready` to leave Blocked; this hart then marks itself `Running` again. - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); info!("cpu {} resumed from suspend.", this_cpu_data().id); } @@ -359,7 +359,7 @@ pub fn wait_for_other_vcpus_suspend(cpu_set: CpuSet) { continue; } loop { - if get_cpu_data(target_cpu_id).vcpu_state.is_blocked() { + if get_cpu_data(target_cpu_id).vcpu.state.is_blocked() { break; } core::hint::spin_loop(); @@ -376,7 +376,8 @@ pub fn signal_other_vcpus_resume(cpu_set: CpuSet) { continue; } get_cpu_data(target_cpu_id) - .vcpu_state + .vcpu + .state .store(VcpuState::Ready); } } diff --git a/src/device/irqchip/gicv2/gic.rs b/src/device/irqchip/gicv2/gic.rs index 48e4784f..151106a6 100644 --- a/src/device/irqchip/gicv2/gic.rs +++ b/src/device/irqchip/gicv2/gic.rs @@ -164,7 +164,7 @@ fn install_pending_guest_irq(cpu: &crate::cpu_data::PerCpu, irq_id: usize) -> bo pub fn flush_current_guest_irqs() { let cpu = this_cpu_data(); - if cpu.zone.is_none() || cpu.vcpu_state.is_stopped() { + if cpu.zone.is_none() || cpu.vcpu.state.is_stopped() { return; } let state = &cpu.vcpu.irq; diff --git a/src/device/irqchip/gicv3/mod.rs b/src/device/irqchip/gicv3/mod.rs index 97e41f47..fddab23a 100644 --- a/src/device/irqchip/gicv3/mod.rs +++ b/src/device/irqchip/gicv3/mod.rs @@ -250,7 +250,7 @@ fn install_pending_guest_irq(irq_id: usize) -> bool { pub fn flush_current_guest_irqs() { let cpu = this_cpu_data(); - if cpu.zone.is_none() || cpu.vcpu_state.is_stopped() { + if cpu.zone.is_none() || cpu.vcpu.state.is_stopped() { return; } let state = &cpu.vcpu.irq; diff --git a/src/device/irqchip/mod.rs b/src/device/irqchip/mod.rs index f3b86c8d..ac0617bf 100644 --- a/src/device/irqchip/mod.rs +++ b/src/device/irqchip/mod.rs @@ -61,7 +61,7 @@ pub fn post_guest_irq(target_cpu: usize, irq_id: usize, source_cpu: Option HyperCall<'a> { let target_data = get_cpu_data(boot_cpu as _); let _lock = target_data.ctrl_lock.lock(); - if target_data.vcpu_state.is_stopped() { + if target_data.vcpu.state.is_stopped() { info!("boot_cpu: {}", boot_cpu); - target_data.vcpu_state.store(VcpuState::Ready); + target_data.vcpu.state.store(VcpuState::Ready); send_event(boot_cpu, SGI_IPI_ID as _, IPI_EVENT_WAKEUP); } else { error!("hv_zone_start: cpu {} already on", boot_cpu); @@ -350,7 +350,7 @@ impl<'a> HyperCall<'a> { // wait all zone's cpus shutdown (Stopped only: includes Blocked / Ready / Running) while zone_w.cpu_set().iter().any(|cpu_id| { let _lock = get_cpu_data(cpu_id).ctrl_lock.lock(); - let not_stopped = !get_cpu_data(cpu_id).vcpu_state.is_stopped(); + let not_stopped = !get_cpu_data(cpu_id).vcpu.state.is_stopped(); count += 1; if count > MAX_WAIT_TIMES { if not_stopped { From 230497eeca703abc409b7e5b170baabbbbb9a86f Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Sun, 18 Oct 2026 02:39:33 -0400 Subject: [PATCH 07/12] refactor(aarch64): use bit_field for SGI register fields --- src/arch/aarch64/ipi.rs | 71 ++++++++++++++++++++------------ src/device/irqchip/gicv2/gic.rs | 70 +++++++++++++++---------------- src/device/irqchip/gicv2/gich.rs | 7 +++- src/device/irqchip/gicv2/vgic.rs | 25 ++++++----- src/device/irqchip/gicv3/mod.rs | 69 ++++++++++++++++++------------- src/device/irqchip/gicv3/vgic.rs | 27 +++++++++--- 6 files changed, 159 insertions(+), 110 deletions(-) diff --git a/src/arch/aarch64/ipi.rs b/src/arch/aarch64/ipi.rs index 57fb1441..414169c1 100644 --- a/src/arch/aarch64/ipi.rs +++ b/src/arch/aarch64/ipi.rs @@ -16,12 +16,10 @@ use crate::device::irqchip::gic_send_event; +use bit_field::BitField; + #[cfg(irq_gicv2)] -use crate::{ - arch::cpu::this_cpu_id, - cpu_data::this_zone, - device::irqchip::gicv2::gicd::{GICD_SGIR_ROUTING_SHIFT, GICD_SGIR_TARGET_LIST_FILTER_SHIFT}, -}; +use crate::{arch::cpu::this_cpu_id, cpu_data::this_zone}; #[cfg(irq_gicv3)] use crate::{ @@ -29,8 +27,26 @@ use crate::{ cpu_data::this_zone, }; +// ICC_SGI1R_EL1 fields. +const ICC_SGI1R_TARGET_LIST_BITS: core::ops::Range = 0..16; #[cfg(irq_gicv3)] -const ICC_SGI1R_TARGET_LIST_MASK: u64 = 0xffff; +const ICC_SGI1R_AFF1_BITS: core::ops::Range = 16..24; +const ICC_SGI1R_INTID_BITS: core::ops::Range = 24..28; +#[cfg(irq_gicv3)] +const ICC_SGI1R_AFF2_BITS: core::ops::Range = 32..40; +const ICC_SGI1R_IRM_BIT: usize = 40; +#[cfg(irq_gicv3)] +const ICC_SGI1R_RS_BITS: core::ops::Range = 44..48; +#[cfg(irq_gicv3)] +const ICC_SGI1R_AFF3_BITS: core::ops::Range = 48..56; + +// GICD_SGIR fields. +#[cfg(irq_gicv2)] +const GICD_SGIR_INTID_BITS: core::ops::Range = 0..4; +#[cfg(irq_gicv2)] +const GICD_SGIR_TARGET_LIST_BITS: core::ops::Range = 16..24; +#[cfg(irq_gicv2)] +const GICD_SGIR_TARGET_LIST_FILTER_BITS: core::ops::Range = 24..26; #[cfg(irq_gicv3)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -48,13 +64,13 @@ struct Sgi1r { impl Sgi1r { fn decode(value: u64) -> Self { Self { - target_list: (value & ICC_SGI1R_TARGET_LIST_MASK) as u16, - aff1: ((value >> 16) & 0xff) as u8, - intid: ((value >> 24) & 0xf) as usize, - aff2: ((value >> 32) & 0xff) as u8, - irm: value & (1 << 40) != 0, - range_selector: ((value >> 44) & 0xf) as u8, - aff3: ((value >> 48) & 0xff) as u8, + target_list: value.get_bits(ICC_SGI1R_TARGET_LIST_BITS) as u16, + aff1: value.get_bits(ICC_SGI1R_AFF1_BITS) as u8, + intid: value.get_bits(ICC_SGI1R_INTID_BITS) as usize, + aff2: value.get_bits(ICC_SGI1R_AFF2_BITS) as u8, + irm: value.get_bit(ICC_SGI1R_IRM_BIT), + range_selector: value.get_bits(ICC_SGI1R_RS_BITS) as u8, + aff3: value.get_bits(ICC_SGI1R_AFF3_BITS) as u8, } } @@ -69,7 +85,7 @@ impl Sgi1r { && aff1 == self.aff1 as u64 && aff0 >= target_aff0 && aff0 < target_aff0 + 16 - && self.target_list & (1 << (aff0 - target_aff0)) != 0 + && self.target_list.get_bit((aff0 - target_aff0) as usize) } } @@ -99,25 +115,28 @@ pub fn handle_guest_sgi1r(value: u64) { #[cfg(irq_gicv2)] pub fn handle_guest_sgi1r(value: u64) { - let target_list = (value & 0xffff) as usize; - let routing_mode = if value & (1 << 40) != 0 { 1 } else { 0 }; - let irq_id = ((value >> 24) & 0xf) as usize; - handle_guest_sgir( - irq_id as u32 - | ((target_list as u32) << GICD_SGIR_TARGET_LIST_FILTER_SHIFT) - | ((routing_mode as u32) << GICD_SGIR_ROUTING_SHIFT), + post_guest_sgi( + value.get_bits(ICC_SGI1R_INTID_BITS) as usize, + value.get_bits(ICC_SGI1R_TARGET_LIST_BITS) as u8, + value.get_bit(ICC_SGI1R_IRM_BIT) as u32, ); } #[cfg(irq_gicv2)] pub fn handle_guest_sgir(value: u32) { + post_guest_sgi( + value.get_bits(GICD_SGIR_INTID_BITS) as usize, + value.get_bits(GICD_SGIR_TARGET_LIST_BITS) as u8, + value.get_bits(GICD_SGIR_TARGET_LIST_FILTER_BITS), + ); +} + +#[cfg(irq_gicv2)] +fn post_guest_sgi(irq_id: usize, target_list: u8, filter: u32) { let source_cpu = this_cpu_id(); - let irq_id = (value & 0xf) as usize; - let target_list = ((value >> GICD_SGIR_TARGET_LIST_FILTER_SHIFT) & 0xff) as usize; - let routing_mode = (value >> GICD_SGIR_ROUTING_SHIFT) & 0x3; for target_cpu in this_zone().cpu_set().iter() { - let selected = match routing_mode { - 0 => target_list & (1 << target_cpu) != 0, + let selected = match filter { + 0 => target_cpu < 8 && target_list.get_bit(target_cpu), 1 => target_cpu != source_cpu, 2 => target_cpu == source_cpu, _ => false, diff --git a/src/device/irqchip/gicv2/gic.rs b/src/device/irqchip/gicv2/gic.rs index 151106a6..79a1530d 100644 --- a/src/device/irqchip/gicv2/gic.rs +++ b/src/device/irqchip/gicv2/gic.rs @@ -18,8 +18,9 @@ use crate::cpu_data::this_cpu_data; use crate::device::irqchip::gicv2::gicc::GICC; use crate::device::irqchip::gicv2::gicd::GICV2_SGIS_NUM; use crate::device::irqchip::gicv2::gich::{ - GICH, GICV2_GICH_HCR_UIE, GICV2_GICH_LR_ACTIVE_STATE, GICV2_GICH_LR_CPUID_SHIFT, - GICV2_GICH_LR_HW, GICV2_GICH_LR_PENDING_STATE, GICV2_GICH_LR_PHYSID_SHIFT, + GicHypervisorInterface, GICH, GICV2_GICH_HCR_UIE, GICV2_GICH_LR_ACTIVE_BIT, + GICV2_GICH_LR_CPUID_BITS, GICV2_GICH_LR_EOI_BIT, GICV2_GICH_LR_HW, GICV2_GICH_LR_PENDING_BIT, + GICV2_GICH_LR_PENDING_STATE, GICV2_GICH_LR_PHYSID_SHIFT, GICV2_GICH_LR_VIRTUALID_BITS, }; use crate::event::{check_events, dispatch_host_events}; use crate::hypercall::SGI_IPI_ID; @@ -29,6 +30,8 @@ use crate::hypercall::SGI_IPI_ID; /// 1. gicv2 spec : https://www.cl.cam.ac.uk/research/srg/han/ACS-P35/zynq/arm_gic_architecture_specification.pdf use alloc::collections::VecDeque; use alloc::vec::Vec; +use bit_field::BitField; +use core::sync::atomic::Ordering; use spin::{Mutex, Once}; // gicv2 support max 8 cpus @@ -185,11 +188,9 @@ pub fn clear_guest_sgi_pending_sources(target_cpu: usize, irq_id: usize, source_ } let target = crate::cpu_data::get_cpu_data(target_cpu); let control = target.ctrl_lock.lock(); - target.vcpu.arch.sgi_source[irq_id] - .fetch_and(!source_mask, core::sync::atomic::Ordering::AcqRel); - let pending_cleared = - target.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire) == 0 - && target.vcpu.irq.clear(irq_id); + target.vcpu.arch.sgi_source[irq_id].fetch_and(!source_mask, Ordering::AcqRel); + let pending_cleared = target.vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) == 0 + && target.vcpu.irq.clear(irq_id); drop(control); if target_cpu == this_cpu_id() { clear_lr_pending_sgi(irq_id, source_mask); @@ -199,23 +200,30 @@ pub fn clear_guest_sgi_pending_sources(target_cpu: usize, irq_id: usize, source_ fn clear_lr_pending_sgi(irq_id: usize, source_mask: u8) { let gich = GICH.get().unwrap(); - let elrsr = (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64; - for index in 0..gich.get_lrs_num() as usize { - if elrsr & (1u64 << index) != 0 { + let empty_lrs = empty_lr_mask(gich); + for index in 0..=gich.get_lr_num() as usize { + if empty_lrs.get_bit(index) { continue; } - let lr = gich.get_lr(index) as usize; - let source = (lr >> GICV2_GICH_LR_CPUID_SHIFT) & 0x7; - if lr & VINTID_MASK != irq_id - || lr & GICV2_GICH_LR_PENDING_STATE == 0 - || source_mask & (1 << source) == 0 + let mut lr = gich.get_lr(index); + let source = lr.get_bits(GICV2_GICH_LR_CPUID_BITS); + if lr.get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize != irq_id + || !lr.get_bit(GICV2_GICH_LR_PENDING_BIT) + || !source_mask.get_bit(source as usize) { continue; } - if lr & GICV2_GICH_LR_ACTIVE_STATE != 0 { - gich.set_lr(index, (lr & !GICV2_GICH_LR_PENDING_STATE) as u32); + if lr.get_bit(GICV2_GICH_LR_ACTIVE_BIT) { + lr.set_bit(GICV2_GICH_LR_PENDING_BIT, false); + gich.set_lr(index, lr); } else { gich.set_lr(index, 0); + // The freed LR may have carried the EOI request of a deferred + // source, so hand that source back to the pending bitmap. + let cpu = this_cpu_data(); + if cpu.vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } } } } @@ -227,36 +235,26 @@ pub(crate) struct PendingSgiLrSummary { pub(crate) fn read_pending_sgi_lr_summary() -> PendingSgiLrSummary { let gich = GICH.get().unwrap(); - let elrsr = (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64; + let empty_lrs = empty_lr_mask(gich); let mut summary = PendingSgiLrSummary { pending_mask: 0, source_masks: [0; GICV2_SGIS_NUM], }; - for index in 0..gich.get_lrs_num() as usize { - if elrsr & (1u64 << index) != 0 { + for index in 0..=gich.get_lr_num() as usize { + if empty_lrs.get_bit(index) { continue; } - let lr = gich.get_lr(index) as usize; - let irq_id = lr & VINTID_MASK; - if irq_id < GICV2_SGIS_NUM && lr & GICV2_GICH_LR_PENDING_STATE != 0 { - summary.pending_mask |= 1 << irq_id; - summary.source_masks[irq_id] |= 1 << ((lr >> GICV2_GICH_LR_CPUID_SHIFT) & 0x7); + let lr = gich.get_lr(index); + let irq_id = lr.get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize; + if irq_id < GICV2_SGIS_NUM && lr.get_bit(GICV2_GICH_LR_PENDING_BIT) { + summary.pending_mask.set_bit(irq_id, true); + summary.source_masks[irq_id] + .set_bit(lr.get_bits(GICV2_GICH_LR_CPUID_BITS) as usize, true); } } summary } -pub fn read_lr_pending_sgi_source_mask(irq_id: usize) -> u8 { - if irq_id >= GICV2_SGIS_NUM { - return 0; - } - read_pending_sgi_lr_summary().source_masks[irq_id] -} - -pub fn read_lr_pending_sgi_mask() -> u32 { - read_pending_sgi_lr_summary().pending_mask -} - pub fn inject_irq(irq_id: usize, is_sgi: bool) -> bool { let elrsr: u64 = (GICH.get().unwrap().get_elrsr(1) as u64) << 32 | GICH.get().unwrap().get_elrsr(0) as u64; diff --git a/src/device/irqchip/gicv2/gich.rs b/src/device/irqchip/gicv2/gich.rs index 3180df39..baf9f55f 100644 --- a/src/device/irqchip/gicv2/gich.rs +++ b/src/device/irqchip/gicv2/gich.rs @@ -30,8 +30,13 @@ pub const GICV2_GICH_VMCR_VMGRP1EN: u32 = 0x1 << 1; pub const GICV2_GICH_VMCR_VMGRP0EN: u32 = 0x1; pub const GICV2_GICH_LR_GRP1: u32 = 0x1 << 30; pub const GICV2_GICH_LR_PENDING_STATE: usize = 0x1 << 28; -pub const GICV2_GICH_LR_ACTIVE_STATE: usize = 0x1 << 29; pub const GICV2_GICH_LR_HW: usize = 0x1 << 31; +// GICH_LR field positions, for use with `bit_field::BitField`. +pub const GICV2_GICH_LR_VIRTUALID_BITS: core::ops::Range = 0..10; +pub const GICV2_GICH_LR_CPUID_BITS: core::ops::Range = 10..13; +pub const GICV2_GICH_LR_EOI_BIT: usize = 19; +pub const GICV2_GICH_LR_PENDING_BIT: usize = 28; +pub const GICV2_GICH_LR_ACTIVE_BIT: usize = 29; pub const GICV2_GICH_HCR_REG_OFFSET: usize = 0x0000; pub const GICV2_GICH_VTR_REG_OFFSET: usize = 0x0004; pub const GICV2_GICH_VMCR_REG_OFFSET: usize = 0x0008; diff --git a/src/device/irqchip/gicv2/vgic.rs b/src/device/irqchip/gicv2/vgic.rs index 640b4219..0795256f 100644 --- a/src/device/irqchip/gicv2/vgic.rs +++ b/src/device/irqchip/gicv2/vgic.rs @@ -25,8 +25,8 @@ use crate::device::irqchip::gicv2::gicd::{ GICD_ISACTIVER_REG_OFFSET, GICD_ISENABLER_REG_OFFSET, GICD_ISPENDR_REG_OFFSET, GICD_ITARGETSR_REG_OFFSET, GICD_LOCK, GICD_SGIR_REG_OFFSET, GICD_SGIR_ROUTING_SHIFT, GICD_SGIR_TARGET_LIST_FILTER_SHIFT, GICD_SPENDSGIR_REG_OFFSET, GICD_TYPER_REG_OFFSET, - GICV2_CONFIG_REGS_NUM, GICV2_INT_REGS_NUM, GICV2_PRIO_REGS_NUM, GICV2_SGI_REGS_NUM, - GICV2_TARGET_REGS_NUM, + GICV2_CONFIG_REGS_NUM, GICV2_INT_REGS_NUM, GICV2_PRIO_REGS_NUM, GICV2_SGIS_NUM, + GICV2_SGI_REGS_NUM, GICV2_TARGET_REGS_NUM, }; use crate::device::irqchip::gicv2::GICV2; use crate::error::HvResult; @@ -36,8 +36,12 @@ use crate::memory::{mmio_perform_access, MMIOAccess, MemFlags, MemoryRegion}; /// reference: /// 1. gicv2 spec : https://www.cl.cam.ac.uk/research/srg/han/ACS-P35/zynq/arm_gic_architecture_specification.pdf use crate::zone::Zone; +use bit_field::BitField; +use core::sync::atomic::Ordering; const GICV2_REG_WIDTH: usize = 4; +/// SGI bits of GICD_ISPENDR0/ICPENDR0; bits 16..32 are physical PPIs. +const SGI_PENDING_BITS: core::ops::Range = 0..16; impl Zone { // trap all Guest OS accesses to the GIC Distributor registers. @@ -317,7 +321,7 @@ fn handle_gicd_sgi_pending_access(mmio: &mut MMIOAccess, is_clear: bool) -> HvRe ); } else { for source in 0..8 { - if source_mask & (1 << source) != 0 { + if source_mask.get_bit(source) { crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(source)); } } @@ -326,19 +330,14 @@ fn handle_gicd_sgi_pending_access(mmio: &mut MMIOAccess, is_clear: bool) -> HvRe return Ok(()); } - let register_base = if is_clear { - GICD_CPENDSGIR_REG_OFFSET - } else { - GICD_SPENDSGIR_REG_OFFSET - }; - let word_index = ((mmio.address & !3) - register_base) / 4; let lr_summary = crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary(); let mut value = 0u32; for byte in 0..4 { - let irq_id = word_index * 4 + byte; - value |= (read_guest_sgi_source_mask(target_cpu, irq_id, lr_summary.source_masks[irq_id]) - as u32) - << (byte * 8); + let irq_id = first_irq + byte; + value.set_bits( + byte * 8..byte * 8 + 8, + read_guest_sgi_source_mask(target_cpu, irq_id, lr_summary.source_masks[irq_id]) as u32, + ); } mmio.value = ((value & width_mask) >> shift) as usize; Ok(()) diff --git a/src/device/irqchip/gicv3/mod.rs b/src/device/irqchip/gicv3/mod.rs index fddab23a..52c3d4fd 100644 --- a/src/device/irqchip/gicv3/mod.rs +++ b/src/device/irqchip/gicv3/mod.rs @@ -27,6 +27,7 @@ use core::sync::atomic::AtomicU64; use alloc::collections::vec_deque::VecDeque; use alloc::vec::Vec; +use bit_field::BitField; use gicr::init_lpi_prop; use gits::gits_init; use spin::{Lazy, Mutex, Once}; @@ -158,13 +159,20 @@ pub fn gicv3_handle_irq_el1() { trace!("handle done") } -const LR_VINTID_MASK: u64 = (1 << 32) - 1; -const LR_PENDING: u64 = 1 << 62; -const LR_ACTIVE: u64 = 1 << 63; -const LR_STATE_MASK: u64 = LR_PENDING | LR_ACTIVE; +// ICH_LR_EL2 and ICH_VTR_EL2 fields, for use with `bit_field::BitField`. +const ICH_LR_VINTID_BITS: core::ops::Range = 0..32; +const ICH_LR_GROUP_BIT: usize = 60; +const ICH_LR_PENDING_BIT: usize = 62; +const ICH_LR_ACTIVE_BIT: usize = 63; +const ICH_VTR_LIST_REGS_BITS: core::ops::Range = 0..5; +const SGIS_NUM: usize = 16; + +fn list_regs_num() -> usize { + read_sysreg!(ich_vtr_el2).get_bits(ICH_VTR_LIST_REGS_BITS) as usize + 1 +} pub fn clear_guest_sgi(target_cpu: usize, irq_id: usize) -> bool { - if irq_id >= 16 { + if irq_id >= SGIS_NUM { return false; } let target = crate::cpu_data::get_cpu_data(target_cpu); @@ -181,43 +189,47 @@ pub fn clear_guest_sgi(target_cpu: usize, irq_id: usize) -> bool { fn try_install_software_sgi(irq_id: usize) -> bool { let elsr = read_sysreg!(ich_elrsr_el2); - let lr_num = (read_sysreg!(ich_vtr_el2) as usize & 0xf) + 1; let mut free_lr = None; - for index in 0..lr_num { - if (1 << index) & elsr != 0 { + for index in 0..list_regs_num() { + if elsr.get_bit(index) { free_lr.get_or_insert(index); continue; } - let lr = read_lr(index); - if lr & LR_VINTID_MASK != irq_id as u64 { + let mut lr = read_lr(index); + if lr.get_bits(ICH_LR_VINTID_BITS) as usize != irq_id { continue; } - let state = lr & LR_STATE_MASK; - if state == 0 || state == LR_ACTIVE { - write_lr(index, lr | LR_PENDING); + // Active-only LRs become pending; pending ones coalesce. + if !lr.get_bit(ICH_LR_PENDING_BIT) { + lr.set_bit(ICH_LR_PENDING_BIT, true); + write_lr(index, lr); } return true; } let Some(index) = free_lr else { return false; }; - write_lr(index, irq_id as u64 | (1 << 60) | LR_PENDING); + let mut lr = 0u64; + lr.set_bits(ICH_LR_VINTID_BITS, irq_id as u64); + lr.set_bit(ICH_LR_GROUP_BIT, true); + lr.set_bit(ICH_LR_PENDING_BIT, true); + write_lr(index, lr); true } pub(crate) fn clear_lr_pending_irq(irq_id: usize) { let elsr = read_sysreg!(ich_elrsr_el2); - let lr_num = (read_sysreg!(ich_vtr_el2) as usize & 0xf) + 1; - for index in 0..lr_num { - if (1 << index) & elsr != 0 { + for index in 0..list_regs_num() { + if elsr.get_bit(index) { continue; } - let lr = read_lr(index); - if lr & LR_VINTID_MASK != irq_id as u64 || lr & LR_PENDING == 0 { + let mut lr = read_lr(index); + if lr.get_bits(ICH_LR_VINTID_BITS) as usize != irq_id || !lr.get_bit(ICH_LR_PENDING_BIT) { continue; } - if lr & LR_ACTIVE != 0 { - write_lr(index, lr & !LR_PENDING); + if lr.get_bit(ICH_LR_ACTIVE_BIT) { + lr.set_bit(ICH_LR_PENDING_BIT, false); + write_lr(index, lr); } else { write_lr(index, 0); } @@ -226,23 +238,22 @@ pub(crate) fn clear_lr_pending_irq(irq_id: usize) { pub(crate) fn lr_pending_sgi_mask() -> u32 { let elsr = read_sysreg!(ich_elrsr_el2); - let lr_num = (read_sysreg!(ich_vtr_el2) as usize & 0xf) + 1; - let mut pending = 0; - for index in 0..lr_num { - if (1 << index) & elsr != 0 { + let mut pending = 0u32; + for index in 0..list_regs_num() { + if elsr.get_bit(index) { continue; } let lr = read_lr(index); - let irq_id = (lr & LR_VINTID_MASK) as usize; - if irq_id < 16 && lr & LR_PENDING != 0 { - pending |= 1 << irq_id; + let irq_id = lr.get_bits(ICH_LR_VINTID_BITS) as usize; + if irq_id < SGIS_NUM && lr.get_bit(ICH_LR_PENDING_BIT) { + pending.set_bit(irq_id, true); } } pending } fn install_pending_guest_irq(irq_id: usize) -> bool { - if irq_id >= 16 { + if irq_id >= SGIS_NUM { return false; } try_install_software_sgi(irq_id) diff --git a/src/device/irqchip/gicv3/vgic.rs b/src/device/irqchip/gicv3/vgic.rs index e877196b..1ddca5fd 100644 --- a/src/device/irqchip/gicv3/vgic.rs +++ b/src/device/irqchip/gicv3/vgic.rs @@ -31,6 +31,11 @@ use crate::{ zone::{this_zone_id, Zone}, }; use alloc::sync::Arc; +use bit_field::BitField; + +/// SGI bits of GICR_ISPENDR0/ICPENDR0; bits 16..32 are physical PPIs. +const SGI_PENDING_BITS: core::ops::Range = 0..16; + pub fn reg_range(base: usize, n: usize, size: usize) -> core::ops::Range { base..(base + n * size) } @@ -275,10 +280,22 @@ pub fn vgicv3_redist_handler(mmio: &mut MMIOAccess, cpu: usize) -> HvResult { let is_sgi_set_pending = reg == GICR_SGI_BASE + GICR_ISPENDR; let is_sgi_clear_pending = reg == GICR_SGI_BASE + GICR_ICPENDR; if is_sgi_set_pending || is_sgi_clear_pending { + // GICR_ISPENDR0/ICPENDR0 only support aligned 32-bit + // accesses; others would misplace the SGI/PPI split. + if mmio.size != 4 || mmio.address % 4 != 0 { + warn!( + "ignore unsupported GICR SGI pending access at {:#x} with size {}", + mmio.address, mmio.size + ); + if !mmio.is_write { + mmio.value = 0; + } + return HvResult::Ok(()); + } if mmio.is_write { - let mask = mmio.value as u32 & 0xffff; - for irq_id in 0..16 { - if mask & (1 << irq_id) == 0 { + let sgis = mmio.value.get_bits(SGI_PENDING_BITS); + for irq_id in SGI_PENDING_BITS { + if !sgis.get_bit(irq_id) { continue; } if is_sgi_set_pending { @@ -288,7 +305,7 @@ pub fn vgicv3_redist_handler(mmio: &mut MMIOAccess, cpu: usize) -> HvResult { } } // Preserve physical PPI writes in bits 16..31 only. - mmio.value &= !0xffff; + mmio.value.set_bits(SGI_PENDING_BITS, 0); mmio_perform_access(gicr_base, mmio); } else { mmio_perform_access(gicr_base, mmio); @@ -296,7 +313,7 @@ pub fn vgicv3_redist_handler(mmio: &mut MMIOAccess, cpu: usize) -> HvResult { if cpu == crate::arch::cpu::this_cpu_id() { value |= super::lr_pending_sgi_mask(); } - mmio.value = (mmio.value & !0xffff) | value as usize; + mmio.value.set_bits(SGI_PENDING_BITS, value as usize); } return HvResult::Ok(()); } From bba7c4025ca58cd5a8552c49a7f49654eca1a8df Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Sun, 18 Oct 2026 02:40:41 -0400 Subject: [PATCH 08/12] fix(gic): correct LR enumeration and consolidate helpers --- src/device/irqchip/gicv2/gich.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/device/irqchip/gicv2/gich.rs b/src/device/irqchip/gicv2/gich.rs index baf9f55f..0c893080 100644 --- a/src/device/irqchip/gicv2/gich.rs +++ b/src/device/irqchip/gicv2/gich.rs @@ -83,14 +83,8 @@ pub fn gich_init(gich_base: usize) { } impl GicHypervisorInterface { - // init GICH for each CPU. - pub fn get_lrs_num(&self) -> u32 { - let vtr = self.VTR.get(); - (vtr & 0b11111) + 1 - } - pub fn clear_all_lr(&self) { - for i in 0..self.get_lrs_num() as usize { + for i in 0..=self.get_lr_num() as usize { self.LR[i].set(0); } self.APR.set(0); @@ -104,6 +98,8 @@ impl GicHypervisorInterface { self.VMCR.set(value); } + /// Returns `GICH_VTR.ListRegs`, the index of the last List register + /// (number of List registers minus one). pub fn get_lr_num(&self) -> u32 { self.VTR.get() & 0b11111 } From cb13f51c125273d6953233c21c8d8dd44dba52c2 Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Sun, 18 Oct 2026 02:41:47 -0400 Subject: [PATCH 09/12] fix(gicv2): keep SGI VirtualIDs unique across LRs Defer additional SGI sources through EOI maintenance and preserve their visibility in pending register reads. --- src/device/irqchip/gicv2/gic.rs | 120 ++++++++++++++++++++++--------- src/device/irqchip/gicv2/gich.rs | 4 ++ 2 files changed, 90 insertions(+), 34 deletions(-) diff --git a/src/device/irqchip/gicv2/gic.rs b/src/device/irqchip/gicv2/gic.rs index 79a1530d..11b8594d 100644 --- a/src/device/irqchip/gicv2/gic.rs +++ b/src/device/irqchip/gicv2/gic.rs @@ -88,6 +88,7 @@ pub fn change_underflow_maintenance(is_enable: bool) { fn handle_maintenace_interrupt() { info!("handle_maintenace_interrupt"); + handle_sgi_eoi_maintenance(); flush_current_guest_irqs(); while let Some((irq_id, is_sgi)) = PENDING_VIRQS.get().unwrap().fetch_irq() { let is_injected: bool = inject_irq(irq_id, is_sgi); @@ -107,62 +108,113 @@ fn has_pending_maintenance_work() -> bool { this_cpu_data().vcpu.irq.has_pending() || PENDING_VIRQS.get().unwrap().has_pending() } -const VINTID_MASK: usize = 0x3ff; +fn empty_lr_mask(gich: &GicHypervisorInterface) -> u64 { + (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64 +} + +/// GICH_LR value for a pending SGI from `source_cpu` (HW=0, no EOI request). +fn encode_pending_sgi_lr(irq_id: usize, source_cpu: usize) -> u32 { + let mut lr = 0u32; + lr.set_bits(GICV2_GICH_LR_VIRTUALID_BITS, irq_id as u32); + lr.set_bits(GICV2_GICH_LR_CPUID_BITS, source_cpu as u32); + lr.set_bit(GICV2_GICH_LR_PENDING_BIT, true); + lr +} + +enum SgiInstall { + Installed, + /// Another source of the same SGI occupies an LR; EOI maintenance was + /// requested on that LR. + Deferred, + NoFreeLr, +} -fn try_install_software_sgi(irq_id: usize, source_cpu: usize) -> bool { +/// Install one SGI source into the local LRs. +/// +/// GICv2 requires every valid LR of a virtual CPU interface to have a unique +/// VirtualID, so the same SGI from another source cannot get its own LR. That +/// source waits for the guest to deactivate the current LR, which raises an EOI +/// maintenance interrupt because of `GICH_LR.EOI`. +fn try_install_software_sgi(irq_id: usize, source_cpu: usize) -> SgiInstall { let gich = GICH.get().unwrap(); - let elrsr = (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64; - let lr_num = gich.get_lrs_num() as usize; + let empty_lrs = empty_lr_mask(gich); let mut free_lr = None; - for index in 0..lr_num { - if elrsr & (1u64 << index) != 0 { + for index in 0..=gich.get_lr_num() as usize { + if empty_lrs.get_bit(index) { free_lr.get_or_insert(index); continue; } - let lr = gich.get_lr(index) as usize; - let lr_source = (lr >> GICV2_GICH_LR_CPUID_SHIFT) & 0x7; - if lr & VINTID_MASK != irq_id || lr_source != source_cpu { + let mut lr = gich.get_lr(index); + if lr.get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize != irq_id { continue; } - if lr & GICV2_GICH_LR_PENDING_STATE != 0 { - return true; + if lr.get_bits(GICV2_GICH_LR_CPUID_BITS) as usize == source_cpu { + lr.set_bit(GICV2_GICH_LR_PENDING_BIT, true); + gich.set_lr(index, lr); + return SgiInstall::Installed; } - gich.set_lr( - index, - (lr | GICV2_GICH_LR_PENDING_STATE | (source_cpu << GICV2_GICH_LR_CPUID_SHIFT)) as u32, - ); - return true; + lr.set_bit(GICV2_GICH_LR_EOI_BIT, true); + gich.set_lr(index, lr); + return SgiInstall::Deferred; } let Some(index) = free_lr else { - return false; + return SgiInstall::NoFreeLr; }; - gich.set_lr( - index, - (irq_id | GICV2_GICH_LR_PENDING_STATE | (source_cpu << GICV2_GICH_LR_CPUID_SHIFT)) as u32, - ); - true + gich.set_lr(index, encode_pending_sgi_lr(irq_id, source_cpu)); + SgiInstall::Installed } fn install_pending_guest_irq(cpu: &crate::cpu_data::PerCpu, irq_id: usize) -> bool { if irq_id >= GICV2_SGIS_NUM { return false; } - let source_mask = cpu.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire); - let source_cpu = source_mask.trailing_zeros() as usize; + let sources = &cpu.vcpu.arch.sgi_source[irq_id]; + let source_cpu = sources.load(Ordering::Acquire).trailing_zeros() as usize; if source_cpu >= 8 { - return false; + // Every source was cleared after the SGI was posted. + return true; } - cpu.vcpu.arch.sgi_source[irq_id] - .fetch_and(!(1u8 << source_cpu), core::sync::atomic::Ordering::AcqRel); - if !try_install_software_sgi(irq_id, source_cpu) { - cpu.vcpu.arch.sgi_source[irq_id] - .fetch_or(1u8 << source_cpu, core::sync::atomic::Ordering::AcqRel); - return false; + let source_bit = 1u8 << source_cpu; + sources.fetch_and(!source_bit, Ordering::AcqRel); + match try_install_software_sgi(irq_id, source_cpu) { + SgiInstall::Installed => { + if sources.load(Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } + true + } + SgiInstall::Deferred => { + // The source stays in `sgi_source`; EOI maintenance reposts it. + sources.fetch_or(source_bit, Ordering::AcqRel); + true + } + SgiInstall::NoFreeLr => { + sources.fetch_or(source_bit, Ordering::AcqRel); + false + } } - if cpu.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire) != 0 { - cpu.vcpu.irq.set(irq_id); +} + +/// Handle EOI maintenance for SGI LRs: free each deactivated SGI LR that +/// requested it for a deferred source, and repost that SGI if any source is +/// still pending. +fn handle_sgi_eoi_maintenance() { + let cpu = this_cpu_data(); + let gich = GICH.get().unwrap(); + let eoi_lrs = (gich.get_eisr(1) as u64) << 32 | gich.get_eisr(0) as u64; + for index in 0..=gich.get_lr_num() as usize { + if !eoi_lrs.get_bit(index) { + continue; + } + let irq_id = gich.get_lr(index).get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize; + if irq_id >= GICV2_SGIS_NUM { + continue; + } + gich.set_lr(index, 0); + if cpu.vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } } - true } pub fn flush_current_guest_irqs() { diff --git a/src/device/irqchip/gicv2/gich.rs b/src/device/irqchip/gicv2/gich.rs index 0c893080..b9a5080e 100644 --- a/src/device/irqchip/gicv2/gich.rs +++ b/src/device/irqchip/gicv2/gich.rs @@ -108,6 +108,10 @@ impl GicHypervisorInterface { self.ELRSR[index].get() } + pub fn get_eisr(&self, index: usize) -> u32 { + self.EISR[index].get() + } + pub fn get_lr(&self, index: usize) -> u32 { self.LR[index].get() } From b66bba0b4cf009ddee5cc05da0e9bb5526163b88 Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Sun, 18 Oct 2026 02:42:17 -0400 Subject: [PATCH 10/12] fix(aarch64): validate SGI pending register accesses --- src/device/irqchip/gicv2/vgic.rs | 93 +++++++++++++++++--------------- 1 file changed, 50 insertions(+), 43 deletions(-) diff --git a/src/device/irqchip/gicv2/vgic.rs b/src/device/irqchip/gicv2/vgic.rs index 0795256f..8da4bef4 100644 --- a/src/device/irqchip/gicv2/vgic.rs +++ b/src/device/irqchip/gicv2/vgic.rs @@ -278,41 +278,47 @@ pub fn set_sgi_irq(irq_id: usize, target_list: usize, routing_mode: usize) { GICD.get().unwrap().set_sgir(val as u32); } +/// Pending SGI sources not held by a local LR. A source deferred behind an LR +/// of the same SGI stays here while its pending bitmap bit is clear. fn read_guest_sgi_source_mask(target_cpu: usize, irq_id: usize, lr_source_mask: u8) -> u8 { - let target = get_cpu_data(target_cpu); - let software = if target.vcpu.irq.pending_mask(irq_id, 1) != 0 { - target.vcpu.arch.sgi_source[irq_id].load(core::sync::atomic::Ordering::Acquire) - } else { - 0 - }; - software | lr_source_mask + get_cpu_data(target_cpu).vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) | lr_source_mask +} + +fn ignore_unsupported_gicd_access(mmio: &mut MMIOAccess) -> HvResult { + warn!( + "ignore unsupported GICD access at {:#x} with size {}", + mmio.address, mmio.size + ); + if !mmio.is_write { + mmio.value = 0; + } + Ok(()) } fn handle_gicd_sgi_pending_access(mmio: &mut MMIOAccess, is_clear: bool) -> HvResult { + // Reject unaligned or wider accesses such as a 64-bit one. + if !matches!(mmio.size, 1 | 2 | 4) || mmio.address % mmio.size != 0 { + return ignore_unsupported_gicd_access(mmio); + } + let shift = (mmio.address % GICV2_REG_WIDTH) * 8; + let width_mask = (u32::MAX >> (32 - mmio.size * 8)) << shift; let target_cpu = this_cpu_id(); - let word_offset = mmio.address & 0x3; - let shift = word_offset * 8; - let width_mask = if mmio.size == 4 { - u32::MAX + let register_base = if is_clear { + GICD_CPENDSGIR_REG_OFFSET } else { - ((1u32 << (mmio.size * 8)) - 1) << shift + GICD_SPENDSGIR_REG_OFFSET }; + // Each register holds four SGIs, one byte of source CPU bits per SGI. + let first_irq = (mmio.address & !0x3) - register_base; if mmio.is_write { let value = ((mmio.value as u32) << shift) & width_mask; for byte in 0..4 { - if width_mask & (0xff << (byte * 8)) == 0 { - continue; - } - let source_mask = ((value >> (byte * 8)) & 0xff) as u8; - let base = if is_clear { - GICD_CPENDSGIR_REG_OFFSET - } else { - GICD_SPENDSGIR_REG_OFFSET - }; - let irq_id = ((mmio.address & !3) - base) / 4 * 4 + byte; - if irq_id >= 16 { + let lane = byte * 8..byte * 8 + 8; + let irq_id = first_irq + byte; + if width_mask.get_bits(lane.clone()) == 0 || irq_id >= GICV2_SGIS_NUM { continue; } + let source_mask = value.get_bits(lane) as u8; if is_clear { crate::device::irqchip::gicv2::gic::clear_guest_sgi_pending_sources( target_cpu, @@ -356,17 +362,15 @@ fn handle_gicd_intid_pending_access( is_clear: bool, gicd_base: usize, ) -> HvResult { + // GICD_ISPENDR0/ICPENDR0 only support aligned 32-bit accesses. + if mmio.size != GICV2_REG_WIDTH || mmio.address % GICV2_REG_WIDTH != 0 { + return ignore_unsupported_gicd_access(mmio); + } let target_cpu = this_cpu_id(); - let shift = (mmio.address & 0x3) * 8; - let width_mask = if mmio.size == 4 { - u32::MAX - } else { - ((1u32 << (mmio.size * 8)) - 1) << shift - }; if mmio.is_write { - let value = ((mmio.value as u32) << shift) & width_mask; - for irq_id in 0..16 { - if value & (1 << irq_id) == 0 { + let mut value = mmio.value as u32; + for irq_id in 0..GICV2_SGIS_NUM { + if !value.get_bit(irq_id) { continue; } if is_clear { @@ -375,20 +379,23 @@ fn handle_gicd_intid_pending_access( crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(target_cpu)); } } - if mmio.size == 4 && shift == 0 { - mmio.value = (value & !0xffff) as usize; - mmio_perform_access(gicd_base, mmio); - } else if shift >= 16 { - mmio_perform_access(gicd_base, mmio); - } + // Forward only the physical PPI bits. + value.set_bits(SGI_PENDING_BITS, 0); + mmio.value = value as usize; + mmio_perform_access(gicd_base, mmio); return Ok(()); } mmio_perform_access(gicd_base, mmio); - let physical_value = ((mmio.value as u32) << shift) & width_mask; - let lr_pending = crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary(); - let mut value = get_cpu_data(target_cpu).vcpu.irq.pending_mask(0, 16) as u32; - value |= lr_pending.pending_mask; - mmio.value = ((((physical_value & !0xffff) | value) & width_mask) >> shift) as usize; + let mut sgi_pending = + crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary().pending_mask; + for irq_id in 0..GICV2_SGIS_NUM { + if read_guest_sgi_source_mask(target_cpu, irq_id, 0) != 0 { + sgi_pending.set_bit(irq_id, true); + } + } + let mut value = mmio.value as u32; + value.set_bits(SGI_PENDING_BITS, sgi_pending); + mmio.value = value as usize; Ok(()) } From 1910e6427479e01ba4e4662f341746682c05bc5a Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Mon, 19 Oct 2026 08:10:27 -0400 Subject: [PATCH 11/12] fix(gicv2): filter PPI bits of GICD_ISPENDR0/ICPENDR0 by zone --- src/device/irqchip/gicv2/vgic.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/device/irqchip/gicv2/vgic.rs b/src/device/irqchip/gicv2/vgic.rs index 8da4bef4..bfb5376c 100644 --- a/src/device/irqchip/gicv2/vgic.rs +++ b/src/device/irqchip/gicv2/vgic.rs @@ -17,6 +17,7 @@ use crate::arch::cpu::this_cpu_id; use crate::arch::zone::{GicConfig, Gicv2Config, HvArchZoneConfig}; use crate::config::{BitmapWord, CONFIG_INTERRUPTS_BITMAP_BITS_PER_WORD}; use crate::cpu_data::{get_cpu_data, this_zone}; +use crate::device::irqchip::gicv2::gic::MAINTENACE_INTERRUPT; use crate::device::irqchip::gicv2::gicd::{ get_max_int_num, GICD, GICD_CPENDSGIR_REG_OFFSET, GICD_CTRL_REG_OFFSET, GICD_ICACTIVER_REG_OFFSET, GICD_ICENABLER_REG_OFFSET, GICD_ICFGR_REG_OFFSET, @@ -284,6 +285,18 @@ fn read_guest_sgi_source_mask(target_cpu: usize, irq_id: usize, lr_source_mask: get_cpu_data(target_cpu).vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) | lr_source_mask } +fn zone_owned_ppi_mask() -> u32 { + let zone = this_zone(); + let zone_r = zone.read(); + let mut mask = 0u32; + for irq in GICV2_SGIS_NUM..32 { + if irq != MAINTENACE_INTERRUPT as usize && zone_r.irq_in_zone(irq as u32) { + mask.set_bit(irq, true); + } + } + mask +} + fn ignore_unsupported_gicd_access(mmio: &mut MMIOAccess) -> HvResult { warn!( "ignore unsupported GICD access at {:#x} with size {}", @@ -379,8 +392,9 @@ fn handle_gicd_intid_pending_access( crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(target_cpu)); } } - // Forward only the physical PPI bits. + // Forward only the physical PPI bits owned by this zone. value.set_bits(SGI_PENDING_BITS, 0); + value &= zone_owned_ppi_mask(); mmio.value = value as usize; mmio_perform_access(gicd_base, mmio); return Ok(()); @@ -393,7 +407,8 @@ fn handle_gicd_intid_pending_access( sgi_pending.set_bit(irq_id, true); } } - let mut value = mmio.value as u32; + // Hide host-owned PPI pending state, then overlay the software SGI state. + let mut value = (mmio.value as u32) & zone_owned_ppi_mask(); value.set_bits(SGI_PENDING_BITS, sgi_pending); mmio.value = value as usize; Ok(()) From ec311ce6d9bbcccab7dcebfeaa7ae7f22be506de Mon Sep 17 00:00:00 2001 From: Chenhao Pei Date: Mon, 19 Oct 2026 08:11:08 -0400 Subject: [PATCH 12/12] fix(gicv2): scan the last list register in inject_irq() --- src/device/irqchip/gicv2/gic.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/device/irqchip/gicv2/gic.rs b/src/device/irqchip/gicv2/gic.rs index 11b8594d..0511c5d2 100644 --- a/src/device/irqchip/gicv2/gic.rs +++ b/src/device/irqchip/gicv2/gic.rs @@ -313,7 +313,7 @@ pub fn inject_irq(irq_id: usize, is_sgi: bool) -> bool { let lr_num: isize = GICH.get().unwrap().get_lr_num() as isize; let lr_pint_mask: usize = 0x3ff << 10; let mut free_lr: isize = -1; - for i in 0..lr_num { + for i in 0..=lr_num { if (1 << i) & elrsr > 0 { free_lr = i; continue; @@ -327,7 +327,7 @@ pub fn inject_irq(irq_id: usize, is_sgi: bool) -> bool { } if free_lr == -1 { warn!("no free lr"); - for i in 0..lr_num { + for i in 0..=lr_num { let lr = GICH.get().unwrap().get_lr(i as usize) as usize; warn!("lr[{}]: {:#x}", i, lr); }