diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 8c3a1c81f..f2d31924d 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -199,12 +199,13 @@ impl ArchCpu { // Return to AArch32 Supervisor (SVC) mode, disable IRQ, FIQ, ABT SPSR_EL2.set(0x1D3); } - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); info!( "cpu {} started at {:#x?}", self.cpuid, this_cpu_data().cpu_on_entry ); + crate::device::irqchip::flush_current_guest_irqs(); unsafe { vmreturn(self.guest_reg() as *mut _ as usize); } @@ -215,7 +216,7 @@ impl ArchCpu { assert!(this_cpu_id() == self.cpuid); let cpu_data = this_cpu_data(); let _lock = cpu_data.ctrl_lock.lock(); - cpu_data.vcpu_state.store(VcpuState::Stopped); + cpu_data.vcpu.state.store(VcpuState::Stopped); drop(_lock); // reset current cpu -> pc = 0x0 (wfi) diff --git a/src/arch/aarch64/ipi.rs b/src/arch/aarch64/ipi.rs index 1904e61a5..414169c19 100644 --- a/src/arch/aarch64/ipi.rs +++ b/src/arch/aarch64/ipi.rs @@ -16,6 +16,79 @@ use crate::device::irqchip::gic_send_event; +use bit_field::BitField; + +#[cfg(irq_gicv2)] +use crate::{arch::cpu::this_cpu_id, cpu_data::this_zone}; + +#[cfg(irq_gicv3)] +use crate::{ + arch::cpu::{cpuid_to_mpidr_affinity, this_cpu_id}, + cpu_data::this_zone, +}; + +// ICC_SGI1R_EL1 fields. +const ICC_SGI1R_TARGET_LIST_BITS: core::ops::Range = 0..16; +#[cfg(irq_gicv3)] +const ICC_SGI1R_AFF1_BITS: core::ops::Range = 16..24; +const ICC_SGI1R_INTID_BITS: core::ops::Range = 24..28; +#[cfg(irq_gicv3)] +const ICC_SGI1R_AFF2_BITS: core::ops::Range = 32..40; +const ICC_SGI1R_IRM_BIT: usize = 40; +#[cfg(irq_gicv3)] +const ICC_SGI1R_RS_BITS: core::ops::Range = 44..48; +#[cfg(irq_gicv3)] +const ICC_SGI1R_AFF3_BITS: core::ops::Range = 48..56; + +// GICD_SGIR fields. +#[cfg(irq_gicv2)] +const GICD_SGIR_INTID_BITS: core::ops::Range = 0..4; +#[cfg(irq_gicv2)] +const GICD_SGIR_TARGET_LIST_BITS: core::ops::Range = 16..24; +#[cfg(irq_gicv2)] +const GICD_SGIR_TARGET_LIST_FILTER_BITS: core::ops::Range = 24..26; + +#[cfg(irq_gicv3)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct Sgi1r { + target_list: u16, + aff1: u8, + intid: usize, + aff2: u8, + irm: bool, + range_selector: u8, + aff3: u8, +} + +#[cfg(irq_gicv3)] +impl Sgi1r { + fn decode(value: u64) -> Self { + Self { + target_list: value.get_bits(ICC_SGI1R_TARGET_LIST_BITS) as u16, + aff1: value.get_bits(ICC_SGI1R_AFF1_BITS) as u8, + intid: value.get_bits(ICC_SGI1R_INTID_BITS) as usize, + aff2: value.get_bits(ICC_SGI1R_AFF2_BITS) as u8, + irm: value.get_bit(ICC_SGI1R_IRM_BIT), + range_selector: value.get_bits(ICC_SGI1R_RS_BITS) as u8, + aff3: value.get_bits(ICC_SGI1R_AFF3_BITS) as u8, + } + } + + fn targets(self, cpu_id: usize, source_cpu: usize) -> bool { + if self.irm { + return cpu_id != source_cpu; + } + let (aff3, aff2, aff1, aff0) = cpuid_to_mpidr_affinity(cpu_id as u64); + let target_aff0 = self.range_selector as u64 * 16; + aff3 == self.aff3 as u64 + && aff2 == self.aff2 as u64 + && aff1 == self.aff1 as u64 + && aff0 >= target_aff0 + && aff0 < target_aff0 + 16 + && self.target_list.get_bit((aff0 - target_aff0) as usize) + } +} + pub fn arch_send_event(cpu_id: u64, sgi_num: u64) { gic_send_event(cpu_id, sgi_num); } @@ -24,6 +97,57 @@ pub fn arch_notify_event(cpu_id: u64, sgi_num: u64, _event_id: usize, _queue_was arch_send_event(cpu_id, sgi_num); } +#[cfg(irq_gicv3)] +pub fn handle_guest_sgi1r(value: u64) { + let sgi = Sgi1r::decode(value); + let source_cpu = this_cpu_id(); + for target_cpu in this_zone().cpu_set().iter() { + if sgi.targets(target_cpu, source_cpu) { + if !crate::device::irqchip::post_guest_irq(target_cpu, sgi.intid, None) { + error!( + "failed to post guest SGI {} to CPU{}", + sgi.intid, target_cpu + ); + } + } + } +} + +#[cfg(irq_gicv2)] +pub fn handle_guest_sgi1r(value: u64) { + post_guest_sgi( + value.get_bits(ICC_SGI1R_INTID_BITS) as usize, + value.get_bits(ICC_SGI1R_TARGET_LIST_BITS) as u8, + value.get_bit(ICC_SGI1R_IRM_BIT) as u32, + ); +} + +#[cfg(irq_gicv2)] +pub fn handle_guest_sgir(value: u32) { + post_guest_sgi( + value.get_bits(GICD_SGIR_INTID_BITS) as usize, + value.get_bits(GICD_SGIR_TARGET_LIST_BITS) as u8, + value.get_bits(GICD_SGIR_TARGET_LIST_FILTER_BITS), + ); +} + +#[cfg(irq_gicv2)] +fn post_guest_sgi(irq_id: usize, target_list: u8, filter: u32) { + let source_cpu = this_cpu_id(); + for target_cpu in this_zone().cpu_set().iter() { + let selected = match filter { + 0 => target_cpu < 8 && target_list.get_bit(target_cpu), + 1 => target_cpu != source_cpu, + 2 => target_cpu == source_cpu, + _ => false, + }; + if selected && !crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(source_cpu)) + { + warn!("failed to post guest SGI {} to CPU{}", irq_id, target_cpu); + } + } +} + pub fn arch_check_events(event: Option) { match event { _ => { diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index f3ab85ad0..ed0d81066 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -27,9 +27,11 @@ pub mod s2pt; pub mod sysreg; pub mod time; pub mod trap; +pub mod vcpu; pub mod zone; pub use s2pt::stage2_mode_detect; pub use s2pt::Stage2PageTable; +pub use vcpu::ArchVcpu; #[cfg(aarch64_uefi_support)] pub mod uefi; diff --git a/src/arch/aarch64/trap.rs b/src/arch/aarch64/trap.rs index 26eab6d97..5a26a3fc8 100644 --- a/src/arch/aarch64/trap.rs +++ b/src/arch/aarch64/trap.rs @@ -21,10 +21,7 @@ use super::cpu::GeneralRegisters; use crate::arch::sysreg::smc_call; use crate::zone::zone_error; use crate::{ - arch::{ - cpu::mpidr_to_cpuid, - sysreg::{read_sysreg, write_sysreg}, - }, + arch::{cpu::mpidr_to_cpuid, sysreg::read_sysreg}, cpu_data::{get_cpu_data, this_cpu_data, this_zone, VcpuState}, device::irqchip::gic_handle_irq, event::{send_event, IPI_EVENT_SHUTDOWN, IPI_EVENT_WAKEUP}, @@ -122,6 +119,7 @@ pub fn arch_handle_exit(regs: &mut GeneralRegisters) -> ! { ExceptionType::EXIT_REASON_EL2_IRQ => irqchip_handle_irq2(), _ => arch_dump_exit(regs.exit_reason), } + crate::device::irqchip::flush_current_guest_irqs(); unsafe { vmreturn(regs as *const _ as usize) } } @@ -341,11 +339,11 @@ fn handle_sysreg(regs: &mut GeneralRegisters) { let val = if rt == 31 { 0 } else { regs.usr[rt] }; trace!("esr_el2 rt{}: {:#x?}", rt, val); let sgi_id: u64 = (val & (0xf << 24)) >> 24; - if !this_cpu_data().vcpu_state.is_running() { + if !this_cpu_data().vcpu.state.is_running() { trace!("skip send sgi {:#x?}", sgi_id); } else { - trace!("send sgi {:#x?}", sgi_id); - write_sysreg!(icc_sgi1r_el1, val); + trace!("post guest sgi {:#x?}", sgi_id); + crate::arch::ipi::handle_guest_sgi1r(val); } arch_skip_instruction(regs); //skip sgi write } else { @@ -432,9 +430,9 @@ fn psci_emulate_cpu_on(regs: &mut GeneralRegisters) -> u64 { let target_data = get_cpu_data(cpu as _); let _lock = target_data.ctrl_lock.lock(); - if target_data.vcpu_state.is_stopped() { + if target_data.vcpu.state.is_stopped() { target_data.cpu_on_entry = regs.usr[2] as _; - target_data.vcpu_state.store(VcpuState::Ready); + target_data.vcpu.state.store(VcpuState::Ready); send_event(cpu as _, SGI_IPI_ID as _, IPI_EVENT_WAKEUP); } else { error!("psci: cpu {} already on", cpu); @@ -463,7 +461,7 @@ fn handle_psci_smc( todo!(); } PsciFnId::PSCI_AFFINITY_INFO_32 | PsciFnId::PSCI_AFFINITY_INFO_64 => { - !get_cpu_data(arg0 as _).vcpu_state.is_online() as _ + !get_cpu_data(arg0 as _).vcpu.state.is_online() as _ } PsciFnId::PSCI_MIG_INFO_TYPE => PSCI_TOS_NOT_PRESENT_MP, PsciFnId::PSCI_FEATURES => psci_emulate_features_info(regs.usr[1]), @@ -476,10 +474,12 @@ fn handle_psci_smc( for cpu_id in zone.read().cpu_set().iter_except(this_cpu_data().id) { let target_cpu = get_cpu_data(cpu_id); let _lock = target_cpu.ctrl_lock.lock(); + target_cpu.vcpu.clear_irq_state(); target_cpu.zone = None; send_event(cpu_id, SGI_IPI_ID as _, IPI_EVENT_SHUTDOWN); } + this_cpu_data().vcpu.clear_irq_state(); this_cpu_data().zone = None; drop(zone); remove_zone(zone_id); diff --git a/src/arch/aarch64/vcpu.rs b/src/arch/aarch64/vcpu.rs new file mode 100644 index 000000000..e27a8ce81 --- /dev/null +++ b/src/arch/aarch64/vcpu.rs @@ -0,0 +1,41 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// +#[cfg(irq_gicv2)] +use core::sync::atomic::{AtomicU8, Ordering}; + +/// Architecture-private vCPU state, separate from the host pCPU's `ArchCpu`. +pub struct ArchVcpu { + #[cfg(irq_gicv2)] + pub sgi_source: [AtomicU8; 16], +} + +impl ArchVcpu { + pub const fn new() -> Self { + #[cfg(irq_gicv2)] + const ZERO_SOURCE: AtomicU8 = AtomicU8::new(0); + Self { + #[cfg(irq_gicv2)] + sgi_source: [ZERO_SOURCE; 16], + } + } + + pub fn clear_irq_state(&self) { + #[cfg(irq_gicv2)] + for source in &self.sgi_source { + source.store(0, Ordering::Release); + } + } +} diff --git a/src/arch/loongarch64/cpu.rs b/src/arch/loongarch64/cpu.rs index 7281572ac..5bf779b7b 100644 --- a/src/arch/loongarch64/cpu.rs +++ b/src/arch/loongarch64/cpu.rs @@ -62,7 +62,7 @@ impl ArchCpu { pub fn run(&mut self) -> ! { assert!(this_cpu_id() == self.get_cpuid()); this_cpu_data().activate_gpm(); - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); if !self.init { self.init(this_cpu_data().cpu_on_entry, this_cpu_data().id, 0); self.init = true; @@ -131,7 +131,7 @@ impl ArchCpu { ); } info!("loongarch64: ArchCpu::idle: cpuid={}", self.get_cpuid()); - this_cpu_data().vcpu_state.store(VcpuState::Stopped); + this_cpu_data().vcpu.state.store(VcpuState::Stopped); // enable ipi on ecfg ecfg_ipi_enable(); // The trap vector is installed with interrupts disabled. Enable them only diff --git a/src/arch/loongarch64/mod.rs b/src/arch/loongarch64/mod.rs index 26f045ade..283471822 100644 --- a/src/arch/loongarch64/mod.rs +++ b/src/arch/loongarch64/mod.rs @@ -29,7 +29,9 @@ pub mod s1pt; pub mod s2pt; pub mod time; pub mod trap; +pub mod vcpu; pub mod zone; +pub use vcpu::ArchVcpu; pub use s1pt::Stage1PageTable; pub use s2pt::stage2_mode_detect; diff --git a/src/arch/loongarch64/trap.rs b/src/arch/loongarch64/trap.rs index 5e0591ab5..25112cbc5 100644 --- a/src/arch/loongarch64/trap.rs +++ b/src/arch/loongarch64/trap.rs @@ -23,7 +23,7 @@ use crate::consts::MAX_CPU_NUM; use crate::cpu_data::this_cpu_data; use crate::device::irqchip::inject_irq; use crate::device::irqchip::ls7a2000::chip::*; -use crate::event::{dump_events, handle_next_event, has_pending_events}; +use crate::event::{dispatch_host_events, dump_events, handle_next_event, has_pending_events}; use crate::hypercall::{SGI_IPI_ID, *}; use crate::memory::{addr, mmio_handle_access, MMIOAccess}; use crate::zone::Zone; @@ -1302,15 +1302,17 @@ fn handle_interrupt(is: usize) { } if ipi_status & event_doorbell != 0 { - // Clear before each fetch. If the fetch observes an empty queue while a - // producer enqueues concurrently, its doorbell remains pending and - // re-fires. Clearing after the fetch could lose that coalesced wakeup. + // Clear the observed host doorbell before checking logical state. + // A concurrent producer is either drained by the dispatcher or + // leaves/re-arms a later notification. clear_ipi_bits(event_doorbell); - if has_pending_events(cpu_id) { - while handle_next_event() { - clear_ipi_bits(event_doorbell); + dispatch_host_events(|| { + if has_pending_events(cpu_id) { + while handle_next_event() { + clear_ipi_bits(event_doorbell); + } } - } + }); } let handled_mask = event_doorbell | virtual_ipi_doorbell; diff --git a/src/arch/loongarch64/vcpu.rs b/src/arch/loongarch64/vcpu.rs new file mode 100644 index 000000000..4c8e838d8 --- /dev/null +++ b/src/arch/loongarch64/vcpu.rs @@ -0,0 +1,25 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// +/// Architecture-private vCPU state placeholder. +pub struct ArchVcpu; + +impl ArchVcpu { + pub const fn new() -> Self { + Self + } + + pub fn clear_irq_state(&self) {} +} diff --git a/src/arch/loongarch64/zone.rs b/src/arch/loongarch64/zone.rs index e692f672a..569263b67 100644 --- a/src/arch/loongarch64/zone.rs +++ b/src/arch/loongarch64/zone.rs @@ -686,13 +686,13 @@ fn virtual_ipi_send(target_guest_cpu: usize, ipi_bits: u32, blocking: bool) -> H let entry = VIRTUAL_IPI_MAILBOX[target_cpu * 4].load(Ordering::Acquire) as usize; let target_data = get_cpu_data(target_cpu); let _lock = target_data.ctrl_lock.lock(); - if target_data.vcpu_state.is_stopped() { + if target_data.vcpu.state.is_stopped() { debug!( "loongarch64: guest boots vCPU{}(pCPU{}) through IPI, entry={:#x}", target_guest_cpu, target_cpu, entry ); target_data.cpu_on_entry = entry; - target_data.vcpu_state.store(VcpuState::Ready); + target_data.vcpu.state.store(VcpuState::Ready); send_event(target_cpu, SGI_IPI_ID as usize, IPI_EVENT_WAKEUP); } else { warn!( diff --git a/src/arch/riscv64/cpu.rs b/src/arch/riscv64/cpu.rs index 9e595e5f2..8b1c666b0 100644 --- a/src/arch/riscv64/cpu.rs +++ b/src/arch/riscv64/cpu.rs @@ -200,7 +200,7 @@ impl ArchCpu { } assert!(this_cpu_id() == self.cpuid); - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); self.init_interrupt(); // reset all registers related @@ -223,7 +223,7 @@ impl ArchCpu { fn vcpu_arch_entry() -> !; } assert!(this_cpu_id() == self.cpuid); - this_cpu_data().vcpu_state.store(VcpuState::Stopped); + this_cpu_data().vcpu.state.store(VcpuState::Stopped); PARKING_MEMORY_SET.call_once(|| { let parking_code: [u8; 8] = [0x73, 0x00, 0x50, 0x10, 0x6F, 0xF0, 0xDF, 0xFF]; // 1: wfi; b 1b diff --git a/src/arch/riscv64/mod.rs b/src/arch/riscv64/mod.rs index 6288400b2..fe7a3615f 100644 --- a/src/arch/riscv64/mod.rs +++ b/src/arch/riscv64/mod.rs @@ -31,7 +31,9 @@ pub mod s2pt; pub mod sbi; pub mod time; pub mod trap; +pub mod vcpu; pub mod zone; +pub use vcpu::ArchVcpu; // pub use s1pt::Stage1PageTable; pub use s2pt::stage2_mode_detect; diff --git a/src/arch/riscv64/sbi.rs b/src/arch/riscv64/sbi.rs index 1c95e12b1..6d3dde894 100644 --- a/src/arch/riscv64/sbi.rs +++ b/src/arch/riscv64/sbi.rs @@ -347,12 +347,12 @@ pub fn sbi_hsm_start_handler(current_cpu: &mut ArchCpu) -> SbiRet { ); let target_cpu = get_cpu_data(cpuid); let _lock = target_cpu.ctrl_lock.lock(); - if !target_cpu.vcpu_state.is_stopped() { + if !target_cpu.vcpu.state.is_stopped() { sbi_ret.error = RET_ERR_FAILED; } else { target_cpu.cpu_on_entry = start_addr; target_cpu.dtb_ipa = opaque; - target_cpu.vcpu_state.store(VcpuState::Ready); + target_cpu.vcpu.state.store(VcpuState::Ready); send_event(cpuid, 0, IPI_EVENT_WAKEUP); } drop(_lock); diff --git a/src/arch/riscv64/trap.rs b/src/arch/riscv64/trap.rs index 4c8de184f..c87bd02b5 100644 --- a/src/arch/riscv64/trap.rs +++ b/src/arch/riscv64/trap.rs @@ -17,7 +17,7 @@ use super::cpu::ArchCpu; use crate::arch::sbi::sbi_vs_handler; #[cfg(plic)] use crate::device::irqchip::plic::{inject_irq, plic_get_hwirq}; -use crate::event::check_events; +use crate::event::{check_events, dispatch_host_events}; use crate::memory::GuestPhysAddr; use crate::memory::{mmio_handle_access, MMIOAccess}; use core::arch::{asm, global_asm}; @@ -407,12 +407,10 @@ pub fn handle_timer_interrupt(_current_cpu: &mut ArchCpu) { /// Handle supervisor software interrupt. pub fn handle_software_interrupt(_current_cpu: &mut ArchCpu) { - while check_events() { - // Get next event to handle, it is handled in check_events function. - } unsafe { riscv::register::sip::clear_ssoft(); } + dispatch_host_events(|| while check_events() {}); } /// Handle supervisor external interrupt. diff --git a/src/arch/riscv64/vcpu.rs b/src/arch/riscv64/vcpu.rs new file mode 100644 index 000000000..4c8e838d8 --- /dev/null +++ b/src/arch/riscv64/vcpu.rs @@ -0,0 +1,25 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// +/// Architecture-private vCPU state placeholder. +pub struct ArchVcpu; + +impl ArchVcpu { + pub const fn new() -> Self { + Self + } + + pub fn clear_irq_state(&self) {} +} diff --git a/src/arch/x86_64/cpu.rs b/src/arch/x86_64/cpu.rs index 5741dce7e..8ab75c215 100644 --- a/src/arch/x86_64/cpu.rs +++ b/src/arch/x86_64/cpu.rs @@ -226,7 +226,7 @@ impl ArchCpu { assert!(this_cpu_id() == self.cpuid); - this_cpu_data().vcpu_state.store(VcpuState::Stopped); + this_cpu_data().vcpu.state.store(VcpuState::Stopped); self.activate_vmx().unwrap(); // info!("idle! cpuid: {:x}", self.cpuid); @@ -268,7 +268,7 @@ impl ArchCpu { } pub fn run(&mut self) { - if this_cpu_data().vcpu_state.is_running() { + if this_cpu_data().vcpu.state.is_running() { // x86 wake up cpu will send ipi twice, but we only want once return; } @@ -280,7 +280,7 @@ impl ArchCpu { // info!("run! cpuid: {:x}", self.cpuid); - per_cpu.vcpu_state.store(VcpuState::Running); + per_cpu.vcpu.state.store(VcpuState::Running); self.activate_vmx().unwrap(); if !per_cpu.boot_cpu { @@ -475,7 +475,7 @@ impl ArchCpu { // pass-through exceptions, set I/O bitmap and MSR bitmaps VmcsControl32::EXCEPTION_BITMAP.write(0)?; - if this_cpu_data().vcpu_state.is_running() { + if this_cpu_data().vcpu.state.is_running() { let pio_bitmap = get_pio_bitmap(this_zone_id()); VmcsControl64::IO_BITMAP_A_ADDR.write(pio_bitmap.a.start_paddr() as _)?; VmcsControl64::IO_BITMAP_B_ADDR.write(pio_bitmap.b.start_paddr() as _)?; @@ -542,7 +542,7 @@ impl ArchCpu { VmcsGuest64::IA32_EFER.write(0)?; // for AP start up, set CS_BASE to entry address, and RIP to 0. - if this_cpu_data().vcpu_state.is_running() && !this_cpu_data().boot_cpu { + if this_cpu_data().vcpu.state.is_running() && !this_cpu_data().boot_cpu { VmcsGuestNW::RIP.write(0)?; VmcsGuestNW::CS_BASE.write(entry)?; } @@ -589,7 +589,7 @@ impl ArchCpu { fn vmexit_handler(&mut self) { crate::arch::trap::handle_vmexit(self).unwrap(); - if this_cpu_data().vcpu_state.is_running() { + if this_cpu_data().vcpu.state.is_running() { check_pending_vectors(self.cpuid); } } diff --git a/src/arch/x86_64/ipi.rs b/src/arch/x86_64/ipi.rs index 67a1588d1..d99da9caf 100644 --- a/src/arch/x86_64/ipi.rs +++ b/src/arch/x86_64/ipi.rs @@ -149,13 +149,7 @@ pub fn arch_notify_event(dest: u64, vector: u64, _event_id: usize, _queue_was_em } pub fn handle_virt_ipi() { - // this may never return! - loop { - let ret = event::check_events(); - if !ret { - break; - } - } + event::dispatch_host_events(|| while event::check_events() {}); } pub fn arch_check_events(event: Option) { diff --git a/src/arch/x86_64/mod.rs b/src/arch/x86_64/mod.rs index 6f775e21a..fc24aa6c6 100644 --- a/src/arch/x86_64/mod.rs +++ b/src/arch/x86_64/mod.rs @@ -36,9 +36,11 @@ pub mod s1pt; pub mod s2pt; pub mod time; pub mod trap; +pub mod vcpu; pub mod vmcs; pub mod vmx; pub mod zone; +pub use vcpu::ArchVcpu; pub use s1pt::Stage1PageTable; pub use s2pt::stage2_mode_detect; diff --git a/src/arch/x86_64/s2pt.rs b/src/arch/x86_64/s2pt.rs index 7d62106ca..7204620c2 100644 --- a/src/arch/x86_64/s2pt.rs +++ b/src/arch/x86_64/s2pt.rs @@ -262,7 +262,7 @@ impl PagingInstr for S2PTInstr { unsafe { invs2pt(InvS2PTType::SingleContext, s2ptp) }; // if this cpu is boot cpu and it is running - if this_cpu_data().vcpu_state.is_running() && this_cpu_data().boot_cpu { + if this_cpu_data().vcpu.state.is_running() && this_cpu_data().boot_cpu { #[cfg(intel_vtd)] iommu::fill_dma_translation_tables(this_zone_id(), root_paddr); } diff --git a/src/arch/x86_64/trap.rs b/src/arch/x86_64/trap.rs index 7c7fccd4c..2a193b06d 100644 --- a/src/arch/x86_64/trap.rs +++ b/src/arch/x86_64/trap.rs @@ -112,7 +112,7 @@ fn handle_irq(vector: u8) { IdtVector::I8042_KEYBOARD_VECTOR if !is_timer => {} IdtVector::APIC_SPURIOUS_VECTOR | IdtVector::APIC_ERROR_VECTOR if !is_timer => {} _ => { - if vector >= 0x20 && this_cpu_data().vcpu_state.is_running() { + if vector >= 0x20 && this_cpu_data().vcpu.state.is_running() { inject_vector(this_cpu_id(), vector, None, false); } } diff --git a/src/arch/x86_64/vcpu.rs b/src/arch/x86_64/vcpu.rs new file mode 100644 index 000000000..4c8e838d8 --- /dev/null +++ b/src/arch/x86_64/vcpu.rs @@ -0,0 +1,25 @@ +// Copyright (c) 2025 Syswonder +// hvisor is licensed under Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +// FIT FOR A PARTICULAR PURPOSE. +// See the Mulan PSL v2 for more details. +// +// Syswonder Website: +// https://www.syswonder.org +// +// Authors: +// +/// Architecture-private vCPU state placeholder. +pub struct ArchVcpu; + +impl ArchVcpu { + pub const fn new() -> Self { + Self + } + + pub fn clear_irq_state(&self) {} +} diff --git a/src/cpu_data.rs b/src/cpu_data.rs index fc1519191..04f490545 100644 --- a/src/cpu_data.rs +++ b/src/cpu_data.rs @@ -14,12 +14,140 @@ // Authors: // use alloc::sync::Arc; +use core::sync::atomic::AtomicU64; use spin::Mutex; -use crate::arch::cpu::{store_cpu_pointer_to_reg, this_cpu_id, ArchCpu}; +use crate::arch::{ + cpu::{store_cpu_pointer_to_reg, this_cpu_id, ArchCpu}, + ArchVcpu, +}; use crate::consts::{INVALID_ADDRESS, PER_CPU_ARRAY_PTR, PER_CPU_SIZE}; use crate::memory::addr::VirtAddr; use crate::zone::Zone; + +const GUEST_IRQ_WORD_BITS: usize = u64::BITS as usize; +const GUEST_IRQ_WORDS: usize = (crate::config::CONFIG_MAX_INTERRUPTS + 63) / 64; + +/// Pending software IRQ state owned by one guest CPU. +/// +/// Bits are guest INTIDs directly. Architecture-specific LR metadata remains +/// in `ArchVcpu`; this state is the common producer/consumer contract. +pub struct GuestCpuIrqState { + pub pending: [AtomicU64; GUEST_IRQ_WORDS], +} + +impl GuestCpuIrqState { + pub const fn new() -> Self { + const ZERO_WORD: AtomicU64 = AtomicU64::new(0); + Self { + pending: [ZERO_WORD; GUEST_IRQ_WORDS], + } + } + + pub fn set(&self, irq_id: usize) -> bool { + if irq_id >= crate::config::CONFIG_MAX_INTERRUPTS { + return false; + } + self.pending[irq_id / GUEST_IRQ_WORD_BITS].fetch_or( + 1u64 << (irq_id % GUEST_IRQ_WORD_BITS), + core::sync::atomic::Ordering::Release, + ); + true + } + + pub fn clear(&self, irq_id: usize) -> bool { + if irq_id >= crate::config::CONFIG_MAX_INTERRUPTS { + return false; + } + self.pending[irq_id / GUEST_IRQ_WORD_BITS].fetch_and( + !(1u64 << (irq_id % GUEST_IRQ_WORD_BITS)), + core::sync::atomic::Ordering::AcqRel, + ); + true + } + + pub fn clear_all(&self) { + for word in &self.pending { + word.store(0, core::sync::atomic::Ordering::Release); + } + } + + pub fn pending_mask(&self, first_irq: usize, count: usize) -> u64 { + if count == 0 || count > GUEST_IRQ_WORD_BITS { + return 0; + } + let word = first_irq / GUEST_IRQ_WORD_BITS; + let offset = first_irq % GUEST_IRQ_WORD_BITS; + let mut value = self + .pending + .get(word) + .map(|pending| pending.load(core::sync::atomic::Ordering::Acquire) >> offset) + .unwrap_or(0); + if offset != 0 { + value |= self + .pending + .get(word + 1) + .map(|pending| { + pending.load(core::sync::atomic::Ordering::Acquire) + << (GUEST_IRQ_WORD_BITS - offset) + }) + .unwrap_or(0); + } + if count == GUEST_IRQ_WORD_BITS { + value + } else { + value & ((1u64 << count) - 1) + } + } + + /// Atomically claim the lowest pending INTID in one bitmap word. + pub fn claim_pending_intid(&self, word_index: usize) -> Option { + let word = self.pending.get(word_index)?; + loop { + let old = word.load(core::sync::atomic::Ordering::Acquire); + if old == 0 { + return None; + } + let bit = old.trailing_zeros() as usize; + let mask = 1u64 << bit; + if word + .compare_exchange( + old, + old & !mask, + core::sync::atomic::Ordering::AcqRel, + core::sync::atomic::Ordering::Acquire, + ) + .is_ok() + { + return Some(word_index * GUEST_IRQ_WORD_BITS + bit); + } + } + } + + pub fn has_pending(&self) -> bool { + self.pending + .iter() + .any(|word| word.load(core::sync::atomic::Ordering::Acquire) != 0) + } + + /// Claim pending INTIDs and let the controller adapter install each one. + /// + /// The adapter owns INTID classes and LR encoding. Returning `false` means + /// the adapter could not install the claimed INTID, so this helper puts it + /// back and reports that maintenance must remain armed. + pub fn flush_pending_with(&self, mut install: impl FnMut(usize) -> bool) -> bool { + for word_index in 0..self.pending.len() { + while let Some(irq_id) = self.claim_pending_intid(word_index) { + if install(irq_id) { + continue; + } + self.set(irq_id); + return true; + } + } + false + } +} use crate::ENTERED_CPUS; use core::fmt::{Debug, Formatter, Result}; use core::sync::atomic::{AtomicU8, Ordering}; @@ -100,12 +228,38 @@ impl Debug for VcpuStateCell { } } +/// vCPU state temporarily colocated with its pinned host pCPU. +/// +/// This object, including its architecture-private `arch` member, is the unit +/// that can later move out of `PerCpu` when fixed 1:N vCPU scheduling returns. +pub struct Vcpu { + pub state: VcpuStateCell, + pub irq: GuestCpuIrqState, + pub arch: ArchVcpu, +} + +impl Vcpu { + pub const fn new() -> Self { + Self { + state: VcpuStateCell::new(VcpuState::Stopped), + irq: GuestCpuIrqState::new(), + arch: ArchVcpu::new(), + } + } + + pub fn clear_irq_state(&self) { + self.irq.clear_all(); + self.arch.clear_irq_state(); + } +} + #[repr(C)] pub struct PerCpu { pub id: usize, pub cpu_on_entry: usize, pub dtb_ipa: usize, - pub vcpu_state: VcpuStateCell, + /// Current pCPU:vCPU=1:1 temporary colocation; move out for fixed 1:N. + pub vcpu: Vcpu, pub arch_cpu: ArchCpu, pub zone: Option>, pub ctrl_lock: Mutex<()>, @@ -123,7 +277,7 @@ impl PerCpu { id: arch_cpu.cpuid, cpu_on_entry: INVALID_ADDRESS, dtb_ipa: INVALID_ADDRESS, - vcpu_state: VcpuStateCell::new(VcpuState::Stopped), + vcpu: Vcpu::new(), arch_cpu, zone: None, ctrl_lock: Mutex::new(()), @@ -179,21 +333,20 @@ pub fn this_cpu_data<'a>() -> &'a mut PerCpu { pub fn this_zone() -> Arc { this_cpu_data().zone.clone().unwrap() } - /// Enter blocked state and wait until another CPU resumes it. #[allow(unused)] pub fn vcpu_suspend() { info!("cpu {} suspending...", this_cpu_data().id); - this_cpu_data().vcpu_state.store(VcpuState::Blocked); + this_cpu_data().vcpu.state.store(VcpuState::Blocked); loop { // TODO: use wfi to optimize the loop - if !this_cpu_data().vcpu_state.is_blocked() { + if !this_cpu_data().vcpu.state.is_blocked() { break; } core::hint::spin_loop(); } // Remote sets `Ready` to leave Blocked; this hart then marks itself `Running` again. - this_cpu_data().vcpu_state.store(VcpuState::Running); + this_cpu_data().vcpu.state.store(VcpuState::Running); info!("cpu {} resumed from suspend.", this_cpu_data().id); } @@ -206,7 +359,7 @@ pub fn wait_for_other_vcpus_suspend(cpu_set: CpuSet) { continue; } loop { - if get_cpu_data(target_cpu_id).vcpu_state.is_blocked() { + if get_cpu_data(target_cpu_id).vcpu.state.is_blocked() { break; } core::hint::spin_loop(); @@ -223,7 +376,8 @@ pub fn signal_other_vcpus_resume(cpu_set: CpuSet) { continue; } get_cpu_data(target_cpu_id) - .vcpu_state + .vcpu + .state .store(VcpuState::Ready); } } diff --git a/src/device/irqchip/gicv2/gic.rs b/src/device/irqchip/gicv2/gic.rs index d5e2b6a0e..11b8594d4 100644 --- a/src/device/irqchip/gicv2/gic.rs +++ b/src/device/irqchip/gicv2/gic.rs @@ -14,13 +14,15 @@ // Authors: // Hangqi Ren <2572131118@qq.com> use crate::arch::cpu::this_cpu_id; +use crate::cpu_data::this_cpu_data; use crate::device::irqchip::gicv2::gicc::GICC; use crate::device::irqchip::gicv2::gicd::GICV2_SGIS_NUM; use crate::device::irqchip::gicv2::gich::{ - GICH, GICV2_GICH_HCR_UIE, GICV2_GICH_LR_CPUID_SHIFT, GICV2_GICH_LR_HW, - GICV2_GICH_LR_PENDING_STATE, GICV2_GICH_LR_PHYSID_SHIFT, + GicHypervisorInterface, GICH, GICV2_GICH_HCR_UIE, GICV2_GICH_LR_ACTIVE_BIT, + GICV2_GICH_LR_CPUID_BITS, GICV2_GICH_LR_EOI_BIT, GICV2_GICH_LR_HW, GICV2_GICH_LR_PENDING_BIT, + GICV2_GICH_LR_PENDING_STATE, GICV2_GICH_LR_PHYSID_SHIFT, GICV2_GICH_LR_VIRTUALID_BITS, }; -use crate::event::check_events; +use crate::event::{check_events, dispatch_host_events}; use crate::hypercall::SGI_IPI_ID; /// This file defines and implements the functional functions of physical gicv2. /// author: ForeverYolo @@ -28,6 +30,8 @@ use crate::hypercall::SGI_IPI_ID; /// 1. gicv2 spec : https://www.cl.cam.ac.uk/research/srg/han/ACS-P35/zynq/arm_gic_architecture_specification.pdf use alloc::collections::VecDeque; use alloc::vec::Vec; +use bit_field::BitField; +use core::sync::atomic::Ordering; use spin::{Mutex, Once}; // gicv2 support max 8 cpus @@ -36,17 +40,13 @@ pub const MAINTENACE_INTERRUPT: u64 = 25; pub fn gicv2_handle_irq() { if let Some(irq_id) = get_pending_irq() { - if irq_id < 8 { + if irq_id < GICV2_SGIS_NUM { deactivate_irq(irq_id); - let mut ipi_handled = false; if irq_id == SGI_IPI_ID as _ { - ipi_handled = check_events(); + dispatch_host_events(|| while check_events() {}); + } else { + warn!("unexpected physical SGI {}", irq_id); } - if !ipi_handled { - inject_irq(irq_id, true); - } - } else if irq_id < GICV2_SGIS_NUM { - deactivate_irq(irq_id); } else if irq_id == MAINTENACE_INTERRUPT as usize { handle_maintenace_interrupt(); } else { @@ -88,6 +88,8 @@ pub fn change_underflow_maintenance(is_enable: bool) { fn handle_maintenace_interrupt() { info!("handle_maintenace_interrupt"); + handle_sgi_eoi_maintenance(); + flush_current_guest_irqs(); while let Some((irq_id, is_sgi)) = PENDING_VIRQS.get().unwrap().fetch_irq() { let is_injected: bool = inject_irq(irq_id, is_sgi); if is_injected { @@ -99,7 +101,210 @@ fn handle_maintenace_interrupt() { return; } } - change_underflow_maintenance(false); + change_underflow_maintenance(has_pending_maintenance_work()); +} + +fn has_pending_maintenance_work() -> bool { + this_cpu_data().vcpu.irq.has_pending() || PENDING_VIRQS.get().unwrap().has_pending() +} + +fn empty_lr_mask(gich: &GicHypervisorInterface) -> u64 { + (gich.get_elrsr(1) as u64) << 32 | gich.get_elrsr(0) as u64 +} + +/// GICH_LR value for a pending SGI from `source_cpu` (HW=0, no EOI request). +fn encode_pending_sgi_lr(irq_id: usize, source_cpu: usize) -> u32 { + let mut lr = 0u32; + lr.set_bits(GICV2_GICH_LR_VIRTUALID_BITS, irq_id as u32); + lr.set_bits(GICV2_GICH_LR_CPUID_BITS, source_cpu as u32); + lr.set_bit(GICV2_GICH_LR_PENDING_BIT, true); + lr +} + +enum SgiInstall { + Installed, + /// Another source of the same SGI occupies an LR; EOI maintenance was + /// requested on that LR. + Deferred, + NoFreeLr, +} + +/// Install one SGI source into the local LRs. +/// +/// GICv2 requires every valid LR of a virtual CPU interface to have a unique +/// VirtualID, so the same SGI from another source cannot get its own LR. That +/// source waits for the guest to deactivate the current LR, which raises an EOI +/// maintenance interrupt because of `GICH_LR.EOI`. +fn try_install_software_sgi(irq_id: usize, source_cpu: usize) -> SgiInstall { + let gich = GICH.get().unwrap(); + let empty_lrs = empty_lr_mask(gich); + let mut free_lr = None; + for index in 0..=gich.get_lr_num() as usize { + if empty_lrs.get_bit(index) { + free_lr.get_or_insert(index); + continue; + } + let mut lr = gich.get_lr(index); + if lr.get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize != irq_id { + continue; + } + if lr.get_bits(GICV2_GICH_LR_CPUID_BITS) as usize == source_cpu { + lr.set_bit(GICV2_GICH_LR_PENDING_BIT, true); + gich.set_lr(index, lr); + return SgiInstall::Installed; + } + lr.set_bit(GICV2_GICH_LR_EOI_BIT, true); + gich.set_lr(index, lr); + return SgiInstall::Deferred; + } + let Some(index) = free_lr else { + return SgiInstall::NoFreeLr; + }; + gich.set_lr(index, encode_pending_sgi_lr(irq_id, source_cpu)); + SgiInstall::Installed +} + +fn install_pending_guest_irq(cpu: &crate::cpu_data::PerCpu, irq_id: usize) -> bool { + if irq_id >= GICV2_SGIS_NUM { + return false; + } + let sources = &cpu.vcpu.arch.sgi_source[irq_id]; + let source_cpu = sources.load(Ordering::Acquire).trailing_zeros() as usize; + if source_cpu >= 8 { + // Every source was cleared after the SGI was posted. + return true; + } + let source_bit = 1u8 << source_cpu; + sources.fetch_and(!source_bit, Ordering::AcqRel); + match try_install_software_sgi(irq_id, source_cpu) { + SgiInstall::Installed => { + if sources.load(Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } + true + } + SgiInstall::Deferred => { + // The source stays in `sgi_source`; EOI maintenance reposts it. + sources.fetch_or(source_bit, Ordering::AcqRel); + true + } + SgiInstall::NoFreeLr => { + sources.fetch_or(source_bit, Ordering::AcqRel); + false + } + } +} + +/// Handle EOI maintenance for SGI LRs: free each deactivated SGI LR that +/// requested it for a deferred source, and repost that SGI if any source is +/// still pending. +fn handle_sgi_eoi_maintenance() { + let cpu = this_cpu_data(); + let gich = GICH.get().unwrap(); + let eoi_lrs = (gich.get_eisr(1) as u64) << 32 | gich.get_eisr(0) as u64; + for index in 0..=gich.get_lr_num() as usize { + if !eoi_lrs.get_bit(index) { + continue; + } + let irq_id = gich.get_lr(index).get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize; + if irq_id >= GICV2_SGIS_NUM { + continue; + } + gich.set_lr(index, 0); + if cpu.vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } + } +} + +pub fn flush_current_guest_irqs() { + let cpu = this_cpu_data(); + if cpu.zone.is_none() || cpu.vcpu.state.is_stopped() { + return; + } + let state = &cpu.vcpu.irq; + crate::device::irqchip::flush_guest_irq_state( + state, + |irq_id| install_pending_guest_irq(cpu, irq_id), + || change_underflow_maintenance(true), + ); +} + +pub fn clear_guest_sgi_pending(target_cpu: usize, irq_id: usize) -> bool { + clear_guest_sgi_pending_sources(target_cpu, irq_id, u8::MAX) +} + +pub fn clear_guest_sgi_pending_sources(target_cpu: usize, irq_id: usize, source_mask: u8) -> bool { + if irq_id >= GICV2_SGIS_NUM { + return false; + } + let target = crate::cpu_data::get_cpu_data(target_cpu); + let control = target.ctrl_lock.lock(); + target.vcpu.arch.sgi_source[irq_id].fetch_and(!source_mask, Ordering::AcqRel); + let pending_cleared = target.vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) == 0 + && target.vcpu.irq.clear(irq_id); + drop(control); + if target_cpu == this_cpu_id() { + clear_lr_pending_sgi(irq_id, source_mask); + } + pending_cleared +} + +fn clear_lr_pending_sgi(irq_id: usize, source_mask: u8) { + let gich = GICH.get().unwrap(); + let empty_lrs = empty_lr_mask(gich); + for index in 0..=gich.get_lr_num() as usize { + if empty_lrs.get_bit(index) { + continue; + } + let mut lr = gich.get_lr(index); + let source = lr.get_bits(GICV2_GICH_LR_CPUID_BITS); + if lr.get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize != irq_id + || !lr.get_bit(GICV2_GICH_LR_PENDING_BIT) + || !source_mask.get_bit(source as usize) + { + continue; + } + if lr.get_bit(GICV2_GICH_LR_ACTIVE_BIT) { + lr.set_bit(GICV2_GICH_LR_PENDING_BIT, false); + gich.set_lr(index, lr); + } else { + gich.set_lr(index, 0); + // The freed LR may have carried the EOI request of a deferred + // source, so hand that source back to the pending bitmap. + let cpu = this_cpu_data(); + if cpu.vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) != 0 { + cpu.vcpu.irq.set(irq_id); + } + } + } +} + +pub(crate) struct PendingSgiLrSummary { + pub pending_mask: u32, + pub source_masks: [u8; GICV2_SGIS_NUM], +} + +pub(crate) fn read_pending_sgi_lr_summary() -> PendingSgiLrSummary { + let gich = GICH.get().unwrap(); + let empty_lrs = empty_lr_mask(gich); + let mut summary = PendingSgiLrSummary { + pending_mask: 0, + source_masks: [0; GICV2_SGIS_NUM], + }; + for index in 0..=gich.get_lr_num() as usize { + if empty_lrs.get_bit(index) { + continue; + } + let lr = gich.get_lr(index); + let irq_id = lr.get_bits(GICV2_GICH_LR_VIRTUALID_BITS) as usize; + if irq_id < GICV2_SGIS_NUM && lr.get_bit(GICV2_GICH_LR_PENDING_BIT) { + summary.pending_mask.set_bit(irq_id, true); + summary.source_masks[irq_id] + .set_bit(lr.get_bits(GICV2_GICH_LR_CPUID_BITS) as usize, true); + } + } + summary } pub fn inject_irq(irq_id: usize, is_sgi: bool) -> bool { @@ -186,4 +391,11 @@ impl PendingIrqs { _ => None, } } + + fn has_pending(&self) -> bool { + self.inner + .get(this_cpu_id()) + .map(|pending_irqs| !pending_irqs.lock().is_empty()) + .unwrap_or(false) + } } diff --git a/src/device/irqchip/gicv2/gich.rs b/src/device/irqchip/gicv2/gich.rs index 982ff7ad3..b9a5080e5 100644 --- a/src/device/irqchip/gicv2/gich.rs +++ b/src/device/irqchip/gicv2/gich.rs @@ -31,6 +31,12 @@ pub const GICV2_GICH_VMCR_VMGRP0EN: u32 = 0x1; pub const GICV2_GICH_LR_GRP1: u32 = 0x1 << 30; pub const GICV2_GICH_LR_PENDING_STATE: usize = 0x1 << 28; pub const GICV2_GICH_LR_HW: usize = 0x1 << 31; +// GICH_LR field positions, for use with `bit_field::BitField`. +pub const GICV2_GICH_LR_VIRTUALID_BITS: core::ops::Range = 0..10; +pub const GICV2_GICH_LR_CPUID_BITS: core::ops::Range = 10..13; +pub const GICV2_GICH_LR_EOI_BIT: usize = 19; +pub const GICV2_GICH_LR_PENDING_BIT: usize = 28; +pub const GICV2_GICH_LR_ACTIVE_BIT: usize = 29; pub const GICV2_GICH_HCR_REG_OFFSET: usize = 0x0000; pub const GICV2_GICH_VTR_REG_OFFSET: usize = 0x0004; pub const GICV2_GICH_VMCR_REG_OFFSET: usize = 0x0008; @@ -77,14 +83,8 @@ pub fn gich_init(gich_base: usize) { } impl GicHypervisorInterface { - // init GICH for each CPU. - pub fn get_lrs_num(&self) -> u32 { - let vtr = self.VTR.get(); - (vtr & 0b11111) + 1 - } - pub fn clear_all_lr(&self) { - for i in 0..self.get_lrs_num() as usize { + for i in 0..=self.get_lr_num() as usize { self.LR[i].set(0); } self.APR.set(0); @@ -98,6 +98,8 @@ impl GicHypervisorInterface { self.VMCR.set(value); } + /// Returns `GICH_VTR.ListRegs`, the index of the last List register + /// (number of List registers minus one). pub fn get_lr_num(&self) -> u32 { self.VTR.get() & 0b11111 } @@ -106,6 +108,10 @@ impl GicHypervisorInterface { self.ELRSR[index].get() } + pub fn get_eisr(&self, index: usize) -> u32 { + self.EISR[index].get() + } + pub fn get_lr(&self, index: usize) -> u32 { self.LR[index].get() } diff --git a/src/device/irqchip/gicv2/vgic.rs b/src/device/irqchip/gicv2/vgic.rs index 5016c330d..8da4bef47 100644 --- a/src/device/irqchip/gicv2/vgic.rs +++ b/src/device/irqchip/gicv2/vgic.rs @@ -13,18 +13,20 @@ // // Authors: // Hangqi Ren <2572131118@qq.com> +use crate::arch::cpu::this_cpu_id; use crate::arch::zone::{GicConfig, Gicv2Config, HvArchZoneConfig}; use crate::config::{BitmapWord, CONFIG_INTERRUPTS_BITMAP_BITS_PER_WORD}; -use crate::cpu_data::this_zone; +use crate::cpu_data::{get_cpu_data, this_zone}; use crate::device::irqchip::gicv2::gicd::{ - get_max_int_num, GICD, GICD_CTRL_REG_OFFSET, GICD_ICACTIVER_REG_OFFSET, - GICD_ICENABLER_REG_OFFSET, GICD_ICFGR_REG_OFFSET, GICD_ICPENDR_REG_OFFSET, - GICD_IDENTIFICATION_NUM, GICD_IDENTIFICATION_OFFSET, GICD_IGROUPR_REG_OFFSET, - GICD_IIDR_REG_OFFSET, GICD_IPRIORITYR_REG_OFFSET, GICD_ISACTIVER_REG_OFFSET, - GICD_ISENABLER_REG_OFFSET, GICD_ISPENDR_REG_OFFSET, GICD_ITARGETSR_REG_OFFSET, GICD_LOCK, - GICD_SGIR_REG_OFFSET, GICD_SGIR_ROUTING_SHIFT, GICD_SGIR_TARGET_LIST_FILTER_SHIFT, - GICD_TYPER_REG_OFFSET, GICV2_CONFIG_REGS_NUM, GICV2_INT_REGS_NUM, GICV2_PRIO_REGS_NUM, - GICV2_TARGET_REGS_NUM, + get_max_int_num, GICD, GICD_CPENDSGIR_REG_OFFSET, GICD_CTRL_REG_OFFSET, + GICD_ICACTIVER_REG_OFFSET, GICD_ICENABLER_REG_OFFSET, GICD_ICFGR_REG_OFFSET, + GICD_ICPENDR_REG_OFFSET, GICD_IDENTIFICATION_NUM, GICD_IDENTIFICATION_OFFSET, + GICD_IGROUPR_REG_OFFSET, GICD_IIDR_REG_OFFSET, GICD_IPRIORITYR_REG_OFFSET, + GICD_ISACTIVER_REG_OFFSET, GICD_ISENABLER_REG_OFFSET, GICD_ISPENDR_REG_OFFSET, + GICD_ITARGETSR_REG_OFFSET, GICD_LOCK, GICD_SGIR_REG_OFFSET, GICD_SGIR_ROUTING_SHIFT, + GICD_SGIR_TARGET_LIST_FILTER_SHIFT, GICD_SPENDSGIR_REG_OFFSET, GICD_TYPER_REG_OFFSET, + GICV2_CONFIG_REGS_NUM, GICV2_INT_REGS_NUM, GICV2_PRIO_REGS_NUM, GICV2_SGIS_NUM, + GICV2_SGI_REGS_NUM, GICV2_TARGET_REGS_NUM, }; use crate::device::irqchip::gicv2::GICV2; use crate::error::HvResult; @@ -34,8 +36,12 @@ use crate::memory::{mmio_perform_access, MMIOAccess, MemFlags, MemoryRegion}; /// reference: /// 1. gicv2 spec : https://www.cl.cam.ac.uk/research/srg/han/ACS-P35/zynq/arm_gic_architecture_specification.pdf use crate::zone::Zone; +use bit_field::BitField; +use core::sync::atomic::Ordering; const GICV2_REG_WIDTH: usize = 4; +/// SGI bits of GICD_ISPENDR0/ICPENDR0; bits 16..32 are physical PPIs. +const SGI_PENDING_BITS: core::ops::Range = 0..16; impl Zone { // trap all Guest OS accesses to the GIC Distributor registers. @@ -272,6 +278,135 @@ pub fn set_sgi_irq(irq_id: usize, target_list: usize, routing_mode: usize) { GICD.get().unwrap().set_sgir(val as u32); } +/// Pending SGI sources not held by a local LR. A source deferred behind an LR +/// of the same SGI stays here while its pending bitmap bit is clear. +fn read_guest_sgi_source_mask(target_cpu: usize, irq_id: usize, lr_source_mask: u8) -> u8 { + get_cpu_data(target_cpu).vcpu.arch.sgi_source[irq_id].load(Ordering::Acquire) | lr_source_mask +} + +fn ignore_unsupported_gicd_access(mmio: &mut MMIOAccess) -> HvResult { + warn!( + "ignore unsupported GICD access at {:#x} with size {}", + mmio.address, mmio.size + ); + if !mmio.is_write { + mmio.value = 0; + } + Ok(()) +} + +fn handle_gicd_sgi_pending_access(mmio: &mut MMIOAccess, is_clear: bool) -> HvResult { + // Reject unaligned or wider accesses such as a 64-bit one. + if !matches!(mmio.size, 1 | 2 | 4) || mmio.address % mmio.size != 0 { + return ignore_unsupported_gicd_access(mmio); + } + let shift = (mmio.address % GICV2_REG_WIDTH) * 8; + let width_mask = (u32::MAX >> (32 - mmio.size * 8)) << shift; + let target_cpu = this_cpu_id(); + let register_base = if is_clear { + GICD_CPENDSGIR_REG_OFFSET + } else { + GICD_SPENDSGIR_REG_OFFSET + }; + // Each register holds four SGIs, one byte of source CPU bits per SGI. + let first_irq = (mmio.address & !0x3) - register_base; + if mmio.is_write { + let value = ((mmio.value as u32) << shift) & width_mask; + for byte in 0..4 { + let lane = byte * 8..byte * 8 + 8; + let irq_id = first_irq + byte; + if width_mask.get_bits(lane.clone()) == 0 || irq_id >= GICV2_SGIS_NUM { + continue; + } + let source_mask = value.get_bits(lane) as u8; + if is_clear { + crate::device::irqchip::gicv2::gic::clear_guest_sgi_pending_sources( + target_cpu, + irq_id, + source_mask, + ); + } else { + for source in 0..8 { + if source_mask.get_bit(source) { + crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(source)); + } + } + } + } + return Ok(()); + } + + let lr_summary = crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary(); + let mut value = 0u32; + for byte in 0..4 { + let irq_id = first_irq + byte; + value.set_bits( + byte * 8..byte * 8 + 8, + read_guest_sgi_source_mask(target_cpu, irq_id, lr_summary.source_masks[irq_id]) as u32, + ); + } + mmio.value = ((value & width_mask) >> shift) as usize; + Ok(()) +} + +fn handle_gicd_spendsgir_access(mmio: &mut MMIOAccess) -> HvResult { + handle_gicd_sgi_pending_access(mmio, false) +} + +fn handle_gicd_cpendsgir_access(mmio: &mut MMIOAccess) -> HvResult { + handle_gicd_sgi_pending_access(mmio, true) +} + +fn handle_gicd_intid_pending_access( + mmio: &mut MMIOAccess, + is_clear: bool, + gicd_base: usize, +) -> HvResult { + // GICD_ISPENDR0/ICPENDR0 only support aligned 32-bit accesses. + if mmio.size != GICV2_REG_WIDTH || mmio.address % GICV2_REG_WIDTH != 0 { + return ignore_unsupported_gicd_access(mmio); + } + let target_cpu = this_cpu_id(); + if mmio.is_write { + let mut value = mmio.value as u32; + for irq_id in 0..GICV2_SGIS_NUM { + if !value.get_bit(irq_id) { + continue; + } + if is_clear { + crate::device::irqchip::gicv2::gic::clear_guest_sgi_pending(target_cpu, irq_id); + } else { + crate::device::irqchip::post_guest_irq(target_cpu, irq_id, Some(target_cpu)); + } + } + // Forward only the physical PPI bits. + value.set_bits(SGI_PENDING_BITS, 0); + mmio.value = value as usize; + mmio_perform_access(gicd_base, mmio); + return Ok(()); + } + mmio_perform_access(gicd_base, mmio); + let mut sgi_pending = + crate::device::irqchip::gicv2::gic::read_pending_sgi_lr_summary().pending_mask; + for irq_id in 0..GICV2_SGIS_NUM { + if read_guest_sgi_source_mask(target_cpu, irq_id, 0) != 0 { + sgi_pending.set_bit(irq_id, true); + } + } + let mut value = mmio.value as u32; + value.set_bits(SGI_PENDING_BITS, sgi_pending); + mmio.value = value as usize; + Ok(()) +} + +fn handle_gicd_ispendr_access(mmio: &mut MMIOAccess, gicd_base: usize) -> HvResult { + handle_gicd_intid_pending_access(mmio, false, gicd_base) +} + +fn handle_gicd_icpendr_access(mmio: &mut MMIOAccess, gicd_base: usize) -> HvResult { + handle_gicd_intid_pending_access(mmio, true, gicd_base) +} + // Handle GIC Distributor register accesses. pub fn vgicv2_dist_handler(mmio: &mut MMIOAccess, _arg: usize) -> HvResult { let gicd_base = GICV2.get().unwrap().gicd_base; @@ -282,9 +417,33 @@ pub fn vgicv2_dist_handler(mmio: &mut MMIOAccess, _arg: usize) -> HvResult { if !mmio.is_write { return Ok(()); } - mmio_perform_access(gicd_base, mmio); + crate::arch::ipi::handle_guest_sgir(mmio.value as u32); Ok(()) } + reg if reg_range(GICD_ISPENDR_REG_OFFSET, 1, GICV2_REG_WIDTH).contains(®) => { + handle_gicd_ispendr_access(mmio, gicd_base) + } + reg if reg_range(GICD_ICPENDR_REG_OFFSET, 1, GICV2_REG_WIDTH).contains(®) => { + handle_gicd_icpendr_access(mmio, gicd_base) + } + reg if reg_range( + GICD_CPENDSGIR_REG_OFFSET, + GICV2_SGI_REGS_NUM, + GICV2_REG_WIDTH, + ) + .contains(®) => + { + handle_gicd_cpendsgir_access(mmio) + } + reg if reg_range( + GICD_SPENDSGIR_REG_OFFSET, + GICV2_SGI_REGS_NUM, + GICV2_REG_WIDTH, + ) + .contains(®) => + { + handle_gicd_spendsgir_access(mmio) + } reg if reg_range( GICD_ITARGETSR_REG_OFFSET, GICV2_TARGET_REGS_NUM, diff --git a/src/device/irqchip/gicv3/mod.rs b/src/device/irqchip/gicv3/mod.rs index 4767e678a..52c3d4fd7 100644 --- a/src/device/irqchip/gicv3/mod.rs +++ b/src/device/irqchip/gicv3/mod.rs @@ -27,6 +27,7 @@ use core::sync::atomic::AtomicU64; use alloc::collections::vec_deque::VecDeque; use alloc::vec::Vec; +use bit_field::BitField; use gicr::init_lpi_prop; use gits::gits_init; use spin::{Lazy, Mutex, Once}; @@ -38,9 +39,10 @@ use crate::arch::cpu::this_cpu_id; use crate::arch::zone::GicConfig; use crate::config::root_zone_config; use crate::consts::{self, MAX_CPU_NUM}; +use crate::cpu_data::this_cpu_data; use crate::device::irqchip::gicv3::gits::gits_reset; -use crate::event::check_events; +use crate::event::{check_events, dispatch_host_events}; use crate::hypercall::SGI_IPI_ID; use crate::zone::Zone; @@ -96,20 +98,15 @@ const TIMER_INTERRUPT_PRINT_INTERVAL: u64 = 50; pub fn gicv3_handle_irq_el1() { while let Some(irq_id) = pending_irq() { - if irq_id < 8 { - trace!("sgi get {}, try to handle...", irq_id); + if irq_id < 16 { deactivate_irq(irq_id); - let mut ipi_handled = false; - if irq_id == SGI_IPI_ID as _ { - ipi_handled = check_events(); - } - if !ipi_handled { - trace!("sgi get {}, inject", irq_id); - inject_irq(irq_id, false); + if irq_id == SGI_IPI_ID as usize { + // SGI 7 is the reserved physical host doorbell. Guest SGIs, + // including guest SGI 7, are posted through software state. + dispatch_host_events(|| while check_events() {}); + } else { + warn!("unexpected physical SGI {}", irq_id); } - } else if irq_id < 16 { - warn!("skip sgi {}", irq_id); - deactivate_irq(irq_id); } else { #[cfg(all(dwc_pcie, dwc_msi))] let mut is_dwc_msi_irq = false; @@ -162,6 +159,117 @@ pub fn gicv3_handle_irq_el1() { trace!("handle done") } +// ICH_LR_EL2 and ICH_VTR_EL2 fields, for use with `bit_field::BitField`. +const ICH_LR_VINTID_BITS: core::ops::Range = 0..32; +const ICH_LR_GROUP_BIT: usize = 60; +const ICH_LR_PENDING_BIT: usize = 62; +const ICH_LR_ACTIVE_BIT: usize = 63; +const ICH_VTR_LIST_REGS_BITS: core::ops::Range = 0..5; +const SGIS_NUM: usize = 16; + +fn list_regs_num() -> usize { + read_sysreg!(ich_vtr_el2).get_bits(ICH_VTR_LIST_REGS_BITS) as usize + 1 +} + +pub fn clear_guest_sgi(target_cpu: usize, irq_id: usize) -> bool { + if irq_id >= SGIS_NUM { + return false; + } + let target = crate::cpu_data::get_cpu_data(target_cpu); + let control = target.ctrl_lock.lock(); + if !target.vcpu.irq.clear(irq_id) { + return false; + } + drop(control); + if target_cpu == this_cpu_id() { + clear_lr_pending_irq(irq_id); + } + true +} + +fn try_install_software_sgi(irq_id: usize) -> bool { + let elsr = read_sysreg!(ich_elrsr_el2); + let mut free_lr = None; + for index in 0..list_regs_num() { + if elsr.get_bit(index) { + free_lr.get_or_insert(index); + continue; + } + let mut lr = read_lr(index); + if lr.get_bits(ICH_LR_VINTID_BITS) as usize != irq_id { + continue; + } + // Active-only LRs become pending; pending ones coalesce. + if !lr.get_bit(ICH_LR_PENDING_BIT) { + lr.set_bit(ICH_LR_PENDING_BIT, true); + write_lr(index, lr); + } + return true; + } + let Some(index) = free_lr else { + return false; + }; + let mut lr = 0u64; + lr.set_bits(ICH_LR_VINTID_BITS, irq_id as u64); + lr.set_bit(ICH_LR_GROUP_BIT, true); + lr.set_bit(ICH_LR_PENDING_BIT, true); + write_lr(index, lr); + true +} + +pub(crate) fn clear_lr_pending_irq(irq_id: usize) { + let elsr = read_sysreg!(ich_elrsr_el2); + for index in 0..list_regs_num() { + if elsr.get_bit(index) { + continue; + } + let mut lr = read_lr(index); + if lr.get_bits(ICH_LR_VINTID_BITS) as usize != irq_id || !lr.get_bit(ICH_LR_PENDING_BIT) { + continue; + } + if lr.get_bit(ICH_LR_ACTIVE_BIT) { + lr.set_bit(ICH_LR_PENDING_BIT, false); + write_lr(index, lr); + } else { + write_lr(index, 0); + } + } +} + +pub(crate) fn lr_pending_sgi_mask() -> u32 { + let elsr = read_sysreg!(ich_elrsr_el2); + let mut pending = 0u32; + for index in 0..list_regs_num() { + if elsr.get_bit(index) { + continue; + } + let lr = read_lr(index); + let irq_id = lr.get_bits(ICH_LR_VINTID_BITS) as usize; + if irq_id < SGIS_NUM && lr.get_bit(ICH_LR_PENDING_BIT) { + pending.set_bit(irq_id, true); + } + } + pending +} + +fn install_pending_guest_irq(irq_id: usize) -> bool { + if irq_id >= SGIS_NUM { + return false; + } + try_install_software_sgi(irq_id) +} + +pub fn flush_current_guest_irqs() { + let cpu = this_cpu_data(); + if cpu.zone.is_none() || cpu.vcpu.state.is_stopped() { + return; + } + let state = &cpu.vcpu.irq; + crate::device::irqchip::flush_guest_irq_state(state, install_pending_guest_irq, || { + enable_maintenace_interrupt(true) + }); +} + fn pending_irq() -> Option { let iar = read_sysreg!(icc_iar1_el1) as usize; if iar == 0x3ff { @@ -268,6 +376,13 @@ impl PendingIrqs { _ => None, } } + + fn has_pending(&self) -> bool { + self.inner + .get(this_cpu_id()) + .map(|pending_irqs| !pending_irqs.lock().is_empty()) + .unwrap_or(false) + } } // Enable or disable an underflow maintenace interrupt. @@ -285,6 +400,7 @@ fn enable_maintenace_interrupt(is_enable: bool) { fn handle_maintenace_interrupt() { trace!("handle_maintenace_interrupt"); + flush_current_guest_irqs(); let pending_irqs = PENDING_VIRQS.get().unwrap(); while let Some((irq_id, is_hardware)) = pending_irqs.fetch_irq() { let is_injected: bool = inject_irq(irq_id, is_hardware); @@ -297,7 +413,11 @@ fn handle_maintenace_interrupt() { return; } } - enable_maintenace_interrupt(false); + enable_maintenace_interrupt(has_pending_maintenance_work()); +} + +fn has_pending_maintenance_work() -> bool { + this_cpu_data().vcpu.irq.has_pending() || PENDING_VIRQS.get().unwrap().has_pending() } /// Inject virtual interrupt to vCPU, return whether it not needs to add pending queue. diff --git a/src/device/irqchip/gicv3/vgic.rs b/src/device/irqchip/gicv3/vgic.rs index b20cf2b71..1ddca5fdf 100644 --- a/src/device/irqchip/gicv3/vgic.rs +++ b/src/device/irqchip/gicv3/vgic.rs @@ -31,6 +31,11 @@ use crate::{ zone::{this_zone_id, Zone}, }; use alloc::sync::Arc; +use bit_field::BitField; + +/// SGI bits of GICR_ISPENDR0/ICPENDR0; bits 16..32 are physical PPIs. +const SGI_PENDING_BITS: core::ops::Range = 0..16; + pub fn reg_range(base: usize, n: usize, size: usize) -> core::ops::Range { base..(base + n * size) } @@ -271,6 +276,47 @@ pub fn vgicv3_redist_handler(mmio: &mut MMIOAccess, cpu: usize) -> HvResult { mmio.value &= !(1 << MAINTENACE_INTERRUPT); mmio.value &= !(1 << SGI_IPI_ID); } + + let is_sgi_set_pending = reg == GICR_SGI_BASE + GICR_ISPENDR; + let is_sgi_clear_pending = reg == GICR_SGI_BASE + GICR_ICPENDR; + if is_sgi_set_pending || is_sgi_clear_pending { + // GICR_ISPENDR0/ICPENDR0 only support aligned 32-bit + // accesses; others would misplace the SGI/PPI split. + if mmio.size != 4 || mmio.address % 4 != 0 { + warn!( + "ignore unsupported GICR SGI pending access at {:#x} with size {}", + mmio.address, mmio.size + ); + if !mmio.is_write { + mmio.value = 0; + } + return HvResult::Ok(()); + } + if mmio.is_write { + let sgis = mmio.value.get_bits(SGI_PENDING_BITS); + for irq_id in SGI_PENDING_BITS { + if !sgis.get_bit(irq_id) { + continue; + } + if is_sgi_set_pending { + crate::device::irqchip::post_guest_irq(cpu, irq_id, None); + } else { + super::clear_guest_sgi(cpu, irq_id); + } + } + // Preserve physical PPI writes in bits 16..31 only. + mmio.value.set_bits(SGI_PENDING_BITS, 0); + mmio_perform_access(gicr_base, mmio); + } else { + mmio_perform_access(gicr_base, mmio); + let mut value = get_cpu_data(cpu).vcpu.irq.pending_mask(0, 16) as u32; + if cpu == crate::arch::cpu::this_cpu_id() { + value |= super::lr_pending_sgi_mask(); + } + mmio.value.set_bits(SGI_PENDING_BITS, value as usize); + } + return HvResult::Ok(()); + } // ignore access to foreign redistributors mmio_perform_access(gicr_base, mmio); } else { diff --git a/src/device/irqchip/mod.rs b/src/device/irqchip/mod.rs index baa707747..ac0617bfb 100644 --- a/src/device/irqchip/mod.rs +++ b/src/device/irqchip/mod.rs @@ -15,6 +15,8 @@ // use crate::arch::zone::HvArchZoneConfig; use crate::config::HvZoneConfig; +use crate::cpu_data::{get_cpu_data, this_cpu_data, GuestCpuIrqState}; +use crate::hypercall::SGI_IPI_ID; use crate::zone::Zone; #[cfg(target_arch = "loongarch64")] @@ -27,6 +29,79 @@ pub fn handle_virtio_irq_event() { crate::device::virtio_trampoline::handle_virtio_irq(); } +/// Synchronize posted IRQ state owned by the current vCPU. +/// +#[inline] +pub fn flush_current_guest_irqs() { + #[cfg(all(irq_gicv2, target_arch = "aarch64"))] + gicv2::gic::flush_current_guest_irqs(); + #[cfg(all(irq_gicv3, target_arch = "aarch64"))] + gicv3::flush_current_guest_irqs(); +} + +/// Run the controller-independent pending claim/retry flow. +pub fn flush_guest_irq_state( + state: &GuestCpuIrqState, + install: impl FnMut(usize) -> bool, + rearm_maintenance: impl FnOnce(), +) { + if state.flush_pending_with(install) { + rearm_maintenance(); + } +} + +/// Publish a guest software IRQ in the target guest CPU's common pending +/// bitmap and kick its fixed home pCPU. `source_cpu` is used only by GICv2 to +/// reconstruct the SGI source field in the virtual LR. +pub fn post_guest_irq(target_cpu: usize, irq_id: usize, source_cpu: Option) -> bool { + if irq_id >= crate::config::CONFIG_MAX_INTERRUPTS { + return false; + } + let target = get_cpu_data(target_cpu); + #[cfg(not(irq_gicv2))] + let _ = source_cpu; + let control = target.ctrl_lock.lock(); + if target.zone.is_none() || target.vcpu.state.is_stopped() { + return false; + } + #[cfg(irq_gicv2)] + if irq_id < 16 { + let Some(source_cpu) = source_cpu.filter(|source_cpu| *source_cpu < 8) else { + return false; + }; + let source_bit = 1u8 << source_cpu; + let previous_sources = target.vcpu.arch.sgi_source[irq_id] + .fetch_or(source_bit, core::sync::atomic::Ordering::AcqRel); + // A repeated SGI from the same source coalesces with the occurrence + // already represented by the source bit, including while it is being + // claimed by the target CPU. + if previous_sources & source_bit != 0 { + return true; + } + } + if !target.vcpu.irq.set(irq_id) { + return false; + } + drop(control); + kick_vcpu(target_cpu); + true +} + +/// Ring the host doorbell for the pCPU to which the target vCPU is pinned. +/// +/// Current mapping is pCPU:vCPU=1:1. A local target relies on the common +/// exit/entry flush hook and does not receive a self IPI. A future scheduler +/// can replace `target_pcpu_id` with a resolved `VcpuTarget`. +pub fn kick_vcpu(target_pcpu_id: usize) { + let target = get_cpu_data(target_pcpu_id); + if target.zone.is_none() || target.vcpu.state.is_stopped() { + return; + } + if target_pcpu_id != this_cpu_data().id { + crate::arch::ipi::arch_send_event(target_pcpu_id as u64, SGI_IPI_ID); + } +} + #[cfg(all(irq_gicv2, target_arch = "aarch64"))] pub mod gicv2; #[cfg(all(irq_gicv2, target_arch = "aarch64"))] diff --git a/src/event.rs b/src/event.rs index ba94a6168..4a8fdaa19 100644 --- a/src/event.rs +++ b/src/event.rs @@ -37,9 +37,9 @@ pub const IPI_EVENT_WAKEUP: usize = 0; pub const IPI_EVENT_SHUTDOWN: usize = 1; pub const IPI_EVENT_VIRTIO_INJECT_IRQ: usize = 2; pub const IPI_EVENT_WAKEUP_VIRTIO_DEVICE: usize = 3; -pub const IPI_EVENT_VIRTIO_PCI_CONFIG: usize = 7; -pub const IPI_EVENT_VIRTIO_PCI_DATA: usize = 8; -pub const IPI_EVENT_VIRTIO_PCI_DONE: usize = 9; +pub const IPI_EVENT_VIRTIO_PCI_CONFIG: usize = 10; +pub const IPI_EVENT_VIRTIO_PCI_DATA: usize = 11; +pub const IPI_EVENT_VIRTIO_PCI_DONE: usize = 12; #[percpu::def_percpu] static PERCPU_EVENTS: Mutex> = Mutex::new(VecDeque::new()); @@ -173,6 +173,19 @@ pub fn check_events() -> bool { handle_event(fetch_event(this_cpu_data().id)) } +/// Run the migration-time IRQ flush hooks around an architecture's existing +/// legacy event dispatch behavior. +/// +/// New posted IRQ state and the legacy FIFO intentionally coexist during the +/// migration. Each architecture retains its existing legacy FIFO drain +/// behavior through `drain_legacy_events`; this layer adds no batching or +/// scheduling policy. The IRQ flush is controller-specific. +pub fn dispatch_host_events(drain_legacy_events: impl FnOnce() -> T) -> T { + crate::device::irqchip::flush_current_guest_irqs(); + let result = drain_legacy_events(); + result +} + /// Handle one queued event, returning whether an event was present. pub fn handle_next_event() -> bool { let event = fetch_event(this_cpu_data().id); diff --git a/src/hypercall/mod.rs b/src/hypercall/mod.rs index 36587d3d8..eeebd96ea 100644 --- a/src/hypercall/mod.rs +++ b/src/hypercall/mod.rs @@ -293,9 +293,9 @@ impl<'a> HyperCall<'a> { let target_data = get_cpu_data(boot_cpu as _); let _lock = target_data.ctrl_lock.lock(); - if target_data.vcpu_state.is_stopped() { + if target_data.vcpu.state.is_stopped() { info!("boot_cpu: {}", boot_cpu); - target_data.vcpu_state.store(VcpuState::Ready); + target_data.vcpu.state.store(VcpuState::Ready); send_event(boot_cpu, SGI_IPI_ID as _, IPI_EVENT_WAKEUP); } else { error!("hv_zone_start: cpu {} already on", boot_cpu); @@ -350,7 +350,7 @@ impl<'a> HyperCall<'a> { // wait all zone's cpus shutdown (Stopped only: includes Blocked / Ready / Running) while zone_w.cpu_set().iter().any(|cpu_id| { let _lock = get_cpu_data(cpu_id).ctrl_lock.lock(); - let not_stopped = !get_cpu_data(cpu_id).vcpu_state.is_stopped(); + let not_stopped = !get_cpu_data(cpu_id).vcpu.state.is_stopped(); count += 1; if count > MAX_WAIT_TIMES { if not_stopped { @@ -363,6 +363,7 @@ impl<'a> HyperCall<'a> { zone_w.cpu_set().iter().for_each(|cpu_id| { let _lock = get_cpu_data(cpu_id).ctrl_lock.lock(); + get_cpu_data(cpu_id).vcpu.clear_irq_state(); get_cpu_data(cpu_id).zone = None; });