diff --git a/libkernel/src/arch/arm64/memory/pg_descriptors.rs b/libkernel/src/arch/arm64/memory/pg_descriptors.rs index 55270e42..31fb1a93 100644 --- a/libkernel/src/arch/arm64/memory/pg_descriptors.rs +++ b/libkernel/src/arch/arm64/memory/pg_descriptors.rs @@ -34,6 +34,10 @@ pub enum MemoryType { Normal, } +/// Level-independent ARM64 mapping attributes, excluding the contiguous hint. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MappingAttributes(u64); + macro_rules! define_descriptor { ( $(#[$outer:meta])* @@ -172,6 +176,24 @@ macro_rules! define_descriptor { impl PaMapper for $name { type MemoryType = MemoryType; + type Attributes = MappingAttributes; + + fn mapping_attributes(self) -> Option { + if self.0 & 0b11 != $map_bits { + return None; + } + // Bits [47:12] encode addresses (or are reserved at block + // levels). Contiguous describes a group, not one mapping. + let address_mask = ((1u64 << 48) - 1) & !0xfff; + Some(MappingAttributes(self.0 & !(address_mask | 0b11 | (1 << 52)))) + } + + fn new_mapping(pa: PA, attributes: Self::Attributes) -> Self { + assert_eq!(pa.value() & ((1 << Self::MAP_SHIFT) - 1), 0, + "Cannot map non-aligned physical address"); + let address_mask = ((1u64 << 48) - 1) & !((1 << Self::MAP_SHIFT) - 1); + Self((pa.value() as u64 & address_mask) | attributes.0 | $map_bits) + } fn could_map(region: PhysMemoryRegion, va: VA) -> bool { let is_aligned = |addr: usize| (addr & ((1 << Self::MAP_SHIFT) - 1)) == 0; @@ -278,6 +300,47 @@ define_descriptor!( }, ); +#[cfg(test)] +mod attribute_tests { + use super::*; + + #[test] + fn splitting_preserves_attributes_and_clears_contiguous_hint() { + let pa = PA::from_value(0x8000_0000); + let block = L1Descriptor::from_raw( + L1Descriptor::new_map_pa(pa, MemoryType::Device, PtePermissions::ro(true)).as_raw() + | (1 << 11) + | (1 << 52) + | (1 << 55), + ); + let small = + L2Descriptor::new_mapping(pa.add_bytes(3 << 21), block.mapping_attributes().unwrap()); + let page = L3Descriptor::new_mapping( + pa.add_bytes((3 << 21) + (5 << 12)), + small.mapping_attributes().unwrap(), + ); + assert_eq!(small.mapped_address(), Some(pa.add_bytes(3 << 21))); + assert_eq!( + page.mapped_address(), + Some(pa.add_bytes((3 << 21) + (5 << 12))) + ); + assert_eq!(page.permissions(), block.permissions()); + let attrs = ((1 << 12) - 1) & !0b11 | (1 << 53) | (1 << 54) | (1 << 55); + assert_eq!(page.as_raw() & attrs, block.as_raw() & attrs); + assert_eq!(small.as_raw() & (1 << 52), 0); + assert_eq!(page.as_raw() & (1 << 52), 0); + assert_eq!(page.as_raw() & 0b11, 0b11); + assert_eq!(page.mapping_attributes(), block.mapping_attributes()); + assert!(L2Descriptor::invalid().mapping_attributes().is_none()); + assert!(page.mark_as_swapped().mapping_attributes().is_none()); + assert!( + L2Descriptor::new_next_table(TPA::from_value(0x1000)) + .mapping_attributes() + .is_none() + ); + } +} + /// The decoded state of an L3 page descriptor. pub enum L3DescriptorState { /// The entry is not present. diff --git a/libkernel/src/arch/arm64/memory/pg_tables.rs b/libkernel/src/arch/arm64/memory/pg_tables.rs index af78b15d..b7e91d62 100644 --- a/libkernel/src/arch/arm64/memory/pg_tables.rs +++ b/libkernel/src/arch/arm64/memory/pg_tables.rs @@ -7,8 +7,9 @@ use crate::{ PAGE_SIZE, address::{TPA, TVA, VA}, paging::{ - PaMapper, PageAllocator, PageTableEntry, PageTableMapper, PgTable, PgTableArray, - TLBInvalidator, TableMapper, TableMapperTable, permissions::PtePermissions, + NullTlbInvalidator, PaMapper, PageAllocator, PageTableEntry, PageTableMapper, + PagingContext, PgTable, PgTableArray, TLBInvalidator, TableMapper, TableMapperTable, + permissions::PtePermissions, }, region::{PhysMemoryRegion, VirtMemoryRegion}, }, @@ -45,11 +46,11 @@ macro_rules! impl_pgtable { self.get_idx(Self::pg_index(va)) } - fn set_desc(self, va: VA, desc: Self::Descriptor, _invalidator: &dyn TLBInvalidator) { + fn set_desc(self, va: VA, desc: Self::Descriptor, invalidator: &impl TLBInvalidator) { unsafe { - self.base - .add(Self::pg_index(va)) - .write_volatile(PageTableEntry::as_raw(desc)) + crate::memory::paging::update_descriptor( + self.base.add(Self::pg_index(va)), va, desc, invalidator, + ) }; } } @@ -95,23 +96,6 @@ pub struct MapAttributes { pub perms: PtePermissions, } -/// A collection of context required to create page tables. -pub struct MappingContext<'a, PA, PM> -where - PA: PageAllocator + 'a, - PM: PageTableMapper + 'a, -{ - /// An implementation of `PageAllocator` used to request new, zeroed page - /// tables when descending the hierarchy. - pub allocator: &'a mut PA, - /// An implementation of `PageTableMapper` that provides safe, temporary CPU - /// access to page tables at their physical addresses. - pub mapper: &'a mut PM, - /// An object responsible for issuing TLB invalidation instructions after a - /// mapping is successfully changed. - pub invalidator: &'a dyn TLBInvalidator, -} - /// Maps a contiguous physical memory region to a virtual memory region. /// /// This function walks the page table hierarchy starting from the provided root @@ -139,8 +123,9 @@ where /// zeroed page tables. /// - `mapper`: An implementation of `PageTableMapper` that provides safe, /// temporary CPU access to page tables. -/// - `invalidator`: An object responsible for issuing TLB invalidation -/// instructions after a mapping is changed. +/// - `invalidator`: An object responsible for synchronous TLB maintenance +/// on descriptor changes. Newly allocated tables are ordered ahead of +/// their link through `prepare()`; see `PagingContext`. /// /// # Returns /// @@ -182,14 +167,15 @@ where /// 4. Memory Lifecycle: The caller is responsible for managing the lifecycle of /// the physical memory described by `attrs.phys`. This function only creates /// a *mapping* to the memory; it does not take ownership of it. -pub fn map_range( +pub fn map_range( l0_table: TPA>, mut attrs: MapAttributes, - ctx: &mut MappingContext, + ctx: &mut PagingContext<'_, PA, PM, I>, ) -> Result<()> where PA: PageAllocator, PM: PageTableMapper, + I: TLBInvalidator, { if attrs.phys.size() != attrs.virt.size() { Err(MapError::SizeMismatch)?; @@ -234,17 +220,18 @@ where Ok(()) } -fn try_map_pa( +fn try_map_pa( table: TPA>, va: VA, phys_region: PhysMemoryRegion, attrs: &MapAttributes, - ctx: &mut MappingContext, + ctx: &mut PagingContext<'_, PA, PM, I>, ) -> Result> where L: PgTable>, PA: PageAllocator, PM: PageTableMapper, + I: TLBInvalidator, { if L::Descriptor::could_map(phys_region, va) { unsafe { @@ -267,7 +254,11 @@ where attrs.mem_type, attrs.perms, ), - ctx.invalidator, + // The slot is verified empty and the descriptor store is + // itself single-copy atomic, so there is neither a stale + // translation to invalidate nor an ordering another core + // must observe. + &NullTlbInvalidator {}, ); })?; } @@ -278,15 +269,16 @@ where } } -pub(super) fn map_at_level( +pub(super) fn map_at_level( table: TPA>, va: VA, - ctx: &mut MappingContext, + ctx: &mut PagingContext<'_, PA, PM, I>, ) -> Result::NextLevel>>> where L: TableMapperTable, PA: PageAllocator, PM: PageTableMapper, + I: TLBInvalidator, { unsafe { let desc = ctx @@ -321,6 +313,11 @@ where L::from_ptr(pgtable).set_desc( va, L::Descriptor::new_next_table(new_pa), + // There is nothing to invalidate here (the slot was verified + // empty), but the invalidator's prepare() orders the zero-fill + // above ahead of this link. Without it, a walker on another + // core following a speculatively fetched link could observe the + // new table page before its contents. ctx.invalidator, ); })?; @@ -338,7 +335,10 @@ pub mod tests { error::KernelError, memory::{ address::{PA, VA}, - paging::test::{MockPageAllocator, PassthroughMapper}, + paging::{ + test::{MockPageAllocator, MockTLBInvalidator, PassthroughMapper}, + walk::WalkContext, + }, }, }; @@ -355,8 +355,8 @@ pub mod tests { pub fn create_map_ctx( &mut self, - ) -> MappingContext<'_, MockPageAllocator, PassthroughMapper> { - MappingContext { + ) -> PagingContext<'_, MockPageAllocator, PassthroughMapper, MockTLBInvalidator> { + PagingContext { allocator: &mut self.inner.allocator, mapper: &mut self.inner.mapper, invalidator: &self.inner.invalidator, @@ -399,6 +399,12 @@ pub mod tests { .unwrap(); assert_eq!(perms_found, Some(expected_perms)); } + + pub fn create_walk_ctx( + &mut self, + ) -> WalkContext<'_, PassthroughMapper, MockTLBInvalidator> { + self.inner.create_walk_ctx() + } } pub type TestHarness = Arm64PagingTestHarness; diff --git a/libkernel/src/arch/arm64/memory/pg_tear_down.rs b/libkernel/src/arch/arm64/memory/pg_tear_down.rs index faa8df3d..fb28d5ec 100644 --- a/libkernel/src/arch/arm64/memory/pg_tear_down.rs +++ b/libkernel/src/arch/arm64/memory/pg_tear_down.rs @@ -3,6 +3,7 @@ use super::pg_descriptors::L0Descriptor; use super::pg_tables::{L0Table, L1Table, L3Table}; use crate::error::Result; +use crate::memory::paging::TLBInvalidator; use crate::memory::region::{PhysMemoryRegion, VirtMemoryRegion}; use crate::memory::{ PAGE_SIZE, @@ -16,9 +17,9 @@ use crate::memory::{ // Implementation for L3 (Leaf Table) impl RecursiveTeardownWalker for L3Table { - fn tear_down( + fn tear_down( table_pa: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, base_va: VA, depth: u8, control: &mut Control, @@ -26,6 +27,7 @@ impl RecursiveTeardownWalker for L3Table { ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, Dealloc: FnMut(PhysMemoryRegion), { @@ -60,10 +62,11 @@ impl RecursiveTeardownWalker for L3Table { if matches!(action, TeardownAction::FreeAndClear) { unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { - L3Table::from_ptr(pgtable) - .to_raw_ptr() - .add(found_idx) - .write_volatile(0u64); + L3Table::from_ptr(pgtable).set_desc( + entry_va, + Self::Descriptor::invalid(), + ctx.invalidator, + ); })?; } } @@ -103,14 +106,15 @@ impl RecursiveTeardownWalker for L3Table { /// /// Block mappings (2 MiB / 1 GiB) are reported as /// [`EntryKind::Mapping`] with the appropriate region size. -pub fn tear_down_address_space( +pub fn tear_down_address_space( l0_table: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, mut control: Control, mut deallocator: Dealloc, ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, Dealloc: FnMut(PhysMemoryRegion), { @@ -147,10 +151,11 @@ where if matches!(action, TeardownAction::FreeAndClear) { unsafe { ctx.mapper.with_page_table(l0_table, |pgtable| { - L0Table::from_ptr(pgtable) - .to_raw_ptr() - .add(idx) - .write_volatile(0u64); + L0Table::from_ptr(pgtable).set_desc( + entry_va, + L0Descriptor::invalid(), + ctx.invalidator, + ); })?; } } @@ -193,21 +198,22 @@ mod tests { use std::collections::HashMap; /// Tear down `l0_table` freeing every frame the walker visits. - fn capture_freed_pages( + fn capture_freed_pages( l0_table: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, ) -> HashMap { capture_freed_pages_filtered(l0_table, ctx, |_| TeardownAction::Free) } /// Tear down `l0_table` using a custom `control` closure. - fn capture_freed_pages_filtered( + fn capture_freed_pages_filtered( l0_table: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, control: Control, ) -> HashMap where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, { let mut freed_map = HashMap::new(); diff --git a/libkernel/src/arch/arm64/memory/pg_walk.rs b/libkernel/src/arch/arm64/memory/pg_walk.rs index 59b48d22..8c2000f3 100644 --- a/libkernel/src/arch/arm64/memory/pg_walk.rs +++ b/libkernel/src/arch/arm64/memory/pg_walk.rs @@ -10,22 +10,49 @@ use crate::{ PAGE_SIZE, address::{TPA, VA}, paging::{ - NullTlbInvalidator, PageTableEntry, PageTableMapper, PgTable, PgTableArray, - walk::{RecursiveWalker, WalkContext}, + NullTlbInvalidator, PageAllocator, PageTableEntry, PageTableMapper, PagingContext, + PgTable, PgTableArray, TLBInvalidator, + walk::{ + RecursiveWalker, WalkContext, WalkOperation, validate_allocating_region, + walk_table_level, + }, }, region::VirtMemoryRegion, }, }; +impl RecursiveWalker for L0Table { + fn walk_with_allocator( + table_pa: TPA>, + region: VirtMemoryRegion, + ctx: &mut WalkContext, + allocator: &mut A, + operation: &mut WalkOperation, + ) -> Result<()> + where + PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, + F: FnMut(VA, L3Descriptor) -> L3Descriptor, + { + walk_table_level(table_pa, region, ctx, allocator, operation, |_| { + Err(MapError::InvalidDescriptor.into()) + }) + } +} + impl RecursiveWalker for L3Table { - fn walk( + fn walk_with_allocator( table_pa: TPA>, region: VirtMemoryRegion, - ctx: &mut WalkContext, - modifier: &mut F, + ctx: &mut WalkContext, + _allocator: &mut A, + operation: &mut WalkOperation, ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, F: FnMut(VA, L3Descriptor) -> L3Descriptor, { unsafe { @@ -34,7 +61,7 @@ impl RecursiveWalker for L3Table { for va in region.iter_pages() { let desc = table.get_desc(va); if desc.is_valid() { - table.set_desc(va, modifier(va, desc), ctx.invalidator); + table.set_desc(va, operation.apply(va, desc), ctx.invalidator); } } }) @@ -59,18 +86,17 @@ impl RecursiveWalker for L3Table { /// /// # Errors /// - `MapError::VirtNotAligned`: The provided `region` is not page-aligned. -/// - `MapError::NotMapped`: Part of the `region` is not mapped down to the L3 -/// level. -/// - `MapError::NotAnL3Mapping`: Part of the `region` is covered by a larger +/// - `MapError::NotL3Mapped`: Part of the `region` is covered by a larger /// block mapping (1GiB or 2MiB), which cannot be modified at the L3 level. -pub fn walk_and_modify_region( +pub fn walk_and_modify_region( l0_table: TPA>, region: VirtMemoryRegion, - ctx: &mut WalkContext, + ctx: &mut WalkContext, mut modifier: F, // Pass closure as a mutable ref to be used across recursive calls ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, F: FnMut(VA, L3Descriptor) -> L3Descriptor, { if !region.is_page_aligned() { @@ -84,6 +110,39 @@ where L0Table::walk(l0_table, region, ctx, &mut modifier) } +/// Removes mappings in a page-aligned region, leaving surrounding mappings intact. +/// +/// Fully covered pages and blocks are cleared directly. Partially covered blocks +/// are demoted using `ctx.allocator`; unmapped entries are skipped. Physical data +/// frames and empty page tables are not freed. +/// +/// Allocation and mapper errors propagate; earlier changes are not rolled back. +/// The caller must serialize access to the page tables and supply synchronous +/// TLB maintenance for active mappings through `ctx.invalidator`. +/// +/// Returns `MapError::VirtNotAligned` for an unaligned start or length, and +/// `KernelError::InvalidValue` if the exclusive end overflows. +pub fn punch_hole( + l0_table: TPA>, + region: VirtMemoryRegion, + ctx: &mut PagingContext, +) -> Result<()> { + validate_allocating_region(region)?; + if region.size() == 0 { + return Ok(()); + } + L0Table::walk_with_allocator( + l0_table, + region, + &mut WalkContext { + mapper: ctx.mapper, + invalidator: ctx.invalidator, + }, + ctx.allocator, + &mut WalkOperation:: L3Descriptor>::PunchHole, + ) +} + /// Obtain the PTE that mapps the VA into the current address space. pub fn get_pte( l0_table: TPA>, @@ -124,6 +183,16 @@ mod tests { use crate::memory::paging::permissions::PtePermissions; use std::sync::atomic::{AtomicUsize, Ordering}; + crate::memory::paging::test::block_split_tests!( + L1Table, + L2Table, + L3Table, + crate::arch::arm64::memory::pg_descriptors::L1Descriptor, + L2Descriptor, + L3Descriptor, + MemoryType::Device + ); + #[test] fn walk_modify_single_page() { let mut harness = TestHarness::new(10); diff --git a/libkernel/src/arch/x86_64/memory/pg_descriptors.rs b/libkernel/src/arch/x86_64/memory/pg_descriptors.rs index 4db2034c..15488016 100644 --- a/libkernel/src/arch/x86_64/memory/pg_descriptors.rs +++ b/libkernel/src/arch/x86_64/memory/pg_descriptors.rs @@ -25,6 +25,14 @@ pub enum MemoryType { WB, } +/// Level-independent x86 mapping attributes. PAT is stored separately because +/// its descriptor bit position depends on the mapping size. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MappingAttributes { + flags: u64, + pat: bool, +} + macro_rules! define_descriptor { ( $(#[$outer:meta])* @@ -111,10 +119,6 @@ macro_rules! define_descriptor { Self(reg.get()) } - - fn address(self) -> PA { - PA::from_value((self.0 & ADDR_MASK) as usize) - } } } } @@ -149,8 +153,38 @@ macro_rules! impl_pa_mapper { ($($name:ident),+ ; marker: $marker:expr) => { $( paste! { + impl $name { + /// The physical address mapped by a block/page descriptor. Bits + /// below MAP_SHIFT are attributes; not part of the address. + fn address(self) -> PA { + PA::from_value( + (self.0 & ADDR_MASK) as usize & !((1usize << Self::MAP_SHIFT) - 1) + ) + } + } + impl PaMapper for $name { type MemoryType = MemoryType; + type Attributes = MappingAttributes; + + fn mapping_attributes(self) -> Option { + if self.0 & $marker != $marker { + return None; + } + let pat_bit = if Self::MAP_SHIFT == 12 { 7 } else { 12 }; + Some(MappingAttributes { + flags: self.0 & !(ADDR_MASK | 1 | (1 << 7)), + pat: self.0 & (1 << pat_bit) != 0, + }) + } + + fn new_mapping(pa: PA, attributes: Self::Attributes) -> Self { + assert_eq!(pa.value() & ((1 << Self::MAP_SHIFT) - 1), 0, + "Cannot map non-aligned physical address"); + let pat_bit = if Self::MAP_SHIFT == 12 { 7 } else { 12 }; + Self((pa.value() as u64 & ADDR_MASK) | attributes.flags + | ((attributes.pat as u64) << pat_bit) | $marker) + } fn could_map(region: PhysMemoryRegion, va: VA) -> bool { let is_aligned = |addr: usize| (addr & ((1 << Self::MAP_SHIFT) - 1)) == 0; @@ -227,7 +261,7 @@ macro_rules! impl_table_mapper { let reg = InMemoryRegister::new(self.0); if reg.matches_all(BlockPageFields::P::Present + BlockPageFields::PS::MapTable) { - Some(self.address().cast()) + Some(TPA::from_value((self.0 & ADDR_MASK) as usize)) } else { None } @@ -272,6 +306,47 @@ mod tests { const KERNEL_PERMS: bool = false; const USER_PERMS: bool = true; + #[test] + fn splitting_preserves_pat_and_mapping_attributes() { + let pa = PA::from_value(0x8000_0000); + let attrs = (1 << 12) | (1 << 5) | (1 << 6) | (1 << 8) | (1 << 9) | (1 << 59); + let block = PDPE::from_raw( + PDPE::new_map_pa(pa, MemoryType::WT, PtePermissions::ro(true)).as_raw() | attrs, + ); + let small = PDE::new_mapping(pa.add_bytes(3 << 21), block.mapping_attributes().unwrap()); + let page = PTE::new_mapping( + pa.add_bytes((3 << 21) + (5 << 12)), + small.mapping_attributes().unwrap(), + ); + assert_eq!(small.mapped_address(), Some(pa.add_bytes(3 << 21))); + assert_eq!( + page.mapped_address(), + Some(pa.add_bytes((3 << 21) + (5 << 12))) + ); + assert_eq!(page.permissions(), block.permissions()); + let common = + (1 << 3) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 8) | (1 << 9) | (1 << 59) | (1 << 63); + assert_eq!(page.as_raw() & common, block.as_raw() & common); + assert_ne!(small.as_raw() & (1 << 12), 0); + assert_ne!(page.as_raw() & (1 << 7), 0); + let no_pat = PDE::new_map_pa(pa, MemoryType::WB, PtePermissions::rw(false)); + assert_eq!( + PTE::new_mapping(pa, no_pat.mapping_attributes().unwrap()).as_raw() & (1 << 7), + 0 + ); + assert_eq!(page.mapping_attributes(), block.mapping_attributes()); + assert_eq!( + PDPE::new_mapping(pa, page.mapping_attributes().unwrap()), + block + ); + assert!(PDE::invalid().mapping_attributes().is_none()); + assert!( + PDE::new_next_table(TPA::from_value(0x1000)) + .mapping_attributes() + .is_none() + ); + } + #[test] fn test_invalid_descriptor() { let d = PML4E::invalid(); diff --git a/libkernel/src/arch/x86_64/memory/pg_tables.rs b/libkernel/src/arch/x86_64/memory/pg_tables.rs index b038963a..954f398e 100644 --- a/libkernel/src/arch/x86_64/memory/pg_tables.rs +++ b/libkernel/src/arch/x86_64/memory/pg_tables.rs @@ -7,8 +7,9 @@ use crate::{ PAGE_SIZE, address::{TPA, TVA, VA}, paging::{ - PaMapper, PageAllocator, PageTableEntry, PageTableMapper, PgTable, PgTableArray, - TLBInvalidator, TableMapper, TableMapperTable, permissions::PtePermissions, + NullTlbInvalidator, PaMapper, PageAllocator, PageTableEntry, PageTableMapper, + PagingContext, PgTable, PgTableArray, TLBInvalidator, TableMapper, TableMapperTable, + permissions::PtePermissions, }, region::{PhysMemoryRegion, VirtMemoryRegion}, }, @@ -45,11 +46,11 @@ macro_rules! impl_pgtable { self.get_idx(Self::pg_index(va)) } - fn set_desc(self, va: VA, desc: Self::Descriptor, _invalidator: &dyn TLBInvalidator) { + fn set_desc(self, va: VA, desc: Self::Descriptor, invalidator: &impl TLBInvalidator) { unsafe { - self.base - .add(Self::pg_index(va)) - .write_volatile(PageTableEntry::as_raw(desc)) + crate::memory::paging::update_descriptor( + self.base.add(Self::pg_index(va)), va, desc, invalidator, + ) }; } } @@ -92,23 +93,6 @@ pub struct MapAttributes { pub perms: PtePermissions, } -/// A collection of context required to create page tables. -pub struct MappingContext<'a, PA, PM> -where - PA: PageAllocator + 'a, - PM: PageTableMapper + 'a, -{ - /// An implementation of `PageAllocator` used to request new, zeroed page - /// tables when descending the hierarchy. - pub allocator: &'a mut PA, - /// An implementation of `PageTableMapper` that provides safe, temporary CPU - /// access to page tables at their physical addresses. - pub mapper: &'a mut PM, - /// An object responsible for issuing TLB invalidation instructions after a - /// mapping is successfully changed. - pub invalidator: &'a dyn TLBInvalidator, -} - /// Maps a contiguous physical memory region to a virtual memory region. /// /// This function walks the page table hierarchy starting from the provided root @@ -179,14 +163,15 @@ where /// 4. Memory Lifecycle: The caller is responsible for managing the lifecycle of /// the physical memory described by `attrs.phys`. This function only creates /// a *mapping* to the memory; it does not take ownership of it. -pub fn map_range( +pub fn map_range( pml4_table: TPA>, mut attrs: MapAttributes, - ctx: &mut MappingContext, + ctx: &mut PagingContext<'_, PA, PM, I>, ) -> Result<()> where PA: PageAllocator, PM: PageTableMapper, + I: TLBInvalidator, { if attrs.phys.size() != attrs.virt.size() { Err(MapError::SizeMismatch)?; @@ -238,17 +223,18 @@ where Ok(()) } -fn try_map_pa( +fn try_map_pa( table: TPA>, va: VA, phys_region: PhysMemoryRegion, attrs: &MapAttributes, - ctx: &mut MappingContext, + ctx: &mut PagingContext<'_, PA, PM, I>, ) -> Result> where L: PgTable>, PA: PageAllocator, PM: PageTableMapper, + I: TLBInvalidator, { if L::Descriptor::could_map(phys_region, va) { unsafe { @@ -271,7 +257,11 @@ where attrs.mem_type, attrs.perms, ), - ctx.invalidator, + // The slot is verified empty and the descriptor store is + // itself single-copy atomic, so there is neither a stale + // translation to invalidate nor an ordering another core + // must observe. + &NullTlbInvalidator {}, ); })?; } @@ -282,15 +272,16 @@ where } } -pub(super) fn map_at_level( +pub(super) fn map_at_level( table: TPA>, va: VA, - ctx: &mut MappingContext, + ctx: &mut PagingContext<'_, PA, PM, I>, ) -> Result::NextLevel>>> where L: TableMapperTable, PA: PageAllocator, PM: PageTableMapper, + I: TLBInvalidator, { unsafe { let desc = ctx @@ -325,6 +316,11 @@ where L::from_ptr(pgtable).set_desc( va, L::Descriptor::new_next_table(new_pa), + // There is nothing to invalidate here (the slot was verified + // empty), but the invalidator's prepare() orders the zero-fill + // above ahead of this link. Without it, a walker on another + // core following a speculatively fetched link could observe the + // new table page before its contents. ctx.invalidator, ); })?; @@ -342,7 +338,10 @@ pub mod tests { error::KernelError, memory::{ address::{PA, VA}, - paging::test::{MockPageAllocator, PassthroughMapper}, + paging::{ + test::{MockPageAllocator, MockTLBInvalidator, PassthroughMapper}, + walk::WalkContext, + }, }, }; @@ -359,8 +358,8 @@ pub mod tests { pub fn create_map_ctx( &mut self, - ) -> MappingContext<'_, MockPageAllocator, PassthroughMapper> { - MappingContext { + ) -> PagingContext<'_, MockPageAllocator, PassthroughMapper, MockTLBInvalidator> { + PagingContext { allocator: &mut self.inner.allocator, mapper: &mut self.inner.mapper, invalidator: &self.inner.invalidator, @@ -403,6 +402,12 @@ pub mod tests { .unwrap(); assert_eq!(perms_found, Some(expected_perms)); } + + pub fn create_walk_ctx( + &mut self, + ) -> WalkContext<'_, PassthroughMapper, MockTLBInvalidator> { + self.inner.create_walk_ctx() + } } pub type TestHarness = X86_64PagingTestHarness; diff --git a/libkernel/src/arch/x86_64/memory/pg_tear_down.rs b/libkernel/src/arch/x86_64/memory/pg_tear_down.rs index 292dd9f0..31bb6166 100644 --- a/libkernel/src/arch/x86_64/memory/pg_tear_down.rs +++ b/libkernel/src/arch/x86_64/memory/pg_tear_down.rs @@ -3,7 +3,7 @@ use super::pg_descriptors::PML4E; use super::pg_tables::{PDPTable, PML4Table, PTable}; use crate::error::Result; -use crate::memory::paging::TableMapper; +use crate::memory::paging::{TLBInvalidator, TableMapper}; use crate::memory::region::{PhysMemoryRegion, VirtMemoryRegion}; use crate::memory::{ PAGE_SIZE, @@ -17,9 +17,9 @@ use crate::memory::{ // Implementation for PTable (Leaf Table) impl RecursiveTeardownWalker for PTable { - fn tear_down( + fn tear_down( table_pa: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, base_va: VA, depth: u8, control: &mut Control, @@ -27,6 +27,7 @@ impl RecursiveTeardownWalker for PTable { ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, Dealloc: FnMut(PhysMemoryRegion), { @@ -61,10 +62,11 @@ impl RecursiveTeardownWalker for PTable { if matches!(action, TeardownAction::FreeAndClear) { unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { - PTable::from_ptr(pgtable) - .to_raw_ptr() - .add(found_idx) - .write_volatile(0u64); + PTable::from_ptr(pgtable).set_desc( + entry_va, + Self::Descriptor::invalid(), + ctx.invalidator, + ); })?; } } @@ -107,14 +109,15 @@ impl RecursiveTeardownWalker for PTable { /// /// Block mappings (2 MiB / 1 GiB) are reported as /// [`EntryKind::Mapping`] with the appropriate region size. -pub fn tear_down_address_space( +pub fn tear_down_address_space( pml4_table: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, mut control: Control, mut deallocator: Dealloc, ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, Dealloc: FnMut(PhysMemoryRegion), { @@ -156,10 +159,11 @@ where if matches!(action, TeardownAction::FreeAndClear) { unsafe { ctx.mapper.with_page_table(pml4_table, |pml4_tbl| { - PML4Table::from_ptr(pml4_tbl) - .to_raw_ptr() - .add(idx) - .write_volatile(0u64); + PML4Table::from_ptr(pml4_tbl).set_desc( + entry_va, + PML4E::invalid(), + ctx.invalidator, + ); })?; } } @@ -202,21 +206,22 @@ mod tests { use std::collections::HashMap; /// Tear down `root_table` freeing every frame the walker visits. - fn capture_freed_pages( + fn capture_freed_pages( root_table: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, ) -> HashMap { capture_freed_pages_filtered(root_table, ctx, |_| TeardownAction::Free) } /// Tear down `root_table` using a custom `control` closure. - fn capture_freed_pages_filtered( + fn capture_freed_pages_filtered( root_table: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, control: Control, ) -> HashMap where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, { let mut freed_map = HashMap::new(); diff --git a/libkernel/src/arch/x86_64/memory/pg_walk.rs b/libkernel/src/arch/x86_64/memory/pg_walk.rs index db730849..748d55f8 100644 --- a/libkernel/src/arch/x86_64/memory/pg_walk.rs +++ b/libkernel/src/arch/x86_64/memory/pg_walk.rs @@ -1,15 +1,17 @@ //! Page table walking and per-entry modification. - use crate::{ error::{MapError, Result}, memory::{ PAGE_SIZE, address::{Address, TPA, VA}, paging::{ - NullTlbInvalidator, PaMapper, PageTableEntry, PageTableMapper, PgTable, PgTableArray, - TableMapper, + NullTlbInvalidator, PaMapper, PageAllocator, PageTableEntry, PageTableMapper, + PagingContext, PgTable, PgTableArray, TLBInvalidator, TableMapper, permissions::PtePermissions, - walk::{RecursiveWalker, Translatable, Translator, WalkContext}, + walk::{ + RecursiveWalker, Translatable, Translator, WalkContext, WalkOperation, + validate_allocating_region, walk_table_level, + }, }, region::{MemoryRegion, VirtMemoryRegion}, }, @@ -20,15 +22,38 @@ use super::{ pg_tables::{PDPTable, PML4Table, PTable}, }; +impl RecursiveWalker for PML4Table { + fn walk_with_allocator( + table_pa: TPA>, + region: VirtMemoryRegion, + ctx: &mut WalkContext, + allocator: &mut A, + operation: &mut WalkOperation, + ) -> Result<()> + where + PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, + F: FnMut(VA, PTE) -> PTE, + { + walk_table_level(table_pa, region, ctx, allocator, operation, |_| { + Err(MapError::InvalidDescriptor.into()) + }) + } +} + impl RecursiveWalker for PTable { - fn walk( + fn walk_with_allocator( table_pa: TPA>, region: VirtMemoryRegion, - ctx: &mut WalkContext, - modifier: &mut F, + ctx: &mut WalkContext, + _allocator: &mut A, + operation: &mut WalkOperation, ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, F: FnMut(VA, PTE) -> PTE, { unsafe { @@ -37,7 +62,7 @@ impl RecursiveWalker for PTable { for va in region.iter_pages() { let desc = table.get_desc(va); if desc.is_valid() { - table.set_desc(va, modifier(va, desc), ctx.invalidator); + table.set_desc(va, operation.apply(va, desc), ctx.invalidator); } } }) @@ -62,18 +87,17 @@ impl RecursiveWalker for PTable { /// /// # Errors /// - `MapError::VirtNotAligned`: The provided `region` is not page-aligned. -/// - `MapError::NotMapped`: Part of the `region` is not mapped down to the L3 -/// level. -/// - `MapError::NotAnL3Mapping`: Part of the `region` is covered by a larger +/// - `MapError::NotL3Mapped`: Part of the `region` is covered by a larger /// block mapping (1GiB or 2MiB), which cannot be modified at the L3 level. -pub fn walk_and_modify_region( +pub fn walk_and_modify_region( pml4_table: TPA>, region: VirtMemoryRegion, - ctx: &mut WalkContext, + ctx: &mut WalkContext, mut modifier: F, // Pass closure as a mutable ref to be used across recursive calls ) -> Result<()> where PM: PageTableMapper, + I: TLBInvalidator, F: FnMut(VA, PTE) -> PTE, { if !region.is_page_aligned() { @@ -87,6 +111,39 @@ where PML4Table::walk(pml4_table, region, ctx, &mut modifier) } +/// Removes mappings in a page-aligned region, leaving surrounding mappings intact. +/// +/// Fully covered pages and blocks are cleared directly. Partially covered +/// blocks are demoted using `ctx.allocator`; unmapped entries are skipped. +/// Physical data frames and empty page tables are not freed. +/// +/// Allocation and mapper errors propagate; earlier changes are not rolled back. +/// The caller must serialize access to the page tables and supply synchronous +/// TLB maintenance for active mappings through `ctx.invalidator`. +/// +/// Returns `MapError::VirtNotAligned` for an unaligned start or length, and +/// `KernelError::InvalidValue` if the exclusive end overflows. +pub fn punch_hole( + pml4_table: TPA>, + region: VirtMemoryRegion, + ctx: &mut PagingContext, +) -> Result<()> { + validate_allocating_region(region)?; + if region.size() == 0 { + return Ok(()); + } + PML4Table::walk_with_allocator( + pml4_table, + region, + &mut WalkContext { + mapper: ctx.mapper, + invalidator: ctx.invalidator, + }, + ctx.allocator, + &mut WalkOperation:: PTE>::PunchHole, + ) +} + /// Obtain the PTE that mapps the VA into the current address space. pub fn get_pte( pml4_table: TPA>, @@ -115,10 +172,10 @@ pub fn get_pte( } impl Translator for PML4Table { - fn translate>( + fn translate, I: TLBInvalidator>( table_pa: Address>, va: Address, - ctx: &mut WalkContext, + ctx: &mut WalkContext, ) -> crate::error::Result, usize, PtePermissions)>> { let desc = unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { @@ -137,10 +194,10 @@ impl Translator for PML4Table { } impl Translator for PTable { - fn translate>( + fn translate, I: TLBInvalidator>( table_pa: Address>, va: Address, - ctx: &mut WalkContext, + ctx: &mut WalkContext, ) -> crate::error::Result, usize, PtePermissions)>> { let desc = unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { @@ -207,6 +264,16 @@ mod tests { }; use std::sync::atomic::{AtomicUsize, Ordering}; + crate::memory::paging::test::block_split_tests!( + PDPTable, + PDTable, + PTable, + crate::arch::x86_64::memory::pg_descriptors::PDPE, + PDE, + PTE, + MemoryType::WT + ); + #[test] fn walk_modify_single_page() { let mut harness = TestHarness::new(10); diff --git a/libkernel/src/memory/paging/mod.rs b/libkernel/src/memory/paging/mod.rs index 87b5c81f..c73eb5e5 100644 --- a/libkernel/src/memory/paging/mod.rs +++ b/libkernel/src/memory/paging/mod.rs @@ -60,6 +60,21 @@ pub trait PaMapper: PageTableEntry { /// The memory attribute type for this descriptor's architecture. type MemoryType: Copy; + /// Level-independent mapping attributes for this architecture, including + /// permissions, cache policy, hardware state and software-defined bits. + /// Does not include the output address, descriptor type or grouping hints. + type Attributes: Copy; + + /// Extracts attributes from a valid physical mapping, or returns `None` + /// for table and non-present descriptors. + fn mapping_attributes(self) -> Option; + + /// Constructs a valid mapping with attributes obtained from any mapping + /// level of the same architecture. Encodes level-specific bits as needed. + /// + /// Panics if `pa` is not aligned to this descriptor's mapping size. + fn new_mapping(pa: PA, attributes: Self::Attributes) -> Self; + /// Constructs a new valid page descriptor that maps a physical address. fn new_map_pa(page_address: PA, memory_type: Self::MemoryType, perms: PtePermissions) -> Self; @@ -130,7 +145,7 @@ pub trait PgTable: Clone + Copy { fn get_idx(self, idx: usize) -> Self::Descriptor; /// Set the value of the descriptor for a particular VA. - fn set_desc(self, va: VA, desc: Self::Descriptor, invalidator: &dyn TLBInvalidator); + fn set_desc(self, va: VA, desc: Self::Descriptor, invalidator: &impl TLBInvalidator); } /// A page-aligned array of raw page table entries for a given table level. @@ -218,10 +233,96 @@ pub trait PageAllocator { fn allocate_page_table(&mut self) -> crate::error::Result>>; } -/// Trait for invalidating TLB entries after page table modifications. -pub trait TLBInvalidator {} +/// Describes a descriptor update, including the entire old mapping's coverage. +#[derive(Debug, Clone, Copy)] +pub struct TranslationChange { + /// Virtual base of the affected entry, aligned to `size`. + pub va: VA, + /// Entry coverage in bytes (4 KiB, 2 MiB, 1 GiB, or 512 GiB). + pub size: usize, + /// Architecture-specific bits of the original descriptor. + pub old_descriptor: u64, + /// Architecture-specific bits of the replacement descriptor. + pub new_descriptor: u64, +} + +/// Synchronous architecture-specific maintenance for descriptor updates. +/// +/// Implementations carry any address-space/ASID context and must complete +/// required cross-CPU shootdowns before returning. These hooks are ran inside +/// the mapper's table access window; they must not re-enter that mapper. +pub trait TLBInvalidator { + /// Called before changing the entry. Order prior stores to newly populated + /// child tables so they are visible before a table descriptor is installed. + fn prepare(&self, change: &TranslationChange); + + /// Called after a valid old descriptor has been cleared, before the + /// replacement is written (break-before-make). + /// + /// Order the clearing store, invalidate translations and walk caches for + /// the entire old entry's coverage, and wait for completion. A table entry + /// requires invalidation of its subtree, not just a single leaf. + fn invalidate(&self, change: &TranslationChange); + + /// Called after the replacement has been written, including invalid + /// replacements. Order descriptor/table stores and synchronize instruction + /// execution as required before the updated mapping is used. + fn publish(&self, change: &TranslationChange); +} + +pub(crate) unsafe fn update_descriptor>( + slot: *mut u64, + va: VA, + new_desc: D, + invalidator: &impl TLBInvalidator, +) { + let old = D::from_raw(unsafe { slot.read_volatile() }); + if old.as_raw() == new_desc.as_raw() { + return; + } + let size = 1 << D::MAP_SHIFT; + let change = TranslationChange { + va: va.align(size), + size, + old_descriptor: old.as_raw(), + new_descriptor: new_desc.as_raw(), + }; + invalidator.prepare(&change); + if old.is_valid() { + unsafe { slot.write_volatile(D::INVALID) }; + invalidator.invalidate(&change); + } + unsafe { slot.write_volatile(new_desc.as_raw()) }; + invalidator.publish(&change); +} /// A no-op TLB invalidator used when invalidation is unnecessary. pub struct NullTlbInvalidator {} -impl TLBInvalidator for NullTlbInvalidator {} +impl TLBInvalidator for NullTlbInvalidator { + fn prepare(&self, _change: &TranslationChange) {} + fn invalidate(&self, _change: &TranslationChange) {} + fn publish(&self, _change: &TranslationChange) {} +} + +/// A collection of context required to modify page tables. +/// +/// Bundles the services shared by the page-table modification operations, such +/// as mapping a region and punching holes out of one: an allocator for new page +/// tables, a mapper providing temporary CPU access to table pages, and +/// synchronous TLB maintenance for descriptor changes. +pub struct PagingContext<'a, PA, PM, I> +where + PA: PageAllocator + 'a, + PM: PageTableMapper + 'a, + I: TLBInvalidator + 'a, +{ + /// An implementation of `PageAllocator` used to request new, zeroed page + /// tables when descending the hierarchy or splitting block mappings. + pub allocator: &'a mut PA, + /// An implementation of `PageTableMapper` that provides safe, temporary CPU + /// access to page tables at their physical addresses. + pub mapper: &'a mut PM, + /// Synchronous TLB maintenance for descriptor updates. + pub invalidator: &'a I, +} diff --git a/libkernel/src/memory/paging/tear_down.rs b/libkernel/src/memory/paging/tear_down.rs index 0dce9eb7..2982ab67 100644 --- a/libkernel/src/memory/paging/tear_down.rs +++ b/libkernel/src/memory/paging/tear_down.rs @@ -4,8 +4,8 @@ use crate::memory::{ PAGE_SIZE, address::{PA, TPA, VA}, paging::{ - PaMapper, PageTableEntry, PageTableMapper, PgTable, PgTableArray, TableMapper, - TableMapperTable, walk::WalkContext, + PaMapper, PageTableEntry, PageTableMapper, PgTable, PgTableArray, TLBInvalidator, + TableMapper, TableMapperTable, walk::WalkContext, }, region::{PhysMemoryRegion, VirtMemoryRegion}, }; @@ -52,9 +52,8 @@ pub enum TeardownAction { /// `deallocator`. Free, /// Like [`TeardownAction::Free`], but also zero the page table entry in the - /// parent table *before* calling `deallocator`. The zeroing uses - /// `write_volatile` and happens before the frame is released to the - /// allocator, closing the stale-PTE window. + /// parent table *before* calling `deallocator`. The clearing completes the + /// invalidator's synchronous TLB maintenance before the frame is released. FreeAndClear, /// Skip this entry entirely. For [`EntryKind::IntermediateTable`] entries, /// the subtree is not walked. `deallocator` is not called. @@ -81,9 +80,9 @@ pub trait RecursiveTeardownWalker: PgTable + Sized { /// free physical memory — use `deallocator` for that. /// - `deallocator`: called by the walker (never inside a live /// `with_page_table` window) to physically release a frame. - fn tear_down( + fn tear_down( table_pa: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, base_va: VA, depth: u8, control: &mut Control, @@ -91,6 +90,7 @@ pub trait RecursiveTeardownWalker: PgTable + Sized { ) -> crate::error::Result<()> where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, Dealloc: FnMut(PhysMemoryRegion); } @@ -107,9 +107,9 @@ where T::Descriptor: PaMapper, ::NextLevel: RecursiveTeardownWalker, { - fn tear_down( + fn tear_down( table_pa: TPA>, - ctx: &mut WalkContext, + ctx: &mut WalkContext, base_va: VA, depth: u8, control: &mut Control, @@ -117,6 +117,7 @@ where ) -> crate::error::Result<()> where PM: PageTableMapper, + I: TLBInvalidator, Control: FnMut(&TeardownEntry) -> TeardownAction, Dealloc: FnMut(PhysMemoryRegion), { @@ -169,10 +170,11 @@ where if matches!(action, TeardownAction::FreeAndClear) { unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { - Self::from_ptr(pgtable) - .to_raw_ptr() - .add(found_idx) - .write_volatile(0u64); + Self::from_ptr(pgtable).set_desc( + entry_va, + Self::Descriptor::invalid(), + ctx.invalidator, + ); })?; } } @@ -199,10 +201,11 @@ where if matches!(action, TeardownAction::FreeAndClear) { unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { - Self::from_ptr(pgtable) - .to_raw_ptr() - .add(found_idx) - .write_volatile(0u64); + Self::from_ptr(pgtable).set_desc( + entry_va, + Self::Descriptor::invalid(), + ctx.invalidator, + ); })?; } } diff --git a/libkernel/src/memory/paging/test.rs b/libkernel/src/memory/paging/test.rs index d7c04fa5..bf5e2959 100644 --- a/libkernel/src/memory/paging/test.rs +++ b/libkernel/src/memory/paging/test.rs @@ -6,12 +6,17 @@ use crate::{ }; use super::{ - PageAllocator, PageTableMapper, PgTable, PgTableArray, TLBInvalidator, walk::WalkContext, + PageAllocator, PageTableMapper, PgTable, PgTableArray, TLBInvalidator, TranslationChange, + walk::WalkContext, }; /// A mock TLB invalidator that does nothing for unit testing. pub struct MockTLBInvalidator; -impl TLBInvalidator for MockTLBInvalidator {} +impl TLBInvalidator for MockTLBInvalidator { + fn prepare(&self, _change: &TranslationChange) {} + fn invalidate(&self, _change: &TranslationChange) {} + fn publish(&self, _change: &TranslationChange) {} +} /// Mock page allocator that allocates on the host heap and uses a counter /// to simulate memory limits. @@ -87,10 +92,877 @@ impl TestHarness { } } - pub fn create_walk_ctx(&mut self) -> WalkContext<'_, PassthroughMapper> { + pub fn create_walk_ctx(&mut self) -> WalkContext<'_, PassthroughMapper, MockTLBInvalidator> { WalkContext { mapper: &mut self.mapper, invalidator: &self.invalidator, } } } + +macro_rules! block_split_tests { + ($middle:ty, $small:ty, $leaf:ty, $big_desc:ty, $small_desc:ty, $page_desc:ty, $mem:expr) => { + fn read_table< + T: PgTable, + P: PgTable>, + >( + harness: &mut TestHarness, + parent: TPA>, + va: VA, + ) -> TPA> { + use crate::memory::paging::TableMapper; + unsafe { + harness + .inner + .mapper + .with_page_table(parent, |ptr| { + P::from_ptr(ptr).get_desc(va).next_table_address().unwrap() + }) + .unwrap() + } + } + + #[test] + fn punch_blocks_preserves_surroundings() { + use crate::memory::paging::PaMapper; + for (large, hole_idx) in [ + (false, 0), + (false, 257), + (false, 511), + (true, 0), + (true, 257), + (true, 511), + ] { + let mut harness = TestHarness::new(10); + let va = VA::from_value(0x4_0000_0000); + let pa = PA::from_value(0x8000_0000); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let perms = PtePermissions::ro(true); + let original = if large { + let desc = <$big_desc>::new_map_pa(pa, $mem, perms); + unsafe { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + <$middle>::from_ptr(ptr).set_desc( + va, + desc, + &harness.inner.invalidator, + ); + }) + .unwrap(); + } + desc.as_raw() + } else { + let small = map_at_level(middle, va, &mut harness.create_map_ctx()).unwrap(); + let desc = <$small_desc>::new_map_pa(pa, $mem, perms); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + <$small>::from_ptr(ptr).set_desc( + va, + desc, + &harness.inner.invalidator, + ); + }) + .unwrap(); + } + desc.as_raw() + }; + let before = harness.inner.allocator.pages_allocated; + let hole = va.add_pages(hole_idx); + punch_hole( + root, + VirtMemoryRegion::new(hole, PAGE_SIZE), + &mut harness.create_map_ctx(), + ) + .unwrap(); + assert_eq!( + harness.inner.allocator.pages_allocated - before, + if large { 2 } else { 1 } + ); + let small = read_table::<$small, $middle>(&mut harness, middle, va); + let leaf = read_table::<$leaf, $small>(&mut harness, small, va); + let parent = if large { + <$small_desc>::new_mapping( + pa, + <$big_desc>::from_raw(original) + .mapping_attributes() + .unwrap(), + ) + } else { + <$small_desc>::from_raw(original) + }; + unsafe { + harness + .inner + .mapper + .with_page_table(leaf, |ptr| { + let table = <$leaf>::from_ptr(ptr); + for idx in 0..512 { + let expected = if idx == hole_idx { + <$page_desc>::invalid() + } else { + <$page_desc>::new_mapping( + pa.add_pages(idx), + parent.mapping_attributes().unwrap(), + ) + }; + assert_eq!(table.get_idx(idx).as_raw(), expected.as_raw()); + } + }) + .unwrap(); + if large { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + let table = <$small>::from_ptr(ptr); + for idx in 1..512 { + assert_eq!( + table.get_idx(idx).as_raw(), + <$small_desc>::new_mapping( + pa.add_bytes(idx << 21), + <$big_desc>::from_raw(original) + .mapping_attributes() + .unwrap(), + ) + .as_raw() + ); + } + }) + .unwrap(); + } + } + let allocated = harness.inner.allocator.pages_allocated; + walk_and_modify_region( + root, + VirtMemoryRegion::new(hole, PAGE_SIZE), + &mut harness.create_walk_ctx(), + |_, _| panic!("Hole must be skipped"), + ) + .unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, allocated); + } + } + + #[test] + fn split_allocation_failure_preserves_block() { + let mut harness = TestHarness::new(3); + let va = VA::from_value(0x4_0000_0000); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let small = map_at_level(middle, va, &mut harness.create_map_ctx()).unwrap(); + let desc = <$small_desc>::new_map_pa( + PA::from_value(0x8000_0000), + $mem, + PtePermissions::rw(false), + ); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + <$small>::from_ptr(ptr).set_desc(va, desc, &harness.inner.invalidator); + }) + .unwrap(); + } + + let result = punch_hole( + root, + VirtMemoryRegion::new(va.add_pages(1), PAGE_SIZE), + &mut harness.create_map_ctx(), + ); + assert!(matches!(result, Err(crate::error::KernelError::NoMemory))); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + assert_eq!(<$small>::from_ptr(ptr).get_desc(va).as_raw(), desc.as_raw()); + }) + .unwrap(); + } + } + + #[test] + fn punch_across_blocks_reports_full_invalidation_ranges() { + use crate::memory::paging::{TLBInvalidator, TableMapper, TranslationChange}; + use std::cell::RefCell; + + struct RecordingInvalidator(RefCell>); + impl TLBInvalidator for RecordingInvalidator { + fn prepare(&self, _change: &TranslationChange) {} + fn invalidate(&self, change: &TranslationChange) { + self.0.borrow_mut().push(*change); + } + fn publish(&self, _change: &TranslationChange) {} + } + let mut harness = TestHarness::new(8); + let va = VA::from_value(0x4_0000_0000); + let pa = PA::from_value(0x8000_0000); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + unsafe { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + for idx in 0..2 { + <$middle>::from_ptr(ptr).set_desc( + va.add_bytes(idx << 30), + <$big_desc>::new_map_pa( + pa.add_bytes(idx << 30), + $mem, + PtePermissions::ro(true), + ), + &harness.inner.invalidator, + ); + } + }) + .unwrap(); + } + let invalidator = RecordingInvalidator(RefCell::new(Vec::new())); + let hole = VA::from_value(va.value() + (1 << 30) - PAGE_SIZE); + let ctx = harness.create_map_ctx(); + let mut ctx = PagingContext { + allocator: ctx.allocator, + mapper: ctx.mapper, + invalidator: &invalidator, + }; + punch_hole(root, VirtMemoryRegion::new(hole, 2 * PAGE_SIZE), &mut ctx).unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, 6); + let changes = invalidator.0.borrow(); + assert_eq!( + changes.iter().map(|c| c.size).collect::>(), + [1 << 30, 1 << 21, PAGE_SIZE, 1 << 30, 1 << 21, PAGE_SIZE] + ); + for change in changes.iter() { + let mapped_pa = pa.add_bytes(change.va.value() - va.value()); + let perms = PtePermissions::ro(true); + let (old, new) = if change.size == PAGE_SIZE { + ( + <$page_desc>::new_map_pa(mapped_pa, $mem, perms).as_raw(), + <$page_desc>::invalid().as_raw(), + ) + } else { + let small = read_table::<$small, $middle>(&mut harness, middle, change.va); + if change.size == 1 << 21 { + let leaf = read_table::<$leaf, $small>(&mut harness, small, change.va); + ( + <$small_desc>::new_map_pa(mapped_pa, $mem, perms).as_raw(), + <$small_desc>::new_next_table(leaf).as_raw(), + ) + } else { + ( + <$big_desc>::new_map_pa(mapped_pa, $mem, perms).as_raw(), + <$big_desc>::new_next_table(small).as_raw(), + ) + } + }; + assert_eq!(change.old_descriptor, old); + assert_eq!(change.new_descriptor, new); + assert_eq!(change.va.value() & (change.size - 1), 0); + } + assert_eq!(changes[0].va, va); + assert_eq!(changes[3].va, va.add_bytes(1 << 30)); + for idx in 0..2 { + let address = if idx == 0 { hole } else { hole.add_pages(1) }; + let small = read_table::<$small, $middle>(&mut harness, middle, address); + let leaf = read_table::<$leaf, $small>(&mut harness, small, address); + unsafe { + harness + .inner + .mapper + .with_page_table(leaf, |ptr| { + let table = <$leaf>::from_ptr(ptr); + assert!(!table.get_desc(address).is_valid()); + let neighbor = if idx == 0 { + VA::from_value(address.value() - PAGE_SIZE) + } else { + address.add_pages(1) + }; + assert_eq!( + table.get_desc(neighbor).mapped_address().unwrap().value(), + pa.value() + neighbor.value() - va.value() + ); + }) + .unwrap(); + } + } + } + + #[test] + fn punch_whole_blocks_without_allocating() { + use crate::memory::paging::{TLBInvalidator, TranslationChange}; + use std::cell::RefCell; + + struct Recorder(RefCell>); + impl TLBInvalidator for Recorder { + fn prepare(&self, _: &TranslationChange) {} + fn invalidate(&self, change: &TranslationChange) { + self.0.borrow_mut().push(*change); + } + fn publish(&self, _: &TranslationChange) {} + } + for large in [false, true] { + let mut harness = TestHarness::new(if large { 2 } else { 3 }); + let va = VA::from_value(0x4_0000_0000); + let pa = PA::from_value(0x8000_0000); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let size = if large { 1 << 30 } else { 1 << 21 }; + let small = if large { + None + } else { + Some(map_at_level(middle, va, &mut harness.create_map_ctx()).unwrap()) + }; + unsafe { + if let Some(small) = small { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + for idx in 0..3 { + <$small>::from_ptr(ptr).set_desc( + va.add_bytes(idx * size), + <$small_desc>::new_map_pa( + pa.add_bytes(idx * size), + $mem, + PtePermissions::ro(true), + ), + &harness.inner.invalidator, + ); + } + }) + .unwrap(); + } else { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + for idx in 0..3 { + <$middle>::from_ptr(ptr).set_desc( + va.add_bytes(idx * size), + <$big_desc>::new_map_pa( + pa.add_bytes(idx * size), + $mem, + PtePermissions::ro(true), + ), + &harness.inner.invalidator, + ); + } + }) + .unwrap(); + } + } + let allocated = harness.inner.allocator.pages_allocated; + let recorder = Recorder(RefCell::new(Vec::new())); + let ctx = harness.create_map_ctx(); + let mut ctx = PagingContext { + allocator: ctx.allocator, + mapper: ctx.mapper, + invalidator: &recorder, + }; + punch_hole( + root, + VirtMemoryRegion::new(va.add_bytes(size), size), + &mut ctx, + ) + .unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, allocated); + let changes = recorder.0.borrow(); + assert_eq!(changes.len(), 1); + assert_eq!(changes[0].va, va.add_bytes(size)); + assert_eq!(changes[0].size, size); + let old = if large { + <$big_desc>::new_map_pa(pa.add_bytes(size), $mem, PtePermissions::ro(true)) + .as_raw() + } else { + <$small_desc>::new_map_pa(pa.add_bytes(size), $mem, PtePermissions::ro(true)) + .as_raw() + }; + assert_eq!(changes[0].old_descriptor, old); + assert_eq!(changes[0].new_descriptor, 0); + unsafe { + if let Some(small) = small { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + let table = <$small>::from_ptr(ptr); + for idx in 0..3 { + let expected = if idx == 1 { + <$small_desc>::invalid() + } else { + <$small_desc>::new_map_pa( + pa.add_bytes(idx * size), + $mem, + PtePermissions::ro(true), + ) + }; + assert_eq!( + table.get_desc(va.add_bytes(idx * size)).as_raw(), + expected.as_raw() + ); + } + }) + .unwrap(); + } else { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + let table = <$middle>::from_ptr(ptr); + for idx in 0..3 { + let expected = if idx == 1 { + <$big_desc>::invalid() + } else { + <$big_desc>::new_map_pa( + pa.add_bytes(idx * size), + $mem, + PtePermissions::ro(true), + ) + }; + assert_eq!( + table.get_desc(va.add_bytes(idx * size)).as_raw(), + expected.as_raw() + ); + } + }) + .unwrap(); + } + } + } + } + + #[test] + fn punch_partial_gigabyte_demotes_only_boundary_blocks() { + let mut harness = TestHarness::new(5); + let root = harness.inner.root_table; + let va = VA::from_value(0xffff_8000_0000_0000); + let pa = PA::from_value(0x8000_0000); + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let original = <$big_desc>::new_map_pa(pa, $mem, PtePermissions::ro(true)); + unsafe { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + <$middle>::from_ptr(ptr).set_desc(va, original, &harness.inner.invalidator); + }) + .unwrap(); + } + // Keep one page at each end. All 510 interior 2 MiB blocks should + // be removed directly, leaving just the two boundary page tables. + let hole = VirtMemoryRegion::new(va.add_pages(1), (1 << 30) - 2 * PAGE_SIZE); + punch_hole(root, hole, &mut harness.create_map_ctx()).unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, 5); + let small = read_table::<$small, $middle>(&mut harness, middle, va); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + for idx in 1..511 { + assert!(!<$small>::from_ptr(ptr).get_idx(idx).is_valid()); + } + }) + .unwrap(); + } + for idx in [0, 511] { + let base = va.add_bytes(idx << 21); + let leaf = read_table::<$leaf, $small>(&mut harness, small, base); + unsafe { + harness + .inner + .mapper + .with_page_table(leaf, |ptr| { + for page in 0..512 { + let desc = <$leaf>::from_ptr(ptr).get_idx(page); + if (idx, page) == (0, 0) || (idx, page) == (511, 511) { + assert_eq!( + desc.mapped_address(), + Some(pa.add_bytes((idx << 21) + page * PAGE_SIZE)) + ); + assert_eq!( + desc.mapping_attributes(), + original.mapping_attributes() + ); + } else { + assert!(!desc.is_valid()); + } + } + }) + .unwrap(); + } + } + punch_hole(root, hole, &mut harness.create_map_ctx()).unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, 5); + punch_hole( + root, + VirtMemoryRegion::new(va, 1 << 30), + &mut harness.create_map_ctx(), + ) + .unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, 5); + for idx in [0, 511] { + let leaf = + read_table::<$leaf, $small>(&mut harness, small, va.add_bytes(idx << 21)); + unsafe { + harness + .inner + .mapper + .with_page_table(leaf, |ptr| { + for page in 0..512 { + assert!(!<$leaf>::from_ptr(ptr).get_idx(page).is_valid()); + } + }) + .unwrap(); + } + } + } + + #[test] + fn punch_aligned_subblock_needs_only_one_table() { + let mut harness = TestHarness::new(3); + let root = harness.inner.root_table; + let va = VA::from_value(0x4_0000_0000); + let pa = PA::from_value(0x8000_0000); + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let original = <$big_desc>::new_map_pa(pa, $mem, PtePermissions::ro(true)); + unsafe { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + <$middle>::from_ptr(ptr).set_desc(va, original, &harness.inner.invalidator); + }) + .unwrap(); + } + punch_hole( + root, + VirtMemoryRegion::new(va.add_bytes(123 << 21), 1 << 21), + &mut harness.create_map_ctx(), + ) + .unwrap(); + assert_eq!(harness.inner.allocator.pages_allocated, 3); + let small = read_table::<$small, $middle>(&mut harness, middle, va); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + for idx in 0..512 { + let desc = <$small>::from_ptr(ptr).get_idx(idx); + if idx == 123 { + assert!(!desc.is_valid()); + } else { + assert_eq!(desc.mapped_address(), Some(pa.add_bytes(idx << 21))); + assert_eq!( + desc.mapping_attributes(), + original.mapping_attributes() + ); + } + } + }) + .unwrap(); + } + // A subsequent partial-page request cannot allocate another table. + let result = punch_hole( + root, + VirtMemoryRegion::new(va.add_pages(1), PAGE_SIZE), + &mut harness.create_map_ctx(), + ); + assert!(matches!(result, Err(crate::error::KernelError::NoMemory))); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + assert_eq!( + <$small>::from_ptr(ptr).get_idx(0).mapped_address(), + Some(pa) + ); + }) + .unwrap(); + } + } + + #[test] + fn punch_at_top_of_address_space_preserves_final_page() { + use crate::memory::paging::PaMapper; + + // Final page of the 64-bit address space. + const LAST_PAGE: usize = usize::MAX - PAGE_SIZE + 1; + + for large in [false, true] { + let block_size: usize = if large { 1 << 30 } else { 1 << 21 }; + // A block occupying the very top of the address space, so its + // descriptor is the last entry of its table. + let va = VA::from_value(usize::MAX - block_size + 1); + let pa = PA::from_value(0x8000_0000); + let perms = PtePermissions::ro(true); + + let mut harness = TestHarness::new(16); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let attrs = if large { + let desc = <$big_desc>::new_map_pa(pa, $mem, perms); + unsafe { + harness + .inner + .mapper + .with_page_table(middle, |ptr| { + <$middle>::from_ptr(ptr).set_desc( + va, + desc, + &harness.inner.invalidator, + ); + }) + .unwrap(); + } + desc.mapping_attributes().unwrap() + } else { + let small = map_at_level(middle, va, &mut harness.create_map_ctx()).unwrap(); + let desc = <$small_desc>::new_map_pa(pa, $mem, perms); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + <$small>::from_ptr(ptr).set_desc( + va, + desc, + &harness.inner.invalidator, + ); + }) + .unwrap(); + } + desc.mapping_attributes().unwrap() + }; + + // Punch the whole block except its final page. The block is + // therefore *not* fully covered and must be split, not zapped. + punch_hole( + root, + VirtMemoryRegion::new(va, block_size - PAGE_SIZE), + &mut harness.create_map_ctx(), + ) + .unwrap(); + + // The final page must survive, still mapping its original frame. + let survivor = get_pte(root, VA::from_value(LAST_PAGE), &mut harness.inner.mapper) + .unwrap() + .expect("final page must remain mapped"); + assert_eq!( + survivor.as_raw(), + <$page_desc>::new_mapping(pa.add_bytes(block_size - PAGE_SIZE), attrs).as_raw(), + "final page was altered (large={large})" + ); + + // Everything below it in the block must be gone. + for offset in [0, PAGE_SIZE, block_size - (2 * PAGE_SIZE)] { + assert!( + get_pte(root, va.add_bytes(offset), &mut harness.inner.mapper) + .unwrap() + .is_none(), + "offset {offset:#x} should have been punched (large={large})" + ); + } + } + } + + /// Punching a single page out of the last block must split it, leaving + /// every other page in the block intact. + #[test] + fn punch_single_page_in_top_block_preserves_surroundings() { + use crate::memory::paging::PaMapper; + + let block_size: usize = 1 << 21; + let va = VA::from_value(usize::MAX - block_size + 1); + let pa = PA::from_value(0x8000_0000); + let perms = PtePermissions::ro(true); + + let mut harness = TestHarness::new(16); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let small = map_at_level(middle, va, &mut harness.create_map_ctx()).unwrap(); + let desc = <$small_desc>::new_map_pa(pa, $mem, perms); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + <$small>::from_ptr(ptr).set_desc(va, desc, &harness.inner.invalidator); + }) + .unwrap(); + } + let attrs = desc.mapping_attributes().unwrap(); + + // Punch one page in the middle of the top block. + let hole_idx = 300; + punch_hole( + root, + VirtMemoryRegion::new(va.add_pages(hole_idx), PAGE_SIZE), + &mut harness.create_map_ctx(), + ) + .unwrap(); + + let leaf = read_table::<$leaf, $small>(&mut harness, small, va); + unsafe { + harness + .inner + .mapper + .with_page_table(leaf, |ptr| { + let table = <$leaf>::from_ptr(ptr); + for idx in 0..512 { + let expected = if idx == hole_idx { + <$page_desc>::invalid() + } else { + <$page_desc>::new_mapping(pa.add_pages(idx), attrs) + }; + assert_eq!( + table.get_idx(idx).as_raw(), + expected.as_raw(), + "page {idx} mismatch" + ); + } + }) + .unwrap(); + } + } + + /// A block that genuinely *is* fully covered must still be zapped in + /// place, without allocating a table to split it into. + #[test] + fn punch_whole_top_block_zaps_without_allocating() { + let block_size: usize = 1 << 21; + // Second-to-last block: the whole range can be expressed without + // its exclusive end overflowing. + let va = VA::from_value(usize::MAX - (2 * block_size) + 1); + let pa = PA::from_value(0x8000_0000); + + let mut harness = TestHarness::new(16); + let root = harness.inner.root_table; + let middle = map_at_level(root, va, &mut harness.create_map_ctx()).unwrap(); + let small = map_at_level(middle, va, &mut harness.create_map_ctx()).unwrap(); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + <$small>::from_ptr(ptr).set_desc( + va, + <$small_desc>::new_map_pa(pa, $mem, PtePermissions::ro(true)), + &harness.inner.invalidator, + ); + }) + .unwrap(); + } + + let before = harness.inner.allocator.pages_allocated; + punch_hole( + root, + VirtMemoryRegion::new(va, block_size), + &mut harness.create_map_ctx(), + ) + .unwrap(); + + assert_eq!( + harness.inner.allocator.pages_allocated, before, + "a fully covered block must be zapped, not split" + ); + unsafe { + harness + .inner + .mapper + .with_page_table(small, |ptr| { + assert_eq!( + <$small>::from_ptr(ptr).get_desc(va).as_raw(), + <$small_desc>::invalid().as_raw() + ); + }) + .unwrap(); + } + } + + /// A region whose exclusive end would wrap past the top of the address + /// space is rejected rather than silently truncated. + #[test] + fn punch_region_wrapping_address_space_is_rejected() { + let mut harness = TestHarness::new(16); + let root = harness.inner.root_table; + let err = punch_hole( + root, + VirtMemoryRegion::new(VA::from_value(usize::MAX - PAGE_SIZE + 1), PAGE_SIZE), + &mut harness.create_map_ctx(), + ) + .unwrap_err(); + assert_eq!(err, KernelError::InvalidValue); + } + }; +} + +pub(crate) use block_split_tests; + +#[test] +fn descriptor_update_reports_block_and_orders_break_make() { + use crate::arch::x86_64::memory::pg_descriptors::PDE; + use crate::memory::{ + address::{PA, VA}, + paging::{PaMapper, PageTableEntry, TableMapper, update_descriptor}, + }; + use std::cell::Cell; + + struct RecordingInvalidator { + slot: *const u64, + old: u64, + new: u64, + phase: Cell, + } + impl TLBInvalidator for RecordingInvalidator { + fn prepare(&self, change: &TranslationChange) { + assert_eq!(self.phase.replace(1), 0); + assert_eq!(unsafe { self.slot.read_volatile() }, self.old); + assert_eq!(change.va.value(), 0x4000_0000); + assert_eq!(change.size, 1 << 21); + assert_eq!(change.old_descriptor, self.old); + assert_eq!(change.new_descriptor, self.new); + } + fn invalidate(&self, _change: &TranslationChange) { + assert_eq!(self.phase.replace(2), 1); + assert_eq!(unsafe { self.slot.read_volatile() }, 0); + } + fn publish(&self, _change: &TranslationChange) { + assert_eq!(self.phase.replace(3), 2); + assert_eq!(unsafe { self.slot.read_volatile() }, self.new); + } + } + let old = PDE::new_map_pa( + PA::from_value(0x8000_0000), + crate::arch::x86_64::memory::pg_descriptors::MemoryType::WB, + super::permissions::PtePermissions::rw(false), + ); + let new = PDE::new_next_table(TPA::from_value(0x1000)); + let mut slot = old.as_raw(); + let invalidator = RecordingInvalidator { + slot: &slot, + old: old.as_raw(), + new: new.as_raw(), + phase: Cell::new(0), + }; + unsafe { + update_descriptor(&mut slot, VA::from_value(0x4000_1000), new, &invalidator); + } + assert_eq!(invalidator.phase.get(), 3); + // Reading via get_pte / an identity modifier must not trigger maintenance. + unsafe { + update_descriptor(&mut slot, VA::from_value(0x4000_1000), new, &invalidator); + } + assert_eq!(invalidator.phase.get(), 3); +} diff --git a/libkernel/src/memory/paging/walk.rs b/libkernel/src/memory/paging/walk.rs index 62d21038..d8d66cce 100644 --- a/libkernel/src/memory/paging/walk.rs +++ b/libkernel/src/memory/paging/walk.rs @@ -3,14 +3,14 @@ use crate::{ error::MapError, memory::{ - address::{Address, GuestPhysical, GuestVirtual, MemKind, Physical, TPA, VA, Virtual}, + address::{Address, GuestPhysical, GuestVirtual, MemKind, PA, Physical, TPA, VA, Virtual}, region::VirtMemoryRegion, }, }; use super::{ - PaMapper, PageTableEntry, PageTableMapper, PgTable, PgTableArray, TLBInvalidator, TableMapper, - TableMapperTable, permissions::PtePermissions, + NullTlbInvalidator, PaMapper, PageAllocator, PageTableEntry, PageTableMapper, PgTable, + PgTableArray, TLBInvalidator, TableMapper, TableMapperTable, permissions::PtePermissions, }; /// A virtual address kind that can be translated through page tables. @@ -28,93 +28,248 @@ impl Translatable for GuestVirtual { } /// A collection of context required to modify page tables. -pub struct WalkContext<'a, PM> { +pub struct WalkContext<'a, PM, I: TLBInvalidator> { /// The mapper used to temporarily access page tables by physical address. pub mapper: &'a mut PM, - /// The TLB invalidator invoked after modifying page table entries. - pub invalidator: &'a dyn TLBInvalidator, + /// Synchronous break/make maintenance for descriptor changes. + pub invalidator: &'a I, +} + +pub(crate) fn validate_allocating_region(region: VirtMemoryRegion) -> crate::error::Result<()> { + if !region.is_page_aligned() || !region.size().is_multiple_of(crate::memory::PAGE_SIZE) { + return Err(MapError::VirtNotAligned.into()); + } + if region + .start_address() + .value() + .checked_add(region.size()) + .is_none() + { + return Err(crate::error::KernelError::InvalidValue); + } + Ok(()) +} + +pub(crate) enum WalkOperation<'a, F> { + Modify(&'a mut F), + PunchHole, +} + +impl WalkOperation<'_, F> { + pub(crate) fn apply(&mut self, va: VA, desc: D) -> D + where + F: FnMut(VA, D) -> D, + { + match self { + Self::Modify(modifier) => modifier(va, desc), + Self::PunchHole => D::invalid(), + } + } } pub(crate) trait RecursiveWalker: PgTable + Sized { - fn walk( + fn walk( table_pa: TPA>, region: VirtMemoryRegion, - ctx: &mut WalkContext, + ctx: &mut WalkContext, modifier: &mut F, ) -> crate::error::Result<()> where PM: PageTableMapper, + I: TLBInvalidator, + F: FnMut(VA, LeafDesc) -> LeafDesc, + { + Self::walk_with_allocator( + table_pa, + region, + ctx, + &mut RejectBlockAllocator, + &mut WalkOperation::Modify(modifier), + ) + } + + fn walk_with_allocator( + table_pa: TPA>, + region: VirtMemoryRegion, + ctx: &mut WalkContext, + allocator: &mut A, + operation: &mut WalkOperation, + ) -> crate::error::Result<()> + where + PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, F: FnMut(VA, LeafDesc) -> LeafDesc; } +struct RejectBlockAllocator; + +impl PageAllocator for RejectBlockAllocator { + fn allocate_page_table(&mut self) -> crate::error::Result>> { + Err(MapError::NotL3Mapped.into()) + } +} + impl RecursiveWalker for T where T: TableMapperTable, + T::Descriptor: PaMapper, + NextDescriptor: PaMapper::Attributes>, ::NextLevel: RecursiveWalker, { - fn walk( + fn walk_with_allocator( table_pa: TPA>, region: VirtMemoryRegion, - ctx: &mut WalkContext, - modifier: &mut F, + ctx: &mut WalkContext, + allocator: &mut A, + operation: &mut WalkOperation, ) -> crate::error::Result<()> where PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, F: FnMut(VA, LeafDesc) -> LeafDesc, { - let table_coverage = 1 << T::Descriptor::MAP_SHIFT; - - let start_idx = Self::pg_index(region.start_address()); - let end_idx = Self::pg_index(region.end_address_inclusive()); - - // Calculate the base address of the *entire* table. - let table_base_va = region - .start_address() - .align(1 << (T::Descriptor::MAP_SHIFT + 9)); - - for idx in start_idx..=end_idx { - let entry_va = table_base_va.add_bytes(idx * table_coverage); - - let desc = unsafe { - ctx.mapper - .with_page_table(table_pa, |pgtable| T::from_ptr(pgtable).get_desc(entry_va))? - }; - - if let Some(next_desc) = desc.next_table_address() { - // `entry_va + table_coverage` can overflow for the last entry - // at each level (e.g. PML4 entry 511 on x86_64). Compute the - // intersection with saturating arithmetic instead of - // constructing an unrepresentable VirtMemoryRegion. - let entry_end = entry_va.value().saturating_add(table_coverage); - let sub_start = region.start_address().value().max(entry_va.value()); - let sub_end = region.end_address().value().min(entry_end); - let sub_region = VirtMemoryRegion::from_start_end_address( - VA::from_value(sub_start), - VA::from_value(sub_end), - ); - - ::NextLevel::walk( - next_desc, sub_region, ctx, modifier, - )?; - } else if desc.is_valid() { - Err(MapError::NotL3Mapped)?; - } else { - // Permit sparse mappings. + walk_table_level(table_pa, region, ctx, allocator, operation, |desc| { + let pa = desc.mapped_address().ok_or(MapError::InvalidDescriptor)?; + let attributes = desc + .mapping_attributes() + .ok_or(MapError::InvalidDescriptor)?; + Ok((pa, attributes)) + }) + } +} + +type NextDescriptor = + <<::Descriptor as TableMapper>::NextLevel as PgTable>::Descriptor; + +pub(crate) fn walk_table_level( + table_pa: TPA>, + region: VirtMemoryRegion, + ctx: &mut WalkContext, + allocator: &mut A, + operation: &mut WalkOperation, + mut mapping: S, +) -> crate::error::Result<()> +where + T: TableMapperTable, + NextDescriptor: PaMapper, + LeafDesc: PageTableEntry, + ::NextLevel: RecursiveWalker, + PM: PageTableMapper, + I: TLBInvalidator, + A: PageAllocator, + F: FnMut(VA, LeafDesc) -> LeafDesc, + S: FnMut( + T::Descriptor, + ) -> crate::error::Result<(PA, as PaMapper>::Attributes)>, +{ + let table_coverage = 1 << T::Descriptor::MAP_SHIFT; + + let start_idx = T::pg_index(region.start_address()); + let end_idx = T::pg_index(region.end_address_inclusive()); + + // Calculate the base address of the *entire* table. + let table_base_va = region + .start_address() + .align(1 << (T::Descriptor::MAP_SHIFT + 9)); + + for idx in start_idx..=end_idx { + let entry_va = table_base_va.add_bytes(idx * table_coverage); + let entry_va_region = VirtMemoryRegion::new(entry_va, table_coverage); + + let desc = unsafe { + ctx.mapper + .with_page_table(table_pa, |pgtable| T::from_ptr(pgtable).get_desc(entry_va))? + }; + + let next_desc = if let Some(next_desc) = desc.next_table_address() { + // Point to another table; simply recurse down. + next_desc + } else if desc.is_valid() { + // If descriptor is valid and doesn't point to a next table, it + // *must* map map a PA region. + let (pa, attributes) = mapping(desc)?; + + // See if we can simply punch the whole descriptor out here. + if matches!(operation, WalkOperation::PunchHole) && region.contains(entry_va_region) { + unsafe { + ctx.mapper.with_page_table(table_pa, |pgtable| { + T::from_ptr(pgtable).set_desc( + entry_va, + T::Descriptor::invalid(), + ctx.invalidator, + ); + })?; + } continue; } - } - Ok(()) + // We couldn't. Either we're not hole punching, or the region didn't + // fully overlap. Let's split, for two reasons: + // + // - Arbitrary descriptor updates (WalkOperation::Modify) can only + // be applied to leaf entries. + // - If the region wasn't fully covered, for a hole punch, split to + // gain more granularity. + let next_desc = + allocator.allocate_page_table::<::NextLevel>()?; + let coverage = 1 << NextDescriptor::::MAP_SHIFT; + let entries = ::NextLevel::DESCRIPTORS_PER_PAGE; + // Populate the whole replacement before making it reachable. + unsafe { + ctx.mapper.with_page_table(next_desc, |pgtable| { + let table = ::NextLevel::from_ptr(pgtable); + for idx in 0..entries { + let child_desc = NextDescriptor::::new_mapping( + pa.add_bytes(idx * coverage), + attributes, + ); + // We can use the NullTLB invalidator here as this page + // table isn't linked in yet. + table.set_desc( + entry_va.add_bytes(idx * coverage), + child_desc, + &NullTlbInvalidator {}, + ); + } + })?; + + // Now set the new descriptor (pointing to the new table) in + // place. + ctx.mapper.with_page_table(table_pa, |pgtable| { + T::from_ptr(pgtable).set_desc( + entry_va, + T::Descriptor::new_next_table(next_desc), + ctx.invalidator, + ); + })?; + } + next_desc + } else { + // Permit sparse mappings without allocating tables for holes. + continue; + }; + + let sub_region = entry_va_region + .intersection(region) + .expect("operation region should overlap with descriptor region"); + + ::NextLevel::walk_with_allocator( + next_desc, sub_region, ctx, allocator, operation, + )?; } + + Ok(()) } pub(crate) type TranslatorResult

= (Address, usize, PtePermissions); pub(crate) trait Translator: PgTable + Sized { - fn translate>( + fn translate, I: TLBInvalidator>( table_pa: Address>, va: Address, - ctx: &mut WalkContext, + ctx: &mut WalkContext, ) -> crate::error::Result>>; } @@ -124,10 +279,10 @@ where T::Descriptor: PaMapper, ::NextLevel: Translator, { - fn translate>( + fn translate, I: TLBInvalidator>( table_pa: Address>, va: Address, - ctx: &mut WalkContext, + ctx: &mut WalkContext, ) -> crate::error::Result, usize, PtePermissions)>> { let desc = unsafe { ctx.mapper.with_page_table(table_pa, |pgtable| { diff --git a/libkernel/src/memory/region.rs b/libkernel/src/memory/region.rs index 7bc95394..74193a5f 100644 --- a/libkernel/src/memory/region.rs +++ b/libkernel/src/memory/region.rs @@ -253,7 +253,7 @@ impl MemoryRegion { /// Returns `true` if this region fully contains `other`. pub fn contains(self, other: Self) -> bool { self.start_address().value() <= other.start_address().value() - && self.end_address().value() >= other.end_address().value() + && self.end_address_inclusive().value() >= other.end_address_inclusive().value() } /// Returns `true` if this region contains the given address. @@ -325,17 +325,22 @@ impl MemoryRegion { /// assert_eq!(intersection.size(), 0x1000); /// ``` pub fn intersection(self, other: Self) -> Option { + if self.is_empty() || other.is_empty() { + return None; + } + // Determine the latest start address and the earliest end address. let intersection_start = core::cmp::max(self.start_address(), other.start_address()); - let intersection_end = core::cmp::min(self.end_address(), other.end_address()); + let intersection_end = + core::cmp::min(self.end_address_inclusive(), other.end_address_inclusive()); // A valid, non-empty overlap exists only if the start of the // potential intersection is before its end. - if intersection_start < intersection_end { - Some(Self::from_start_end_address( - intersection_start, - intersection_end, - )) + if intersection_start <= intersection_end { + // +1 since we used `inclusive()` arithmatic above to avoid + // overflows. + let intersection_length = (intersection_end.value() - intersection_start.value()) + 1; + Some(Self::new(intersection_start, intersection_length)) } else { None } @@ -403,9 +408,8 @@ impl MemoryRegion { let pages_count = self.size >> PAGE_SHIFT; core::iter::from_fn(move || { - let addr = self.start_address().add_pages(count); - if count < pages_count { + let addr = self.start_address().add_pages(count); count += 1; Some(addr) } else { @@ -633,6 +637,14 @@ mod tests { assert_eq!(region.iter_pages().count(), num_pages); } + #[test] + fn iter_pages_top_of_address_space() { + let top = VA::from_value(usize::MAX - (PAGE_SIZE * 3)).align_up(PAGE_SIZE); + let rgn = VirtMemoryRegion::new(top, PAGE_SIZE * 3); + + assert_eq!(rgn.iter_pages().count(), 3); + } + #[test] fn test_punch_hole_middle() { // Region: [0x1000 ... 0x5000) (size 0x4000) @@ -722,6 +734,251 @@ mod tests { assert_eq!(main.punch_hole(hole), (Some(main), None)); } + fn assert_intersection( + a: PhysMemoryRegion, + b: PhysMemoryRegion, + expected: Option, + ) { + assert_eq!(a.intersection(b), expected, "a.intersection(b)"); + assert_eq!(b.intersection(a), expected, "b.intersection(a)"); + } + + #[test] + fn intersection_partial_overlap() { + // A: [0x1000 ... 0x3000) + // B: [0x2000 ... 0x4000) + // Expect:[0x2000 ... 0x3000) + let a = region(0x1000, 0x2000); + let b = region(0x2000, 0x2000); + let i = a.intersection(b).unwrap(); + + assert_eq!(i.start_address().value(), 0x2000); + assert_eq!(i.end_address().value(), 0x3000); + assert_eq!(i.size(), 0x1000); + } + + #[test] + fn intersection_is_commutative() { + let a = region(0x1000, 0x2000); + let b = region(0x2000, 0x2000); + + assert_eq!(a.intersection(b), b.intersection(a)); + assert_eq!(b.intersection(a), Some(region(0x2000, 0x1000))); + } + + #[test] + fn intersection_identical_is_same() { + let a = region(0x1000, 0x1000); + + assert_intersection(a, a, Some(a)); + } + + #[test] + fn intersection_fully_contained() { + let outer = region(0x1000, 0x4000); + let inner = region(0x2000, 0x1000); + + assert_intersection(outer, inner, Some(inner)); + } + + #[test] + fn intersection_shared_start() { + let a = region(0x1000, 0x1000); + let b = region(0x1000, 0x3000); + + assert_intersection(a, b, Some(a)); + } + + #[test] + fn intersection_shared_end() { + let a = region(0x3000, 0x1000); + let b = region(0x1000, 0x3000); + + assert_intersection(a, b, Some(a)); + } + + #[test] + fn intersection_unaligned() { + let a = region(0x1234, 0xcc); + let b = region(0x12ab, 0xd56); + + assert_intersection(a, b, Some(region(0x12ab, 0x55))); + } + + #[test] + fn intersection_starting_at_zero() { + let a = region(0, 0x2000); + let b = region(0, 0x1000); + + assert_intersection(a, b, Some(b)); + } + + #[test] + fn intersection_adjacent_is_none() { + let a = region(0x1000, 0x1000); + let b = region(0x2000, 0x1000); + + assert_intersection(a, b, None); + } + + #[test] + fn intersection_disjoint_is_none() { + let a = region(0x1000, 0x1000); + let b = region(0x5000, 0x1000); + + assert_intersection(a, b, None); + } + + #[test] + fn intersection_one_byte_gap_is_none() { + let a = region(0x1000, 0xfff); + let b = region(0x2000, 0x1000); + + assert_intersection(a, b, None); + } + + // --- Single-byte overlaps -------------------------------------------- + + #[test] + fn intersection_single_byte_overlap() { + let a = region(0x1000, 0x1000); + let b = region(0x1fff, 0x1001); + + assert_intersection(a, b, Some(region(0x1fff, 1))); + } + + #[test] + fn intersection_single_byte_region_inside_larger() { + let outer = region(0x1000, 0x1000); + + for addr in [0x1000, 0x1800, 0x1fff] { + let byte = region(addr, 1); + assert_intersection(outer, byte, Some(byte)); + } + } + + #[test] + fn intersection_identical_single_byte_regions() { + let a = region(0x1000, 1); + + assert_intersection(a, a, Some(a)); + } + + #[test] + fn intersection_single_byte_region_just_outside_is_none() { + let outer = region(0x1000, 0x1000); + + assert_intersection(outer, region(0xfff, 1), None); + assert_intersection(outer, region(0x2000, 1), None); + } + + #[test] + fn intersection_with_empty_region_is_none() { + let a = region(0x1000, 0x1000); + + for addr in 0x0..=0x8000 { + assert_intersection(a, region(addr, 0), None); + } + } + + #[test] + fn intersection_of_two_empty_regions_is_none() { + assert_intersection(region(0x1000, 0), region(0x1000, 0), None); + assert_intersection(region(0x1000, 0), region(0x2000, 0), None); + assert_intersection(PhysMemoryRegion::empty(), PhysMemoryRegion::empty(), None); + } + + #[test] + fn intersection_at_top_of_address_space() { + // A: last page. B: last half page. + let a = region(usize::MAX - 0xfff, 0x1000); + let b = region(usize::MAX - 0x7ff, 0x800); + + let i = a.intersection(b).unwrap(); + + assert_eq!(i, b); + assert_eq!(i.end_address_inclusive().value(), usize::MAX); + assert_eq!(b.intersection(a), Some(b)); + } + + #[test] + fn intersection_partial_overlap_at_top_of_address_space() { + let a = region(usize::MAX - 0x1fff, 0x1800); + let b = region(usize::MAX - 0xfff, 0x1000); + + assert_intersection(a, b, Some(region(usize::MAX - 0xfff, 0x800))); + } + + #[test] + fn intersection_identical_at_top_of_address_space() { + let a = region(usize::MAX - 0xfff, 0x1000); + + assert_intersection(a, a, Some(a)); + } + + #[test] + fn intersection_adjacent_at_top_of_address_space_is_none() { + let a = region(usize::MAX - 0x1fff, 0x1000); + let b = region(usize::MAX - 0xfff, 0x1000); + + assert_intersection(a, b, None); + } + + #[test] + fn intersection_last_byte_of_address_space() { + let top = region(usize::MAX - 0xfff, 0x1000); + let last_byte = region(usize::MAX, 1); + + assert_intersection(top, last_byte, Some(last_byte)); + } + + #[test] + fn intersection_huge_regions() { + // [0 ... MAX-1] ∩ [1 ... MAX] == [1 ... MAX-1] + let a = region(0, usize::MAX); + let b = region(1, usize::MAX); + + assert_intersection(a, b, Some(region(1, usize::MAX - 1))); + assert_intersection(a, a, Some(a)); + assert_intersection(b, b, Some(b)); + } + + #[test] + fn intersection_result_is_contained_in_both_inputs() { + let a = region(0x1000, 0x3000); + let b = region(0x2800, 0x4000); + + let i = a.intersection(b).unwrap(); + + assert!(a.contains(i)); + assert!(b.contains(i)); + assert!(!i.is_empty()); + } + + #[test] + fn intersection_is_idempotent() { + let a = region(0x1000, 0x3000); + let b = region(0x2800, 0x4000); + + let i = a.intersection(b).unwrap(); + + assert_eq!(i.intersection(a), Some(i)); + assert_eq!(i.intersection(b), Some(i)); + } + + #[test] + fn intersection_is_associative() { + let a = region(0x1000, 0x4000); + let b = region(0x2000, 0x4000); + let c = region(0x3000, 0x4000); + + let ab_c = a.intersection(b).and_then(|ab| ab.intersection(c)); + let a_bc = b.intersection(c).and_then(|bc| a.intersection(bc)); + + assert_eq!(ab_c, a_bc); + assert_eq!(ab_c, Some(region(0x3000, 0x2000))); + } + #[test] fn is_empty_zero_size() { let r = region(0x1000, 0); diff --git a/src/arch/arm64/boot/logical_map.rs b/src/arch/arm64/boot/logical_map.rs index aef3989f..d235ce63 100644 --- a/src/arch/arm64/boot/logical_map.rs +++ b/src/arch/arm64/boot/logical_map.rs @@ -3,17 +3,19 @@ use crate::memory::{INITAL_ALLOCATOR, PageOffsetTranslator}; use super::super::memory::{ fixmap::{FIXMAPS, Fixmap}, mmu::smalloc_page_allocator::SmallocPageAlloc, - tlb::AllEl1TlbInvalidator, }; use libkernel::{ arch::arm64::memory::{ pg_descriptors::MemoryType, - pg_tables::{L0Table, MapAttributes, MappingContext, map_range}, + pg_tables::{L0Table, MapAttributes, map_range}, }, error::Result, memory::{ address::{TPA, TVA}, - paging::{PageTableMapper, PgTable, PgTableArray, permissions::PtePermissions}, + paging::{ + NullTlbInvalidator, PageTableMapper, PagingContext, PgTable, PgTableArray, + permissions::PtePermissions, + }, }, }; @@ -44,10 +46,14 @@ pub fn setup_logical_map(pgtbl_base: TPA>) -> Result<()> { }; let mut pg_alloc = SmallocPageAlloc::new(alloc); - let mut ctx = MappingContext { + // Boot is single-core and this address space is not yet live to any + // page-table walker, so no ordering or invalidation service is needed. If + // multicore boot ever overlaps with page-table construction, these sites + // need a real invalidator. + let mut ctx = PagingContext { allocator: &mut pg_alloc, mapper: &mut mapper, - invalidator: &AllEl1TlbInvalidator::new(), + invalidator: &NullTlbInvalidator {}, }; for mem_region in mem_list.iter() { diff --git a/src/arch/arm64/boot/memory.rs b/src/arch/arm64/boot/memory.rs index 2edce388..5e6a4197 100644 --- a/src/arch/arm64/boot/memory.rs +++ b/src/arch/arm64/boot/memory.rs @@ -1,20 +1,19 @@ use crate::arch::arm64::memory::{ mmu::{page_mapper::PageOffsetPgTableMapper, smalloc_page_allocator::SmallocPageAlloc}, set_kimage_start, - tlb::AllEl1TlbInvalidator, }; use crate::memory::INITAL_ALLOCATOR; use core::ptr::NonNull; use libkernel::{ arch::arm64::memory::{ pg_descriptors::MemoryType, - pg_tables::{L0Table, MapAttributes, MappingContext, map_range}, + pg_tables::{L0Table, MapAttributes, map_range}, }, error::{KernelError, Result}, memory::{ PAGE_SIZE, address::{PA, TPA, VA}, - paging::{PgTableArray, permissions::PtePermissions}, + paging::{NullTlbInvalidator, PagingContext, PgTableArray, permissions::PtePermissions}, region::{PhysMemoryRegion, VirtMemoryRegion}, }, }; @@ -132,10 +131,12 @@ pub fn setup_stack_and_heap(pgtbl_base: TPA>) -> Result Result let kernel_range = PhysMemoryRegion::new(image_addr, image_size); let mut translator = IdmapTranslator {}; - let invalidator = NullTlbInvalidator {}; let highmem_l0 = bump_alloc.allocate_page_table::()?; - let mut bootstrap_ctx = MappingContext { + // The MMU is disabled and boot is single-core, so no ordering or + // invalidation service is needed; see PagingContext::invalidator. + let mut bootstrap_ctx = PagingContext { allocator: &mut bump_alloc, mapper: &mut translator, - invalidator: &invalidator, + invalidator: &NullTlbInvalidator {}, }; map_range( diff --git a/src/arch/arm64/memory/address_space.rs b/src/arch/arm64/memory/address_space.rs index c054e68e..8e775f76 100644 --- a/src/arch/arm64/memory/address_space.rs +++ b/src/arch/arm64/memory/address_space.rs @@ -12,7 +12,7 @@ use alloc::vec::Vec; use libkernel::{ arch::arm64::memory::{ pg_descriptors::{L3Descriptor, MemoryType}, - pg_tables::{L0Table, MapAttributes, MappingContext, map_range}, + pg_tables::{L0Table, MapAttributes, map_range}, pg_tear_down::tear_down_address_space, pg_walk::{get_pte, walk_and_modify_region}, }, @@ -23,8 +23,8 @@ use libkernel::{ address_space::{PageInfo, UserAddressSpace}, page::PageFrame, paging::{ - PaMapper, PageAllocator, PageTableEntry, PgTableArray, permissions::PtePermissions, - tear_down::TeardownAction, walk::WalkContext, + PaMapper, PageAllocator, PageTableEntry, PagingContext, PgTableArray, + permissions::PtePermissions, tear_down::TeardownAction, walk::WalkContext, }, region::{PhysMemoryRegion, VirtMemoryRegion}, }, @@ -63,7 +63,7 @@ impl UserAddressSpace for Arm64ProcessAddressSpace { } fn map_page(&mut self, page: PageFrame, va: VA, perms: PtePermissions) -> Result<()> { - let mut ctx = MappingContext { + let mut ctx = PagingContext { allocator: &mut PageTableAllocator::new(), mapper: &mut PageOffsetPgTableMapper {}, invalidator: &AllEl0TlbInvalidator::new(), @@ -159,9 +159,10 @@ impl UserAddressSpace for Arm64ProcessAddressSpace { where Self: Sized, { + let invalidator = AllEl0TlbInvalidator::new(); let mut walk_ctx = WalkContext { mapper: &mut PageOffsetPgTableMapper {}, - invalidator: &AllEl0TlbInvalidator::new(), + invalidator: &invalidator, }; walk_and_modify_region(self.l0_table, region, &mut walk_ctx, |va, pgd| { @@ -176,10 +177,10 @@ impl UserAddressSpace for Arm64ProcessAddressSpace { alloc1.clone().leak(); alloc1.leak(); - let mut ctx = MappingContext { + let mut ctx = PagingContext { allocator: &mut PageTableAllocator::new(), mapper: &mut PageOffsetPgTableMapper {}, - invalidator: &AllEl0TlbInvalidator::new(), + invalidator: &invalidator, }; map_range( diff --git a/src/arch/arm64/memory/mmu.rs b/src/arch/arm64/memory/mmu.rs index 4a6341f8..f7e3f6f1 100644 --- a/src/arch/arm64/memory/mmu.rs +++ b/src/arch/arm64/memory/mmu.rs @@ -3,14 +3,14 @@ use crate::sync::{OnceLock, SpinLock}; use libkernel::{ arch::arm64::memory::{ pg_descriptors::MemoryType, - pg_tables::{L0Table, MapAttributes, MappingContext, map_range}, + pg_tables::{L0Table, MapAttributes, map_range}, pg_walk::get_pte, }, error::Result, memory::{ address::{PA, TPA, VA}, address_space::KernAddressSpace, - paging::{PaMapper, PgTableArray, permissions::PtePermissions}, + paging::{PaMapper, PagingContext, PgTableArray, permissions::PtePermissions}, region::{PhysMemoryRegion, VirtMemoryRegion}, }, }; @@ -30,7 +30,7 @@ pub struct Arm64KernelAddressSpace { impl Arm64KernelAddressSpace { fn do_map(&self, map_attrs: MapAttributes) -> Result<()> { - let mut ctx = MappingContext { + let mut ctx = PagingContext { allocator: &mut PageTableAllocator::new(), mapper: &mut PageOffsetPgTableMapper {}, invalidator: &AllEl1TlbInvalidator::new(), diff --git a/src/arch/arm64/memory/tlb.rs b/src/arch/arm64/memory/tlb.rs index cbace9c9..b807c326 100644 --- a/src/arch/arm64/memory/tlb.rs +++ b/src/arch/arm64/memory/tlb.rs @@ -1,6 +1,6 @@ use core::arch::asm; -use libkernel::memory::paging::TLBInvalidator; +use libkernel::memory::paging::{TLBInvalidator, TranslationChange}; pub struct AllEl1TlbInvalidator; @@ -12,24 +12,23 @@ impl AllEl1TlbInvalidator { impl Drop for AllEl1TlbInvalidator { fn drop(&mut self) { - unsafe { - asm!( - // Data Synchronization Barrier, Inner Shareable, write to - // read/write. - "dsb ishst", - // Invalidate TLB by VA, for EL1, Inner Shareable - "tlbi vmalle1is", - // Data Synchronization Barrier, Inner Shareable. - "dsb ish", - // Instruction Synchronization Barrier. - "isb", - options(nostack, preserves_flags) - ); - } + invalidate_all(); } } -impl TLBInvalidator for AllEl1TlbInvalidator {} +impl TLBInvalidator for AllEl1TlbInvalidator { + fn prepare(&self, _change: &TranslationChange) { + order_table_stores(); + } + + fn invalidate(&self, _change: &TranslationChange) { + invalidate_all(); + } + + fn publish(&self, _change: &TranslationChange) { + publish_tables(); + } +} pub struct AllEl0TlbInvalidator; @@ -41,21 +40,46 @@ impl AllEl0TlbInvalidator { impl Drop for AllEl0TlbInvalidator { fn drop(&mut self) { - unsafe { - asm!( - // Data Synchronization Barrier, Inner Shareable, write to - // read/write. - "dsb ishst", - // Invalidate TLB by VA, for EL1, Inner Shareable - "tlbi vmalle1is", - // Data Synchronization Barrier, Inner Shareable. - "dsb ish", - // Instruction Synchronization Barrier. - "isb", - options(nostack, preserves_flags) - ); - } - } -} - -impl TLBInvalidator for AllEl0TlbInvalidator {} + invalidate_all(); + } +} + +impl TLBInvalidator for AllEl0TlbInvalidator { + fn prepare(&self, _change: &TranslationChange) { + order_table_stores(); + } + + fn invalidate(&self, _change: &TranslationChange) { + invalidate_all(); + } + + fn publish(&self, _change: &TranslationChange) { + publish_tables(); + } +} + +fn invalidate_all() { + // Conservatively invalidate all levels and ASIDs, including cached blocks + // and table walks. Range-aware implementations can use TranslationChange. + unsafe { + asm!( + "dsb ishst", + "tlbi vmalle1is", + "dsb ish", + "isb", + options(nostack, preserves_flags) + ); + } +} + +fn publish_tables() { + unsafe { + asm!("dsb ishst", "isb", options(nostack, preserves_flags)); + } +} + +fn order_table_stores() { + unsafe { + asm!("dsb ishst", options(nostack, preserves_flags)); + } +}