libkernel: paging: implement block-level hole punching

libkernel/src/arch/arm64/memory/pg_descriptors.rs

@@ -34,6 +34,10 @@ pub enum MemoryType {
     Normal,
 }
 
+/// Level-independent ARM64 mapping attributes, excluding the contiguous hint.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct MappingAttributes(u64);
+
 macro_rules! define_descriptor {
     (
         $(#[$outer:meta])*
@@ -172,6 +176,24 @@ macro_rules! define_descriptor {
 
             impl PaMapper for $name {
                 type MemoryType = MemoryType;
+                type Attributes = MappingAttributes;
+
+                fn mapping_attributes(self) -> Option<Self::Attributes> {
+                    if self.0 & 0b11 != $map_bits {
+                        return None;
+                    }
+                    // Bits [47:12] encode addresses (or are reserved at block
+                    // levels). Contiguous describes a group, not one mapping.
+                    let address_mask = ((1u64 << 48) - 1) & !0xfff;
+                    Some(MappingAttributes(self.0 & !(address_mask | 0b11 | (1 << 52))))
+                }
+
+                fn new_mapping(pa: PA, attributes: Self::Attributes) -> Self {
+                    assert_eq!(pa.value() & ((1 << Self::MAP_SHIFT) - 1), 0,
+                        "Cannot map non-aligned physical address");
+                    let address_mask = ((1u64 << 48) - 1) & !((1 << Self::MAP_SHIFT) - 1);
+                    Self((pa.value() as u64 & address_mask) | attributes.0 | $map_bits)
+                }
 
                 fn could_map(region: PhysMemoryRegion, va: VA) -> bool {
                     let is_aligned = |addr: usize| (addr & ((1 << Self::MAP_SHIFT) - 1)) == 0;
@@ -278,6 +300,47 @@ define_descriptor!(
     },
 );
 
+#[cfg(test)]
+mod attribute_tests {
+    use super::*;
+
+    #[test]
+    fn splitting_preserves_attributes_and_clears_contiguous_hint() {
+        let pa = PA::from_value(0x8000_0000);
+        let block = L1Descriptor::from_raw(
+            L1Descriptor::new_map_pa(pa, MemoryType::Device, PtePermissions::ro(true)).as_raw()
+                | (1 << 11)
+                | (1 << 52)
+                | (1 << 55),
+        );
+        let small =
+            L2Descriptor::new_mapping(pa.add_bytes(3 << 21), block.mapping_attributes().unwrap());
+        let page = L3Descriptor::new_mapping(
+            pa.add_bytes((3 << 21) + (5 << 12)),
+            small.mapping_attributes().unwrap(),
+        );
+        assert_eq!(small.mapped_address(), Some(pa.add_bytes(3 << 21)));
+        assert_eq!(
+            page.mapped_address(),
+            Some(pa.add_bytes((3 << 21) + (5 << 12)))
+        );
+        assert_eq!(page.permissions(), block.permissions());
+        let attrs = ((1 << 12) - 1) & !0b11 | (1 << 53) | (1 << 54) | (1 << 55);
+        assert_eq!(page.as_raw() & attrs, block.as_raw() & attrs);
+        assert_eq!(small.as_raw() & (1 << 52), 0);
+        assert_eq!(page.as_raw() & (1 << 52), 0);
+        assert_eq!(page.as_raw() & 0b11, 0b11);
+        assert_eq!(page.mapping_attributes(), block.mapping_attributes());
+        assert!(L2Descriptor::invalid().mapping_attributes().is_none());
+        assert!(page.mark_as_swapped().mapping_attributes().is_none());
+        assert!(
+            L2Descriptor::new_next_table(TPA::from_value(0x1000))
+                .mapping_attributes()
+                .is_none()
+        );
+    }
+}
+
 /// The decoded state of an L3 page descriptor.
 pub enum L3DescriptorState {
     /// The entry is not present.

libkernel/src/arch/arm64/memory/pg_tables.rs

@@ -7,8 +7,9 @@ use crate::{
         PAGE_SIZE,
         address::{TPA, TVA, VA},
         paging::{
-            PaMapper, PageAllocator, PageTableEntry, PageTableMapper, PgTable, PgTableArray,
-            TLBInvalidator, TableMapper, TableMapperTable, permissions::PtePermissions,
+            NullTlbInvalidator, PaMapper, PageAllocator, PageTableEntry, PageTableMapper,
+            PagingContext, PgTable, PgTableArray, TLBInvalidator, TableMapper, TableMapperTable,
+            permissions::PtePermissions,
         },
         region::{PhysMemoryRegion, VirtMemoryRegion},
     },
@@ -45,11 +46,11 @@ macro_rules! impl_pgtable {
                 self.get_idx(Self::pg_index(va))
             }
 
-            fn set_desc(self, va: VA, desc: Self::Descriptor, _invalidator: &dyn TLBInvalidator) {
+            fn set_desc(self, va: VA, desc: Self::Descriptor, invalidator: &impl TLBInvalidator) {
                 unsafe {
-                    self.base
-                        .add(Self::pg_index(va))
-                        .write_volatile(PageTableEntry::as_raw(desc))
+                    crate::memory::paging::update_descriptor(
+                        self.base.add(Self::pg_index(va)), va, desc, invalidator,
+                    )
                 };
             }
         }
@@ -95,23 +96,6 @@ pub struct MapAttributes {
     pub perms: PtePermissions,
 }
 
-/// A collection of context required to create page tables.
-pub struct MappingContext<'a, PA, PM>
-where
-    PA: PageAllocator + 'a,
-    PM: PageTableMapper + 'a,
-{
-    /// An implementation of `PageAllocator` used to request new, zeroed page
-    /// tables when descending the hierarchy.
-    pub allocator: &'a mut PA,
-    /// An implementation of `PageTableMapper` that provides safe, temporary CPU
-    /// access to page tables at their physical addresses.
-    pub mapper: &'a mut PM,
-    /// An object responsible for issuing TLB invalidation instructions after a
-    /// mapping is successfully changed.
-    pub invalidator: &'a dyn TLBInvalidator,
-}
-
 /// Maps a contiguous physical memory region to a virtual memory region.
 ///
 /// This function walks the page table hierarchy starting from the provided root
@@ -139,8 +123,9 @@ where
 ///     zeroed page tables.
 ///   - `mapper`: An implementation of `PageTableMapper` that provides safe,
 ///     temporary CPU access to page tables.
-///   - `invalidator`: An object responsible for issuing TLB invalidation
-///     instructions after a mapping is changed.
+///   - `invalidator`: An object responsible for synchronous TLB maintenance
+///     on descriptor changes. Newly allocated tables are ordered ahead of
+///     their link through `prepare()`; see `PagingContext`.
 ///
 /// # Returns
 ///
@@ -182,14 +167,15 @@ where
 /// 4. Memory Lifecycle: The caller is responsible for managing the lifecycle of
 ///    the physical memory described by `attrs.phys`. This function only creates
 ///    a *mapping* to the memory; it does not take ownership of it.
-pub fn map_range<PA, PM>(
+pub fn map_range<PA, PM, I>(
     l0_table: TPA<PgTableArray<L0Table>>,
     mut attrs: MapAttributes,
-    ctx: &mut MappingContext<PA, PM>,
+    ctx: &mut PagingContext<'_, PA, PM, I>,
 ) -> Result<()>
 where
     PA: PageAllocator,
     PM: PageTableMapper,
+    I: TLBInvalidator,
 {
     if attrs.phys.size() != attrs.virt.size() {
         Err(MapError::SizeMismatch)?;
@@ -234,17 +220,18 @@ where
     Ok(())
 }
 
-fn try_map_pa<L, PA, PM>(
+fn try_map_pa<L, PA, PM, I>(
     table: TPA<PgTableArray<L>>,
     va: VA,
     phys_region: PhysMemoryRegion,
     attrs: &MapAttributes,
-    ctx: &mut MappingContext<PA, PM>,
+    ctx: &mut PagingContext<'_, PA, PM, I>,
 ) -> Result<Option<usize>>
 where
     L: PgTable<Descriptor: PaMapper<MemoryType = MemoryType>>,
     PA: PageAllocator,
     PM: PageTableMapper,
+    I: TLBInvalidator,
 {
     if L::Descriptor::could_map(phys_region, va) {
         unsafe {
@@ -267,7 +254,11 @@ where
                         attrs.mem_type,
                         attrs.perms,
                     ),
-                    ctx.invalidator,
+                    // The slot is verified empty and the descriptor store is
+                    // itself single-copy atomic, so there is neither a stale
+                    // translation to invalidate nor an ordering another core
+                    // must observe.
+                    &NullTlbInvalidator {},
                 );
             })?;
         }
@@ -278,15 +269,16 @@ where
     }
 }
 
-pub(super) fn map_at_level<L, PA, PM>(
+pub(super) fn map_at_level<L, PA, PM, I>(
     table: TPA<PgTableArray<L>>,
     va: VA,
-    ctx: &mut MappingContext<PA, PM>,
+    ctx: &mut PagingContext<'_, PA, PM, I>,
 ) -> Result<TPA<PgTableArray<<L::Descriptor as TableMapper>::NextLevel>>>
 where
     L: TableMapperTable,
     PA: PageAllocator,
     PM: PageTableMapper,
+    I: TLBInvalidator,
 {
     unsafe {
         let desc = ctx
@@ -321,6 +313,11 @@ where
             L::from_ptr(pgtable).set_desc(
                 va,
                 L::Descriptor::new_next_table(new_pa),
+                // There is nothing to invalidate here (the slot was verified
+                // empty), but the invalidator's prepare() orders the zero-fill
+                // above ahead of this link. Without it, a walker on another
+                // core following a speculatively fetched link could observe the
+                // new table page before its contents.
                 ctx.invalidator,
             );
         })?;
@@ -338,7 +335,10 @@ pub mod tests {
         error::KernelError,
         memory::{
             address::{PA, VA},
-            paging::test::{MockPageAllocator, PassthroughMapper},
+            paging::{
+                test::{MockPageAllocator, MockTLBInvalidator, PassthroughMapper},
+                walk::WalkContext,
+            },
         },
     };
 
@@ -355,8 +355,8 @@ pub mod tests {
 
         pub fn create_map_ctx(
             &mut self,
-        ) -> MappingContext<'_, MockPageAllocator, PassthroughMapper> {
-            MappingContext {
+        ) -> PagingContext<'_, MockPageAllocator, PassthroughMapper, MockTLBInvalidator> {
+            PagingContext {
                 allocator: &mut self.inner.allocator,
                 mapper: &mut self.inner.mapper,
                 invalidator: &self.inner.invalidator,
@@ -399,6 +399,12 @@ pub mod tests {
             .unwrap();
             assert_eq!(perms_found, Some(expected_perms));
         }
+
+        pub fn create_walk_ctx(
+            &mut self,
+        ) -> WalkContext<'_, PassthroughMapper, MockTLBInvalidator> {
+            self.inner.create_walk_ctx()
+        }
     }
 
     pub type TestHarness = Arm64PagingTestHarness;

libkernel/src/arch/arm64/memory/pg_tear_down.rs

@@ -3,6 +3,7 @@
 use super::pg_descriptors::L0Descriptor;
 use super::pg_tables::{L0Table, L1Table, L3Table};
 use crate::error::Result;
+use crate::memory::paging::TLBInvalidator;
 use crate::memory::region::{PhysMemoryRegion, VirtMemoryRegion};
 use crate::memory::{
     PAGE_SIZE,
@@ -16,16 +17,17 @@ use crate::memory::{
 
 // Implementation for L3 (Leaf Table)
 impl RecursiveTeardownWalker for L3Table {
-    fn tear_down<Control, Dealloc, PM>(
+    fn tear_down<Control, Dealloc, PM, I>(
         table_pa: TPA<PgTableArray<Self>>,
-        ctx: &mut WalkContext<PM>,
+        ctx: &mut WalkContext<PM, I>,
         base_va: VA,
         depth: u8,
         control: &mut Control,
         deallocator: &mut Dealloc,
     ) -> Result<()>
     where
         PM: PageTableMapper,
+        I: TLBInvalidator,
         Control: FnMut(&TeardownEntry) -> TeardownAction,
         Dealloc: FnMut(PhysMemoryRegion),
     {
@@ -60,10 +62,11 @@ impl RecursiveTeardownWalker for L3Table {
                         if matches!(action, TeardownAction::FreeAndClear) {
                             unsafe {
                                 ctx.mapper.with_page_table(table_pa, |pgtable| {
-                                    L3Table::from_ptr(pgtable)
-                                        .to_raw_ptr()
-                                        .add(found_idx)
-                                        .write_volatile(0u64);
+                                    L3Table::from_ptr(pgtable).set_desc(
+                                        entry_va,
+                                        Self::Descriptor::invalid(),
+                                        ctx.invalidator,
+                                    );
                                 })?;
                             }
                         }
@@ -103,14 +106,15 @@ impl RecursiveTeardownWalker for L3Table {
 ///
 /// Block mappings (2 MiB / 1 GiB) are reported as
 /// [`EntryKind::Mapping`] with the appropriate region size.
-pub fn tear_down_address_space<Control, Dealloc, PM>(
+pub fn tear_down_address_space<Control, Dealloc, PM, I>(
     l0_table: TPA<PgTableArray<L0Table>>,
-    ctx: &mut WalkContext<PM>,
+    ctx: &mut WalkContext<PM, I>,
     mut control: Control,
     mut deallocator: Dealloc,
 ) -> Result<()>
 where
     PM: PageTableMapper,
+    I: TLBInvalidator,
     Control: FnMut(&TeardownEntry) -> TeardownAction,
     Dealloc: FnMut(PhysMemoryRegion),
 {
@@ -147,10 +151,11 @@ where
                     if matches!(action, TeardownAction::FreeAndClear) {
                         unsafe {
                             ctx.mapper.with_page_table(l0_table, |pgtable| {
-                                L0Table::from_ptr(pgtable)
-                                    .to_raw_ptr()
-                                    .add(idx)
-                                    .write_volatile(0u64);
+                                L0Table::from_ptr(pgtable).set_desc(
+                                    entry_va,
+                                    L0Descriptor::invalid(),
+                                    ctx.invalidator,
+                                );
                             })?;
                         }
                     }
@@ -193,21 +198,22 @@ mod tests {
     use std::collections::HashMap;
 
     /// Tear down `l0_table` freeing every frame the walker visits.
-    fn capture_freed_pages<PM: PageTableMapper>(
+    fn capture_freed_pages<PM: PageTableMapper, I: TLBInvalidator>(
         l0_table: TPA<PgTableArray<L0Table>>,
-        ctx: &mut WalkContext<PM>,
+        ctx: &mut WalkContext<PM, I>,
     ) -> HashMap<usize, usize> {
         capture_freed_pages_filtered(l0_table, ctx, |_| TeardownAction::Free)
     }
 
     /// Tear down `l0_table` using a custom `control` closure.
-    fn capture_freed_pages_filtered<PM, Control>(
+    fn capture_freed_pages_filtered<PM, Control, I>(
         l0_table: TPA<PgTableArray<L0Table>>,
-        ctx: &mut WalkContext<PM>,
+        ctx: &mut WalkContext<PM, I>,
         control: Control,
     ) -> HashMap<usize, usize>
     where
         PM: PageTableMapper,
+        I: TLBInvalidator,
         Control: FnMut(&TeardownEntry) -> TeardownAction,
     {
         let mut freed_map = HashMap::new();