systemd part 7: implement shared mmap maps

libkernel/src/arch/arm64/memory/pg_descriptors.rs

@@ -170,7 +170,23 @@ macro_rules! define_descriptor {
                         reg.modify(BlockPageFields::COW::NotCowShared)
                     }
 
-                    Self(reg.get())
+                    // User mappings are leaf pages. PROT_NONE keeps their
+                    // frame but is invalid to hardware; CoW must stay valid so
+                    // writes trigger protection faults, not demand faults.
+                    // Preserve existing block descriptor semantics: swapped
+                    // blocks cannot yet be demoted by the generic walker.
+                    let map_bits = if $map_bits == 0b11
+                        && !perms.is_read()
+                        && !perms.is_write()
+                        && !perms.is_execute()
+                        && !perms.is_cow()
+                    {
+                        0b10
+                    } else {
+                        $map_bits
+                    };
+
+                    Self((reg.get() & !0b11) | map_bits)
                 }
             }
 

libkernel/src/memory/proc_vm/memory_map/mod.rs

@@ -103,19 +103,42 @@ impl<AS: UserAddressSpace> MemoryMap<AS> {
     pub fn mmap(
         &mut self,
         requested_address: AddressRequest,
-        mut len: usize,
+        len: usize,
         perms: VMAPermissions,
         kind: VMAreaKind,
         name: String,
     ) -> Result<VA> {
+        self.mmap_impl(requested_address, len, perms, kind, name, false)
+            .map(|(address, _)| address)
+    }
+
+    /// Creates fresh backing, returning replaced frames for the caller to release.
+    pub fn mmap_replacing(
+        &mut self,
+        requested_address: AddressRequest,
+        len: usize,
+        perms: VMAPermissions,
+        kind: VMAreaKind,
+        name: String,
+    ) -> Result<(VA, Vec<PageFrame>)> {
+        self.mmap_impl(requested_address, len, perms, kind, name, true)
+    }
+
+    fn mmap_impl(
+        &mut self,
+        requested_address: AddressRequest,
+        mut len: usize,
+        perms: VMAPermissions,
+        kind: VMAreaKind,
+        name: String,
+        replace: bool,
+    ) -> Result<(VA, Vec<PageFrame>)> {
         if len == 0 {
             return Err(KernelError::InvalidValue);
         }
 
         // Ensure the length is page-aligned.
-        if len & PAGE_MASK != 0 {
-            len = (len & !PAGE_MASK) + PAGE_SIZE;
-        }
+        len = len.checked_add(PAGE_MASK).ok_or(KernelError::NoMemory)? & !PAGE_MASK;
 
         let region = match requested_address {
             AddressRequest::Any => self.find_free_region(len).ok_or(KernelError::NoMemory)?,
@@ -153,15 +176,21 @@ impl<AS: UserAddressSpace> MemoryMap<AS> {
             }
         };
 
-        // At this point, `start_addr` points to a valid, free region.
-        // We can now create and insert the new VMA, handling merges.
+        // Fresh mmap replaces old pages, unlike VMA permission updates.
+        let pages = if replace {
+            self.unmap_region(region, None)?
+        } else {
+            Vec::new()
+        };
+
+        // Create and insert the new VMA, handling merges.
         let mut new_vma = VMArea::new(region, kind, perms);
 
         new_vma.set_name(name);
 
         self.insert_and_merge(new_vma);
 
-        Ok(region.start_address())
+        Ok((region.start_address(), pages))
     }
 
     /// Unmaps a region of memory, similar to the `munmap` syscall.
@@ -208,20 +237,22 @@ impl<AS: UserAddressSpace> MemoryMap<AS> {
             .map(|x| x.region.start_address())
             .ok_or(KernelError::NoMemory)?;
 
-        let affected_vma = self
-            .vmas
-            .remove(&affected_vma_addr)
-            .expect("Should have the same key as the start address");
+        let affected_vma = self.vmas[&affected_vma_addr].clone();
+        // Reject unsupported ranges without deleting the existing VMA.
+        if !affected_vma.region.contains(protect_region) {
+            return Err(KernelError::NoMemory);
+        }
+        self.address_space
+            .protect_range(protect_region, new_perms.into())?;
+        self.vmas.remove(&affected_vma_addr);
 
         // Easy case, the entire VMA is changing.
         if affected_vma.region == protect_region {
             let old_vma = affected_vma.clone();
             let mut new_vma = old_vma.clone();
             new_vma.permissions = new_perms;
 
-            self.insert_and_merge(new_vma.clone());
-            self.address_space
-                .protect_range(protect_region, new_perms.into())?;
+            self.insert_and_merge(new_vma);
 
             return Ok(());
         }
@@ -236,8 +267,6 @@ impl<AS: UserAddressSpace> MemoryMap<AS> {
                 self.insert_and_merge(affected_vma.shrink_to(left));
             }
 
-            self.address_space
-                .protect_range(protect_region, new_perms.into())?;
             self.insert_and_merge(new_vma);
 
             if let Some(right) = right {
@@ -753,8 +782,9 @@ impl<AS: UserAddressSpace> MemoryMap<AS> {
         for vma in new_vmas.values() {
             let mut pte_perms = PtePermissions::from(vma.permissions);
 
-            // Mark all writable pages as CoW.
-            if pte_perms.is_write() {
+            // Shared mappings stay shared across fork; private writable
+            // mappings become CoW.
+            if !vma.is_shared() && pte_perms.is_write() {
                 pte_perms = pte_perms.into_cow();
             }
 

libkernel/src/memory/proc_vm/vmarea.rs

@@ -12,9 +12,10 @@
 use core::cmp;
 
 use crate::{
+    error::{KernelError, Result},
     fs::{Inode, InodeId},
     memory::{
-        PAGE_MASK, PAGE_SIZE, address::VA, address_space::AccessKind,
+        PAGE_MASK, PAGE_SIZE, address::VA, address_space::AccessKind, page::PageFrame,
         paging::permissions::PtePermissions, region::VirtMemoryRegion,
     },
 };
@@ -141,6 +142,44 @@ impl VMFileMapping {
     }
 }
 
+/// A demand-paged backing object shared by anonymous mappings across fork.
+pub trait SharedAnonBacking: Send + Sync {
+    /// Claims one owned frame reference at a page-aligned byte offset.
+    ///
+    /// The backing retains its own reference. The caller must either install
+    /// the returned reference in a page table or release it on failure.
+    fn claim_page(&self, offset: usize) -> Result<PageFrame>;
+}
+
+/// A slice of a shared anonymous backing object.
+#[derive(Clone)]
+pub struct VMSharedMapping {
+    backing: Arc<dyn SharedAnonBacking>,
+    offset: usize,
+}
+
+impl PartialEq for VMSharedMapping {
+    fn eq(&self, other: &Self) -> bool {
+        Arc::ptr_eq(&self.backing, &other.backing) && self.offset == other.offset
+    }
+}
+
+impl VMSharedMapping {
+    /// Creates a mapping starting at the beginning of a backing object.
+    pub fn new(backing: Arc<dyn SharedAnonBacking>) -> Self {
+        Self { backing, offset: 0 }
+    }
+
+    /// Claims a frame at a page-aligned byte offset relative to this mapping.
+    pub fn claim_page(&self, offset: usize) -> Result<PageFrame> {
+        let offset = self
+            .offset
+            .checked_add(offset)
+            .ok_or(KernelError::InvalidValue)?;
+        self.backing.claim_page(offset)
+    }
+}
+
 /// Defines the backing source for a `VMArea`.
 #[derive(Clone, PartialEq)]
 pub enum VMAreaKind {
@@ -158,6 +197,9 @@ pub enum VMAreaKind {
     /// physical page that has been zero-filled. This is used for the heap,
     /// and the stack.
     Anon,
+
+    /// The VMA shares a demand-paged anonymous backing object.
+    SharedAnon(VMSharedMapping),
 }
 
 impl VMAreaKind {
@@ -324,7 +366,7 @@ impl VMArea {
     pub fn resolve_fault(&self, faulting_addr: VA) -> Option<VMAFileRead> {
         // Match on the kind of VMA. If it's anonymous, there's no file to read from.
         let mapping = match &self.kind {
-            VMAreaKind::Anon => return None,
+            VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => return None,
             VMAreaKind::File(mapping) => mapping,
         };
 
@@ -403,6 +445,11 @@ impl VMArea {
         match (&self.kind, &other.kind) {
             (VMAreaKind::Anon, VMAreaKind::Anon) => true,
 
+            (VMAreaKind::SharedAnon(left), VMAreaKind::SharedAnon(right)) => {
+                Arc::ptr_eq(&left.backing, &right.backing)
+                    && left.offset.checked_add(self.region.size()) == Some(right.offset)
+            }
+
             (VMAreaKind::File(self_map), VMAreaKind::File(other_map)) => {
                 // Check that they point to the same inode.
                 let same_file = Arc::ptr_eq(&self_map.file, &other_map.file);
@@ -423,6 +470,10 @@ impl VMArea {
     pub(super) fn clone_with_new_region(&self, new_region: VirtMemoryRegion) -> Self {
         let mut clone = self.clone();
 
+        if let VMAreaKind::SharedAnon(mapping) = &mut clone.kind {
+            mapping.offset +=
+                new_region.start_address().value() - self.region.start_address().value();
+        }
         clone.region = new_region;
 
         clone
@@ -434,6 +485,11 @@ impl VMArea {
         matches!(self.kind, VMAreaKind::File(_))
     }
 
+    /// Returns whether this VMA was created as a shared mapping.
+    pub fn is_shared(&self) -> bool {
+        matches!(self.kind, VMAreaKind::SharedAnon(_))
+    }
+
     /// Shrink this VMA's region to `new_region`, recalculating file offsets,
     /// for file mappings.
     #[must_use]
@@ -466,7 +522,7 @@ impl VMArea {
 
                 new_vma
             }
-            VMAreaKind::Anon => new_vma,
+            VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => new_vma,
         }
     }
 
@@ -479,15 +535,15 @@ impl VMArea {
     pub fn file_offset(&self) -> Option<u64> {
         match self.kind {
             VMAreaKind::File(ref vmfile_mapping) => Some(vmfile_mapping.offset()),
-            VMAreaKind::Anon => None,
+            VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => None,
         }
     }
 
     /// Returns the inode ID of the backing file, or `None` for anonymous mappings.
     pub fn inode_id(&self) -> Option<InodeId> {
         match self.kind {
             VMAreaKind::File(ref vmfile_mapping) => Some(vmfile_mapping.file().id()),
-            VMAreaKind::Anon => None,
+            VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => None,
         }
     }
 

src/arch/arm64/memory/address_space.rs

@@ -92,10 +92,7 @@ impl UserAddressSpace for Arm64ProcessAddressSpace {
         };
 
         walk_and_modify_region(self.l0_table, va_range, &mut walk_ctx, |_, desc| {
-            match (perms.is_execute(), perms.is_read(), perms.is_write()) {
-                (false, false, false) => desc.mark_as_swapped(),
-                _ => desc.set_permissions(perms),
-            }
+            desc.set_permissions(perms)
         })
     }
 

src/drivers/fs/proc/task/task_file.rs

@@ -229,7 +229,7 @@ Threads:\t{tasks}\n",
                             if vma.permissions().read { "r" } else { "-" },
                             if vma.permissions().write { "w" } else { "-" },
                             if vma.permissions().execute { "x" } else { "-" },
-                            "p", // Don't suport shared mappings... yet!
+                            if vma.is_shared() { "s" } else { "p" },
                             vma.file_offset().unwrap_or_default(),
                             vma.name()
                         ));

src/memory/fault.rs

@@ -7,6 +7,7 @@ use libkernel::{
         address::VA,
         address_space::{AccessKind, PageInfo, UserAddressSpace},
         paging::permissions::PtePermissions,
+        proc_vm::vmarea::VMAreaKind,
     },
 };
 
@@ -54,9 +55,34 @@ pub fn handle_demand_fault(
     }
     .clone();
 
-    let mut new_page = ClaimedPage::alloc_zeroed()?;
     let page_va = faulting_addr.page_aligned();
 
+    if let VMAreaKind::SharedAnon(mapping) = vma.kind() {
+        let offset = page_va.value() - vma.region.start_address().value();
+        let frame = mapping.claim_page(offset)?;
+        // SAFETY: claim_page transfers an owned allocator reference to us.
+        let shared_page = unsafe { ClaimedPage::from_pfn(frame) };
+
+        return match vm.mm_mut().address_space_mut().map_page(
+            frame,
+            page_va,
+            vma.permissions().into(),
+        ) {
+            Ok(()) => {
+                // Transfer only this extra reference to the page table.
+                shared_page.leak();
+                Ok(FaultResolution::Resolved)
+            }
+            Err(KernelError::MappingError(MapError::AlreadyMapped)) => {
+                // The owned extra reference is dropped, not the backing page.
+                Ok(FaultResolution::Resolved)
+            }
+            Err(e) => Err(e),
+        };
+    }
+
+    let mut new_page = ClaimedPage::alloc_zeroed()?;
+
     if let Some(vma_read) = vma.resolve_fault(faulting_addr) {
         drop(vm);