From d4246bfc72d7f01fc8a18809bc0599a5d326d8cd Mon Sep 17 00:00:00 2001 From: Ashwin Naren Date: Thu, 23 Apr 2026 18:57:17 -0700 Subject: [PATCH] implement shared mmap for anonymous maps --- .../src/arch/arm64/memory/pg_descriptors.rs | 18 ++++- .../src/memory/proc_vm/memory_map/mod.rs | 66 ++++++++++++++----- libkernel/src/memory/proc_vm/vmarea.rs | 66 +++++++++++++++++-- src/arch/arm64/memory/address_space.rs | 5 +- src/drivers/fs/proc/task/task_file.rs | 2 +- src/memory/fault.rs | 28 +++++++- src/memory/mmap.rs | 42 +++++++----- src/memory/mod.rs | 1 + src/memory/shared_anon.rs | 57 ++++++++++++++++ src/process/threading/futex/key.rs | 28 ++++++-- 10 files changed, 263 insertions(+), 50 deletions(-) create mode 100644 src/memory/shared_anon.rs diff --git a/libkernel/src/arch/arm64/memory/pg_descriptors.rs b/libkernel/src/arch/arm64/memory/pg_descriptors.rs index 31fb1a93..8628641b 100644 --- a/libkernel/src/arch/arm64/memory/pg_descriptors.rs +++ b/libkernel/src/arch/arm64/memory/pg_descriptors.rs @@ -170,7 +170,23 @@ macro_rules! define_descriptor { reg.modify(BlockPageFields::COW::NotCowShared) } - Self(reg.get()) + // User mappings are leaf pages. PROT_NONE keeps their + // frame but is invalid to hardware; CoW must stay valid so + // writes trigger protection faults, not demand faults. + // Preserve existing block descriptor semantics: swapped + // blocks cannot yet be demoted by the generic walker. + let map_bits = if $map_bits == 0b11 + && !perms.is_read() + && !perms.is_write() + && !perms.is_execute() + && !perms.is_cow() + { + 0b10 + } else { + $map_bits + }; + + Self((reg.get() & !0b11) | map_bits) } } diff --git a/libkernel/src/memory/proc_vm/memory_map/mod.rs b/libkernel/src/memory/proc_vm/memory_map/mod.rs index 880528bd..614f98e4 100644 --- a/libkernel/src/memory/proc_vm/memory_map/mod.rs +++ b/libkernel/src/memory/proc_vm/memory_map/mod.rs @@ -103,19 +103,42 @@ impl MemoryMap { pub fn mmap( &mut self, requested_address: AddressRequest, - mut len: usize, + len: usize, perms: VMAPermissions, kind: VMAreaKind, name: String, ) -> Result { + self.mmap_impl(requested_address, len, perms, kind, name, false) + .map(|(address, _)| address) + } + + /// Creates fresh backing, returning replaced frames for the caller to release. + pub fn mmap_replacing( + &mut self, + requested_address: AddressRequest, + len: usize, + perms: VMAPermissions, + kind: VMAreaKind, + name: String, + ) -> Result<(VA, Vec)> { + self.mmap_impl(requested_address, len, perms, kind, name, true) + } + + fn mmap_impl( + &mut self, + requested_address: AddressRequest, + mut len: usize, + perms: VMAPermissions, + kind: VMAreaKind, + name: String, + replace: bool, + ) -> Result<(VA, Vec)> { if len == 0 { return Err(KernelError::InvalidValue); } // Ensure the length is page-aligned. - if len & PAGE_MASK != 0 { - len = (len & !PAGE_MASK) + PAGE_SIZE; - } + len = len.checked_add(PAGE_MASK).ok_or(KernelError::NoMemory)? & !PAGE_MASK; let region = match requested_address { AddressRequest::Any => self.find_free_region(len).ok_or(KernelError::NoMemory)?, @@ -153,15 +176,21 @@ impl MemoryMap { } }; - // At this point, `start_addr` points to a valid, free region. - // We can now create and insert the new VMA, handling merges. + // Fresh mmap replaces old pages, unlike VMA permission updates. + let pages = if replace { + self.unmap_region(region, None)? + } else { + Vec::new() + }; + + // Create and insert the new VMA, handling merges. let mut new_vma = VMArea::new(region, kind, perms); new_vma.set_name(name); self.insert_and_merge(new_vma); - Ok(region.start_address()) + Ok((region.start_address(), pages)) } /// Unmaps a region of memory, similar to the `munmap` syscall. @@ -208,10 +237,14 @@ impl MemoryMap { .map(|x| x.region.start_address()) .ok_or(KernelError::NoMemory)?; - let affected_vma = self - .vmas - .remove(&affected_vma_addr) - .expect("Should have the same key as the start address"); + let affected_vma = self.vmas[&affected_vma_addr].clone(); + // Reject unsupported ranges without deleting the existing VMA. + if !affected_vma.region.contains(protect_region) { + return Err(KernelError::NoMemory); + } + self.address_space + .protect_range(protect_region, new_perms.into())?; + self.vmas.remove(&affected_vma_addr); // Easy case, the entire VMA is changing. if affected_vma.region == protect_region { @@ -219,9 +252,7 @@ impl MemoryMap { let mut new_vma = old_vma.clone(); new_vma.permissions = new_perms; - self.insert_and_merge(new_vma.clone()); - self.address_space - .protect_range(protect_region, new_perms.into())?; + self.insert_and_merge(new_vma); return Ok(()); } @@ -236,8 +267,6 @@ impl MemoryMap { self.insert_and_merge(affected_vma.shrink_to(left)); } - self.address_space - .protect_range(protect_region, new_perms.into())?; self.insert_and_merge(new_vma); if let Some(right) = right { @@ -753,8 +782,9 @@ impl MemoryMap { for vma in new_vmas.values() { let mut pte_perms = PtePermissions::from(vma.permissions); - // Mark all writable pages as CoW. - if pte_perms.is_write() { + // Shared mappings stay shared across fork; private writable + // mappings become CoW. + if !vma.is_shared() && pte_perms.is_write() { pte_perms = pte_perms.into_cow(); } diff --git a/libkernel/src/memory/proc_vm/vmarea.rs b/libkernel/src/memory/proc_vm/vmarea.rs index adbfa811..6a672241 100644 --- a/libkernel/src/memory/proc_vm/vmarea.rs +++ b/libkernel/src/memory/proc_vm/vmarea.rs @@ -12,9 +12,10 @@ use core::cmp; use crate::{ + error::{KernelError, Result}, fs::{Inode, InodeId}, memory::{ - PAGE_MASK, PAGE_SIZE, address::VA, address_space::AccessKind, + PAGE_MASK, PAGE_SIZE, address::VA, address_space::AccessKind, page::PageFrame, paging::permissions::PtePermissions, region::VirtMemoryRegion, }, }; @@ -141,6 +142,44 @@ impl VMFileMapping { } } +/// A demand-paged backing object shared by anonymous mappings across fork. +pub trait SharedAnonBacking: Send + Sync { + /// Claims one owned frame reference at a page-aligned byte offset. + /// + /// The backing retains its own reference. The caller must either install + /// the returned reference in a page table or release it on failure. + fn claim_page(&self, offset: usize) -> Result; +} + +/// A slice of a shared anonymous backing object. +#[derive(Clone)] +pub struct VMSharedMapping { + backing: Arc, + offset: usize, +} + +impl PartialEq for VMSharedMapping { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.backing, &other.backing) && self.offset == other.offset + } +} + +impl VMSharedMapping { + /// Creates a mapping starting at the beginning of a backing object. + pub fn new(backing: Arc) -> Self { + Self { backing, offset: 0 } + } + + /// Claims a frame at a page-aligned byte offset relative to this mapping. + pub fn claim_page(&self, offset: usize) -> Result { + let offset = self + .offset + .checked_add(offset) + .ok_or(KernelError::InvalidValue)?; + self.backing.claim_page(offset) + } +} + /// Defines the backing source for a `VMArea`. #[derive(Clone, PartialEq)] pub enum VMAreaKind { @@ -158,6 +197,9 @@ pub enum VMAreaKind { /// physical page that has been zero-filled. This is used for the heap, /// and the stack. Anon, + + /// The VMA shares a demand-paged anonymous backing object. + SharedAnon(VMSharedMapping), } impl VMAreaKind { @@ -324,7 +366,7 @@ impl VMArea { pub fn resolve_fault(&self, faulting_addr: VA) -> Option { // Match on the kind of VMA. If it's anonymous, there's no file to read from. let mapping = match &self.kind { - VMAreaKind::Anon => return None, + VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => return None, VMAreaKind::File(mapping) => mapping, }; @@ -403,6 +445,11 @@ impl VMArea { match (&self.kind, &other.kind) { (VMAreaKind::Anon, VMAreaKind::Anon) => true, + (VMAreaKind::SharedAnon(left), VMAreaKind::SharedAnon(right)) => { + Arc::ptr_eq(&left.backing, &right.backing) + && left.offset.checked_add(self.region.size()) == Some(right.offset) + } + (VMAreaKind::File(self_map), VMAreaKind::File(other_map)) => { // Check that they point to the same inode. let same_file = Arc::ptr_eq(&self_map.file, &other_map.file); @@ -423,6 +470,10 @@ impl VMArea { pub(super) fn clone_with_new_region(&self, new_region: VirtMemoryRegion) -> Self { let mut clone = self.clone(); + if let VMAreaKind::SharedAnon(mapping) = &mut clone.kind { + mapping.offset += + new_region.start_address().value() - self.region.start_address().value(); + } clone.region = new_region; clone @@ -434,6 +485,11 @@ impl VMArea { matches!(self.kind, VMAreaKind::File(_)) } + /// Returns whether this VMA was created as a shared mapping. + pub fn is_shared(&self) -> bool { + matches!(self.kind, VMAreaKind::SharedAnon(_)) + } + /// Shrink this VMA's region to `new_region`, recalculating file offsets, /// for file mappings. #[must_use] @@ -466,7 +522,7 @@ impl VMArea { new_vma } - VMAreaKind::Anon => new_vma, + VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => new_vma, } } @@ -479,7 +535,7 @@ impl VMArea { pub fn file_offset(&self) -> Option { match self.kind { VMAreaKind::File(ref vmfile_mapping) => Some(vmfile_mapping.offset()), - VMAreaKind::Anon => None, + VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => None, } } @@ -487,7 +543,7 @@ impl VMArea { pub fn inode_id(&self) -> Option { match self.kind { VMAreaKind::File(ref vmfile_mapping) => Some(vmfile_mapping.file().id()), - VMAreaKind::Anon => None, + VMAreaKind::Anon | VMAreaKind::SharedAnon(_) => None, } } diff --git a/src/arch/arm64/memory/address_space.rs b/src/arch/arm64/memory/address_space.rs index 8e775f76..fc2da625 100644 --- a/src/arch/arm64/memory/address_space.rs +++ b/src/arch/arm64/memory/address_space.rs @@ -92,10 +92,7 @@ impl UserAddressSpace for Arm64ProcessAddressSpace { }; walk_and_modify_region(self.l0_table, va_range, &mut walk_ctx, |_, desc| { - match (perms.is_execute(), perms.is_read(), perms.is_write()) { - (false, false, false) => desc.mark_as_swapped(), - _ => desc.set_permissions(perms), - } + desc.set_permissions(perms) }) } diff --git a/src/drivers/fs/proc/task/task_file.rs b/src/drivers/fs/proc/task/task_file.rs index 4bded60f..966f373d 100644 --- a/src/drivers/fs/proc/task/task_file.rs +++ b/src/drivers/fs/proc/task/task_file.rs @@ -229,7 +229,7 @@ Threads:\t{tasks}\n", if vma.permissions().read { "r" } else { "-" }, if vma.permissions().write { "w" } else { "-" }, if vma.permissions().execute { "x" } else { "-" }, - "p", // Don't suport shared mappings... yet! + if vma.is_shared() { "s" } else { "p" }, vma.file_offset().unwrap_or_default(), vma.name() )); diff --git a/src/memory/fault.rs b/src/memory/fault.rs index f27f6c79..b38b4452 100644 --- a/src/memory/fault.rs +++ b/src/memory/fault.rs @@ -7,6 +7,7 @@ use libkernel::{ address::VA, address_space::{AccessKind, PageInfo, UserAddressSpace}, paging::permissions::PtePermissions, + proc_vm::vmarea::VMAreaKind, }, }; @@ -54,9 +55,34 @@ pub fn handle_demand_fault( } .clone(); - let mut new_page = ClaimedPage::alloc_zeroed()?; let page_va = faulting_addr.page_aligned(); + if let VMAreaKind::SharedAnon(mapping) = vma.kind() { + let offset = page_va.value() - vma.region.start_address().value(); + let frame = mapping.claim_page(offset)?; + // SAFETY: claim_page transfers an owned allocator reference to us. + let shared_page = unsafe { ClaimedPage::from_pfn(frame) }; + + return match vm.mm_mut().address_space_mut().map_page( + frame, + page_va, + vma.permissions().into(), + ) { + Ok(()) => { + // Transfer only this extra reference to the page table. + shared_page.leak(); + Ok(FaultResolution::Resolved) + } + Err(KernelError::MappingError(MapError::AlreadyMapped)) => { + // The owned extra reference is dropped, not the backing page. + Ok(FaultResolution::Resolved) + } + Err(e) => Err(e), + }; + } + + let mut new_page = ClaimedPage::alloc_zeroed()?; + if let Some(vma_read) = vma.resolve_fault(faulting_addr) { drop(vm); diff --git a/src/memory/mmap.rs b/src/memory/mmap.rs index dff372ac..e5dbf300 100644 --- a/src/memory/mmap.rs +++ b/src/memory/mmap.rs @@ -1,17 +1,21 @@ use core::sync::atomic::{AtomicUsize, Ordering}; -use crate::{process::fd_table::Fd, sched::syscall_ctx::ProcessCtx}; +use crate::{ + memory::shared_anon::SharedAnon, process::fd_table::Fd, sched::syscall_ctx::ProcessCtx, +}; use alloc::{ string::{String, ToString}, + sync::Arc, vec::Vec, }; use libkernel::{ error::{KernelError, Result}, memory::{ address::VA, + page::PageFrame, proc_vm::{ memory_map::{AddressRequest, RemapDestination}, - vmarea::{VMAPermissions, VMAreaKind}, + vmarea::{VMAPermissions, VMAreaKind, VMSharedMapping}, }, region::VirtMemoryRegion, }, @@ -65,13 +69,15 @@ pub async fn sys_mmap( return Err(KernelError::InvalidValue); } - // Ensure mapping sharability has been specified: - if (flags & (MAP_SHARED | MAP_PRIVATE)) == 0 { + let mapping_kind = flags & (MAP_SHARED | MAP_PRIVATE); + if mapping_kind == 0 || mapping_kind == (MAP_SHARED | MAP_PRIVATE) { return Err(KernelError::InvalidValue); } - // TODO: Shared Mappings. - if (flags & MAP_SHARED) != 0 { + let is_shared = (flags & MAP_SHARED) != 0; + let is_anon = (flags & (MAP_ANON | MAP_ANONYMOUS)) != 0; + + if is_shared && !is_anon { return Err(KernelError::NotSupported); } @@ -94,8 +100,16 @@ pub async fn sys_mmap( let requested_len = len as usize; - let (kind, name) = if (flags & (MAP_ANON | MAP_ANONYMOUS)) != 0 { - (VMAreaKind::Anon, String::new()) + let (kind, name) = if is_anon { + if offset != 0 { + return Err(KernelError::InvalidValue); + } + let kind = if is_shared { + VMAreaKind::SharedAnon(VMSharedMapping::new(Arc::new(SharedAnon::new()))) + } else { + VMAreaKind::Anon + }; + (kind, String::new()) } else { // File-backed mapping: require a valid fd and use the provided offset. let fd = ctx @@ -135,30 +149,30 @@ pub async fn sys_mmap( // Lock the task and call the core memory manager to perform the mapping. let proc_vm = ctx.shared().vm.shared_vm(); - let new_mapping_addr = proc_vm.lock_save_irq().mm_mut().mmap( + let (new_mapping_addr, pages) = proc_vm.lock_save_irq().mm_mut().mmap_replacing( address_request, requested_len, permissions, kind, name, )?; + free_unmapped_pages(pages)?; Ok(new_mapping_addr.value()) } -fn free_unmapped_pages(pages: Vec) -> Result<()> { +fn free_unmapped_pages(pages: Vec) -> Result<()> { if pages.is_empty() { return Ok(()); } - // The frames returned by unmap/mremap are no longer mapped and belong to this process; - // creating temporary allocations from these regions allows the allocator to reclaim them on drop. + // Each unmapped frame owns one reference, even if other processes share it. let allocator = crate::memory::PAGE_ALLOC .get() .ok_or(KernelError::NoMemory)?; - // Create a temporary allocation from the single-page region and drop it immediately to free. for p in pages { + // SAFETY: unmapping transfers this page-table reference to us. let tmp = unsafe { allocator.alloc_from_region(p.as_phys_range()) }; drop(tmp); } @@ -170,9 +184,7 @@ pub async fn sys_munmap(ctx: &ProcessCtx, addr: VA, len: usize) -> Result let region = VirtMemoryRegion::new(addr, len); let proc_vm = ctx.shared().vm.shared_vm(); let pages = proc_vm.lock_save_irq().mm_mut().munmap(region)?; - free_unmapped_pages(pages)?; - Ok(0) } diff --git a/src/memory/mod.rs b/src/memory/mod.rs index 44aca297..0b24e608 100644 --- a/src/memory/mod.rs +++ b/src/memory/mod.rs @@ -16,6 +16,7 @@ pub mod mincore; pub mod mmap; pub mod page; pub mod process_vm; +pub mod shared_anon; pub mod uaccess; pub type PageOffsetTranslator = diff --git a/src/memory/shared_anon.rs b/src/memory/shared_anon.rs new file mode 100644 index 00000000..04f533ce --- /dev/null +++ b/src/memory/shared_anon.rs @@ -0,0 +1,57 @@ +use alloc::collections::{BTreeMap, btree_map::Entry}; +use libkernel::{ + error::Result, + memory::{page::PageFrame, proc_vm::vmarea::SharedAnonBacking}, +}; + +use crate::sync::SpinLock; + +use super::{PAGE_ALLOC, page::ClaimedPage}; + +/// Lazily allocated pages shared by all mappings of an anonymous backing. +/// +/// The backing retains one canonical reference per page until its last mapping +/// is dropped. Each fault obtains an additional reference for its page table. +pub struct SharedAnon { + pages: SpinLock>, +} + +impl SharedAnon { + pub fn new() -> Self { + Self { + pages: SpinLock::new(BTreeMap::new()), + } + } +} + +impl Default for SharedAnon { + fn default() -> Self { + Self::new() + } +} + +impl SharedAnonBacking for SharedAnon { + fn claim_page(&self, offset: usize) -> Result { + // Faults hold the VM lock first: VM -> backing -> page allocator. + let mut pages = self.pages.lock_save_irq(); + let page = match pages.entry(offset) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => entry.insert(ClaimedPage::alloc_zeroed()?), + }; + + // SAFETY: The canonical page is allocated and held alive under the + // backing lock. This temporary allocation aliases its ownership, so it + // must be leaked rather than dropped. Only clone() adds a reference; + // that extra owned reference is transferred to the caller. + let allocation = unsafe { + PAGE_ALLOC + .get() + .unwrap() + .alloc_from_region(page.pa().to_pfn().as_phys_range()) + }; + let claimed = allocation.clone().leak(); + allocation.leak(); + + Ok(claimed.start_address().to_pfn()) + } +} diff --git a/src/process/threading/futex/key.rs b/src/process/threading/futex/key.rs index aa96a708..9dced5ea 100644 --- a/src/process/threading/futex/key.rs +++ b/src/process/threading/futex/key.rs @@ -1,7 +1,8 @@ -use crate::sched::syscall_ctx::ProcessCtx; +use crate::{memory::page::ClaimedPage, sched::syscall_ctx::ProcessCtx}; use libkernel::error::{KernelError, Result}; use libkernel::memory::address::{TUA, VA}; -use libkernel::memory::address_space::UserAddressSpace; +use libkernel::memory::address_space::{AccessKind, UserAddressSpace}; +use libkernel::memory::proc_vm::vmarea::VMAreaKind; #[derive(PartialEq, Eq, PartialOrd, Ord, Hash, Clone, Copy, Debug)] pub enum FutexKey { @@ -21,11 +22,28 @@ impl FutexKey { pub fn new_shared(ctx: &ProcessCtx, uaddr: TUA) -> Result { let proc_vm = ctx.shared().vm.shared_vm(); - let pg_info = proc_vm - .lock_save_irq() + let mut vm = proc_vm.lock_save_irq(); + let va = VA::from_value(uaddr.value()); + + // A different process may have faulted in this shared page while our + // PTE is still absent. Resolve its canonical frame from the backing. + if let Some(vma) = vm.find_vma_for_fault(va, AccessKind::Read) + && let VMAreaKind::SharedAnon(mapping) = vma.kind() + { + let offset = va.page_aligned().value() - vma.region.start_address().value(); + let frame = mapping.claim_page(offset)?; + // SAFETY: claim_page transfers an owned reference, released on return. + let _page = unsafe { ClaimedPage::from_pfn(frame) }; + return Ok(Self::Shared { + frame: frame.value(), + offset: uaddr.page_offset(), + }); + } + + let pg_info = vm .mm_mut() .address_space_mut() - .translate(VA::from_value(uaddr.value())) + .translate(va) .ok_or(KernelError::Fault)?; Ok(Self::Shared {