diff --git a/Cargo.lock b/Cargo.lock index b1cf5dc3..0de57dab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,9 +307,9 @@ dependencies = [ [[package]] name = "ext4plus" -version = "0.1.0-rc.1" +version = "0.1.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4329a9b9a7c59c6c8277de1447710ccf6f30e501734bb9b0dc8ceb458b0e9a" +checksum = "6c83a722564b63b6ed4265a3bff5888bc50ce66c2f7eccb33e716d384a884144" dependencies = [ "async-lock", "async-trait", diff --git a/libkernel/Cargo.toml b/libkernel/Cargo.toml index 9a69793e..361cb450 100644 --- a/libkernel/Cargo.toml +++ b/libkernel/Cargo.toml @@ -35,7 +35,7 @@ tock-registers = { version = "0.10.1", optional = true } # fs async-trait = { workspace = true, optional = true } -ext4plus = { version = "0.1.0-beta.3", optional = true } +ext4plus = { version = "0.1.0-rc.2", optional = true } # proc_vm object = { version = "0.39.0", default-features = false, features = ["core", "elf", "read_core"], optional = true } diff --git a/libkernel/src/fs/filesystems/ext4/cache.rs b/libkernel/src/fs/filesystems/ext4/cache.rs new file mode 100644 index 00000000..c3377bf9 --- /dev/null +++ b/libkernel/src/fs/filesystems/ext4/cache.rs @@ -0,0 +1,70 @@ +use alloc::collections::BTreeMap; +use alloc::sync::{Arc, Weak}; +use core::num::NonZeroU32; + +/// Shares live inodes, without retaining every inode ever visited. +pub(super) struct InodeCache { + entries: BTreeMap>, +} + +impl InodeCache { + pub(super) fn new() -> Self { + Self { + entries: BTreeMap::new(), + } + } + + pub(super) fn get_or_insert_with( + &mut self, + id: NonZeroU32, + create: impl FnOnce() -> T, + ) -> Arc { + if let Some(existing) = self.entries.get(&id).and_then(Weak::upgrade) { + return existing; + } + // Only scan on a miss. Cache hits (including the root inode) stay cheap. + // A traversal with no retained handles cannot accumulate dead entries. + self.entries.retain(|_, inode| inode.strong_count() != 0); + let inode = Arc::new(create()); + self.entries.insert(id, Arc::downgrade(&inode)); + inode + } + + pub(super) fn get(&self, id: NonZeroU32) -> Option> { + self.entries.get(&id).and_then(Weak::upgrade) + } + + pub(super) fn remove(&mut self, id: NonZeroU32) { + self.entries.remove(&id); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shares_live_inodes_and_reaps_closed_inodes() { + let mut cache = InodeCache::new(); + let root_id = NonZeroU32::new(2).unwrap(); + let root = cache.get_or_insert_with(root_id, || 42); + assert!(Arc::ptr_eq(&root, &cache.get_or_insert_with(root_id, || 0))); + for id in 3..4096 { + let inode = cache.get_or_insert_with(NonZeroU32::new(id).unwrap(), || id); + assert!(cache.entries.len() <= 2); + drop(inode); + } + assert_eq!(*root, 42); + } + + #[test] + fn inode_number_reuse_does_not_share_unlinked_state() { + let mut cache = InodeCache::new(); + let id = NonZeroU32::new(10).unwrap(); + let old = cache.get_or_insert_with(id, || 1); + cache.remove(id); + let new = cache.get_or_insert_with(id, || 2); + assert!(!Arc::ptr_eq(&old, &new)); + assert_eq!((*old, *new), (1, 2)); + } +} diff --git a/libkernel/src/fs/filesystems/ext4/disk.rs b/libkernel/src/fs/filesystems/ext4/disk.rs new file mode 100644 index 00000000..6f16c766 --- /dev/null +++ b/libkernel/src/fs/filesystems/ext4/disk.rs @@ -0,0 +1,283 @@ +//! The small set of on-disk edits not exposed by ext4plus's public API. +//! +//! Allocation, inode freeing, block mapping and directory indexing remain in +//! ext4plus. All offsets below describe the ext4 disk format, never Rust object +//! layout. See https://docs.kernel.org/filesystems/ext4/ondisk/index.html. + +use super::*; +use ext4plus::prelude::read_at; + +pub(super) struct DiskLayout { + pub block_size: u64, + inode_size: usize, + inodes_per_group: u32, + inodes_count: u32, + descriptor_size: usize, + descriptor_start: u64, + blocks_count: u64, + checksum_seed: Option, +} + +pub(super) struct SavedInode { + offset: u64, + bytes: Vec, +} + +impl SavedInode { + pub async fn restore(&self, dev: &Ext4Device) -> Result<()> { + dev.write_at(self.offset, &self.bytes).await + } +} + +fn u16_at(bytes: &[u8], offset: usize) -> u16 { + u16::from_le_bytes(bytes[offset..offset + 2].try_into().unwrap()) +} + +fn u32_at(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes(bytes[offset..offset + 4].try_into().unwrap()) +} + +// ext4 uses the reflected Castagnoli CRC without the final complement. +fn crc32c(mut crc: u32, bytes: &[u8]) -> u32 { + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + crc = (crc >> 1) ^ (0x82f6_3b78 & 0u32.wrapping_sub(crc & 1)); + } + } + crc +} + +impl DiskLayout { + pub async fn load(dev: &Ext4Device) -> Result { + let mut sb = [0; 1024]; + dev.read_at(1024, &mut sb).await?; + let log_block_size = u32_at(&sb, 0x18); + let incompatible = u32_at(&sb, 0x60); + if u16_at(&sb, 0x38) != 0xef53 || log_block_size > 6 { + return Err(FsError::InvalidFs.into()); + } + // META_BG relocates descriptor tables. ext4plus does not support it + // either; do not guess inode locations on such a filesystem. + if incompatible & 0x10 != 0 { + return Err(KernelError::NotSupported); + } + let block_size = 1024u64 << log_block_size; + let is_64bit = incompatible & 0x80 != 0; + let descriptor_size = if is_64bit { + usize::from(u16_at(&sb, 0xfe)) + } else { + 32 + }; + let inode_size = if u32_at(&sb, 0x4c) == 0 { + 128 + } else { + usize::from(u16_at(&sb, 0x58)) + }; + let inodes_per_group = u32_at(&sb, 0x28); + if !matches!(descriptor_size, 32 | 64) + || (is_64bit && descriptor_size != 64) + || inode_size < 128 + || !inode_size.is_power_of_two() + || inode_size as u64 > block_size + || inodes_per_group == 0 + { + return Err(FsError::InvalidFs.into()); + } + let blocks_count = u64::from(u32_at(&sb, 4)) + | if is_64bit { + u64::from(u32_at(&sb, 0x150)) << 32 + } else { + 0 + }; + let checksum_seed = (u32_at(&sb, 0x64) & 0x400 != 0).then(|| { + if incompatible & 0x2000 != 0 { + u32_at(&sb, 0x270) + } else { + crc32c(!0, &sb[0x68..0x78]) + } + }); + Ok(Self { + block_size, + inode_size, + inodes_per_group, + inodes_count: u32_at(&sb, 0), + descriptor_size, + descriptor_start: (u64::from(u32_at(&sb, 0x14)) + 1) * block_size, + blocks_count, + checksum_seed, + }) + } + + async fn read_inode( + &self, + dev: &Ext4Device, + id: NonZeroU32, + ) -> Result<(u64, Vec)> { + if id.get() > self.inodes_count { + return Err(FsError::InvalidFs.into()); + } + let index = id.get() - 1; + let group = index / self.inodes_per_group; + let slot = index % self.inodes_per_group; + let mut descriptor = vec![0; self.descriptor_size]; + dev.read_at( + self.descriptor_start + u64::from(group) * self.descriptor_size as u64, + &mut descriptor, + ) + .await?; + let table = u64::from(u32_at(&descriptor, 8)) + | if self.descriptor_size == 64 { + u64::from(u32_at(&descriptor, 0x28)) << 32 + } else { + 0 + }; + let offset = table + .checked_mul(self.block_size) + .and_then(|start| start.checked_add(u64::from(slot) * self.inode_size as u64)) + .ok_or(FsError::InvalidFs)?; + let end = offset + .checked_add(self.inode_size as u64) + .ok_or(FsError::InvalidFs)?; + let disk_size = self + .blocks_count + .checked_mul(self.block_size) + .ok_or(FsError::InvalidFs)?; + if table == 0 || end > disk_size { + return Err(FsError::InvalidFs.into()); + } + let mut bytes = vec![0; self.inode_size]; + dev.read_at(offset, &mut bytes).await?; + Ok((offset, bytes)) + } + + fn inode_seed(&self, id: NonZeroU32, bytes: &[u8]) -> Option { + self.checksum_seed + .map(|seed| crc32c(crc32c(seed, &id.get().to_le_bytes()), &bytes[0x64..0x68])) + } + + fn checksum_inode(&self, id: NonZeroU32, bytes: &mut [u8]) { + if let Some(seed) = self.inode_seed(id, bytes) { + let high = bytes.len() >= 132 && u16_at(bytes, 0x80) >= 4; + bytes[0x7c..0x7e].fill(0); + if high { + bytes[0x82..0x84].fill(0); + } + let crc = crc32c(seed, bytes).to_le_bytes(); + bytes[0x7c..0x7e].copy_from_slice(&crc[..2]); + if high { + bytes[0x82..0x84].copy_from_slice(&crc[2..]); + } + } + } + + /// Clear the inline target before passing the last unlink to the library, + /// whose generic block-map freeing otherwise treats target bytes as blocks. + /// Caller holds the namespace and inode locks and has flushed the inode. + pub async fn clear_inline_target( + &self, + dev: &Ext4Device, + id: NonZeroU32, + ) -> Result { + let (offset, mut bytes) = self.read_inode(dev, id).await?; + if u16_at(&bytes, 0) & 0xf000 != 0xa000 + || u16_at(&bytes, 0x1a) != 1 + || u32_at(&bytes, 0x20) & 0x80000 != 0 + || !(1..60).contains(&u32_at(&bytes, 4)) + || u32_at(&bytes, 0x6c) != 0 + || u32_at(&bytes, 0x1c) != 0 + || u16_at(&bytes, 0x74) != 0 + { + return Err(FsError::InvalidFs.into()); + } + let saved = SavedInode { + offset, + bytes: bytes.clone(), + }; + bytes[0x28..0x64].fill(0); + self.checksum_inode(id, &mut bytes); + if let Err(error) = dev.write_at(offset, &bytes).await { + saved.restore(dev).await?; + return Err(error); + } + Ok(saved) + } + + /// Change only the existing ".." record, preserving a directory's htree. + pub async fn reparent( + &self, + dev: &Ext4Device, + fs: &Ext4, + inode: &mut ExtInode, + old: NonZeroU32, + new: NonZeroU32, + ) -> Result<()> { + let mut block = vec![0; self.block_size as usize]; + if read_at(fs, inode, &mut block, 0).await? != block.len() + || u32_at(&block, 0) != inode.index.get() + || u16_at(&block, 4) != 12 + || block[6] != 1 + || block[8] != b'.' + || u32_at(&block, 12) != old.get() + || u16_at(&block, 16) < 12 + || usize::from(u16_at(&block, 16)) > block.len() - 12 + || block[18] != 2 + || &block[20..22] != b".." + { + return Err(FsError::InvalidFs.into()); + } + let (_, bytes) = self.read_inode(dev, inode.index).await?; + let seed = self.inode_seed(inode.index, &bytes); + let checksum = |block: &[u8]| -> Result> { + let Some(seed) = seed else { return Ok(None) }; + let len = block.len(); + if inode.flags().contains(InodeFlags::DIRECTORY_HTREE) { + let count = usize::from(u16_at(block, 0x22)); + let limit = usize::from(u16_at(block, 0x20)); + if block[0x1d] != 8 || count == 0 || count > limit || 32 + limit * 8 != len - 8 { + return Err(FsError::InvalidFs.into()); + } + Ok(Some(crc32c( + crc32c( + crc32c(seed, &block[..32 + count * 8]), + &block[len - 8..len - 4], + ), + &[0; 4], + ))) + } else { + if block[len - 12..len - 4] != [0, 0, 0, 0, 12, 0, 0, 0xde] { + return Err(FsError::InvalidFs.into()); + } + Ok(Some(crc32c(seed, &block[..len - 12]))) + } + }; + let end = block.len() - 4; + if let Some(expected) = checksum(&block)? + && u32_at(&block, end) != expected + { + return Err(FsError::InvalidFs.into()); + } + block[12..16].copy_from_slice(&new.get().to_le_bytes()); + if let Some(crc) = checksum(&block)? { + block[end..].copy_from_slice(&crc.to_le_bytes()); + } + if write_at(fs, inode, &block, 0).await? != block.len() { + return Err(KernelError::Other("short directory write")); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn crc32c_ext4_convention_and_incremental_seed() { + assert_eq!(crc32c(!0, b"123456789"), !0xe3069283); + assert_eq!( + crc32c(crc32c(!0, b"1234"), b"56789"), + crc32c(!0, b"123456789") + ); + } +} diff --git a/libkernel/src/fs/filesystems/ext4/mod.rs b/libkernel/src/fs/filesystems/ext4/mod.rs index c1d12310..16a06b86 100644 --- a/libkernel/src/fs/filesystems/ext4/mod.rs +++ b/libkernel/src/fs/filesystems/ext4/mod.rs @@ -36,12 +36,37 @@ use core::time::Duration; use ext4plus::prelude::{ AsyncIterator, AsyncSkip, Dir, DirEntryName, Ext4, Ext4Error, Ext4Read, Ext4Write, File, FollowSymlinks, Inode as ExtInode, InodeCreationOptions, InodeFlags, InodeMode, Metadata, - PathBuf as ExtPathBuf, ReadDir, write_at, + PathBuf as ExtPathBuf, ReadDir, read_at, write_at, }; use log::error; +mod cache; +mod disk; +#[cfg(test)] +mod tests; +use cache::InodeCache; +use disk::DiskLayout; + +// BlockBuffer performs sector read/modify/write. Serialize those writes even +// for distinct inodes, which may occupy the same device sector. +struct Ext4Device { + buffer: BlockBuffer, + writes: Mutex<(), CPU>, +} + +impl Ext4Device { + async fn read_at(&self, offset: u64, bytes: &mut [u8]) -> Result<()> { + self.buffer.read_at(offset, bytes).await + } + + async fn write_at(&self, offset: u64, bytes: &[u8]) -> Result<()> { + let _write = self.writes.lock().await; + self.buffer.write_at(offset, bytes).await + } +} + #[async_trait] -impl Ext4Read for BlockBuffer { +impl Ext4Read for Ext4Device { async fn read( &self, start_byte: u64, @@ -52,7 +77,7 @@ impl Ext4Read for BlockBuffer { } #[async_trait] -impl Ext4Write for BlockBuffer { +impl Ext4Write for Ext4Device { async fn write( &self, start_byte: u64, @@ -194,7 +219,7 @@ impl DerefMut for InodeInner { pub struct Ext4Inode { fs_ref: Weak>, id: NonZeroU32, - inner: Mutex, + inner: Arc>, path: ExtPathBuf, } @@ -259,13 +284,34 @@ where } async fn truncate(&self, size: u64) -> Result<()> { - let inner = self.inner.lock().await; + let mut inner = self.inner.lock().await; if inner.file_type() != ext4plus::FileType::Regular { return Err(KernelError::NotSupported); } let fs = self.fs_ref.upgrade().unwrap(); + // The library frees whole blocks but leaves the retained EOF block's + // tail unchanged. Clear it before shrinking, without allocating holes. + if size < inner.size_in_bytes() && !size.is_multiple_of(fs.disk.block_size) { + let end = size + .checked_add(fs.disk.block_size - size % fs.disk.block_size) + .ok_or(KernelError::TooLarge)?; + let mut view = inner.clone(); + view.set_size_in_bytes(end); + let mut tail = vec![0; (end - size) as usize]; + let read = read_at(&fs.inner, &view, &mut tail, size).await?; + if read != tail.len() { + return Err(KernelError::Other("short truncate tail read")); + } + if tail.iter().any(|&byte| byte != 0) { + tail.fill(0); + if write_at(&fs.inner, &mut inner, &tail, size).await? != tail.len() { + return Err(KernelError::Other("short truncate tail write")); + } + } + } let mut file = File::open_inode(&fs.inner, inner.clone())?; file.truncate(size).await?; + *inner = InodeInner::Regular(file); Ok(()) } @@ -281,6 +327,11 @@ where async fn setattr(&self, attr: FileAttr) -> Result<()> { let mut inner = self.inner.lock().await; + // chmod changes permission bits, never the existing inode's file type. + let permission_mask = FilePermissions::all().bits(); + let mode = + (inner.mode().bits() & !permission_mask) | (attr.permissions.bits() & permission_mask); + inner.set_mode(InodeMode::from_bits_retain(mode))?; inner.set_atime(attr.atime); inner.set_ctime(attr.ctime); inner.set_mtime(attr.mtime); @@ -294,6 +345,7 @@ where async fn lookup(&self, name: &str) -> Result> { let fs = self.fs_ref.upgrade().unwrap(); + let _namespace = fs.namespace.lock().await; let inner = self.inner.lock().await; let child_inode = match &*inner { InodeInner::Directory(d) => { @@ -303,10 +355,14 @@ where _ => return Err(KernelError::NotSupported), }; let child_path = self.path.join(name); + let child_id = child_inode.index; + let child_inner = fs + .shared_inode_inner(InodeInner::new(child_inode, &fs.inner).await) + .await?; Ok(Arc::new(Ext4Inode:: { fs_ref: self.fs_ref.clone(), - id: child_inode.index, - inner: Mutex::new(InodeInner::new(child_inode, &fs.inner).await), + id: child_id, + inner: child_inner, path: child_path, })) } @@ -319,6 +375,7 @@ where time: Option, ) -> Result> { let fs = self.fs_ref.upgrade().unwrap(); + let _namespace = fs.namespace.lock().await; let mut inner = self.inner.lock().await; let inner_dir = match &mut *inner { InodeInner::Directory(d) => d, @@ -338,8 +395,7 @@ where .await?; InodeInner::Regular(File::open_inode(&fs.inner, inode)?) } else if matches!(file_type, FileType::Directory) { - let old_links_count = inner_dir.inode().links_count(); - inner_dir.inode_mut().set_links_count(old_links_count + 1); + // Dir::link accounts for the child's ".." link to this parent. let inode = fs .inner .create_inode(InodeCreationOptions { @@ -363,26 +419,30 @@ where ) .await?; let new_path = self.path.join(name); + let new_inode_id = new_inode.index; + let new_inode_inner = fs.shared_inode_inner(new_inode).await?; Ok(Arc::new(Ext4Inode:: { fs_ref: self.fs_ref.clone(), - id: new_inode.index, - inner: Mutex::new(new_inode), + id: new_inode_id, + inner: new_inode_inner, path: new_path, })) } async fn link(&self, name: &str, inode: Arc) -> Result<()> { + let fs = self.fs_ref.upgrade().unwrap(); + let _namespace = fs.namespace.lock().await; + if inode.id().fs_id() != fs.id() { + return Err(KernelError::Fs(FsError::CrossDevice)); + } + if inode.id() == self.id() { + return Err(KernelError::Fs(FsError::IsADirectory)); + } let mut inner = self.inner.lock().await; let inner_dir = match &mut *inner { InodeInner::Directory(d) => d, _ => return Err(KernelError::NotSupported), }; - let fs = self.fs_ref.upgrade().unwrap(); - // TODO: This forces the other inode out of sync - // Check fs ids match - if inode.id().fs_id() != fs.id() { - return Err(KernelError::Fs(FsError::CrossDevice)); - } let mut other_inode = inode .as_any() .downcast_ref::>() @@ -390,7 +450,9 @@ where .inner .lock() .await; - let file_type = other_inode.file_type(); + if other_inode.file_type() == ext4plus::FileType::Directory { + return Err(KernelError::Fs(FsError::IsADirectory)); + } inner_dir .link( DirEntryName::try_from(name.as_bytes()).unwrap(), @@ -401,16 +463,14 @@ where } async fn unlink(&self, name: &str) -> Result<()> { + let fs = self.fs_ref.upgrade().unwrap(); + let _namespace = fs.namespace.lock().await; let mut inner = self.inner.lock().await; let inner_dir = match &mut *inner { InodeInner::Directory(d) => d, _ => return Err(KernelError::NotSupported), }; - let fs = self.fs_ref.upgrade().unwrap(); - let entry = DirEntryName::try_from(name.as_bytes()).unwrap(); - let child_inode = inner_dir.get_entry(entry).await?; - inner_dir.unlink(entry, child_inode).await?; - Ok(()) + fs.unlink_locked(inner_dir, name).await } async fn readdir(&self, start_offset: u64) -> Result> { @@ -446,125 +506,76 @@ where new_name: &str, no_replace: bool, ) -> Result<()> { - if old_name == new_name && old_parent.id().inode_id() == self.id().inode_id() { - return Ok(()); - } - if old_parent.id().fs_id() != self.id().fs_id() { return Err(KernelError::Fs(FsError::CrossDevice)); } let fs = self.fs_ref.upgrade().unwrap(); - - let old_parent_inode = old_parent + let _namespace = fs.namespace.lock().await; + let old_parent = old_parent .as_any() .downcast_ref::>() - .ok_or(FsError::CrossDevice)? - .inner - .lock() - .await - .clone(); - let mut old_parent_dir = Dir::open_inode(&fs.inner, old_parent_inode)?; - + .ok_or(FsError::CrossDevice)?; let mut inner = self.inner.lock().await; let inner_dir = match &mut *inner { InodeInner::Directory(d) => d, _ => return Err(KernelError::NotSupported), }; - - // inode being moved (source) - let mut child_inode = old_parent_dir - .get_entry( - DirEntryName::try_from(old_name) - .map_err(|_| KernelError::Fs(FsError::InvalidInput))?, - ) - .await?; - - // Check if destination exists and handle overwrite constraints. - let dst_lookup = inner_dir - .get_entry( - DirEntryName::try_from(new_name) - .map_err(|_| KernelError::Fs(FsError::InvalidInput))?, - ) - .await; - - if no_replace && dst_lookup.is_ok() { - return Err(KernelError::Fs(FsError::AlreadyExists)); - } - - if let Ok(target_inode) = dst_lookup { - let target_kind = target_inode.file_type(); - let source_kind = child_inode.file_type(); - - if target_kind == ext4plus::FileType::Directory { - let target_is_empty = fs - .inner - .read_dir(&self.path.join(new_name)) - .await? - .all(|e| { - let Ok(entry) = e else { - // If we fail to read the directory, be conservative and treat it as non-empty. - return false; - }; - let name = entry.file_name().as_str().unwrap(); - name == "." || name == ".." - }) - .await; - - if !target_is_empty { - return Err(KernelError::Fs(FsError::DirectoryNotEmpty)); - } - if source_kind != ext4plus::FileType::Directory { - return Err(KernelError::Fs(FsError::IsADirectory)); - } - } else if source_kind == ext4plus::FileType::Directory { - // Can't replace non-directory with a directory. + if old_parent.id == self.id { + // Both wrappers share one mutex and one directory object. + fs.rename_locked(None, inner_dir, old_name, new_name, no_replace) + .await + } else { + // namespace serializes multi-inode operations, so opposing moves + // cannot acquire these directory locks in opposite orders. + let mut old_inner = old_parent.inner.lock().await; + let InodeInner::Directory(old_dir) = &mut *old_inner else { return Err(KernelError::Fs(FsError::NotADirectory)); - } - - // Overwrite: remove destination entry first. - inner_dir - .unlink( - DirEntryName::try_from(new_name) - .map_err(|_| KernelError::Fs(FsError::InvalidInput))?, - target_inode, - ) - .await?; + }; + fs.rename_locked(Some(old_dir), inner_dir, old_name, new_name, no_replace) + .await } - - // Link into destination parent, then unlink from source parent. - inner_dir - .link( - DirEntryName::try_from(new_name.as_bytes()).unwrap(), - &mut child_inode, - ) - .await?; - old_parent_dir - .unlink( - DirEntryName::try_from(old_name.as_bytes()).unwrap(), - child_inode, - ) - .await?; - *old_parent - .as_any() - .downcast_ref::>() - .ok_or(FsError::CrossDevice)? - .inner - .lock() - .await = InodeInner::Directory(old_parent_dir); - Ok(()) } async fn symlink(&self, name: &str, target: &Path) -> Result<()> { + let fs = self.fs_ref.upgrade().unwrap(); + let _namespace = fs.namespace.lock().await; let mut inner = self.inner.lock().await; let inner_dir = match &mut *inner { InodeInner::Directory(d) => d, _ => return Err(KernelError::NotSupported), }; - let fs = self.fs_ref.upgrade().unwrap(); + let entry = Ext4Filesystem::::entry_name(name)?; + match inner_dir.get_entry(entry).await { + Ok(_) => return Err(FsError::AlreadyExists.into()), + Err(Ext4Error::NotFound) => (), + Err(error) => return Err(error.into()), + } + if target.as_str().len() == 60 { + // ext4 fast symlinks must leave a byte for the terminator. The + // library's creation threshold includes 60, unlike its reader. + let mut inode = fs + .inner + .create_inode(InodeCreationOptions { + file_type: ext4plus::FileType::Regular, + mode: InodeMode::S_IFREG | InodeMode::from_bits_truncate(0o777), + uid: 0, + gid: 0, + time: Duration::ZERO, + flags: InodeFlags::empty(), + }) + .await?; + if write_at(&fs.inner, &mut inode, target.as_str().as_bytes(), 0).await? != 60 { + return Err(KernelError::Other("short symlink write")); + } + inode.set_mode(InodeMode::S_IFLNK | InodeMode::from_bits_truncate(0o777))?; + inode.write(&fs.inner).await?; + inner_dir.link(entry, &mut inode).await?; + return Ok(()); + } fs.inner .symlink( inner_dir, - DirEntryName::try_from(name.as_bytes()).unwrap(), + entry, ExtPathBuf::new(target.as_str().as_bytes()), 0, 0, @@ -631,13 +642,17 @@ where /// An EXT4 filesystem instance. /// -/// For now this struct only stores the underlying block buffer and an ID -/// assigned by the VFS when the filesystem is mounted. +/// Lock order: namespace, directory inode(s), child inode, inode cache. +/// Cache guards are never held while awaiting an inode lock. Single-inode +/// data/attribute operations do not take namespace, keeping file I/O parallel. pub struct Ext4Filesystem { inner: Ext4, id: u64, this: Weak>, - dev: Arc, + namespace: Mutex<(), CPU>, + inode_cache: Mutex>, CPU>, + dev: Arc>, + disk: DiskLayout, _phantom_data: PhantomData, } @@ -645,17 +660,252 @@ impl Ext4Filesystem where CPU: CpuOps + Send + Sync, { + // Caller holds namespace. On a cache miss, refresh the inode after checking + // the cache: a writer could have closed the last handle since the parent's + // get_entry read its snapshot. No new writer can open it during this reload. + async fn shared_inode_inner(&self, inner: InodeInner) -> Result>> { + let inode_id = inner.index; + if let Some(existing) = self.inode_cache.lock().await.get(inode_id) { + return Ok(existing); + } + let fresh = ExtInode::read(&self.inner, inode_id).await?; + let inner = InodeInner::new(fresh, &self.inner).await; + Ok(self + .inode_cache + .lock() + .await + .get_or_insert_with(inode_id, || Mutex::new(inner))) + } + + // Caller holds namespace and parent. Keep the child locked from selecting + // its authoritative inode through disk mutation and cache publication. + async fn unlink_locked(&self, parent: &mut Dir, name: &str) -> Result<()> { + let entry = Self::entry_name(name)?; + let snapshot = parent.get_entry(entry).await?; + if snapshot.index == parent.inode().index { + return Err(FsError::InvalidInput.into()); + } + let id = snapshot.index; + let child = self + .shared_inode_inner(InodeInner::new(snapshot, &self.inner).await) + .await?; + let mut child = child.lock().await; + let is_dir = child.file_type() == ext4plus::FileType::Directory; + if is_dir { + let mut entries = Dir::open_inode(&self.inner, child.clone())?.read_dir()?; + while let Some(entry) = entries.next().await { + let entry = entry?; + if !matches!(entry.file_name().as_str().unwrap(), "." | "..") { + return Err(FsError::DirectoryNotEmpty.into()); + } + } + } + let mut to_unlink = child.clone(); + let mut saved = None; + if is_dir { + // rmdir removes both the parent name and the child's "." link. + to_unlink.set_links_count(1); + } else if child.file_type() == ext4plus::FileType::Symlink + && child.links_count() == 1 + && child.fs_blocks(&self.inner)? == 0 + { + child.write(&self.inner).await?; + let original = self.disk.clear_inline_target(&self.dev, id).await?; + to_unlink = match ExtInode::read(&self.inner, id).await { + Ok(inode) => inode, + Err(error) => { + original.restore(&self.dev).await?; + return Err(error.into()); + } + }; + saved = Some(original); + } + let remaining = match parent.unlink(entry, to_unlink).await { + Ok(remaining) => remaining, + Err(error) => { + // If removal failed before deleting the name, keep its target + // intact. Never restore an inode after the name was removed: + // allocation/freeing may already have progressed. + if let Some(original) = saved + && parent + .get_entry(entry) + .await + .is_ok_and(|inode| inode.index == id) + { + original.restore(&self.dev).await?; + } + return Err(error.into()); + } + }; + if is_dir { + let links = parent.inode().links_count().saturating_sub(1); + parent.inode_mut().set_links_count(links); + parent.inode_mut().write(&self.inner).await?; + } + if let Some(inode) = remaining { + *child = InodeInner::new(inode, &self.inner).await; + } else { + child.set_links_count(0); + // Evict before releasing namespace: a reused inode number must not + // be associated with an older, still-open wrapper. + self.inode_cache.lock().await.remove(id); + } + Ok(()) + } + + fn entry_name(name: &str) -> Result> { + if name == "." || name == ".." { + return Err(FsError::InvalidInput.into()); + } + DirEntryName::try_from(name).map_err(|_| FsError::InvalidInput.into()) + } + + // None means source and destination are the very same directory. + async fn rename_locked( + &self, + mut old_dir: Option<&mut Dir>, + new_dir: &mut Dir, + old_name: &str, + new_name: &str, + no_replace: bool, + ) -> Result<()> { + let old_entry = Self::entry_name(old_name)?; + let new_entry = Self::entry_name(new_name)?; + let source = old_dir + .as_deref() + .unwrap_or(new_dir) + .get_entry(old_entry) + .await?; + let target = match new_dir.get_entry(new_entry).await { + Ok(inode) => Some(inode), + Err(Ext4Error::NotFound) => None, + Err(e) => return Err(e.into()), + }; + if no_replace && target.is_some() { + return Err(FsError::AlreadyExists.into()); + } + if target + .as_ref() + .is_some_and(|inode| inode.index == source.index) + { + return Ok(()); + } + + // Reject moving a directory into itself or a descendant. Walk inode + // numbers rather than wrapper paths, which can be stale after rename. + if source.file_type() == ext4plus::FileType::Directory { + let mut ancestor = new_dir.inode().clone(); + loop { + if ancestor.index == source.index { + return Err(FsError::InvalidInput.into()); + } + let index = ancestor.index; + ancestor = Dir::open_inode(&self.inner, ancestor)? + .get_entry(DirEntryName::try_from("..").unwrap()) + .await?; + if ancestor.index == index { + break; + } + } + } + if let Some(target) = target { + let source_is_dir = source.file_type() == ext4plus::FileType::Directory; + let target_is_dir = target.file_type() == ext4plus::FileType::Directory; + if source_is_dir != target_is_dir { + return Err(if target_is_dir { + FsError::IsADirectory + } else { + FsError::NotADirectory + } + .into()); + } + if target_is_dir { + let mut entries = Dir::open_inode(&self.inner, target)?.read_dir()?; + while let Some(entry) = entries.next().await { + let entry = entry?; + if !matches!(entry.file_name().as_str().unwrap(), "." | "..") { + return Err(FsError::DirectoryNotEmpty.into()); + } + } + } + self.unlink_locked(new_dir, new_name).await?; + } + let child = self + .shared_inode_inner(InodeInner::new(source, &self.inner).await) + .await?; + let mut child = child.lock().await; + let is_dir = child.file_type() == ext4plus::FileType::Directory; + let child_links = child.links_count(); + let parent_links = new_dir.inode().links_count(); + // Link first so unlink cannot free the source inode. Both parents and + // the shared child stay locked until the final metadata is published. + if let Err(error) = new_dir.link(new_entry, &mut child).await { + // Dir::link updates link counts before inserting the directory + // entry. Undo those counts if insertion failed. + child.set_links_count(child_links); + child.write(&self.inner).await?; + if is_dir { + new_dir.inode_mut().set_links_count(parent_links); + new_dir.inode_mut().write(&self.inner).await?; + } + return Err(error.into()); + } + let source_dir = old_dir.as_deref_mut().unwrap_or(new_dir); + let remaining = match source_dir.unlink(old_entry, child.clone()).await { + Ok(remaining) => remaining.ok_or(FsError::InvalidFs)?, + Err(error) => { + let inode = new_dir + .unlink(new_entry, child.clone()) + .await? + .ok_or(FsError::InvalidFs)?; + *child = InodeInner::new(inode, &self.inner).await; + if is_dir { + new_dir.inode_mut().set_links_count(parent_links); + new_dir.inode_mut().write(&self.inner).await?; + } + return Err(error.into()); + } + }; + *child = InodeInner::new(remaining, &self.inner).await; + if is_dir { + let old_parent_id = source_dir.inode().index; + let old_links = source_dir.inode().links_count().saturating_sub(1); + source_dir.inode_mut().set_links_count(old_links); + source_dir.inode_mut().write(&self.inner).await?; + // The same-parent case just undoes Dir::link's parent increment. + if old_dir.is_some() { + self.disk + .reparent( + &self.dev, + &self.inner, + &mut child, + old_parent_id, + new_dir.inode().index, + ) + .await?; + } + } + Ok(()) + } + /// Construct a new EXT4 filesystem instance. pub async fn new(dev: BlockBuffer, id: u64) -> Result> { - let dev_arc = Arc::new(dev); + let dev_arc = Arc::new(Ext4Device { + buffer: dev, + writes: Mutex::new(()), + }); let inner = Ext4::load_with_writer(Box::new(dev_arc.clone()), Some(Box::new(dev_arc.clone()))) .await?; + let disk = DiskLayout::load(&dev_arc).await?; Ok(Arc::new_cyclic(|weak| Self { inner, id, this: weak.clone(), + namespace: Mutex::new(()), + inode_cache: Mutex::new(InodeCache::new()), dev: dev_arc, + disk, _phantom_data: PhantomData, })) } @@ -677,11 +927,16 @@ where /// Returns the root inode of the mounted EXT4 filesystem. async fn root_inode(&self) -> Result> { + let _namespace = self.namespace.lock().await; let root = self.inner.read_root_inode().await?; + let root_id = root.index; + let root_inner = self + .shared_inode_inner(InodeInner::new(root, &self.inner).await) + .await?; Ok(Arc::new(Ext4Inode:: { fs_ref: self.this.clone(), - id: root.index, - inner: Mutex::new(InodeInner::new(root, &self.inner).await), + id: root_id, + inner: root_inner, path: ExtPathBuf::new("/"), })) } @@ -689,7 +944,7 @@ where /// Flushes any dirty data to the underlying block device. The current /// stub implementation simply forwards the request to `BlockBuffer::sync`. async fn sync(&self) -> Result<()> { - self.dev.sync().await?; + self.dev.buffer.sync().await?; Ok(()) } } diff --git a/libkernel/src/fs/filesystems/ext4/tests.rs b/libkernel/src/fs/filesystems/ext4/tests.rs new file mode 100644 index 00000000..fff9d8c8 --- /dev/null +++ b/libkernel/src/fs/filesystems/ext4/tests.rs @@ -0,0 +1,256 @@ +//! Supplemental disk-format tests of the MOSS driver (not host syscall tests). +use super::*; +use crate::{fs::BlockDevice, test::MockCpuOps}; +use std::{fs, os::unix::fs::FileExt, process::Command}; + +struct ImageDevice(fs::File); + +#[async_trait] +impl BlockDevice for ImageDevice { + async fn read(&self, block: u64, bytes: &mut [u8]) -> Result<()> { + self.0 + .read_exact_at(bytes, block * 512) + .map_err(|_| KernelError::Other("image read")) + } + async fn write(&self, block: u64, bytes: &[u8]) -> Result<()> { + self.0 + .write_all_at(bytes, block * 512) + .map_err(|_| KernelError::Other("image write")) + } + fn block_size(&self) -> usize { + 512 + } + async fn sync(&self) -> Result<()> { + self.0 + .sync_all() + .map_err(|_| KernelError::Other("image sync")) + } +} + +async fn mount(path: &std::path::Path) -> Arc> { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + Ext4Filesystem::new(BlockBuffer::new(Box::new(ImageDevice(file))), 1) + .await + .unwrap() +} + +fn run(command: &mut Command, allowed: &[i32]) { + let output = command + .output() + .expect("install e2fsprogs to run this ignored test"); + assert!( + allowed.contains(&output.status.code().unwrap_or(-1)), + "{command:?}: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test] +#[ignore = "requires mkfs.ext4 and e2fsck; creates isolated images under build/ext4-tests"] +async fn adapter_disk_formats() { + let stamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let base = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../build/ext4-tests") + .join(format!("{}-{stamp}", std::process::id())); + fs::create_dir_all(&base).unwrap(); + // Populate an htree through the standard offline checker, then exercise + // the MOSS driver's directory move on both indexed and linear directories. + let seed = base.join("rootfs"); + fs::create_dir_all(seed.join("indexed")).unwrap(); + for index in 0..200 { + fs::write(seed.join(format!("indexed/entry-{index:04}-padding")), []).unwrap(); + } + + for (kind, block_size, inode_size, features) in [ + ("ext2", "1024", "256", ""), + ("ext4", "4096", "256", ""), + ("ext4", "1024", "256", "metadata_csum_seed,^64bit"), + ] { + let image = base.join(format!("{kind}-{block_size}.img")); + let file = fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&image) + .unwrap(); + file.set_len(32 * 1024 * 1024).unwrap(); + drop(file); + let mut mkfs = Command::new("mkfs.ext4"); + mkfs.args(["-q", "-F", "-t", kind, "-b", block_size, "-I", inode_size]); + if !features.is_empty() { + mkfs.args(["-O", features]); + } + run(mkfs.arg("-d").arg(&seed).arg(&image), &[0]); + run(Command::new("e2fsck").args(["-fyD"]).arg(&image), &[0, 1]); + + let fs = mount(&image).await; + let root = fs.root_inode().await.unwrap(); + let mode = FilePermissions::from_bits_truncate(0o755); + // Permission updates must survive closing all wrappers and remounting, + // including special mode bits, without changing the inode's file type. + for (name, kind, permissions) in [ + ("mode-file", FileType::File, 0o6751), + ("mode-dir", FileType::Directory, 0o1750), + ] { + let inode = root.create(name, kind, mode, None).await.unwrap(); + let alias = root.lookup(name).await.unwrap(); + let mut attr = inode.getattr().await.unwrap(); + // Defensively ignore type bits even from a direct VFS caller. + attr.permissions = FilePermissions::from_bits_retain(permissions | 0xf000); + inode.setattr(attr).await.unwrap(); + let observed = alias.getattr().await.unwrap(); + assert_eq!(observed.permissions.bits(), permissions); + assert_eq!(observed.file_type, kind); + } + let left = root + .create("left", FileType::Directory, mode, None) + .await + .unwrap(); + let right = root + .create("right", FileType::Directory, mode, None) + .await + .unwrap(); + let linear = left + .create("linear", FileType::Directory, mode, None) + .await + .unwrap(); + let left_links = left.getattr().await.unwrap().nlinks; + right + .rename_from(left.clone(), "linear", "moved", false) + .await + .unwrap(); + assert_eq!(linear.lookup("..").await.unwrap().id(), right.id()); + assert_eq!(left.getattr().await.unwrap().nlinks, left_links - 1); + let right_links = right.getattr().await.unwrap().nlinks; + right + .rename_from(right.clone(), "moved", "again", false) + .await + .unwrap(); + assert_eq!(right.getattr().await.unwrap().nlinks, right_links); + right.unlink("again").await.unwrap(); + assert_eq!(right.getattr().await.unwrap().nlinks, right_links - 1); + + let indexed = root.lookup("indexed").await.unwrap(); + let ext_inode = indexed + .as_any() + .downcast_ref::>() + .unwrap(); + assert!( + ext_inode + .inner + .lock() + .await + .flags() + .contains(InodeFlags::DIRECTORY_HTREE) + ); + right + .rename_from(root.clone(), "indexed", "indexed", false) + .await + .unwrap(); + assert_eq!(indexed.lookup("..").await.unwrap().id(), right.id()); + assert!(right.unlink("indexed").await.is_err()); + for index in 0..200 { + indexed + .unlink(&format!("entry-{index:04}-padding")) + .await + .unwrap(); + } + right.unlink("indexed").await.unwrap(); + + for length in [1, 13, 59, 60, 61, 120] { + let target = "t".repeat(length); + root.symlink("sym", Path::new(&target)).await.unwrap(); + let link = root.lookup("sym").await.unwrap(); + assert_eq!(link.readlink().await.unwrap().as_str(), target); + root.link("sym-hard", link.clone()).await.unwrap(); + root.unlink("sym").await.unwrap(); + assert_eq!(link.getattr().await.unwrap().nlinks, 1); + assert_eq!(link.readlink().await.unwrap().as_str(), target); + root.unlink("sym-hard").await.unwrap(); + } + + let file = root + .create("file", FileType::File, mode, None) + .await + .unwrap(); + root.link("hard", file.clone()).await.unwrap(); + let hard = root.lookup("hard").await.unwrap(); + let data = vec![0x5a; 16385]; + for size in [5, 1023, 1024, 4095, 4096, 4097, 0] { + file.truncate(0).await.unwrap(); + let mut written = 0; + while written < data.len() { + let count = file + .write_at(written as u64, &data[written..]) + .await + .unwrap(); + assert!(count > 0); + written += count; + } + hard.truncate(size).await.unwrap(); + assert_eq!(file.getattr().await.unwrap().size, size); + file.truncate(data.len() as u64).await.unwrap(); + let mut bytes = vec![0xff; data.len() - size as usize]; + assert_eq!(hard.read_at(size, &mut bytes).await.unwrap(), bytes.len()); + assert!(bytes.iter().all(|&byte| byte == 0)); + } + // A shrink inside a hole must not allocate a new block to clear it. + file.truncate(0).await.unwrap(); + file.truncate(16385).await.unwrap(); + let before = file + .as_any() + .downcast_ref::>() + .unwrap() + .inner + .lock() + .await + .blocks(); + file.truncate(4097).await.unwrap(); + assert_eq!( + file.as_any() + .downcast_ref::>() + .unwrap() + .inner + .lock() + .await + .blocks(), + before + ); + root.unlink("file").await.unwrap(); + root.unlink("hard").await.unwrap(); + root.unlink("left").await.unwrap(); + root.unlink("right").await.unwrap(); + fs.sync().await.unwrap(); + drop(fs); + run(Command::new("e2fsck").args(["-fn"]).arg(&image), &[0]); + let reopened = mount(&image).await; + let root = reopened.root_inode().await.unwrap(); + assert!(root.lookup("right").await.is_err()); + for (name, kind, permissions) in [ + ("mode-file", FileType::File, 0o6751), + ("mode-dir", FileType::Directory, 0o1750), + ] { + let inode = root.lookup(name).await.unwrap(); + let mut attr = inode.getattr().await.unwrap(); + assert_eq!(attr.permissions.bits(), permissions); + assert_eq!(attr.file_type, kind); + // Also check that clearing permission bits is not an OR-only update. + attr.permissions = FilePermissions::from_bits_retain(0o600); + inode.setattr(attr).await.unwrap(); + assert_eq!(inode.getattr().await.unwrap().permissions.bits(), 0o600); + root.unlink(name).await.unwrap(); + } + reopened.sync().await.unwrap(); + run(Command::new("e2fsck").args(["-fn"]).arg(&image), &[0]); + std::println!( + "verified {kind}, block={block_size}, inode={inode_size}, features={features}" + ); + } +} diff --git a/libkernel/src/fs/mod.rs b/libkernel/src/fs/mod.rs index c66cdf0c..be4e80e7 100644 --- a/libkernel/src/fs/mod.rs +++ b/libkernel/src/fs/mod.rs @@ -18,6 +18,7 @@ pub mod blk; pub mod filesystems; pub mod path; pub mod pathbuf; +pub mod stats; use core::any::Any; @@ -48,6 +49,11 @@ mod _open_flags { const O_DIRECTORY = 0o200000; const O_APPEND = 0o2000; const O_NONBLOCK = 0o4000; + // AArch64 overrides the asm-generic value (compat with AArch32). + #[cfg(target_arch = "aarch64")] + const O_LARGEFILE = 0o400000; + #[cfg(not(target_arch = "aarch64"))] + const O_LARGEFILE = 0o100000; const O_CLOEXEC = 0o2000000; } } @@ -63,6 +69,8 @@ pub const PROCFS_ID: u64 = 2; pub const SYSFS_ID: u64 = 3; /// Filesystem instance ID for the cgroup filesystem. pub const CGROUPFS_ID: u64 = 4; +/// Filesystem instance ID for the internal pidfd filesystem. +pub const PIDFS_ID: u64 = 5; /// Starting ID for user-mounted filesystem instances. pub const FS_ID_START: u64 = 10; @@ -79,6 +87,15 @@ pub trait Filesystem: Send + Sync { /// Get magic fn magic(&self) -> u64; + /// Queries filesystem-wide statistics, independently of any open file. + async fn statfs(&self) -> Result { + Ok(stats::FilesystemStats { + magic: self.magic(), + id: self.id(), + ..Default::default() + }) + } + /// Flushes all pending data to the underlying storage device(s). /// /// The default implementation is a no-op so that read-only filesystems do @@ -335,6 +352,12 @@ pub trait Inode: Send + Sync + Any { Err(KernelError::NotSupported) } + /// Resolves a magic link directly to an inode, without interpreting its + /// display name as a pathname. Ordinary symlinks return `None`. + async fn follow_link(&self) -> Result>> { + Ok(None) + } + /// Flushes all modified data, including metadata, to the disk device containing the inode. /// /// The default implementation is a no-op so that read-only filesystems do @@ -369,6 +392,11 @@ pub trait SimpleFile { async fn readlink(&self) -> Result { Err(KernelError::NotSupported) } + + /// Resolves a magic link; see [`Inode::follow_link`]. + async fn follow_link(&self) -> Result>> { + Ok(None) + } } #[allow(missing_docs)] @@ -408,6 +436,10 @@ where self.readlink().await } + async fn follow_link(&self) -> Result>> { + self.follow_link().await + } + fn as_any(&self) -> &dyn Any { self } diff --git a/libkernel/src/fs/stats.rs b/libkernel/src/fs/stats.rs new file mode 100644 index 00000000..6f9da484 --- /dev/null +++ b/libkernel/src/fs/stats.rs @@ -0,0 +1,31 @@ +//! Filesystem statistics, separate from the architecture-specific userspace ABI. + +/// Linux marks the statfs mount-flags word as valid with this bit. +pub const ST_VALID: u64 = 0x20; + +/// Statistics belonging to a filesystem instance, not to an individual inode. +#[derive(Debug, Clone, Copy, Default)] +pub struct FilesystemStats { + /// Filesystem type magic. + pub magic: u64, + /// Optimal transfer block size in bytes. + pub block_size: u64, + /// Total data blocks. + pub blocks: u64, + /// Free data blocks. + pub blocks_free: u64, + /// Data blocks available to unprivileged users. + pub blocks_available: u64, + /// Total inodes. + pub files: u64, + /// Free inodes. + pub files_free: u64, + /// Unique filesystem instance ID. + pub id: u64, + /// Maximum filename length in bytes. + pub name_length: u64, + /// Allocation unit size in bytes; zero means use `block_size`. + pub fragment_size: u64, + /// Linux statfs flags for the filesystem's mount. + pub flags: u64, +} diff --git a/src/arch/arm64/boot/mod.rs b/src/arch/arm64/boot/mod.rs index 4a6e1d57..9eaab92d 100644 --- a/src/arch/arm64/boot/mod.rs +++ b/src/arch/arm64/boot/mod.rs @@ -46,7 +46,7 @@ pub(super) mod memory; mod paging_bootstrap; pub(super) mod secondary; -global_asm!(include_str!("start.s")); +global_asm!(include_str!("start.s"), frame_size = const size_of::()); /// Stage 1 Initialize of the system architecture. /// diff --git a/src/arch/arm64/boot/secondary.rs b/src/arch/arm64/boot/secondary.rs index 3ae3a4ba..2940f1e8 100644 --- a/src/arch/arm64/boot/secondary.rs +++ b/src/arch/arm64/boot/secondary.rs @@ -6,6 +6,7 @@ use crate::{ arch_init_secondary, memory::{KERNEL_STACK_PG_ORDER, allocate_kstack_region}, }, + exceptions::ExceptionState, memory::flush_to_ram, psci::{PSCIEntry, PSCIMethod, boot_secondary_psci}, }, @@ -65,12 +66,13 @@ extern "C" fn do_secondary_start(boot_info: *const SecondaryBootInfo) { "ldr x1, [x19, #0x20]", // Load `start_fn` "ldr x2, [x19, #0x8]", // Load kstack addr "mov sp, x2", // Set final stack addr - "sub sp, sp, #(0x10 * 18)", // Allocate a context switch frame + "sub sp, sp, #{frame_size}", // Allocate a context switch frame "mov x0, sp", // ctx switch frame ptr "ldr x2, [x19, #0x28]", // return to exception_ret. "mov lr, x2", "br x1", // branch to Rust entry point - "b ." // Just in case! + "b .", // Just in case! + frame_size = const size_of::(), ) } diff --git a/src/arch/arm64/boot/start.s b/src/arch/arm64/boot/start.s index 9e1fba12..568bc727 100644 --- a/src/arch/arm64/boot/start.s +++ b/src/arch/arm64/boot/start.s @@ -76,7 +76,7 @@ _start: 1: mov sp, x0 // Allocate a context switch frame - sub sp, sp, #(16 * 18) + sub sp, sp, #{frame_size} // Jump to the main kernel bring-up path mov x0, sp // context switch frame ptr. diff --git a/src/arch/arm64/exceptions/exceptions.s b/src/arch/arm64/exceptions/exceptions.s index e43ffe35..6c02a9c6 100644 --- a/src/arch/arm64/exceptions/exceptions.s +++ b/src/arch/arm64/exceptions/exceptions.s @@ -1,3 +1,7 @@ +// Only this context-save assembly uses FP registers. Rust kernel code keeps +// the project's aarch64-unknown-none-softfloat target. +.arch_extension fp + // Load an absolute 64-bit address in a relocatable way .macro adr_a register, symbol movz \register, #:abs_g3:\symbol @@ -33,6 +37,31 @@ stp x2, x3, [sp, #(16 * 16)] str x4, [sp, #(16 * 17)] + // Preserve the complete userspace FP/SIMD context, not just the ABI's + // callee-saved D registers. Interrupts may occur at any instruction. + stp q0, q1, [sp, #({fpsimd} + 32 * 0)] + stp q2, q3, [sp, #({fpsimd} + 32 * 1)] + stp q4, q5, [sp, #({fpsimd} + 32 * 2)] + stp q6, q7, [sp, #({fpsimd} + 32 * 3)] + stp q8, q9, [sp, #({fpsimd} + 32 * 4)] + stp q10, q11, [sp, #({fpsimd} + 32 * 5)] + stp q12, q13, [sp, #({fpsimd} + 32 * 6)] + stp q14, q15, [sp, #({fpsimd} + 32 * 7)] + stp q16, q17, [sp, #({fpsimd} + 32 * 8)] + stp q18, q19, [sp, #({fpsimd} + 32 * 9)] + stp q20, q21, [sp, #({fpsimd} + 32 * 10)] + stp q22, q23, [sp, #({fpsimd} + 32 * 11)] + stp q24, q25, [sp, #({fpsimd} + 32 * 12)] + stp q26, q27, [sp, #({fpsimd} + 32 * 13)] + stp q28, q29, [sp, #({fpsimd} + 32 * 14)] + stp q30, q31, [sp, #({fpsimd} + 32 * 15)] + mrs x1, FPCR + mrs x2, FPSR + str w1, [sp, #{fpcr}] + str w2, [sp, #{fpsr}] + str xzr, [sp, #{reserved}] + str xzr, [sp, #{fp_reserved}] + mov x0, sp // Call handler @@ -45,7 +74,7 @@ .macro vector_handler handler __vector_\handler: - sub sp, sp, #(16 * 18) + sub sp, sp, #{frame_size} b __impl_\handler @@ -57,7 +86,7 @@ __impl_\handler: .macro kvector_handler handler __vector_\handler: - sub sp, sp, #(16 * 18) + sub sp, sp, #{frame_size} // Detect stack overflow without clobbering GP registers. msr SP_EL0, x0 @@ -73,7 +102,7 @@ __vector_\handler: ldr x0, =EMERG_STACK_END ldr x0, [x0] mov sp, x0 - sub sp, sp, #(16 * 18) + sub sp, sp, #{frame_size} mrs x0, TPIDR_EL1 b __impl_\handler @@ -119,7 +148,27 @@ exception_vectors: .section .vectors.impl, "ax" .global restore_ctx_and_eret restore_ctx_and_eret: - add sp, sp, #(0x10 * 18) + add sp, sp, #{frame_size} + ldp q0, q1, [x0, #({fpsimd} + 32 * 0)] + ldp q2, q3, [x0, #({fpsimd} + 32 * 1)] + ldp q4, q5, [x0, #({fpsimd} + 32 * 2)] + ldp q6, q7, [x0, #({fpsimd} + 32 * 3)] + ldp q8, q9, [x0, #({fpsimd} + 32 * 4)] + ldp q10, q11, [x0, #({fpsimd} + 32 * 5)] + ldp q12, q13, [x0, #({fpsimd} + 32 * 6)] + ldp q14, q15, [x0, #({fpsimd} + 32 * 7)] + ldp q16, q17, [x0, #({fpsimd} + 32 * 8)] + ldp q18, q19, [x0, #({fpsimd} + 32 * 9)] + ldp q20, q21, [x0, #({fpsimd} + 32 * 10)] + ldp q22, q23, [x0, #({fpsimd} + 32 * 11)] + ldp q24, q25, [x0, #({fpsimd} + 32 * 12)] + ldp q26, q27, [x0, #({fpsimd} + 32 * 13)] + ldp q28, q29, [x0, #({fpsimd} + 32 * 14)] + ldp q30, q31, [x0, #({fpsimd} + 32 * 15)] + ldr w1, [x0, #{fpcr}] + ldr w2, [x0, #{fpsr}] + msr FPCR, x1 + msr FPSR, x2 ldp lr, x1, [x0, #(16 * 15)] ldp x2, x3, [x0, #(16 * 16)] ldr x4, [x0, #(16 * 17)] @@ -153,3 +202,4 @@ exception_return: adr_a x1 restore_ctx_and_eret br x1 +.arch_extension nofp diff --git a/src/arch/arm64/exceptions/mod.rs b/src/arch/arm64/exceptions/mod.rs index c90c60fd..5d618319 100644 --- a/src/arch/arm64/exceptions/mod.rs +++ b/src/arch/arm64/exceptions/mod.rs @@ -10,8 +10,11 @@ use crate::{ sched::{syscall_ctx::ProcessCtx, uspc_ret::dispatch_userspace_task}, spawn_kernel_work, }; -use aarch64_cpu::registers::{CPACR_EL1, ReadWriteable, VBAR_EL1}; -use core::{arch::global_asm, fmt::Display}; +use aarch64_cpu::{ + asm::barrier::{SY, isb}, + registers::{CPACR_EL1, ReadWriteable, VBAR_EL1}, +}; +use core::{arch::global_asm, fmt::Display, mem::offset_of}; use esr::{Esr, Exception}; use libkernel::{ error::Result, @@ -37,15 +40,33 @@ unsafe extern "C" { pub static EMERG_STACK_END: VA = VA::from_value(0xffff_c000_0000_0000); #[repr(C)] -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Default)] +pub struct FpSimdState { + q: [[u64; 2]; 32], + fpcr: u32, + fpsr: u32, + reserved: u64, +} + +#[repr(C)] +#[derive(Clone, Copy, Default)] pub struct ExceptionState { pub x: [u64; 31], // x0-x30 pub elr_el1: u64, // Exception link register pub spsr_el1: u64, // Saved program status register pub sp_el0: u64, // Stack pointer of EL0 pub tpid_el0: u64, // Thread process ID + // Explicit padding keeps the stack frame and Q-register area 16-byte + // aligned without leaving uninitialized bytes in signal/clone contexts. + pub(crate) reserved: u64, + pub fpsimd: FpSimdState, } +const _: () = { + assert!(size_of::().is_multiple_of(16)); + assert!(offset_of!(ExceptionState, fpsimd).is_multiple_of(16)); +}; + impl Display for ExceptionState { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { writeln!(f, "X0: 0x{:016x}, X1: 0x{:016x}", self.x[0], self.x[1])?; @@ -79,7 +100,15 @@ impl Display for ExceptionState { } } -global_asm!(include_str!("exceptions.s")); +global_asm!( + include_str!("exceptions.s"), + frame_size = const size_of::(), + reserved = const offset_of!(ExceptionState, reserved), + fpsimd = const offset_of!(ExceptionState, fpsimd), + fpcr = const offset_of!(ExceptionState, fpsimd) + offset_of!(FpSimdState, fpcr), + fpsr = const offset_of!(ExceptionState, fpsimd) + offset_of!(FpSimdState, fpsr), + fp_reserved = const offset_of!(ExceptionState, fpsimd) + offset_of!(FpSimdState, reserved), +); pub fn default_handler(state: &ExceptionState) { panic!("Unhandled CPU exception. Program state:\n{state}"); @@ -171,11 +200,6 @@ extern "C" fn el0_sync(state_ptr: *mut ExceptionState) -> *const ExceptionState let mut ctx2 = unsafe { ctx.clone() }; spawn_kernel_work(&mut ctx2, handle_syscall(ctx)); } - Exception::TrappedFP(_) => { - CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); - // TODO: Flag to start saving FP/SIMD context for this task and, - // save the state. - } _ => default_handler(state), } @@ -251,5 +275,10 @@ pub fn exceptions_init() -> Result<()> { } pub fn secondary_exceptions_init() { + // Save FP/SIMD eagerly in every exception frame, including on secondary + // CPUs. The kernel itself remains compiled for the soft-float target. + CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); + isb(SY); VBAR_EL1.set(EXCEPTION_BASE.value() as u64); + isb(SY); } diff --git a/src/arch/arm64/exceptions/syscall.rs b/src/arch/arm64/exceptions/syscall.rs index dd25345c..6e74644b 100644 --- a/src/arch/arm64/exceptions/syscall.rs +++ b/src/arch/arm64/exceptions/syscall.rs @@ -827,6 +827,8 @@ pub async fn handle_syscall(mut ctx: ProcessCtx) { 0x125 => Err(KernelError::NotSupported), 0x1ae => Err(KernelError::NotSupported), 0x1b2 => sys_pidfd_open(&ctx, arg1 as _, arg2 as _).await, + // Let libc fall back to clone(2) until clone3(2) is implemented. + 0x1b3 => Err(KernelError::NotSupported), 0x1b4 => sys_close_range(&ctx, arg1.into(), arg2.into(), arg3 as _).await, 0x1b7 => { sys_faccessat2( diff --git a/src/arch/arm64/mod.rs b/src/arch/arm64/mod.rs index 6711c661..ad3fa4d2 100644 --- a/src/arch/arm64/mod.rs +++ b/src/arch/arm64/mod.rs @@ -96,6 +96,7 @@ impl Arch for Aarch64 { spsr_el1: 0, sp_el0: stack_top.value() as _, tpid_el0: 0, + ..Default::default() } } diff --git a/src/arch/arm64/proc/idle.rs b/src/arch/arm64/proc/idle.rs index 5a683016..7c8eb690 100644 --- a/src/arch/arm64/proc/idle.rs +++ b/src/arch/arm64/proc/idle.rs @@ -52,6 +52,7 @@ pub fn create_idle_task() -> OwnedTask { spsr_el1: 0, sp_el0: 0, tpid_el0: 0, + ..Default::default() }; let code_map = VMArea::new( diff --git a/src/arch/arm64/proc/signal.rs b/src/arch/arm64/proc/signal.rs index 87e0c264..c0492bd2 100644 --- a/src/arch/arm64/proc/signal.rs +++ b/src/arch/arm64/proc/signal.rs @@ -15,11 +15,14 @@ use libkernel::{ }, }; -#[repr(C)] +#[repr(C, align(16))] #[derive(Clone, Copy)] struct RtSigFrame { uctx: ExceptionState, alt_stack_prev_addr: UA, + // Keep SP 16-byte aligned on alternate stacks too, with initialized + // bytes rather than implicit padding in the frame copied to userspace. + reserved: u64, } // SAFETY: The signal frame that's copied to user-space only contains @@ -39,6 +42,7 @@ pub async fn do_signal( let mut frame = RtSigFrame { uctx: saved_state, alt_stack_prev_addr: UA::null(), + reserved: 0, }; // Use the provided restorer trampoline, or the one provided by the VDSO if diff --git a/src/drivers/fs/mod.rs b/src/drivers/fs/mod.rs index 9de76fcc..a48b3916 100644 --- a/src/drivers/fs/mod.rs +++ b/src/drivers/fs/mod.rs @@ -13,6 +13,7 @@ pub mod cgroup; pub mod dev; pub mod ext4; pub mod fat32; +pub mod pidfs; pub mod proc; pub mod sys; pub mod tmpfs; diff --git a/src/drivers/fs/pidfs.rs b/src/drivers/fs/pidfs.rs new file mode 100644 index 00000000..9c9be3e6 --- /dev/null +++ b/src/drivers/fs/pidfs.rs @@ -0,0 +1,94 @@ +//! The single, kernel-only filesystem backing pidfds. + +use crate::{fs::VFS, process::Tid, sync::OnceLock}; +use alloc::{boxed::Box, sync::Arc}; +use async_trait::async_trait; +use libkernel::{ + error::{FsError, KernelError, Result}, + fs::{ + Filesystem, Inode, InodeId, PIDFS_ID, + attr::{FileAttr, FilePermissions}, + stats::{FilesystemStats, ST_VALID}, + }, + memory::PAGE_SIZE, +}; + +const PID_FS_MAGIC: u64 = 0x5049_4446; +const NAME_MAX: u64 = 255; + +pub struct PidFs; + +#[async_trait] +impl Filesystem for PidFs { + async fn root_inode(&self) -> Result> { + // No path-addressable root and no userspace mount driver. + Err(KernelError::NotSupported) + } + + fn id(&self) -> u64 { + PIDFS_ID + } + + fn magic(&self) -> u64 { + PID_FS_MAGIC + } + + async fn statfs(&self) -> Result { + // Linux pidfs uses simple_statfs. Capacity/inode counters are zero; + // they do not count live processes. The internal mount has ST_VALID + // but no user mount flags. VFS fills fragment_size from block_size. + Ok(FilesystemStats { + magic: self.magic(), + id: self.id(), + block_size: PAGE_SIZE as u64, + name_length: NAME_MAX, + flags: ST_VALID, + ..Default::default() + }) + } +} + +static PIDFS_INSTANCE: OnceLock> = OnceLock::new(); + +pub fn new_inode(pid: Tid) -> Arc { + PIDFS_INSTANCE.get_or_init(|| { + let fs = Arc::new(PidFs); + VFS.register_internal_fs(fs.clone()); + fs + }); + Arc::new(PidInode { + // MOSS allocates task IDs monotonically. Keep the inode independent + // of the task table so an open pidfd survives target exit/reaping. + id: InodeId::from_fsid_and_inodeid(PIDFS_ID, pid.value() as u64), + }) +} + +struct PidInode { + id: InodeId, +} + +#[async_trait] +impl Inode for PidInode { + fn id(&self) -> InodeId { + self.id + } + + async fn getattr(&self) -> Result { + Ok(FileAttr { + id: self.id, + block_size: PAGE_SIZE as u32, + permissions: FilePermissions::S_IRUSR + | FilePermissions::S_IWUSR + | FilePermissions::S_IXUSR, + ..Default::default() + }) + } + + async fn lookup(&self, _name: &str) -> Result> { + Err(FsError::NotADirectory.into()) + } + + fn as_any(&self) -> &dyn core::any::Any { + self + } +} diff --git a/src/drivers/fs/proc/task/fd.rs b/src/drivers/fs/proc/task/fd.rs index 96ff7708..77adf103 100644 --- a/src/drivers/fs/proc/task/fd.rs +++ b/src/drivers/fs/proc/task/fd.rs @@ -1,7 +1,6 @@ use crate::drivers::fs::proc::{get_inode_id, procfs}; use crate::process::fd_table::Fd; use crate::process::{Tid, find_task_by_tid}; -use crate::sched::current_work; use alloc::borrow::ToOwned; use alloc::boxed::Box; use alloc::format; @@ -55,7 +54,7 @@ impl Inode for ProcFdInode { async fn lookup(&self, name: &str) -> Result> { let fd: i32 = name.parse().map_err(|_| FsError::NotFound)?; - let task = current_work(); + let task = find_task_by_tid(self.tid).ok_or(FsError::NotFound)?; let fd_table = task.fd_table.lock_save_irq(); if fd_table.get(Fd(fd)).is_none() { return Err(FsError::NotFound.into()); @@ -102,8 +101,6 @@ impl Inode for ProcFdInode { } } -// TODO: Support fd links in /proc/[pid]/fd/ - pub struct ProcFdFile { id: InodeId, attr: FileAttr, @@ -167,10 +164,10 @@ impl SimpleFile for ProcFdFile { if let Some(path) = file.path() { Ok(path.to_owned()) } else { - // TODO: Find file type - todo!( - "Implement readlink for /proc/[pid]/fd/[fd] when fd doesn't refer to a file with an inode" - ) + let (ops, _) = &*file.lock().await; + ops.anonymous_name() + .map(PathBuf::from) + .ok_or(KernelError::NotSupported) } } else { Err(FsError::NotFound.into()) @@ -179,4 +176,17 @@ impl SimpleFile for ProcFdFile { Err(KernelError::NotSupported) } } + + async fn follow_link(&self) -> Result>> { + if self.fd_info { + return Ok(None); + } + let task = find_task_by_tid(self.tid).ok_or(FsError::NotFound)?; + let file = task + .fd_table + .lock_save_irq() + .get(Fd(self.fd)) + .ok_or(FsError::NotFound)?; + Ok(file.inode()) + } } diff --git a/src/fs/fops.rs b/src/fs/fops.rs index 807c911d..0c50b19b 100644 --- a/src/fs/fops.rs +++ b/src/fs/fops.rs @@ -40,6 +40,12 @@ macro_rules! process_iovec { #[async_trait] pub trait FileOps: Send + Sync { + /// Display name for `/proc//fd` links to anonymous files. This is not + /// a pathname: following the link uses the open file's inode instead. + fn anonymous_name(&self) -> Option<&'static str> { + None + } + /// Reads data from the current file position into `buf`. /// The file's cursor is advanced by the number of bytes read. async fn read(&mut self, ctx: &mut FileCtx, buf: UA, count: usize) -> Result { diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 5b8bfb4d..62833524 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -125,6 +125,12 @@ impl VFS { } } + /// Registers a kernel-only filesystem without exposing a userspace mount. + pub(crate) fn register_internal_fs(&self, fs: Arc) { + assert!(fs.id() < FS_ID_START); + self.state.lock_save_irq().filesystems.insert(fs.id(), fs); + } + /// Creates an instance of a filesystem from a registered driver. /// /// This does not mount the filesystem, but prepares an instance that can @@ -277,7 +283,13 @@ impl VFS { let mut current_inode = root; let mut symlink_count = 0; - let mut components: Vec<_> = path.components().map(|s| s.to_owned()).collect(); + // Preserve `.` so a non-directory followed by `/.` fails with ENOTDIR. + let mut components: Vec<_> = path + .as_str() + .split('/') + .filter(|s| !s.is_empty()) + .map(|s| s.to_owned()) + .collect(); components.reverse(); while let Some(component) = components.pop() { @@ -291,6 +303,12 @@ impl VFS { current_inode = mount_root; } + if current_inode.getattr().await?.file_type != FileType::Directory { + return Err(FsError::NotADirectory.into()); + } + if component == "." { + continue; + } let next_inode = current_inode.lookup(&component).await?; let attr = next_inode.getattr().await?; @@ -301,9 +319,21 @@ impl VFS { return Err(FsError::Loop.into()); // prevent infinite looping } + if let Some(target) = next_inode.follow_link().await? { + current_inode = target; + continue; + } + let target = next_inode.readlink().await?; - let mut new_components: Vec<_> = - target.components().map(|s| s.to_owned()).collect(); + let mut new_components: Vec<_> = target + .as_str() + .split('/') + .filter(|s| !s.is_empty()) + .map(|s| s.to_owned()) + .collect(); + if target.as_str().ends_with('/') { + new_components.push(".".to_owned()); + } new_components.reverse(); for comp in new_components { components.push(comp); @@ -330,6 +360,12 @@ impl VFS { current_inode = mount_root; } + if path.as_str().ends_with('/') + && current_inode.getattr().await?.file_type != FileType::Directory + { + return Err(FsError::NotADirectory.into()); + } + Ok(current_inode) } diff --git a/src/fs/open_file.rs b/src/fs/open_file.rs index c4d8560a..c127ad17 100644 --- a/src/fs/open_file.rs +++ b/src/fs/open_file.rs @@ -41,6 +41,11 @@ impl OpenFile { self.path = Some(path); } + /// Attaches an inode for a kernel-created file with no pathname. + pub fn set_inode(&mut self, inode: Arc) { + self.inode = Some(inode); + } + pub fn inode(&self) -> Option> { self.inode.clone() } @@ -53,8 +58,13 @@ impl OpenFile { self.state.lock().await.1.flags } - pub async fn set_flags(&self, flags: OpenFlags) { - self.state.lock().await.1.flags = flags; + pub async fn set_status_flags(&self, flags: OpenFlags) { + // F_SETFL changes supported mutable status bits only. In particular, + // retain the access mode and PIDFD_THREAD (O_EXCL); O_LARGEFILE added + // by musl is an immutable open flag, not an invalid status request. + let mutable = OpenFlags::O_APPEND | OpenFlags::O_NONBLOCK; + let mut state = self.state.lock().await; + state.1.flags = (state.1.flags & !mutable) | (flags & mutable); } pub async fn lock(&self) -> AsyncMutexGuard<'_, (Box, FileCtx)> { diff --git a/src/fs/syscalls/at/chmod.rs b/src/fs/syscalls/at/chmod.rs index 6d44ffca..7f9b6376 100644 --- a/src/fs/syscalls/at/chmod.rs +++ b/src/fs/syscalls/at/chmod.rs @@ -34,7 +34,7 @@ pub async fn sys_fchmodat( let task = ctx.shared().clone(); let path = Path::new(UserCStr::from_ptr(path).copy_from_user(&mut buf).await?); let start_node = resolve_at_start_node(ctx, dirfd, path, flags).await?; - let mode = FilePermissions::from_bits_retain(mode); + let mode = FilePermissions::from_bits_truncate(mode); let node = resolve_path_flags(dirfd, path, start_node, &task, flags).await?; let mut attr = node.getattr().await?; diff --git a/src/fs/syscalls/chdir.rs b/src/fs/syscalls/chdir.rs index 71bc3968..b535c568 100644 --- a/src/fs/syscalls/chdir.rs +++ b/src/fs/syscalls/chdir.rs @@ -25,7 +25,9 @@ pub async fn sys_getcwd(ctx: &ProcessCtx, buf: UA, len: usize) -> Result copy_to_user_slice(slice, buf).await?; - Ok(buf.value()) + // Linux returns the length of the copied pathname, including the trailing + // NUL. libc uses this value to validate the result before returning `buf`. + Ok(slice.len()) } pub async fn sys_chdir(ctx: &ProcessCtx, path: TUA) -> Result { diff --git a/src/fs/syscalls/chmod.rs b/src/fs/syscalls/chmod.rs index 0f824544..93b159d0 100644 --- a/src/fs/syscalls/chmod.rs +++ b/src/fs/syscalls/chmod.rs @@ -16,7 +16,7 @@ pub async fn sys_fchmod(ctx: &ProcessCtx, fd: Fd, mode: u16) -> Result { .lock_save_irq() .get(fd) .ok_or(KernelError::BadFd)?; - let permissions = FilePermissions::from_bits_retain(mode); + let permissions = FilePermissions::from_bits_truncate(mode); let inode = file.inode().ok_or(KernelError::BadFd)?; let mut attr = inode.getattr().await?; diff --git a/src/fs/syscalls/statfs.rs b/src/fs/syscalls/statfs.rs index 3974fd5c..d385babb 100644 --- a/src/fs/syscalls/statfs.rs +++ b/src/fs/syscalls/statfs.rs @@ -5,15 +5,32 @@ use crate::process::fd_table::Fd; use crate::sched::syscall_ctx::ProcessCtx; use alloc::sync::Arc; use core::ffi::c_char; -use libkernel::error::KernelError; +use libkernel::error::{FsError, KernelError}; use libkernel::fs::Inode; use libkernel::fs::path::Path; +use libkernel::fs::stats::FilesystemStats; use libkernel::memory::address::TUA; use libkernel::pod::Pod; -type FswordT = u32; +// `__statfs_word` is `__kernel_long_t` on AArch64. Keep this definition in +// sync with include/uapi/asm-generic/statfs.h rather than with the host ABI. +type FswordT = i64; type FsBlockCntT = u64; +#[repr(C)] +#[derive(Debug, Clone, Copy)] +struct Fsid { + val: [i32; 2], +} + +impl From for Fsid { + fn from(value: u64) -> Self { + Self { + val: [value as u32 as i32, (value >> 32) as u32 as i32], + } + } +} + #[repr(C)] #[derive(Debug, Clone, Copy)] pub struct StatFs { @@ -32,7 +49,7 @@ pub struct StatFs { /// Free inodes in filesystem f_ffree: FsBlockCntT, /// Filesystem ID - f_fsid: u64, + f_fsid: Fsid, /// Maximum length of filenames f_namelen: FswordT, /// Fragment size (since Linux 2.6) @@ -40,7 +57,7 @@ pub struct StatFs { /// Mount flags of filesystem (since Linux 2.6.36) f_flags: FswordT, /// Padding bytes reserved for future use - f_spare: [FswordT; 6], + f_spare: [FswordT; 4], } unsafe impl Pod for StatFs {} @@ -49,20 +66,30 @@ unsafe impl UserCopyable for StatFs {} async fn statfs_impl(inode: Arc) -> libkernel::error::Result { let fs = VFS.get_fs(inode).await?; - Ok(StatFs { - f_type: fs.magic() as _, - f_bsize: 0, - f_blocks: 0, - f_bfree: 0, - f_bavail: 0, - f_files: 0, - f_ffree: 0, - f_fsid: fs.id(), - f_namelen: 0, - f_frsize: 0, - f_flags: 0, - f_spare: [0; 6], - }) + Ok(fs.statfs().await?.into()) +} + +impl From for StatFs { + fn from(stats: FilesystemStats) -> Self { + Self { + f_type: stats.magic as _, + f_bsize: stats.block_size as _, + f_blocks: stats.blocks, + f_bfree: stats.blocks_free, + f_bavail: stats.blocks_available, + f_files: stats.files, + f_ffree: stats.files_free, + f_fsid: stats.id.into(), + f_namelen: stats.name_length as _, + f_frsize: if stats.fragment_size == 0 { + stats.block_size + } else { + stats.fragment_size + } as _, + f_flags: stats.flags as _, + f_spare: [0; 4], + } + } } pub async fn sys_statfs( @@ -72,9 +99,12 @@ pub async fn sys_statfs( ) -> libkernel::error::Result { let mut buf = [0; 1024]; let path = Path::new(UserCStr::from_ptr(path).copy_from_user(&mut buf).await?); - let inode = VFS - .resolve_path(path, VFS.root_inode(), &ctx.shared().clone()) - .await?; + if path.as_str().is_empty() { + return Err(FsError::NotFound.into()); + } + let task = ctx.shared().clone(); + let cwd = task.cwd.lock_save_irq().0.clone(); + let inode = VFS.resolve_path(path, cwd, &task).await?; let statfs = statfs_impl(inode).await?; copy_to_user(stat, statfs).await?; Ok(0) @@ -91,7 +121,64 @@ pub async fn sys_fstatfs( .lock_save_irq() .get(fd) .ok_or(KernelError::BadFd)?; - let statfs = statfs_impl(fd.inode().ok_or(KernelError::InvalidValue)?).await?; + let inode = fd.inode().ok_or(KernelError::InvalidValue)?; + let statfs = statfs_impl(inode).await?; copy_to_user(stat, statfs).await?; Ok(0) } + +#[cfg(test)] +mod tests { + use super::StatFs; + use core::mem::{align_of, offset_of, size_of}; + use libkernel::fs::stats::FilesystemStats; + use moss_macros::ktest; + + #[ktest] + fn statfs_matches_aarch64_uapi_layout() { + assert_eq!(size_of::(), 120); + assert_eq!(align_of::(), 8); + assert_eq!(offset_of!(StatFs, f_type), 0); + assert_eq!(offset_of!(StatFs, f_bsize), 8); + assert_eq!(offset_of!(StatFs, f_blocks), 16); + assert_eq!(offset_of!(StatFs, f_fsid), 56); + assert_eq!(offset_of!(StatFs, f_namelen), 64); + assert_eq!(offset_of!(StatFs, f_flags), 80); + assert_eq!(offset_of!(StatFs, f_spare), 88); + } + + #[ktest] + fn statfs_converts_all_fields_and_clears_reserved_words() { + let stats = FilesystemStats { + magic: 0x5049_4446, + block_size: 4096, + blocks: 1, + blocks_free: 2, + blocks_available: 3, + files: 4, + files_free: 5, + id: 0x89ab_cdef_0123_4567, + name_length: 255, + fragment_size: 512, + flags: 0x20, + }; + let abi = StatFs::from(stats); + assert_eq!(abi.f_type, stats.magic as i64); + assert_eq!(abi.f_bsize, 4096); + assert_eq!((abi.f_blocks, abi.f_bfree, abi.f_bavail), (1, 2, 3)); + assert_eq!((abi.f_files, abi.f_ffree), (4, 5)); + assert_eq!(abi.f_fsid.val, [0x0123_4567, 0x89ab_cdefu32 as i32]); + assert_eq!(abi.f_namelen, 255); + assert_eq!(abi.f_frsize, 512); + assert_eq!(abi.f_flags, 0x20); + assert_eq!(abi.f_spare, [0; 4]); + assert_eq!( + StatFs::from(FilesystemStats { + fragment_size: 0, + ..stats + }) + .f_frsize, + 4096 + ); + } +} diff --git a/src/process/fd_table/fcntl.rs b/src/process/fd_table/fcntl.rs index 1bbdc7d2..5bf86dd6 100644 --- a/src/process/fd_table/fcntl.rs +++ b/src/process/fd_table/fcntl.rs @@ -79,7 +79,7 @@ pub async fn sys_fcntl(ctx: &ProcessCtx, fd: Fd, op: u32, arg: usize) -> Result< fd.file.clone() }; // TODO: Ignore sync/dsync when implemented - open_fd.set_flags(fl).await; + open_fd.set_status_flags(fl).await; Ok(0) } _ => Err(KernelError::InvalidValue), diff --git a/src/process/pidfd.rs b/src/process/pidfd.rs index 2bd03988..bd5342f6 100644 --- a/src/process/pidfd.rs +++ b/src/process/pidfd.rs @@ -1,5 +1,7 @@ +use crate::drivers::fs::pidfs; use crate::fs::fops::FileOps; use crate::fs::open_file::OpenFile; +use crate::process::fd_table::FdFlags; use crate::process::thread_group::pid::PidT; use crate::process::{Tid, find_task_by_tid}; use crate::sched::syscall_ctx::ProcessCtx; @@ -34,15 +36,19 @@ impl PidFile { pub fn new_open_file(pid: Tid, flags: PidfdFlags) -> Arc { let file = PidFile::new(pid, flags); - Arc::new(OpenFile::new( - Box::new(file), - OpenFlags::from_bits(flags.bits()).unwrap(), - )) + let mut open_file = + OpenFile::new(Box::new(file), OpenFlags::from_bits(flags.bits()).unwrap()); + open_file.set_inode(pidfs::new_inode(pid)); + Arc::new(open_file) } } #[async_trait] impl FileOps for PidFile { + fn anonymous_name(&self) -> Option<&'static str> { + Some("anon_inode:[pidfd]") + } + async fn readat(&mut self, _buf: UA, _count: usize, _offset: u64) -> Result { Err(KernelError::InvalidValue) } @@ -53,16 +59,23 @@ impl FileOps for PidFile { } pub async fn sys_pidfd_open(ctx: &ProcessCtx, pid: PidT, flags: u32) -> Result { + if pid <= 0 { + return Err(KernelError::InvalidValue); + } let pid = Tid::from_pid_t(pid); let flags = PidfdFlags::from_bits(flags).ok_or(KernelError::InvalidValue)?; - if !flags.contains(PidfdFlags::PIDFD_THREAD) { - // Ensure the pid exists and is a thread group leader. - let _ = find_task_by_tid(pid).unwrap(); + let task = find_task_by_tid(pid).ok_or(KernelError::NoProcess)?; + if !flags.contains(PidfdFlags::PIDFD_THREAD) && task.process.tgid.value() != pid.value() { + return Err(KernelError::NoProcess); } let file = PidFile::new_open_file(pid, flags); - let fd = ctx.task().fd_table.lock_save_irq().insert(file)?; + let fd = ctx + .task() + .fd_table + .lock_save_irq() + .insert_with_flags(file, FdFlags::CLOEXEC)?; Ok(fd.as_raw() as _) } diff --git a/src/process/thread_group/signal/sigaltstack.rs b/src/process/thread_group/signal/sigaltstack.rs index b2143f12..82f5230d 100644 --- a/src/process/thread_group/signal/sigaltstack.rs +++ b/src/process/thread_group/signal/sigaltstack.rs @@ -65,7 +65,7 @@ impl From> for UserSigAltStack { } else { Self { ss_sp: UA::null(), - ss_flags: SigAltStackFlags::empty(), + ss_flags: SigAltStackFlags::SS_DISABLE, ss_size: 0, } } @@ -90,7 +90,9 @@ pub async fn sys_sigaltstack( let old_ss_value = signals.alt_stack.clone(); if let Some(ss) = ss { - if ss.ss_size < MIN_STACK_SZ { + // Disabling ignores ss_sp/ss_size, including the zero-sized + // disabled state returned when no alternate stack is installed. + if !ss.ss_flags.contains(SigAltStackFlags::SS_DISABLE) && ss.ss_size < MIN_STACK_SZ { Err(KernelError::NoMemory)?; } diff --git a/src/process/threading/futex/mod.rs b/src/process/threading/futex/mod.rs index b7f53d6e..e63d411a 100644 --- a/src/process/threading/futex/mod.rs +++ b/src/process/threading/futex/mod.rs @@ -25,6 +25,7 @@ const FUTEX_WAKE: i32 = 1; const FUTEX_WAIT_BITSET: i32 = 9; const FUTEX_WAKE_BITSET: i32 = 10; const FUTEX_PRIVATE_FLAG: i32 = 128; +const FUTEX_CLOCK_REALTIME: i32 = 256; type FutexTable = BTreeMap; @@ -173,8 +174,14 @@ pub async fn sys_futex( _uaddr2: TUA, val3: u32, ) -> Result { - // Strip PRIVATE flag if present - let cmd = op & !FUTEX_PRIVATE_FLAG; + let cmd = op & !(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME); + let use_realtime_clock = op & FUTEX_CLOCK_REALTIME != 0; + + // Of the operations implemented here, Linux accepts this flag only for + // WAIT_BITSET. Unsupported clock/op combinations return ENOSYS, not EINVAL. + if use_realtime_clock && cmd != FUTEX_WAIT_BITSET { + return Err(KernelError::NotSupported); + } let key = if op & FUTEX_PRIVATE_FLAG != 0 { FutexKey::new_private(ctx, uaddr) @@ -189,8 +196,13 @@ pub async fn sys_futex( } else { let ts = Duration::from(TimeSpec::copy_from_user(timeout).await?); if matches!(cmd, FUTEX_WAIT_BITSET) { - // FUTEX_WAIT_BITSET takes an absolute realtime deadline. - Some(Deadline::Realtime(ts)) + // FUTEX_WAIT_BITSET uses an absolute deadline. Its default + // clock is monotonic unless FUTEX_CLOCK_REALTIME is set. + if use_realtime_clock { + Some(Deadline::Realtime(ts)) + } else { + Some(Deadline::Monotonic(ts)) + } } else { // FUTEX_WAIT takes a relative timeout on the monotonic // clock; convert to an absolute monotonic deadline. @@ -227,6 +239,9 @@ pub async fn sys_futex( Ok(wake_key(val as _, key, mask)) } - _ => Err(KernelError::NotSupported), + _ => { + log::warn!("Unsupported futex operation: op={op:#x}, cmd={cmd:#x}"); + Err(KernelError::NotSupported) + } } } diff --git a/src/sched/mod.rs b/src/sched/mod.rs index 1559381e..32a6c8e7 100644 --- a/src/sched/mod.rs +++ b/src/sched/mod.rs @@ -131,6 +131,16 @@ fn schedule() { drop(deferred); } +/// Request a voluntary yield without switching the in-flight syscall's task. +fn request_yield() { + SCHED_STATE.borrow_mut().run_q.request_yield(); +} + +/// Checked by the dispatcher only after it has finished polling kernel work. +fn yield_requested() -> bool { + SCHED_STATE.borrow().run_q.yield_requested() +} + pub fn spawn_kernel_work(ctx: &mut ProcessCtx, fut: impl Future + 'static + Send) { ctx.task_mut().ctx.put_kernel_work(Box::pin(fut)); } diff --git a/src/sched/runqueue/mod.rs b/src/sched/runqueue/mod.rs index 85f8c710..14e03cc3 100644 --- a/src/sched/runqueue/mod.rs +++ b/src/sched/runqueue/mod.rs @@ -67,6 +67,7 @@ impl PartialOrd for ByDeadline { /// 2. `running_task` is NOT in `queue`. pub struct RunQueue { total_weight: u64, + yield_requested: bool, ineligible: BinaryHeap, eligible: BinaryHeap, pub(super) running_task: Option, @@ -80,6 +81,7 @@ impl RunQueue { Self { total_weight: 0, + yield_requested: false, ineligible: BinaryHeap::new(), eligible: BinaryHeap::new(), running_task: None, @@ -95,6 +97,7 @@ impl RunQueue { /// re-enter `SCHED_STATE` and panic. pub fn schedule(&mut self, now: Instant) -> Vec { self.v_clock.advance(now, self.weight()); + let yielding = core::mem::take(&mut self.yield_requested); let mut prev_task = ptr::null(); let mut next_task = None; @@ -105,6 +108,9 @@ impl RunQueue { let state = cur_task.work.state.load(Ordering::Acquire); match state { TaskState::Running | TaskState::Woken => { + if yielding { + cur_task.yield_slice(); + } if cur_task.tick(now) { // Deadline exceeded — requeue for the next time slice. self.enqueue(cur_task); @@ -229,6 +235,14 @@ impl RunQueue { self.total_weight } + pub fn request_yield(&mut self) { + self.yield_requested = true; + } + + pub fn yield_requested(&self) -> bool { + self.yield_requested + } + #[allow(clippy::borrowed_box)] pub fn current(&self) -> &RunnableTask { self.running_task.as_ref().unwrap_or(&self.idle) diff --git a/src/sched/sched_task/mod.rs b/src/sched/sched_task/mod.rs index 1fab0f59..88f604db 100644 --- a/src/sched/sched_task/mod.rs +++ b/src/sched/sched_task/mod.rs @@ -126,6 +126,12 @@ impl DerefMut for RunnableTask { } impl RunnableTask { + /// Give up the unused portion of this slice. The next tick replenishes + /// the virtual deadline and requeues us, allowing a peer to run. + pub fn yield_slice(&mut self) { + self.v_eligible = self.v_eligible.max(self.v_deadline); + } + /// Re-issue a virtual deadline fn replenish_deadline(&mut self) { let q_ns: u128 = DEFAULT_TIME_SLICE.as_nanos(); diff --git a/src/sched/syscalls/mod.rs b/src/sched/syscalls/mod.rs index 07f360c2..1e0a002f 100644 --- a/src/sched/syscalls/mod.rs +++ b/src/sched/syscalls/mod.rs @@ -3,12 +3,14 @@ use crate::memory::uaccess::{copy_from_user_slice, copy_to_user_slice}; use crate::process::thread_group::pid::PidT; use crate::sched::sched_task::CPU_MASK_SIZE; use crate::sched::syscall_ctx::ProcessCtx; -use crate::sched::{current_work, schedule}; +use crate::sched::{current_work, request_yield}; use alloc::vec; use libkernel::memory::address::UA; pub fn sys_sched_yield() -> libkernel::error::Result { - schedule(); + // The dispatcher is still polling this task's syscall future. Switching + // here would leave its ProcessCtx pointing at the previous task. + request_yield(); Ok(0) } diff --git a/src/sched/uspc_ret.rs b/src/sched/uspc_ret.rs index bdad906a..9ba99c68 100644 --- a/src/sched/uspc_ret.rs +++ b/src/sched/uspc_ret.rs @@ -1,4 +1,4 @@ -use super::{current_work, current_work_waker, schedule}; +use super::{current_work, current_work_waker, schedule, yield_requested}; use crate::{ arch::{Arch, ArchImpl}, process::{ @@ -164,7 +164,11 @@ pub fn dispatch_userspace_task(frame: *mut UserCtx) { // point was hit). We don't need to clear the kernel // context here as we used the *take* function // above. - state = State::ProcessKernelWork; + state = if yield_requested() { + State::PickNewTask + } else { + State::ProcessKernelWork + }; continue; } Poll::Pending => { diff --git a/usertest/src/fpsimd.rs b/usertest/src/fpsimd.rs new file mode 100644 index 00000000..3923bcb7 --- /dev/null +++ b/usertest/src/fpsimd.rs @@ -0,0 +1,335 @@ +use crate::register_test; +use std::{ + arch::{asm, global_asm}, + sync::{ + Barrier, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + thread, +}; + +#[repr(C, align(16))] +#[derive(Clone, Copy, Default)] +struct FpState { + regs: [[u64; 2]; 32], + fpcr: u32, + fpsr: u32, +} + +impl FpState { + fn sentinel(worker: u32) -> Self { + let mut state = Self { + fpcr: (worker & 3) << 22, + fpsr: worker, + ..Default::default() + }; + for (reg, lanes) in state.regs.iter_mut().enumerate() { + *lanes = [ + (u64::from(worker) << 48) | reg as u64, + 0xfedc_ba98_7654_0000 | (u64::from(worker) << 8) | reg as u64, + ]; + } + state + } + + fn assert_preserved(&self, observed: &Self) { + for reg in 0..32 { + assert_eq!(observed.regs[reg], self.regs[reg], "Q{reg} changed"); + } + assert_eq!(observed.fpcr, self.fpcr, "FPCR changed"); + assert_eq!(observed.fpsr, self.fpsr, "FPSR changed"); + } +} + +// No Rust/libc call may occur between loading the sentinels and observing +// them: the calling convention permits callees to clobber SIMD registers. +// Preserve the ABI's callee-saved D8-D15 and floating-point control state. +global_asm!( + r#" + .text + .p2align 2 + .global fpsimd_syscall_probe + .type fpsimd_syscall_probe, %function +fpsimd_syscall_probe: + sub sp, sp, #80 + stp d8, d9, [sp, #0] + stp d10, d11, [sp, #16] + stp d12, d13, [sp, #32] + stp d14, d15, [sp, #48] + mrs x5, fpcr + mrs x6, fpsr + stp x5, x6, [sp, #64] + mov x9, x0 + mov x10, x1 + mov x8, x2 + mov x0, x3 + mov x1, x4 + ldr w2, [x9, #512] + ldr w3, [x9, #516] + msr fpcr, x2 + msr fpsr, x3 + ldp q0, q1, [x9, #0] + ldp q2, q3, [x9, #32] + ldp q4, q5, [x9, #64] + ldp q6, q7, [x9, #96] + ldp q8, q9, [x9, #128] + ldp q10, q11, [x9, #160] + ldp q12, q13, [x9, #192] + ldp q14, q15, [x9, #224] + ldp q16, q17, [x9, #256] + ldp q18, q19, [x9, #288] + ldp q20, q21, [x9, #320] + ldp q22, q23, [x9, #352] + ldp q24, q25, [x9, #384] + ldp q26, q27, [x9, #416] + ldp q28, q29, [x9, #448] + ldp q30, q31, [x9, #480] + mov x2, xzr + mov x3, xzr + mov x4, xzr + svc #0 + stp q0, q1, [x10, #0] + stp q2, q3, [x10, #32] + stp q4, q5, [x10, #64] + stp q6, q7, [x10, #96] + stp q8, q9, [x10, #128] + stp q10, q11, [x10, #160] + stp q12, q13, [x10, #192] + stp q14, q15, [x10, #224] + stp q16, q17, [x10, #256] + stp q18, q19, [x10, #288] + stp q20, q21, [x10, #320] + stp q22, q23, [x10, #352] + stp q24, q25, [x10, #384] + stp q26, q27, [x10, #416] + stp q28, q29, [x10, #448] + stp q30, q31, [x10, #480] + mrs x2, fpcr + mrs x3, fpsr + str w2, [x10, #512] + str w3, [x10, #516] + ldp x2, x3, [sp, #64] + msr fpcr, x2 + msr fpsr, x3 + ldp d8, d9, [sp, #0] + ldp d10, d11, [sp, #16] + ldp d12, d13, [sp, #32] + ldp d14, d15, [sp, #48] + add sp, sp, #80 + ret + .size fpsimd_syscall_probe, .-fpsimd_syscall_probe +"# +); + +unsafe extern "C" { + fn fpsimd_syscall_probe( + input: *const FpState, + output: *mut FpState, + syscall: libc::c_long, + arg0: libc::c_long, + arg1: libc::c_long, + ) -> libc::c_long; +} + +fn test_fpsimd_context() { + // More runnable workers than CPUs in the project's four-CPU SMP setup. + let barrier = Barrier::new(8); + thread::scope(|scope| { + for worker in 1..=8 { + let barrier = &barrier; + scope.spawn(move || { + let expected = FpState::sentinel(worker); + let delay = libc::timespec { + tv_sec: 0, + tv_nsec: 1_000_000, + }; + barrier.wait(); + for _ in 0..32 { + let mut observed = FpState::default(); + assert_eq!( + unsafe { + fpsimd_syscall_probe( + &expected, + &mut observed, + libc::SYS_nanosleep, + (&delay as *const libc::timespec) as libc::c_long, + 0, + ) + }, + 0 + ); + expected.assert_preserved(&observed); + } + }); + } + }); +} + +register_test!(test_fpsimd_context); + +fn test_fpsimd_user_copy() { + // libc/LLVM may keep a memcpy or memset's data in SIMD registers while a + // demand-page fault switches tasks. Exercise this without any filesystem + // I/O so corrupted user buffers cannot be mistaken for an ext4 race. + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + assert!(page_size > 0); + let len = page_size as usize * 16 + 1; + let barrier = Barrier::new(8); + thread::scope(|scope| { + for worker in 1..=8u8 { + let barrier = &barrier; + scope.spawn(move || { + barrier.wait(); + for round in 0..16u8 { + let byte = worker * 17 + round; + let source = vec![byte; len]; + unsafe { + let destination = libc::mmap( + std::ptr::null_mut(), + len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ); + assert_ne!(destination, libc::MAP_FAILED); + // Do not prefault the destination before the copy. + std::ptr::copy_nonoverlapping( + source.as_ptr(), + destination.cast::(), + len, + ); + let observed = std::slice::from_raw_parts(destination.cast::(), len); + let mismatch = observed.iter().position(|&value| value != byte); + assert_eq!(libc::munmap(destination, len), 0); + assert!( + mismatch.is_none(), + "user copy corrupted: worker {worker}, round {round}, offset {mismatch:?}" + ); + } + } + }); + } + }); +} + +register_test!(test_fpsimd_user_copy); + +static SIGNAL_SEEN: AtomicBool = AtomicBool::new(false); +static SIGNAL_SP: AtomicUsize = AtomicUsize::new(0); + +extern "C" fn clobber_fpsimd(_: libc::c_int) { + let stack_pointer: usize; + unsafe { + asm!( + "mov {stack_pointer}, sp", + "movi v0.16b, #0", + "movi v31.16b, #0", + "msr fpcr, xzr", + "msr fpsr, xzr", + stack_pointer = out(reg) stack_pointer, + out("v0") _, + out("v31") _, + options(nostack), + ); + } + SIGNAL_SP.store(stack_pointer, Ordering::Relaxed); + SIGNAL_SEEN.store(true, Ordering::Relaxed); +} + +fn test_fpsimd_signal_return() { + unsafe { + // An unaligned stack top also checks that the kernel aligns SP, + // instead of relying on the caller to provide an aligned buffer. + let mut stack_mem = vec![0u8; 64 * 1024 + 1]; + let stack = libc::stack_t { + ss_sp: stack_mem.as_mut_ptr().add(1).cast(), + ss_flags: 0, + ss_size: 64 * 1024, + }; + let mut old_stack: libc::stack_t = std::mem::zeroed(); + assert_eq!(libc::sigaltstack(&stack, &mut old_stack), 0); + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = clobber_fpsimd as *const () as usize; + assert_eq!(libc::sigemptyset(&mut action.sa_mask), 0); + let expected = FpState::sentinel(1); + for flags in [0, libc::SA_ONSTACK] { + action.sa_flags = flags; + assert_eq!( + libc::sigaction(libc::SIGUSR1, &action, std::ptr::null_mut()), + 0 + ); + let mut observed = FpState::default(); + SIGNAL_SEEN.store(false, Ordering::Relaxed); + assert_eq!( + fpsimd_syscall_probe( + &expected, + &mut observed, + libc::SYS_kill, + libc::getpid().into(), + libc::SIGUSR1.into(), + ), + 0 + ); + assert!(SIGNAL_SEEN.load(Ordering::Relaxed)); + let sp = SIGNAL_SP.load(Ordering::Relaxed); + assert_eq!(sp % 16, 0, "signal handler SP is not aligned"); + if flags == libc::SA_ONSTACK { + let start = stack.ss_sp as usize; + assert!((start..start + stack.ss_size).contains(&sp)); + } + expected.assert_preserved(&observed); + } + let disabled = libc::stack_t { + ss_sp: std::ptr::null_mut(), + ss_flags: libc::SS_DISABLE, + ss_size: 0, + }; + assert_eq!(libc::sigaltstack(&disabled, std::ptr::null_mut()), 0); + let mut queried: libc::stack_t = std::mem::zeroed(); + assert_eq!(libc::sigaltstack(std::ptr::null(), &mut queried), 0); + assert_eq!(queried.ss_flags, libc::SS_DISABLE); + assert_eq!(libc::sigaltstack(&old_stack, std::ptr::null_mut()), 0); + } +} + +register_test!(test_fpsimd_signal_return); + +fn test_fpsimd_clone() { + // The test runner forks before entering each test, so no other threads + // exist here. Use raw clone(SIGCHLD, NULL, NULL, NULL, 0), i.e. fork, + // without a libc call clobbering the sentinels before the syscall. + unsafe { + let expected = FpState::sentinel(3); + let mut observed = FpState::default(); + let pid = fpsimd_syscall_probe( + &expected, + &mut observed, + libc::SYS_clone, + libc::SIGCHLD.into(), + 0, + ); + assert!(pid >= 0, "clone returned {pid}"); + if pid == 0 { + expected.assert_preserved(&observed); + libc::_exit(0); + } + let mut status = 0; + loop { + let waited = libc::waitpid(pid as libc::pid_t, &mut status, 0); + if waited == pid as libc::pid_t { + break; + } + assert_eq!(waited, -1); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::EINTR) + ); + } + assert!(libc::WIFEXITED(status)); + assert_eq!(libc::WEXITSTATUS(status), 0); + expected.assert_preserved(&observed); + } +} + +register_test!(test_fpsimd_clone); diff --git a/usertest/src/fs.rs b/usertest/src/fs.rs index 38969d42..73cf51b3 100644 --- a/usertest/src/fs.rs +++ b/usertest/src/fs.rs @@ -1,7 +1,14 @@ use crate::register_test; use std::ffi::{CStr, CString}; use std::fs; +use std::fs::{File, OpenOptions}; +use std::io::{Read, Seek, Write}; use std::mem::MaybeUninit; +use std::os::fd::AsRawFd; +use std::os::unix::fs::{FileExt, MetadataExt, symlink}; +use std::path::Path; +use std::sync::{Arc, Barrier, mpsc}; +use std::thread; fn test_opendir() { let path = CString::new("/").unwrap(); @@ -63,6 +70,36 @@ fn test_chdir() { register_test!(test_chdir); +fn test_getcwd_syscall_length() { + unsafe { + assert_eq!(libc::chdir(c"/".as_ptr()), 0); + let mut buffer = [0xa5u8; 16]; + let length = libc::syscall(libc::SYS_getcwd, buffer.as_mut_ptr(), buffer.len()); + assert_eq!(length, 2, "getcwd must return the size including the NUL"); + assert_eq!(&buffer[..2], b"/\0"); + assert!(buffer[2..].iter().all(|&byte| byte == 0xa5)); + + let mut short = [0xa5u8; 1]; + assert_eq!( + libc::syscall(libc::SYS_getcwd, short.as_mut_ptr(), short.len()), + -1 + ); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ERANGE) + ); + assert_eq!(short, [0xa5]); + + assert_eq!( + libc::getcwd(buffer.as_mut_ptr().cast(), buffer.len()), + buffer.as_mut_ptr().cast() + ); + assert_eq!(&buffer[..2], b"/\0"); + } +} + +register_test!(test_getcwd_syscall_length); + fn test_fchdir() { let path = CString::new("/dev").unwrap(); let mut buffer = [1u8; 16]; @@ -162,6 +199,46 @@ fn test_fchmod() { register_test!(test_fchmod); +fn test_chmod_preserves_file_type() { + for (name, kind) in [("file", libc::S_IFREG), ("dir", libc::S_IFDIR)] { + let path = format!("/tmp/chmod-type-{}-{name}", std::process::id()); + if kind == libc::S_IFDIR { + fs::create_dir(&path).unwrap(); + } else { + fs::write(&path, b"preserved").unwrap(); + } + let opened = File::open(&path).unwrap(); + let c_path = CString::new(path.clone()).unwrap(); + // Linux only accepts the permission portion of chmod's mode argument. + // Try each file-type pattern, including otherwise invalid combinations. + for type_bits in [libc::S_IFDIR, libc::S_IFREG, libc::S_IFLNK, libc::S_IFMT] { + for (fd_call, permissions) in [(false, 0o751), (true, 0o640)] { + let result = unsafe { + if fd_call { + libc::fchmod(opened.as_raw_fd(), type_bits | permissions) + } else { + libc::chmod(c_path.as_ptr(), type_bits | permissions) + } + }; + assert_eq!(result, 0, "chmod: {}", std::io::Error::last_os_error()); + for metadata in [opened.metadata().unwrap(), fs::metadata(&path).unwrap()] { + assert_eq!(metadata.mode() & libc::S_IFMT, kind); + assert_eq!(metadata.mode() & 0o7777, permissions); + } + } + } + drop(opened); + if kind == libc::S_IFDIR { + fs::remove_dir(&path).unwrap(); + } else { + assert_eq!(fs::read(&path).unwrap(), b"preserved"); + fs::remove_file(&path).unwrap(); + } + } +} + +register_test!(test_chmod_preserves_file_type); + fn test_chown() { let dir_path = "/tmp/chown_test"; let c_dir_path = CString::new(dir_path).unwrap(); @@ -613,6 +690,273 @@ fn test_rust_dir() { register_test!(test_rust_dir); +fn on_test_filesystem(root: &str, name: &str, test: fn(&Path)) { + let c_root = CString::new(root).unwrap(); + let mut stat: libc::statfs = unsafe { std::mem::zeroed() }; + assert_eq!(unsafe { libc::statfs(c_root.as_ptr(), &mut stat) }, 0); + assert_eq!( + stat.f_type as u64, + if root == "/" { 0xef53 } else { 0x0102_1994 } + ); + let dir = Path::new(root).join(name); + fs::create_dir(&dir).unwrap(); + test(&dir); + fs::remove_dir_all(dir).unwrap(); +} + +fn check_unlink_write_race(dir: &Path) { + let path = dir.join("data"); + let alias = dir.join("alias"); + let file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .unwrap(); + thread::scope(|scope| { + // Dropping requests on assertion failure releases the writer instead + // of stranding it at a barrier and hiding the original failure. + let (request, requests) = mpsc::sync_channel::(0); + let (complete, completions) = mpsc::sync_channel(0); + let file_ref = &file; + let writer = scope.spawn(move || { + for round in requests { + let result = file_ref.set_len(0).and_then(|()| { + let bytes = vec![(round + 1) as u8; 8193 + round]; + file_ref.write_all_at(&bytes, 0) + }); + if complete.send(result).is_err() { + break; + } + } + }); + for round in 0..64 { + fs::hard_link(&path, &alias).unwrap(); + request.send(round).unwrap(); + fs::remove_file(&alias).unwrap(); + completions.recv().unwrap().unwrap(); + let expected = vec![(round + 1) as u8; 8193 + round]; + assert_eq!(file.metadata().unwrap().nlink(), 1); + assert_eq!(file.metadata().unwrap().len(), expected.len() as u64); + let observed = fs::read(&path).unwrap(); + assert_eq!(observed.len(), expected.len(), "unlink/write round {round}"); + let mismatch = observed.iter().zip(&expected).position(|(a, b)| a != b); + assert!( + mismatch.is_none(), + "unlink/write round {round}: first differing byte {mismatch:?}" + ); + } + drop(request); + writer.join().unwrap(); + }); +} + +fn test_ext4_unlink_write_race() { + on_test_filesystem("/", "fs-unlink-race", check_unlink_write_race); +} + +register_test!(test_ext4_unlink_write_race); + +fn check_symlink_boundaries(dir: &Path) { + for len in [1, 13, 59, 60, 61, 120] { + let target = "t".repeat(len); + let link = dir.join(format!("symlink-{len}")); + let alias = dir.join(format!("alias-{len}")); + symlink(&target, &link).unwrap(); + assert_eq!(fs::read_link(&link).unwrap(), Path::new(&target)); + fs::hard_link(&link, &alias).unwrap(); + assert_eq!(fs::symlink_metadata(&link).unwrap().nlink(), 2); + fs::remove_file(&link).unwrap(); + assert_eq!(fs::symlink_metadata(&alias).unwrap().nlink(), 1); + assert_eq!(fs::read_link(&alias).unwrap(), Path::new(&target)); + fs::remove_file(&alias).unwrap(); + } + // Reuse freed inode/block numbers after removing the links. + fs::write(dir.join("after-unlink"), vec![0x5a; 8192]).unwrap(); + assert_eq!( + fs::read(dir.join("after-unlink")).unwrap(), + vec![0x5a; 8192] + ); +} + +fn test_ext4_symlink_boundaries() { + on_test_filesystem("/", "fs-symlink", check_symlink_boundaries); +} + +register_test!(test_ext4_symlink_boundaries); + +fn check_rename_inode_consistency(dir: &Path) { + let old = dir.join("old"); + let new = dir.join("new"); + fs::write(&old, b"payload").unwrap(); + let mut opened = File::open(&old).unwrap(); + let ino = opened.metadata().unwrap().ino(); + fs::rename(&old, &new).unwrap(); + assert!(!old.exists()); + assert_eq!(fs::metadata(&new).unwrap().ino(), ino); + let mut bytes = Vec::new(); + opened.read_to_end(&mut bytes).unwrap(); + assert_eq!(bytes, b"payload"); + assert_eq!(opened.metadata().unwrap().nlink(), 1); + + // Renaming two names of the same inode is a no-op, not an unlink. + fs::hard_link(&new, &old).unwrap(); + fs::rename(&old, &new).unwrap(); + assert_eq!(fs::metadata(&old).unwrap().ino(), ino); + assert_eq!(opened.metadata().unwrap().nlink(), 2); + let c_old = CString::new(old.as_os_str().as_encoded_bytes()).unwrap(); + let c_new = CString::new(new.as_os_str().as_encoded_bytes()).unwrap(); + assert_eq!( + unsafe { + libc::syscall( + libc::SYS_renameat2, + libc::AT_FDCWD, + c_old.as_ptr(), + libc::AT_FDCWD, + c_new.as_ptr(), + 1, + ) + }, + -1 + ); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::EEXIST) + ); + fs::remove_file(&old).unwrap(); + assert_eq!(opened.metadata().unwrap().nlink(), 1); + + fs::create_dir(dir.join("other")).unwrap(); + let dst = dir.join("other/dst"); + fs::write(&dst, b"replaced").unwrap(); + fs::hard_link(&dst, &old).unwrap(); + let replaced = File::open(&old).unwrap(); + fs::rename(&new, &dst).unwrap(); + assert!(!new.exists()); + assert_eq!(fs::read(&dst).unwrap(), b"payload"); + assert_eq!(replaced.metadata().unwrap().nlink(), 1); + assert_eq!(fs::read(&old).unwrap(), b"replaced"); + + // Opposing cross-directory moves must not invert directory lock order. + fs::create_dir(dir.join("left")).unwrap(); + fs::create_dir(dir.join("right")).unwrap(); + thread::scope(|scope| { + let barrier = Arc::new(Barrier::new(2)); + for worker in 0..2 { + let barrier = barrier.clone(); + scope.spawn(move || { + let a = dir.join(format!("left/{worker}")); + let b = dir.join(format!("right/{worker}")); + let (a, b) = if worker == 0 { (a, b) } else { (b, a) }; + fs::write(&a, [worker as u8]).unwrap(); + barrier.wait(); + for _ in 0..32 { + fs::rename(&a, &b).unwrap(); + fs::rename(&b, &a).unwrap(); + } + assert_eq!(fs::read(&a).unwrap(), [worker as u8]); + }); + } + }); +} + +fn test_ext4_rename_inode_consistency() { + on_test_filesystem("/", "fs-rename", check_rename_inode_consistency); +} + +register_test!(test_ext4_rename_inode_consistency); + +fn test_ext4_directory_rename() { + on_test_filesystem("/", "fs-directory", |dir| { + let base_links = fs::metadata(dir).unwrap().nlink(); + let source = dir.join("source"); + let parent = dir.join("parent"); + fs::create_dir(&source).unwrap(); + fs::create_dir(&parent).unwrap(); + assert_eq!(fs::metadata(dir).unwrap().nlink(), base_links + 2); + fs::write(source.join("file"), b"directory payload").unwrap(); + let opened = File::open(&source).unwrap(); + let ino = opened.metadata().unwrap().ino(); + let renamed = dir.join("renamed"); + fs::rename(&source, &renamed).unwrap(); + assert_eq!(fs::metadata(&renamed).unwrap().ino(), ino); + let destination = parent.join("child"); + fs::rename(&renamed, &destination).unwrap(); + assert_eq!( + fs::read(destination.join("file")).unwrap(), + b"directory payload" + ); + assert_eq!(fs::metadata(dir).unwrap().nlink(), base_links + 1); + assert_eq!(fs::metadata(&parent).unwrap().nlink(), 3); + // Use the pre-rename directory fd: path normalization cannot fake "..". + let mut stat: libc::stat = unsafe { std::mem::zeroed() }; + assert_eq!( + unsafe { libc::fstatat(opened.as_raw_fd(), c"..".as_ptr(), &mut stat, 0) }, + 0 + ); + assert_eq!(stat.st_ino, fs::metadata(&parent).unwrap().ino()); + assert!(fs::rename(&parent, destination.join("invalid")).is_err()); + assert!(fs::remove_dir(&destination).is_err()); + fs::remove_file(destination.join("file")).unwrap(); + fs::remove_dir(&destination).unwrap(); + assert_eq!(fs::metadata(&parent).unwrap().nlink(), 2); + }); +} + +register_test!(test_ext4_directory_rename); + +fn check_truncate_boundaries(dir: &Path) { + let path = dir.join("file"); + let link = dir.join("link"); + let mut file = OpenOptions::new() + .create_new(true) + .read(true) + .write(true) + .open(&path) + .unwrap(); + fs::hard_link(&path, &link).unwrap(); + let other = File::open(&link).unwrap(); + let data: Vec = (0..16385).map(|n| (n % 251 + 1) as u8).collect(); + // Include EOF in a retained block, exact block boundaries, and zero. + for size in [5, 4095, 4096, 4097, 0] { + file.set_len(0).unwrap(); + file.rewind().unwrap(); + file.write_all(&data).unwrap(); + let c_path = CString::new(link.as_os_str().as_encoded_bytes()).unwrap(); + assert_eq!( + unsafe { libc::truncate(c_path.as_ptr(), size as libc::off_t) }, + 0 + ); + for metadata in [ + file.metadata().unwrap(), + other.metadata().unwrap(), + fs::metadata(&path).unwrap(), + ] { + assert_eq!(metadata.len(), size as u64); + assert_eq!(metadata.nlink(), 2); + } + let mut bytes = vec![0xff; data.len()]; + assert_eq!(other.read_at(&mut bytes, 0).unwrap(), size); + assert_eq!(&bytes[..size], &data[..size]); + assert_eq!(fs::read(&path).unwrap(), data[..size]); + file.set_len(data.len() as u64).unwrap(); + let mut expected = data[..size].to_vec(); + expected.resize(data.len(), 0); + let actual = fs::read(&link).unwrap(); + assert_eq!( + actual.len(), + expected.len(), + "shrink/extend length at {size}" + ); + assert!(actual == expected, "shrink/extend content at {size}"); + } +} + +fn test_ext4_truncate_boundaries() { + on_test_filesystem("/", "fs-truncate", check_truncate_boundaries); +} + +register_test!(test_ext4_truncate_boundaries); fn test_o_append() { use std::io::{Seek, SeekFrom, Write}; let path = "/tmp/o_append_test"; diff --git a/usertest/src/futex.rs b/usertest/src/futex.rs index a1a241f8..38cf189c 100644 --- a/usertest/src/futex.rs +++ b/usertest/src/futex.rs @@ -44,6 +44,127 @@ fn test_futex() { register_test!(test_futex); +fn test_futex_timeout_clocks() { + // No threads are needed: each wait expires with an unchanged futex word. + // A relative WAIT must use monotonic time; WAIT_BITSET uses an absolute + // monotonic deadline unless FUTEX_CLOCK_REALTIME is explicitly selected. + for (operation, clock, absolute) in [ + (libc::FUTEX_WAIT, libc::CLOCK_MONOTONIC, false), + (libc::FUTEX_WAIT_BITSET, libc::CLOCK_MONOTONIC, true), + ( + libc::FUTEX_WAIT_BITSET | libc::FUTEX_CLOCK_REALTIME, + libc::CLOCK_REALTIME, + true, + ), + ] { + for private in [0, libc::FUTEX_PRIVATE_FLAG] { + let word = 0u32; + let mut deadline = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + if absolute { + assert_eq!(unsafe { libc::clock_gettime(clock, &mut deadline) }, 0); + } + deadline.tv_nsec += 100_000_000; + if deadline.tv_nsec >= 1_000_000_000 { + deadline.tv_nsec -= 1_000_000_000; + deadline.tv_sec += 1; + } + let start = std::time::Instant::now(); + let result = unsafe { + libc::syscall( + libc::SYS_futex, + &word, + operation | private, + 0u32, + &deadline, + std::ptr::null::(), + u32::MAX, + ) + }; + let error = std::io::Error::last_os_error(); + assert_eq!(result, -1); + assert_eq!(error.raw_os_error(), Some(libc::ETIMEDOUT)); + assert!( + start.elapsed() >= Duration::from_millis(80), + "timeout expired early" + ); + } + } +} + +register_test!(test_futex_timeout_clocks); + +fn test_futex_unsupported_realtime_ops() { + for operation in [libc::FUTEX_WAIT, libc::FUTEX_WAKE, libc::FUTEX_WAKE_BITSET] { + for private in [0, libc::FUTEX_PRIVATE_FLAG] { + let word = 0u32; + let result = unsafe { + libc::syscall( + libc::SYS_futex, + &word, + operation | private | libc::FUTEX_CLOCK_REALTIME, + 0u32, + std::ptr::null::(), + std::ptr::null::(), + u32::MAX, + ) + }; + assert_eq!(result, -1); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ENOSYS) + ); + } + } +} + +register_test!(test_futex_unsupported_realtime_ops); + +fn test_pthread_cond_timedwait_clocks() { + // Exercise libc's actual timed-wait path, not just the raw futex ABI. + // In glibc, rejecting WAIT_BITSET|CLOCK_REALTIME can abort the process. + for clock in [libc::CLOCK_REALTIME, libc::CLOCK_MONOTONIC] { + unsafe { + let mut mutex: libc::pthread_mutex_t = std::mem::zeroed(); + let mut cond: libc::pthread_cond_t = std::mem::zeroed(); + let mut attr: libc::pthread_condattr_t = std::mem::zeroed(); + assert_eq!(libc::pthread_mutex_init(&mut mutex, std::ptr::null()), 0); + assert_eq!(libc::pthread_condattr_init(&mut attr), 0); + assert_eq!(libc::pthread_condattr_setclock(&mut attr, clock), 0); + assert_eq!(libc::pthread_cond_init(&mut cond, &attr), 0); + assert_eq!(libc::pthread_condattr_destroy(&mut attr), 0); + assert_eq!(libc::pthread_mutex_lock(&mut mutex), 0); + + let mut deadline: libc::timespec = std::mem::zeroed(); + assert_eq!(libc::clock_gettime(clock, &mut deadline), 0); + deadline.tv_nsec += 100_000_000; + if deadline.tv_nsec >= 1_000_000_000 { + deadline.tv_nsec -= 1_000_000_000; + deadline.tv_sec += 1; + } + let start = std::time::Instant::now(); + loop { + let result = libc::pthread_cond_timedwait(&mut cond, &mut mutex, &deadline); + if result == 0 { + // POSIX permits spurious wakeups; retain the same deadline. + continue; + } + assert_eq!(result, libc::ETIMEDOUT, "clock {clock}"); + break; + } + assert!(start.elapsed() >= Duration::from_millis(80)); + assert_eq!(libc::pthread_mutex_trylock(&mut mutex), libc::EBUSY); + assert_eq!(libc::pthread_mutex_unlock(&mut mutex), 0); + assert_eq!(libc::pthread_cond_destroy(&mut cond), 0); + assert_eq!(libc::pthread_mutex_destroy(&mut mutex), 0); + } + } +} + +register_test!(test_pthread_cond_timedwait_clocks); + fn test_futex_bitset() { // Wait on bit 1, Wake on bit 1 { diff --git a/usertest/src/futex2.rs b/usertest/src/futex2.rs index 1f845f77..cce1dc8c 100644 --- a/usertest/src/futex2.rs +++ b/usertest/src/futex2.rs @@ -3,7 +3,7 @@ use crate::register_test; use std::sync::{ - Arc, + Arc, Barrier, atomic::{AtomicU32, Ordering}, }; use std::thread; @@ -368,8 +368,10 @@ fn test_futex2_timeout() { let addr = &word as *const u32; for clockid in [libc::CLOCK_MONOTONIC, libc::CLOCK_REALTIME] { - let ts = abs_deadline(clockid, 50); + // Start measuring before computing the deadline. Otherwise time spent + // between the two clock reads is incorrectly counted as an early wake. let start = Instant::now(); + let ts = abs_deadline(clockid, 50); let ret = unsafe { futex2_wait(addr, 0, MATCH_ANY, FUTEX2_SIZE_U32, &ts, clockid) }; if ret != -1 || errno() != libc::ETIMEDOUT { @@ -720,18 +722,22 @@ fn test_futex2_realtime_retarget() { register_test!(test_futex2_realtime_retarget); fn test_futex2_requeue_timeout_race() { - // Requeue waiters that are about to time out: their timeout-path - // unregistration must find them on the destination queue. + // Requeue timed waiters: their timeout-path unregistration must find them + // on the destination queue, even when they arrive at different times. const NR_THREADS: usize = 4; let f1 = Arc::new(AtomicU32::new(0)); let f2 = Arc::new(AtomicU32::new(0)); + let ready = Arc::new(Barrier::new(NR_THREADS + 1)); let threads: Vec<_> = (0..NR_THREADS) .map(|_| { let f1 = f1.clone(); + let ready = ready.clone(); thread::spawn(move || { - let ts = abs_deadline(libc::CLOCK_MONOTONIC, 150); + // Do not include thread creation time in the wait deadline. + ready.wait(); + let ts = abs_deadline(libc::CLOCK_MONOTONIC, 2_000); let ret = unsafe { futex2_wait( f1.as_ptr() as *const u32, @@ -749,19 +755,31 @@ fn test_futex2_requeue_timeout_race() { }) .collect(); - thread::sleep(Duration::from_millis(50)); + ready.wait(); let pair = [ FutexWaitv::new(f1.as_ptr() as *const u32, 0, FUTEX2_SIZE_U32), FutexWaitv::new(f2.as_ptr() as *const u32, 0, FUTEX2_SIZE_U32), ]; - unsafe { - // Move everyone to f2 without waking anybody. Return counts the - // requeued waiters (0 woken + NR requeued). - let ret = futex2_requeue(pair.as_ptr(), 0, 0, NR_THREADS as i32); - if ret != NR_THREADS as i64 { - panic!("requeue returned {ret}, expected {NR_THREADS} requeued"); + // A fixed sleep cannot prove that every waiter has entered the kernel. + // Count actual migrations, retaining the requirement that all four move. + let start = Instant::now(); + let mut moved = 0; + while moved < NR_THREADS { + let remaining = NR_THREADS - moved; + let ret = unsafe { futex2_requeue(pair.as_ptr(), 0, 0, remaining as i32) }; + assert!( + (0..=remaining as i64).contains(&ret), + "requeue returned {ret}" + ); + moved += ret as usize; + if moved < NR_THREADS { + assert!( + start.elapsed() < Duration::from_secs(2), + "only {moved}/{NR_THREADS} waiters reached the destination" + ); + thread::sleep(Duration::from_millis(1)); } } @@ -776,6 +794,7 @@ fn test_futex2_requeue_timeout_race() { if ret != 0 { panic!("{ret} stale waiters left on f2"); } + assert_eq!(futex2_wake(f1.as_ptr(), MATCH_ANY, 64, FUTEX2_SIZE_U32), 0); } } diff --git a/usertest/src/main.rs b/usertest/src/main.rs index adffa663..e05056f8 100644 --- a/usertest/src/main.rs +++ b/usertest/src/main.rs @@ -6,10 +6,14 @@ use std::{ }; mod epoll; +#[cfg(target_arch = "aarch64")] +mod fpsimd; mod fs; mod futex; mod futex2; mod inotify; +mod pidfd; +mod sched; mod signalfd; mod signals; mod socket; @@ -74,13 +78,37 @@ fn test_fork() { } else { // Parent process let mut status = 0; - libc::waitpid(pid, &mut status, 0); + loop { + let waited = libc::waitpid(pid, &mut status, 0); + if waited == pid { + break; + } + let error = std::io::Error::last_os_error(); + if waited == -1 && error.raw_os_error() == Some(libc::EINTR) { + continue; + } + panic!("waitpid({pid}) failed: {error}"); + } } } } register_test!(test_fork); +fn test_clone3_fallback() { + // Until clone3 is implemented, ENOSYS tells libc to retry with clone. + // Unsupported arguments must not reach the kernel's unknown-syscall panic. + let result = + unsafe { libc::syscall(libc::SYS_clone3, std::ptr::null::(), 0usize) }; + assert_eq!(result, -1); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ENOSYS) + ); +} + +register_test!(test_clone3_fallback); + #[expect(deprecated)] fn test_vfork_exit() { unsafe { @@ -373,7 +401,17 @@ fn run_test(test_fn: fn()) -> Result<(), i32> { } else { // Parent process let mut status = 0; - libc::waitpid(pid, &mut status, 0); + loop { + let waited = libc::waitpid(pid, &mut status, 0); + if waited == pid { + break; + } + let error = std::io::Error::last_os_error(); + if waited == -1 && error.raw_os_error() == Some(libc::EINTR) { + continue; + } + panic!("waitpid({pid}) failed: {error}"); + } if !libc::WIFEXITED(status) || libc::WEXITSTATUS(status) != 0 { Err(status) } else { @@ -390,6 +428,7 @@ fn main() { let filter = args.get(1).map(|s| s.as_str()); let start = std::time::Instant::now(); let mut failures = 0; + let mut executed = 0; for test in inventory::iter:: { if let Some(filter) = filter && !test.test_text.contains(filter) @@ -397,17 +436,36 @@ fn main() { continue; } print!("{} ...", test.test_text); + executed += 1; let _ = stdout().flush(); match run_test(test.test_fn) { Ok(()) => println!("{}", " OK".green()), - Err(code) => { + Err(status) => { println!(" {}", "FAILED".red()); - eprintln!("Test '{}' failed with exit code {}", test.test_text, code); + if libc::WIFSIGNALED(status) { + eprintln!( + "Test '{}' terminated by signal {} (wait status {status})", + test.test_text, + libc::WTERMSIG(status) + ); + } else if libc::WIFEXITED(status) { + eprintln!( + "Test '{}' failed with exit code {}", + test.test_text, + libc::WEXITSTATUS(status) + ); + } else { + eprintln!("Test '{}' failed with wait status {status}", test.test_text); + } failures += 1; } } } let end = std::time::Instant::now(); + if executed == 0 { + eprintln!("No tests matched filter {filter:?}"); + std::process::exit(1); + } if failures > 0 { eprintln!( "{failures} tests failed in {} ms", diff --git a/usertest/src/pidfd.rs b/usertest/src/pidfd.rs new file mode 100644 index 00000000..8f8c8dd7 --- /dev/null +++ b/usertest/src/pidfd.rs @@ -0,0 +1,477 @@ +//! Linux pidfs statfs contract, exercised inside the AArch64 MOSS guest. +use crate::register_test; +use std::{ + ffi::CString, + mem::{size_of, zeroed}, + os::fd::{AsRawFd, FromRawFd, OwnedFd}, + ptr, +}; + +const PID_FS_MAGIC: u64 = 0x5049_4446; +const PIDFD_THREAD: i32 = libc::O_EXCL; +const ST_VALID: i64 = 0x20; + +// asm-generic/statfs.h, not the host's layout or libc's private fsid/spare fields. +#[repr(C)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +struct StatFs { + kind: i64, + bsize: i64, + blocks: u64, + bfree: u64, + bavail: u64, + files: u64, + ffree: u64, + fsid: [i32; 2], + namelen: i64, + frsize: i64, + flags: i64, + spare: [i64; 4], +} + +fn own_fd(fd: i32) -> OwnedFd { + assert!( + fd >= 0, + "fd creation failed: {}", + std::io::Error::last_os_error() + ); + unsafe { OwnedFd::from_raw_fd(fd) } +} + +fn pidfd(pid: i32, flags: i32) -> OwnedFd { + own_fd(unsafe { libc::syscall(libc::SYS_pidfd_open, pid, flags) } as i32) +} + +fn fstatfs(fd: i32) -> StatFs { + let mut stat = StatFs::default(); + assert_eq!( + unsafe { libc::syscall(libc::SYS_fstatfs, fd, &mut stat) }, + 0 + ); + stat +} + +fn statfs(path: &str) -> StatFs { + let path = CString::new(path).unwrap(); + let mut stat = StatFs::default(); + assert_eq!( + unsafe { libc::syscall(libc::SYS_statfs, path.as_ptr(), &mut stat) }, + 0, + "statfs({path:?}): {}", + std::io::Error::last_os_error() + ); + stat +} + +fn check(stat: StatFs) { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + assert!(page_size > 0); + assert_eq!(stat.kind as u64, PID_FS_MAGIC); + assert_eq!(stat.bsize, page_size as i64); + assert_eq!(stat.frsize, page_size as i64); + assert_eq!((stat.blocks, stat.bfree, stat.bavail), (0, 0, 0)); + assert_eq!((stat.files, stat.ffree), (0, 0)); + assert_ne!(stat.fsid, [0; 2]); // An instance ID, not a PID or the magic. + assert_eq!(stat.namelen, 255); + assert_eq!(stat.flags, ST_VALID); + assert_eq!(stat.spare, [0; 4]); +} + +fn expect_error(ret: libc::c_long, errno: i32) { + assert_eq!(ret, -1); + assert_eq!(std::io::Error::last_os_error().raw_os_error(), Some(errno)); +} + +fn wait_child(pid: i32) { + let mut status = 0; + loop { + let ret = unsafe { libc::waitpid(pid, &mut status, 0) }; + if ret == pid { + break; + } + expect_error(ret as _, libc::EINTR); + } + assert!(libc::WIFEXITED(status), "child wait status {status}"); + assert_eq!(libc::WEXITSTATUS(status), 0); +} + +fn test_pidfd_fstatfs() { + let fd = pidfd(unsafe { libc::getpid() }, 0); + let raw = fstatfs(fd.as_raw_fd()); + check(raw); + unsafe { + let mut stat: libc::statfs = zeroed(); + assert_eq!(libc::fstatfs(fd.as_raw_fd(), &mut stat), 0); + assert_eq!(size_of::(), size_of::()); + assert_eq!( + ptr::read_unaligned((&stat as *const libc::statfs).cast::()), + raw + ); + // GNU's large-file API is the same native 64-bit syscall on AArch64. + #[cfg(target_env = "gnu")] + { + let mut large: libc::statfs64 = zeroed(); + assert_eq!(libc::fstatfs64(fd.as_raw_fd(), &mut large), 0); + assert_eq!(size_of::(), size_of::()); + assert_eq!( + ptr::read_unaligned((&large as *const libc::statfs64).cast::()), + raw + ); + } + } +} +register_test!(test_pidfd_fstatfs); + +fn test_pidfd_statfs_abi_bounds() { + assert_eq!(size_of::(), 120); + let fd = pidfd(unsafe { libc::getpid() }, 0); + // Unaligned output, nonzero initial contents, and canaries on both ends. + let mut bytes = [0xa5u8; 122]; + let out = unsafe { bytes.as_mut_ptr().add(1).cast::() }; + assert_eq!( + unsafe { libc::syscall(libc::SYS_fstatfs, fd.as_raw_fd(), out) }, + 0 + ); + check(unsafe { ptr::read_unaligned(out) }); + assert_eq!(bytes[0], 0xa5); + assert_eq!(bytes[121], 0xa5); +} +register_test!(test_pidfd_statfs_abi_bounds); + +fn test_pidfd_statfs_instance_and_flags() { + let pid = unsafe { libc::getpid() }; + let baseline = fstatfs(pidfd(pid, 0).as_raw_fd()); + check(baseline); + for flags in [ + 0, + libc::O_NONBLOCK, + PIDFD_THREAD, + PIDFD_THREAD | libc::O_NONBLOCK, + ] { + let fd = pidfd(pid, flags); + assert_eq!( + unsafe { libc::fcntl(fd.as_raw_fd(), libc::F_GETFD) }, + libc::FD_CLOEXEC + ); + let dup = own_fd(unsafe { libc::dup(fd.as_raw_fd()) }); + assert_eq!(fstatfs(fd.as_raw_fd()), baseline); + drop(fd); + assert_eq!(fstatfs(dup.as_raw_fd()), baseline); + assert_eq!(unsafe { libc::fcntl(dup.as_raw_fd(), libc::F_GETFD) }, 0); + assert_eq!( + unsafe { libc::fcntl(dup.as_raw_fd(), libc::F_SETFL, libc::O_NONBLOCK) }, + 0 + ); + assert_eq!(fstatfs(dup.as_raw_fd()), baseline); + assert_eq!( + unsafe { libc::fcntl(dup.as_raw_fd(), libc::F_GETFL) } & PIDFD_THREAD, + flags & PIDFD_THREAD + ); + } + for path in ["/", "/tmp", "/proc"] { + let other = statfs(path); + assert_ne!(other.kind as u64, PID_FS_MAGIC); + assert_ne!(other.fsid, baseline.fsid); + } +} +register_test!(test_pidfd_statfs_instance_and_flags); + +fn test_pidfd_statfs_paths() { + let fd = pidfd(unsafe { libc::getpid() }, 0); + let baseline = fstatfs(fd.as_raw_fd()); + let path = format!("/proc/self/fd/{}", fd.as_raw_fd()); + assert_eq!(statfs(&path), baseline); + assert_eq!( + statfs(&format!( + "/proc/{}/fd/{}", + unsafe { libc::getpid() }, + fd.as_raw_fd() + )), + baseline + ); + assert_eq!( + std::fs::read_link(&path).unwrap().as_os_str(), + "anon_inode:[pidfd]" + ); + let mut stat = StatFs::default(); + for suffix in ["/", "/.", "/..", "/child"] { + let bad = CString::new(format!("{path}{suffix}")).unwrap(); + expect_error( + unsafe { libc::syscall(libc::SYS_statfs, bad.as_ptr(), &mut stat) }, + libc::ENOTDIR, + ); + } + let link = format!("/tmp/pidfd-statfs-{}", unsafe { libc::getpid() }); + std::os::unix::fs::symlink(&path, &link).unwrap(); + assert_eq!(statfs(&link), baseline); + std::fs::remove_file(&link).unwrap(); + let cwd = std::env::current_dir().unwrap(); + std::env::set_current_dir("/proc/self/fd").unwrap(); + assert_eq!(statfs(&fd.as_raw_fd().to_string()), baseline); + std::env::set_current_dir(cwd).unwrap(); + assert_ne!( + statfs(&format!("/proc/self/fdinfo/{}", fd.as_raw_fd())).kind as u64, + PID_FS_MAGIC + ); + drop(fd); + let path = CString::new(path).unwrap(); + expect_error( + unsafe { libc::syscall(libc::SYS_statfs, path.as_ptr(), &mut stat) }, + libc::ENOENT, + ); +} +register_test!(test_pidfd_statfs_paths); + +fn test_pidfd_statfs_lifetime_and_fork() { + unsafe { + libc::alarm(20); + } + let own = pidfd(unsafe { libc::getpid() }, 0); + let baseline = fstatfs(own.as_raw_fd()); + // A pipe keeps the target alive until its pidfds have been opened. + let mut pipe = [-1; 2]; + assert_eq!(unsafe { libc::pipe(pipe.as_mut_ptr()) }, 0); + let read = own_fd(pipe[0]); + let write = own_fd(pipe[1]); + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + libc::alarm(20); + } + let same = fstatfs(own.as_raw_fd()) == baseline; + let mut byte = 0u8; + let ret = unsafe { libc::read(read.as_raw_fd(), (&mut byte as *mut u8).cast(), 1) }; + unsafe { libc::_exit(if same && ret == 1 { 0 } else { 1 }) }; + } + let target = pidfd(child, 0); + let nonblock = pidfd(child, libc::O_NONBLOCK); + assert_eq!(fstatfs(target.as_raw_fd()), baseline); + assert_eq!(fstatfs(nonblock.as_raw_fd()), baseline); + assert_eq!( + unsafe { libc::write(write.as_raw_fd(), b"x".as_ptr().cast(), 1) }, + 1 + ); + wait_child(child); + // A reaped task must not turn a valid pidfd into ESRCH or change its FS. + assert_eq!(fstatfs(target.as_raw_fd()), baseline); + assert_eq!(fstatfs(nonblock.as_raw_fd()), baseline); + assert_eq!( + statfs(&format!("/proc/self/fd/{}", target.as_raw_fd())), + baseline + ); +} +register_test!(test_pidfd_statfs_lifetime_and_fork); + +fn test_pidfd_statfs_other_process_fd_table() { + unsafe { + libc::alarm(20); + } + // The child has fd 200 but the parent does not. /proc//fd lookup + // must consult the child's table, not the caller's table. + let self_fd = pidfd(unsafe { libc::getpid() }, 0); + let baseline = fstatfs(self_fd.as_raw_fd()); + let mut ready = [-1; 2]; + let mut done = [-1; 2]; + assert_eq!(unsafe { libc::pipe(ready.as_mut_ptr()) }, 0); + assert_eq!(unsafe { libc::pipe(done.as_mut_ptr()) }, 0); + let ready_r = own_fd(ready[0]); + let ready_w = own_fd(ready[1]); + let done_r = own_fd(done[0]); + let done_w = own_fd(done[1]); + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + unsafe { + libc::alarm(20); + } + assert_eq!(unsafe { libc::dup3(self_fd.as_raw_fd(), 200, 0) }, 200); + assert_eq!( + unsafe { libc::write(ready_w.as_raw_fd(), b"x".as_ptr().cast(), 1) }, + 1 + ); + let mut byte = 0u8; + assert_eq!( + unsafe { libc::read(done_r.as_raw_fd(), (&mut byte as *mut u8).cast(), 1) }, + 1 + ); + unsafe { libc::_exit(0) }; + } + let mut byte = 0u8; + assert_eq!( + unsafe { libc::read(ready_r.as_raw_fd(), (&mut byte as *mut u8).cast(), 1) }, + 1 + ); + let mut out = StatFs::default(); + expect_error( + unsafe { libc::syscall(libc::SYS_fstatfs, 200, &mut out) }, + libc::EBADF, + ); + assert_eq!(statfs(&format!("/proc/{child}/fd/200")), baseline); + assert_eq!( + unsafe { libc::write(done_w.as_raw_fd(), b"x".as_ptr().cast(), 1) }, + 1 + ); + wait_child(child); +} +register_test!(test_pidfd_statfs_other_process_fd_table); + +fn test_pidfd_statfs_errors() { + let fd = pidfd(unsafe { libc::getpid() }, 0); + let mut out = StatFs::default(); + for bad_fd in [-1, i32::MAX] { + expect_error( + unsafe { libc::syscall(libc::SYS_fstatfs, bad_fd, &mut out) }, + libc::EBADF, + ); + expect_error( + unsafe { libc::syscall(libc::SYS_fstatfs, bad_fd, ptr::null_mut::()) }, + libc::EBADF, + ); + assert_eq!(out, StatFs::default()); + } + expect_error( + unsafe { libc::syscall(libc::SYS_fstatfs, fd.as_raw_fd(), ptr::null_mut::()) }, + libc::EFAULT, + ); + let path = CString::new(format!("/proc/self/fd/{}", fd.as_raw_fd())).unwrap(); + expect_error( + unsafe { libc::syscall(libc::SYS_statfs, path.as_ptr(), ptr::null_mut::()) }, + libc::EFAULT, + ); + expect_error( + unsafe { libc::syscall(libc::SYS_statfs, ptr::null::(), &mut out) }, + libc::EFAULT, + ); + expect_error( + unsafe { libc::syscall(libc::SYS_statfs, c"".as_ptr(), &mut out) }, + libc::ENOENT, + ); + let closed = fd.as_raw_fd(); + drop(fd); + expect_error( + unsafe { libc::syscall(libc::SYS_fstatfs, closed, &mut out) }, + libc::EBADF, + ); + // Bad targets must not manufacture pidfs inodes or panic in the kernel. + for flags in [0, PIDFD_THREAD] { + expect_error( + unsafe { libc::syscall(libc::SYS_pidfd_open, i32::MAX, flags) }, + libc::ESRCH, + ); + } + for pid in [-1, 0] { + expect_error( + unsafe { libc::syscall(libc::SYS_pidfd_open, pid, 0) }, + libc::EINVAL, + ); + } +} +register_test!(test_pidfd_statfs_errors); + +fn test_pidfd_statfs_faulting_buffers() { + let fd = pidfd(unsafe { libc::getpid() }, 0); + let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize; + unsafe { + let mapping = libc::mmap( + ptr::null_mut(), + page * 2, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ); + assert_ne!(mapping, libc::MAP_FAILED); + let bytes = mapping.cast::(); + ptr::write_bytes(bytes, 0xa5, page * 2); + assert_eq!(libc::mprotect(mapping, page, libc::PROT_READ), 0); + expect_error( + libc::syscall(libc::SYS_fstatfs, fd.as_raw_fd(), mapping), + libc::EFAULT, + ); + assert!( + std::slice::from_raw_parts(bytes, page) + .iter() + .all(|x| *x == 0xa5) + ); + assert_eq!( + libc::mprotect(mapping, page, libc::PROT_READ | libc::PROT_WRITE), + 0 + ); + assert_eq!(libc::munmap(bytes.add(page).cast(), page), 0); + let exact = bytes.add(page - size_of::()).cast::(); + assert_eq!(libc::syscall(libc::SYS_fstatfs, fd.as_raw_fd(), exact), 0); + check(ptr::read_unaligned(exact)); + expect_error( + libc::syscall(libc::SYS_fstatfs, fd.as_raw_fd(), bytes.add(page - 1)), + libc::EFAULT, + ); + assert_eq!(libc::munmap(mapping, page), 0); + } + check(fstatfs(fd.as_raw_fd())); +} +register_test!(test_pidfd_statfs_faulting_buffers); + +fn test_pidfd_statfs_threads() { + unsafe { + libc::alarm(20); + } + let baseline = fstatfs(pidfd(unsafe { libc::getpid() }, 0).as_raw_fd()); + let (tx, rx) = std::sync::mpsc::channel(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let thread = std::thread::spawn(move || { + let tid = unsafe { libc::syscall(libc::SYS_gettid) } as i32; + tx.send(tid).unwrap(); + done_rx.recv().unwrap(); + }); + let tid = rx.recv().unwrap(); + expect_error( + unsafe { libc::syscall(libc::SYS_pidfd_open, tid, 0) }, + libc::ESRCH, + ); + let fd = pidfd(tid, PIDFD_THREAD); + assert_eq!(fstatfs(fd.as_raw_fd()), baseline); + done_tx.send(()).unwrap(); + thread.join().unwrap(); + assert_eq!(fstatfs(fd.as_raw_fd()), baseline); + // Concurrent opens and statfs calls must all observe one pidfs instance. + std::thread::scope(|scope| { + for _ in 0..8 { + scope.spawn(|| { + for _ in 0..32 { + let other = pidfd(unsafe { libc::getpid() }, 0); + assert_eq!(fstatfs(other.as_raw_fd()), baseline); + assert_eq!(fstatfs(fd.as_raw_fd()), baseline); + } + }); + } + }); +} +register_test!(test_pidfd_statfs_threads); + +fn test_pidfd_statvfs_libc() { + let fd = pidfd(unsafe { libc::getpid() }, 0); + let raw = fstatfs(fd.as_raw_fd()); + let path = CString::new(format!("/proc/self/fd/{}", fd.as_raw_fd())).unwrap(); + unsafe { + let mut stat: libc::statvfs = zeroed(); + assert_eq!(libc::fstatvfs(fd.as_raw_fd(), &mut stat), 0); + let mut by_path: libc::statvfs = zeroed(); + assert_eq!(libc::statvfs(path.as_ptr(), &mut by_path), 0); + for v in [stat, by_path] { + assert_eq!(v.f_bsize as i64, raw.bsize); + assert_eq!(v.f_frsize as i64, raw.frsize); + assert_eq!((v.f_blocks, v.f_bfree, v.f_bavail), (0, 0, 0)); + assert_eq!((v.f_files, v.f_ffree, v.f_favail), (0, 0, 0)); + assert_eq!(v.f_namemax, 255); + assert_eq!(v.f_fsid, stat.f_fsid); + // GNU strips ST_VALID; musl preserves it. The raw statfs ABI + // is identical and checked separately, independent of libc. + #[cfg(target_env = "gnu")] + assert_eq!(v.f_flag, 0); + #[cfg(target_env = "musl")] + assert_eq!(v.f_flag, ST_VALID as libc::c_ulong); + } + } +} +register_test!(test_pidfd_statvfs_libc); diff --git a/usertest/src/sched.rs b/usertest/src/sched.rs new file mode 100644 index 00000000..05a9e75d --- /dev/null +++ b/usertest/src/sched.rs @@ -0,0 +1,70 @@ +use crate::register_test; +use std::{ + cell::Cell, + hint::black_box, + sync::{Arc, Barrier}, + thread, +}; + +thread_local! { + static YIELD_MARKER: Cell = const { Cell::new(0) }; +} + +fn test_sched_yield_context() { + // Every registered test runs in a child process. Turn a lost-thread/join + // hang into a failed test, even if the corrupted thread cannot panic. + unsafe { + assert_ne!(libc::signal(libc::SIGALRM, libc::SIG_DFL), libc::SIG_ERR); + let timer = libc::itimerval { + it_interval: libc::timeval { + tv_sec: 0, + tv_usec: 0, + }, + it_value: libc::timeval { + tv_sec: 60, + tv_usec: 0, + }, + }; + assert_eq!( + libc::setitimer(libc::ITIMER_REAL, &timer, std::ptr::null_mut()), + 0 + ); + } + + let barrier = Arc::new(Barrier::new(2)); + let threads: Vec<_> = (1..=2) + .map(|marker| { + let barrier = barrier.clone(); + thread::spawn(move || { + let tid = unsafe { libc::syscall(libc::SYS_gettid) }; + let stack_marker = [marker; 16]; + let stack_address = black_box(stack_marker.as_ptr()); + YIELD_MARKER.set(marker); + barrier.wait(); + + for _ in 0..10_000 { + assert_eq!(unsafe { libc::sched_yield() }, 0); + assert_eq!(unsafe { libc::syscall(libc::SYS_gettid) }, tid); + assert_eq!(YIELD_MARKER.get(), marker); + assert_eq!(black_box(stack_marker.as_ptr()), stack_address); + assert_eq!(black_box(stack_marker), [marker; 16]); + } + marker + }) + }) + .collect(); + + for (index, thread) in threads.into_iter().enumerate() { + assert_eq!(thread.join().unwrap(), index + 1); + } + + unsafe { + let timer: libc::itimerval = std::mem::zeroed(); + assert_eq!( + libc::setitimer(libc::ITIMER_REAL, &timer, std::ptr::null_mut()), + 0 + ); + } +} + +register_test!(test_sched_yield_context); diff --git a/usertest/src/signals.rs b/usertest/src/signals.rs index b51920ff..3d2973d6 100644 --- a/usertest/src/signals.rs +++ b/usertest/src/signals.rs @@ -6,6 +6,49 @@ use std::{ static SIGNAL_CAUGHT: AtomicBool = AtomicBool::new(false); +fn test_sigaltstack_disable_and_query() { + unsafe { + let mut previous: libc::stack_t = std::mem::zeroed(); + assert_eq!(libc::sigaltstack(ptr::null(), &mut previous), 0); + // Rust may already have installed its stack-overflow signal stack. + // Explicitly disable it instead of assuming the initial state. + let disabled = libc::stack_t { + ss_sp: ptr::null_mut(), + ss_flags: libc::SS_DISABLE, + ss_size: 0, + }; + assert_eq!(libc::sigaltstack(&disabled, ptr::null_mut()), 0); + let mut initial: libc::stack_t = std::mem::zeroed(); + assert_eq!(libc::sigaltstack(ptr::null(), &mut initial), 0); + assert_ne!(initial.ss_flags & libc::SS_DISABLE, 0); + + let mut memory = vec![0u8; 64 * 1024]; + let enabled = libc::stack_t { + ss_sp: memory.as_mut_ptr().cast(), + ss_flags: 0, + ss_size: memory.len(), + }; + assert_eq!(libc::sigaltstack(&enabled, ptr::null_mut()), 0); + let mut queried: libc::stack_t = std::mem::zeroed(); + assert_eq!(libc::sigaltstack(ptr::null(), &mut queried), 0); + assert_eq!(queried.ss_flags, 0); + assert_eq!(queried.ss_sp, enabled.ss_sp); + assert_eq!(queried.ss_size, enabled.ss_size); + + let disabled = libc::stack_t { + ss_sp: ptr::null_mut(), + ss_flags: libc::SS_DISABLE, + ss_size: 0, + }; + assert_eq!(libc::sigaltstack(&disabled, ptr::null_mut()), 0); + assert_eq!(libc::sigaltstack(ptr::null(), &mut queried), 0); + assert_ne!(queried.ss_flags & libc::SS_DISABLE, 0); + assert_eq!(libc::sigaltstack(&previous, ptr::null_mut()), 0); + } +} + +register_test!(test_sigaltstack_disable_and_query); + extern "C" fn signal_handler(_: libc::c_int) { SIGNAL_CAUGHT.store(true, Ordering::Relaxed); }