Skip to content

Improve Windows virtual memory fidelity #1229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Weiteng Chen (CvvT) merged 3 commits into from
Aug 27, 2026
+324 −22
Merged

Improve Windows virtual memory fidelity #1229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7161fb7
Improve Windows virtual memory fidelity
CvvT Aug 25, 2026
ec384b9
Fix Windows virtual memory edge cases
CvvT Aug 27, 2026
ab27bcf
clean up code
CvvT Aug 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
289 changes: 268 additions & 21 deletions litebox_shim_windows/src/syscalls/mm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use litebox_common_windows::nt_status::NtStatus;
use rangemap::RangeMap;
use zerocopy::{FromBytes, Immutable, IntoBytes};

use crate::syscalls::ProcessHandle;
use crate::syscalls::{ProcessHandle, section::pagefile_view_protection_is_compatible};
use crate::{
ConstPtr, MutPtr, PAGE_SIZE, ShimFS, ShimPlatform, Task, WindowsPageManager,
WindowsVirtualAllocation, WindowsVirtualAllocations,
Expand All @@ -37,6 +37,7 @@ bitflags::bitflags! {
const PAGE_GUARD = 0x100;
const PAGE_NOCACHE = 0x200;
const PAGE_WRITECOMBINE = 0x400;
const PAGE_TARGETS_INVALID = 0x40000000;
}
}

Expand All @@ -52,10 +53,21 @@ impl PageProtection {
let guard = self.contains(Self::PAGE_GUARD);
let nocache = self.contains(Self::PAGE_NOCACHE);
let writecombine = self.contains(Self::PAGE_WRITECOMBINE);
// TODO(windows-cfg): Enforce target invalidation for allocations and no-update semantics
// for protection changes; currently this operation flag is only validated.
let targets_invalid = self.contains(Self::PAGE_TARGETS_INVALID);
let executable = matches!(
self.base(),
value if value == Self::PAGE_EXECUTE.bits()
|| value == Self::PAGE_EXECUTE_READ.bits()
|| value == Self::PAGE_EXECUTE_READWRITE.bits()
|| value == Self::PAGE_EXECUTE_WRITECOPY.bits()
);

!(noaccess && (guard || nocache || writecombine)
|| guard && (nocache || writecombine)
|| nocache && writecombine)
|| nocache && writecombine
|| targets_invalid && !executable)
}
}

Expand Down Expand Up @@ -303,6 +315,14 @@ impl<Platform: ShimPlatform, FS: ShimFS> Task<Platform, FS> {
|| (zero_bits > 21 && zero_bits < 32)
|| (zero_bits != 0 && base != 0)
{
litebox_util_log::debug!(
base:% = format_args!("{base:#x}"),
size,
zero_bits:% = format_args!("{zero_bits:#x}"),
allocation_type:% = format_args!("{:#x}", allocation_type.bits()),
protect:% = format_args!("{protect:#x}");
"Rejected NtAllocateVirtualMemory parameters"
);
return NtStatus::INVALID_PARAMETER;
}
if allocation_type.contains(AllocationType::MEM_RESET) {
Expand Down Expand Up @@ -452,14 +472,21 @@ impl<Platform: ShimPlatform, FS: ShimFS> Task<Platform, FS> {
base_address: MutPtr<Platform, usize>,
region_size: MutPtr<Platform, usize>,
) -> NtStatus {
if find_private_virtual_allocation(
&self.process.virtual_allocations,
aligned_base,
aligned_len,
)
.is_none()
{
let Some(allocation) =
find_virtual_allocation(&self.process.virtual_allocations, aligned_base, aligned_len)
else {
return NtStatus::INVALID_PARAMETER;
};
if !matches!(
allocation.type_,
MemoryType::MEM_PRIVATE | MemoryType::MEM_MAPPED
) {
return NtStatus::INVALID_PARAMETER;
}
if allocation.type_ == MemoryType::MEM_MAPPED
&& !pagefile_view_protection_is_compatible(allocation.allocation_protect, protect)
{
return NtStatus::SECTION_PROTECTION;
}
if update_permissions(
&self.global.page_manager,
Expand Down Expand Up @@ -519,7 +546,7 @@ impl<Platform: ShimPlatform, FS: ShimFS> Task<Platform, FS> {
}

let Some((aligned_base, aligned_len)) =
free_region(&self.process.virtual_allocations, base, size, free_type)
free_region(&self.process.virtual_allocations, base, size)
else {
return NtStatus::INVALID_PARAMETER;
};
Expand Down Expand Up @@ -548,10 +575,11 @@ impl<Platform: ShimPlatform, FS: ShimFS> Task<Platform, FS> {
if unsafe { self.global.page_manager.remove_pages(ptr, aligned_len) }.is_err() {
return NtStatus::UNABLE_TO_FREE_VM;
}
self.process
.virtual_allocations
.write()
.remove(&aligned_base);
split_virtual_allocation_around_released_region(
&self.process.virtual_allocations,
aligned_base,
aligned_len,
);
}
if base_address.write_at_offset(0, aligned_base).is_none()
|| region_size.write_at_offset(0, aligned_len).is_none()
Expand Down Expand Up @@ -1077,26 +1105,65 @@ fn free_region<Platform: ShimPlatform>(
virtual_allocations: &WindowsVirtualAllocations<Platform>,
base: usize,
size: usize,
free_type: FreeType,
) -> Option<(usize, usize)> {
if size == 0 {
let allocation_base = base & !(PAGE_SIZE - 1);
let allocation = virtual_allocations
.read()
.get(&base)
.get(&allocation_base)
.filter(|allocation| allocation.type_ == MemoryType::MEM_PRIVATE)
.cloned()?;
return Some((allocation.base, allocation.size));
}

if free_type == FreeType::MEM_RELEASE {
return None;
}

let (aligned_base, aligned_len) = page_aligned_region(base, size)?;
find_private_virtual_allocation(virtual_allocations, aligned_base, aligned_len)?;
Some((aligned_base, aligned_len))
}

fn split_virtual_allocation_around_released_region<Platform: ShimPlatform>(
virtual_allocations: &WindowsVirtualAllocations<Platform>,
released_base: usize,
released_size: usize,
) {
let Some(released_end) = released_base.checked_add(released_size) else {
return;
};
let mut allocations = virtual_allocations.write();
let Some((&allocation_key, allocation)) = allocations.range(..=released_base).next_back()
else {
return;
};
let allocation = allocation.clone();
let Some(allocation_end) = allocation.base.checked_add(allocation.size) else {
return;
};
allocations.remove(&allocation_key);

for (base, end) in [
(allocation.base, released_base),
(released_end, allocation_end),
] {
if base >= end {
continue;
}
let mut pages = RangeMap::new();
for (range, protect) in allocation.pages.overlapping(base..end) {
pages.insert(range.start.max(base)..range.end.min(end), *protect);
}
allocations.insert(
base,
WindowsVirtualAllocation {
base,
size: end - base,
allocation_protect: allocation.allocation_protect,
type_: allocation.type_,
pages,
},
);
}
}

fn committed_pages(
base: usize,
size: usize,
Expand Down Expand Up @@ -1232,7 +1299,7 @@ pub(super) fn parse_page_protection(
) -> Option<(PageProtection, MemoryRegionPermissions)> {
let protect = PageProtection::from_bits(protect)?;
let permissions = page_protect_to_permissions(protect)?;
Some((protect, permissions))
Some((protect - PageProtection::PAGE_TARGETS_INVALID, permissions))
}

fn page_protect_to_permissions(protect: PageProtection) -> Option<MemoryRegionPermissions> {
Expand Down Expand Up @@ -1778,6 +1845,36 @@ mod tests {
info
}

#[test]
fn page_targets_invalid_is_not_reported_as_page_protection() {
run_with_test_platform_pointers(|| {
let task = crate::tests::test_task();
let mut base = 0usize;
let mut region_size = PAGE_SIZE;
assert_eq!(
task.sys_nt_allocate_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut base),
0,
mut_ptr(&mut region_size),
(AllocationType::MEM_RESERVE | AllocationType::MEM_COMMIT).bits(),
(PageProtection::PAGE_EXECUTE_READ | PageProtection::PAGE_TARGETS_INVALID)
.bits(),
),
NtStatus::SUCCESS
);

let info = query_basic_information(&task, base);
assert_eq!(
info.allocation_protect,
PageProtection::PAGE_EXECUTE_READ.bits()
);
assert_eq!(info.protect, PageProtection::PAGE_EXECUTE_READ.bits());

release_allocation(&task, base);
});
}

#[test]
fn read_virtual_memory_copies_across_chunks() {
run_with_test_platform_pointers(|| {
Expand Down Expand Up @@ -2237,6 +2334,156 @@ mod tests {
});
}

#[test]
fn free_virtual_memory_partial_release_preserves_suffix_for_commit() {
run_with_test_platform_pointers(|| {
let task = crate::tests::test_task();
let mut first_base = 0usize;
let mut first_size = 0x100000usize;
assert_eq!(
task.sys_nt_allocate_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut first_base),
0,
mut_ptr(&mut first_size),
AllocationType::MEM_RESERVE.bits(),
PageProtection::PAGE_READWRITE.bits(),
),
NtStatus::SUCCESS
);
let mut second_base = 0usize;
let mut second_size = 28080usize;
assert_eq!(
task.sys_nt_allocate_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut second_base),
0,
mut_ptr(&mut second_size),
AllocationType::MEM_RESERVE.bits(),
PageProtection::PAGE_READWRITE.bits(),
),
NtStatus::SUCCESS
);
let mut base = 0usize;
let mut region_size = 0x1d0000usize;
assert_eq!(
task.sys_nt_allocate_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut base),
0,
mut_ptr(&mut region_size),
AllocationType::MEM_RESERVE.bits(),
PageProtection::PAGE_READWRITE.bits(),
),
NtStatus::SUCCESS
);

let mut release_base = base;
let mut release_size = 0x1c0000usize;
assert_eq!(
task.sys_nt_free_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut release_base),
mut_ptr(&mut release_size),
FreeType::MEM_RELEASE.bits(),
),
NtStatus::SUCCESS
);
assert_eq!(release_base, base);
assert_eq!(release_size, 0x1c0000);

let suffix_base = base + 0x1c0000;
let mut commit_base = suffix_base;
let mut commit_size = PAGE_SIZE * 2;
assert_eq!(
task.sys_nt_allocate_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut commit_base),
0,
mut_ptr(&mut commit_size),
AllocationType::MEM_COMMIT.bits(),
PageProtection::PAGE_READWRITE.bits(),
),
NtStatus::SUCCESS
);
assert_eq!(commit_base, suffix_base);
assert_eq!(commit_size, PAGE_SIZE * 2);
let committed = MutPtr::<TestPlatform, u8>::from_usize(commit_base);
assert_eq!(committed.write_at_offset(0, 0x5a), Some(()));
assert_eq!(committed.read_at_offset(0), Some(0x5a));

release_allocation(&task, first_base);
release_allocation(&task, second_base);
release_allocation(&task, suffix_base);
});
}

#[test]
fn free_virtual_memory_middle_release_preserves_fragment_protections() {
run_with_test_platform_pointers(|| {
let task = crate::tests::test_task();
let (base, _) = allocate_committed_rw(&task, PAGE_SIZE * 3);

let mut prefix_base = base;
let mut prefix_size = PAGE_SIZE;
let mut old_protect = 0u32;
assert_eq!(
task.sys_nt_protect_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut prefix_base),
mut_ptr(&mut prefix_size),
PageProtection::PAGE_READONLY.bits(),
mut_ptr(&mut old_protect),
),
NtStatus::SUCCESS
);

let suffix_base = base + PAGE_SIZE * 2;
let mut protect_suffix_base = suffix_base;
let mut suffix_size = PAGE_SIZE;
assert_eq!(
task.sys_nt_protect_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut protect_suffix_base),
mut_ptr(&mut suffix_size),
PageProtection::PAGE_EXECUTE_READ.bits(),
mut_ptr(&mut old_protect),
),
NtStatus::SUCCESS
);

let mut release_base = base + PAGE_SIZE;
let mut release_size = PAGE_SIZE;
assert_eq!(
task.sys_nt_free_virtual_memory(
ProcessHandle::CURRENT,
mut_ptr(&mut release_base),
mut_ptr(&mut release_size),
FreeType::MEM_RELEASE.bits(),
),
NtStatus::SUCCESS
);

let prefix = query_basic_information(&task, base);
assert_eq!(prefix.allocation_base, base);
assert_eq!(prefix.region_size, PAGE_SIZE);
assert_eq!(prefix.protect, PageProtection::PAGE_READONLY.bits());

let released = query_basic_information(&task, base + PAGE_SIZE);
assert_eq!(released.allocation_base, 0);
assert_eq!(released.region_size, PAGE_SIZE);
assert_eq!(released.state, MemoryState::MEM_FREE.bits());

let suffix = query_basic_information(&task, suffix_base);
assert_eq!(suffix.allocation_base, suffix_base);
assert_eq!(suffix.region_size, PAGE_SIZE);
assert_eq!(suffix.protect, PageProtection::PAGE_EXECUTE_READ.bits());

release_allocation(&task, base);
release_allocation(&task, suffix_base);
});
}

#[test]
fn free_virtual_memory_decommit_discards_page_contents() {
run_with_test_platform_pointers(|| {
Expand Down
Loading