Fix some overflows when allocating a max-size fiber stack (#12868)

* Fix some overflows when allocating a max-size fiber stack

These are some minor issues that won't actually surface in practice but
seem good to fix nonetheless. Allocating a max-size fiber stack should
fail, and it shouldn't fail with a panic or a debug assert.

* Fix CI

Author: alexcrichton

crates/fiber/src/lib.rs

@@ -428,4 +428,13 @@ mod tests {
         assert_eq!(fiber.resume(2.0), Err(4));
         assert_eq!(fiber.resume(3.0), Ok("hello".to_string()));
     }
+
+    #[test]
+    fn fiber_stack_max_size() {
+        if cfg!(windows) {
+            return;
+        }
+        assert!(FiberStack::new(usize::MAX, true).is_err());
+        assert!(FiberStack::new(usize::MAX, false).is_err());
+    }
 }

crates/fiber/src/nostd.rs

@@ -67,7 +67,7 @@ impl FiberStack {
         // Round up the size to at least one page.
         let size = core::cmp::max(4096, size);
         let mut storage = Vec::new();
-        storage.reserve_exact(size);
+        storage.try_reserve_exact(size)?;
         if zeroed {
             storage.resize(size, 0);
         }

crates/fiber/src/unix.rs

@@ -175,7 +175,10 @@ impl MmapFiberStack {
         let size = if size == 0 {
             page_size
         } else {
-            (size + (page_size - 1)) & (!(page_size - 1))
+            let with_extra = size
+                .checked_add(page_size - 1)
+                .ok_or(io::ErrorKind::OutOfMemory)?;
+            with_extra & (!(page_size - 1))
         };
 
         unsafe {
@@ -451,7 +454,9 @@ mod asan {
 
     pub fn new_fiber_stack(size: usize) -> std::io::Result<Box<dyn RuntimeFiberStack>> {
         let page_size = host_page_size();
-        let needed_size = size + page_size;
+        let needed_size = size
+            .checked_add(page_size)
+            .ok_or(std::io::ErrorKind::OutOfMemory)?;
         let mut stacks = FIBER_STACKS.lock().unwrap();
 
         let stack = match stacks.iter().position(|i| needed_size <= i.mapping_len) {