optimizations to libcc2rs related with strings

Reduces heap allocations in the common case

Author: nunoplopes

docs/src/runtime/cstr.md

@@ -10,6 +10,17 @@ instead of undefined behavior), `memset`, `memcmp`, and `to_rust_string` for
 crossing into Rust APIs.
 
 `CStringIterator`, returned by `to_c_string_iterator`, walks the bytes of a
-`Ptr<u8>` up to the null terminator; the `string.h` rules are built on it,
-`to_rust_string` collects it into a `String`, and `Display` for `Ptr<u8>` prints
-it, so a C string can be formatted directly.
+`Ptr<u8>` up to the null terminator; the `string.h` rules are built on it, and
+`Display` for `Ptr<u8>` prints it, so a C string can be formatted directly.
+
+Copying a string does not go through the iterator, which would grow its
+destination as it goes. `with_c_str` scans for the null terminator in the
+backing storage and lends the bytes to a closure, so measuring or borrowing a
+string allocates nothing (`c_str_len`, and `count` on a `CStringIterator`, are
+built on it). `to_c_bytes` and `to_rust_string`, and through them `strdup` and
+the `std::string` constructors, copy the string with a single allocation.
+`to_c_bytes` leaves room for one more byte, which callers usually spend on the
+terminator or a newline.
+
+Iterating a `Ptr<T>` reports how many elements are left, so collecting a `Ptr`
+into a `Vec` also allocates once.

libcc2rs/src/alloc.rs

@@ -31,7 +31,7 @@ pub fn calloc_refcount(a0: usize, a1: usize) -> AnyPtr {
 }
 
 pub fn strdup_refcount(a0: Ptr<u8>) -> Ptr<u8> {
-    let mut bytes: Vec<u8> = a0.to_c_string_iterator().collect();
+    let mut bytes = a0.to_c_bytes();
     bytes.push(0);
     Ptr::alloc_array(bytes.into_boxed_slice())
 }

libcc2rs/src/cstr.rs

@@ -117,8 +117,116 @@ impl Ptr<u8> {
         CStringIterator { ptr: self.clone() }
     }
 
+    // Calls `f` with the bytes of the C string, excluding the terminating NUL.
+    // The bytes are borrowed from the allocation, so `f` must not write to it.
+    pub fn with_c_str<R>(&self, f: impl FnOnce(&[u8]) -> R) -> R {
+        fn until_nul(tail: &[u8]) -> &[u8] {
+            match tail.iter().position(|&b| b == 0) {
+                Some(len) => &tail[..len],
+                None => panic!("ub: unterminated string"),
+            }
+        }
+        match &self.kind {
+            PtrKind::Null => panic!("ub: null pointer"),
+            PtrKind::StackSingle(weak) | PtrKind::HeapSingle(weak) => {
+                assert_eq!(self.offset, 0, "ub: invalid offset");
+                let rc = weak.upgrade().expect("ub: dangling pointer");
+                let b = rc.borrow();
+                f(until_nul(std::slice::from_ref(&*b)))
+            }
+            PtrKind::StackArray(weak) | PtrKind::HeapArray(weak) => {
+                let rc = weak.upgrade().expect("ub: dangling pointer");
+                let b = rc.borrow();
+                f(until_nul(&b[self.offset..]))
+            }
+            PtrKind::StackVec(weak) | PtrKind::HeapVec(weak) => {
+                let rc = weak.upgrade().expect("ub: dangling pointer");
+                let b = rc.borrow();
+                f(until_nul(&b[self.offset..]))
+            }
+            PtrKind::Reinterpreted(_) => f(&self.to_c_string_iterator().collect::<Vec<u8>>()),
+        }
+    }
+
+    // The length of the C string, like `strlen`.
+    pub fn c_str_len(&self) -> usize {
+        self.with_c_str(|s| s.len())
+    }
+
+    // Copies the C string, excluding the terminating NUL, using a single
+    // allocation. The vector has room for one more byte, which is what callers
+    // usually append (a NUL or a newline).
+    pub fn to_c_bytes(&self) -> Vec<u8> {
+        self.with_c_str(|s| {
+            let mut bytes = Vec::with_capacity(s.len() + 1);
+            bytes.extend_from_slice(s);
+            bytes
+        })
+    }
+
     pub fn to_rust_string(&self) -> String {
-        let bytes: Vec<u8> = self.to_c_string_iterator().collect();
-        String::from_utf8_lossy(&bytes).into_owned()
+        self.with_c_str(|s| String::from_utf8_lossy(s).into_owned())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::AsPointer;
+
+    // "ab\0cd\0": C string functions must stop at the first NUL.
+    const EMBEDDED_NUL: [u8; 6] = [b'a', b'b', 0, b'c', b'd', 0];
+
+    fn check(p: &Ptr<u8>) {
+        assert_eq!(p.c_str_len(), 2);
+        assert_eq!(p.to_c_bytes(), b"ab");
+        assert_eq!(p.to_rust_string(), "ab");
+        assert_eq!(p.to_c_string_iterator().count(), 2);
+        assert_eq!(p.to_c_string_iterator().collect::<Vec<u8>>(), b"ab");
+        assert_eq!(p.with_c_str(|s| s.to_vec()), b"ab");
+        assert_eq!(p.offset(3).to_c_bytes(), b"cd");
+        assert_eq!(p.offset(2).c_str_len(), 0);
+    }
+
+    #[test]
+    fn stops_at_first_nul_in_array() {
+        let p = Ptr::alloc_array(EMBEDDED_NUL.to_vec().into_boxed_slice());
+        check(&p);
+        p.delete();
+    }
+
+    #[test]
+    fn stops_at_first_nul_in_vec() {
+        let v = std::rc::Rc::new(RefCell::new(EMBEDDED_NUL.to_vec()));
+        check(&v.as_pointer());
+    }
+
+    #[test]
+    fn stops_at_first_nul_in_reinterpreted_view() {
+        let p: Ptr<u16> = Ptr::alloc_array(
+            EMBEDDED_NUL
+                .chunks(2)
+                .map(|c| u16::from_ne_bytes([c[0], c[1]]))
+                .collect::<Vec<u16>>()
+                .into_boxed_slice(),
+        );
+        let bytes = p.reinterpret_cast::<u8>();
+        check(&bytes);
+        bytes.delete();
+    }
+
+    #[test]
+    fn empty_string() {
+        let p = Ptr::<u8>::from_string_literal(b"");
+        assert_eq!(p.c_str_len(), 0);
+        assert!(p.to_c_bytes().is_empty());
+        assert_eq!(p.to_rust_string(), "");
+    }
+
+    #[test]
+    #[should_panic(expected = "ub: unterminated string")]
+    fn unterminated_string_panics() {
+        let p = Ptr::alloc_array(vec![b'a', b'b'].into_boxed_slice());
+        p.c_str_len();
     }
 }

libcc2rs/src/iterators.rs

@@ -232,6 +232,11 @@ pub struct CStringIterator {
 
 impl Iterator for CStringIterator {
     type Item = u8;
+
+    fn count(self) -> usize {
+        self.ptr.c_str_len()
+    }
+
     fn next(&mut self) -> Option<Self::Item> {
         // read until the null terminator
         match self.ptr.read() {

libcc2rs/src/rc.rs

@@ -655,6 +655,13 @@ impl<T> Iterator for Ptr<T> {
             None
         }
     }
+
+    // Lets collections built from the iterator (e.g. `std::string`'s
+    // constructors) allocate their storage once instead of growing it.
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let remaining = self.len().saturating_sub(self.get_offset());
+        (remaining, Some(remaining))
+    }
 }
 
 // Ptr iterator that yields values instead of pointers.

rules/stdio/tgt_refcount.rs

@@ -72,15 +72,15 @@ fn f11(a0: i32, a1: Ptr<CFile>) -> i32 {
 }
 
 fn f12(a0: Ptr<u8>, a1: Ptr<CFile>) -> i32 {
-    let __bytes: Vec<u8> = a0.to_c_string_iterator().collect();
+    let __bytes = a0.to_c_bytes();
     match a1.with_mut(|__f| __f.write(&__bytes)) == __bytes.len() {
         true => 0,
         false => -1,
     }
 }
 
 fn f13(a0: Ptr<u8>) -> i32 {
-    let mut __bytes: Vec<u8> = a0.to_c_string_iterator().collect();
+    let mut __bytes = a0.to_c_bytes();
     __bytes.push(b'\n');
     match libcc2rs::c_stdout().with_mut(|__f| __f.write(&__bytes)) == __bytes.len() {
         true => 0,

rules/string/tgt_refcount.rs

@@ -23,7 +23,7 @@ fn f1(a0: Vec<u8>, a1: usize, a2: usize) -> Vec<u8> {
 fn f3(a0: Vec<u8>, a1: Ptr<u8>) -> Vec<u8> {
     let mut r = a0;
     r.pop();
-    r.extend(a1.to_c_string_iterator());
+    a1.with_c_str(|__s| r.extend_from_slice(__s));
     r.push(0);
     r
 }
@@ -53,9 +53,9 @@ fn f7(a0: Ptr<u8>, a1: usize) -> Vec<u8> {
 }
 
 fn f10(a0: Ptr<u8>) -> Vec<u8> {
-    a0.to_c_string_iterator()
-        .chain(std::iter::once(0))
-        .collect::<Vec<u8>>()
+    let mut __bytes = a0.to_c_bytes();
+    __bytes.push(0);
+    __bytes
 }
 
 fn f11(a0: Ptr<u8>) -> Ptr<u8> {
@@ -83,18 +83,19 @@ fn f15(a0: Ptr<u8>) -> Ptr<u8> {
 }
 
 fn f16(a0: Vec<u8>, a1: Ptr<u8>) -> usize {
-    let __lookup: Vec<u8> = a1.to_c_string_iterator().collect();
-    a0.iter()
-        .take(a0.len().saturating_sub(1))
-        .rposition(|&x| __lookup.contains(&x))
-        .unwrap_or(usize::MAX)
+    a1.with_c_str(|__lookup| {
+        a0.iter()
+            .take(a0.len().saturating_sub(1))
+            .rposition(|&x| __lookup.contains(&x))
+            .unwrap_or(usize::MAX)
+    })
 }
 
 // TODO: This should modify a0 in place
 fn f17(a0: Vec<u8>, a1: Ptr<u8>) -> Vec<u8> {
     let mut __tmp2 = a0;
     __tmp2.pop();
-    __tmp2.extend(a1.to_c_string_iterator());
+    a1.with_c_str(|__s| __tmp2.extend_from_slice(__s));
     __tmp2.push(0);
     __tmp2
 }

tests/unit/c_string_embedded_nul.cpp

@@ -0,0 +1,18 @@
+// C string functions stop at the first NUL.
+#include <cstdio>
+#include <cstring>
+#include <string>
+
+int main() {
+  char buf[] = {'a', 'b', '\0', 'c', 'd', '\0'};
+  fputs(buf, stdout);
+  fputc('|', stdout);
+  puts(buf);
+  std::string s(buf);
+  printf("%zu\n", s.size());
+  printf("%zu %zu\n", strlen(buf), s.find_last_of(buf));
+  const char *lit = "xy\0zw";
+  std::string t(lit);
+  printf("%zu\n", t.size());
+  return 0;
+}

tests/unit/out/refcount/c_string_embedded_nul.rs

@@ -0,0 +1,72 @@
+extern crate libcc2rs;
+use libcc2rs::*;
+use std::cell::RefCell;
+use std::collections::BTreeMap;
+use std::io::prelude::*;
+use std::io::{Read, Seek, Write};
+use std::os::fd::AsFd;
+use std::rc::{Rc, Weak};
+pub fn main() {
+    __cpp2rust_init_globals();
+    std::process::exit(main_0());
+}
+fn main_0() -> i32 {
+    let buf: Value<Box<[u8]>> = Rc::new(RefCell::new(Box::new([
+        ('a' as u8),
+        ('b' as u8),
+        ('\0' as u8),
+        ('c' as u8),
+        ('d' as u8),
+        ('\0' as u8),
+    ])));
+    {
+        let __bytes = (buf.as_pointer() as Ptr<u8>).to_c_bytes();
+        match libcc2rs::c_stdout().with_mut(|__f| __f.write(&__bytes)) == __bytes.len() {
+            true => 0,
+            false => -1,
+        }
+    };
+    {
+        let __c = (('|' as u8) as i32) as u8;
+        match libcc2rs::c_stdout().with_mut(|__f| __f.write(&[__c])) {
+            1 => __c as i32,
+            _ => -1,
+        }
+    };
+    {
+        let mut __bytes = (buf.as_pointer() as Ptr<u8>).to_c_bytes();
+        __bytes.push(b'\n');
+        match libcc2rs::c_stdout().with_mut(|__f| __f.write(&__bytes)) == __bytes.len() {
+            true => 0,
+            false => -1,
+        }
+    };
+    let s: Value<Vec<u8>> = Rc::new(RefCell::new({
+        let mut __bytes = (buf.as_pointer() as Ptr<u8>).to_c_bytes();
+        __bytes.push(0);
+        __bytes
+    }));
+    println!("{}", ((*s.borrow()).len() - 1));
+    println!(
+        "{} {}",
+        (buf.as_pointer() as Ptr::<u8>)
+            .to_c_string_iterator()
+            .count(),
+        (buf.as_pointer() as Ptr::<u8>).with_c_str(|__lookup| {
+            (*s.borrow())
+                .iter()
+                .take((*s.borrow()).len().saturating_sub(1))
+                .rposition(|&x| __lookup.contains(&x))
+                .unwrap_or(usize::MAX)
+        })
+    );
+    let lit: Value<Ptr<u8>> = Rc::new(RefCell::new(Ptr::<u8>::from_string_literal(b"xy\0zw")));
+    let t: Value<Vec<u8>> = Rc::new(RefCell::new({
+        let mut __bytes = (*lit.borrow()).to_c_bytes();
+        __bytes.push(0);
+        __bytes
+    }));
+    println!("{}", ((*t.borrow()).len() - 1));
+    return 0;
+}
+pub fn __cpp2rust_init_globals() {}

tests/unit/out/refcount/char_printing.rs

@@ -13,12 +13,11 @@ pub fn main() {
 fn main_0() -> i32 {
     let vec_: Value<Vec<u8>> = Rc::new(RefCell::new(vec![195_u8, 167_u8]));
     let i: Value<i32> = Rc::new(RefCell::new(27));
-    let str: Value<Vec<u8>> = Rc::new(RefCell::new(
-        Ptr::<u8>::from_string_literal(b"rdas.")
-            .to_c_string_iterator()
-            .chain(std::iter::once(0))
-            .collect::<Vec<u8>>(),
-    ));
+    let str: Value<Vec<u8>> = Rc::new(RefCell::new({
+        let mut __bytes = Ptr::<u8>::from_string_literal(b"rdas.").to_c_bytes();
+        __bytes.push(0);
+        __bytes
+    }));
     write!(libcc2rs::cout(), "{:} a", (*i.borrow()),);
     libcc2rs::cout().write_all(
         &([