Tokenize Type 4 programs with a byte-level lexer

crates/pdf-color-space/src/device_n_color_space.rs

@@ -151,7 +151,7 @@ mod tests {
         let arr = device_n_array(vec![
             ObjectVariant::Array(vec![name("Cyan"), name("Magenta")].into()),
             name("DeviceRGB"),
-            function_stream("pop pop 0.1 0.2 0.3", 3),
+            function_stream("{ pop pop 0.1 0.2 0.3 }", 3),
         ]);
 
         let parsed = read_device_n(arr).unwrap();
@@ -178,7 +178,7 @@ mod tests {
         let arr = device_n_array(vec![
             ObjectVariant::Array(vec![name("Spot")].into()),
             name("DeviceCMYK"),
-            function_stream("pop 0.2 0.4 0.6 0.8", 4),
+            function_stream("{ pop 0.2 0.4 0.6 0.8 }", 4),
             attrs,
         ]);
 
@@ -197,7 +197,7 @@ mod tests {
         let arr = device_n_array(vec![
             ObjectVariant::Array(vec![name("Cyan"), name("Magenta"), name("Yellow")].into()),
             name("DeviceRGB"),
-            function_stream("pop pop pop 0.25 0.5 0.75", 3),
+            function_stream("{ pop pop pop 0.25 0.5 0.75 }", 3),
         ]);
 
         let parsed = read_device_n(arr).unwrap();
@@ -218,7 +218,7 @@ mod tests {
         let arr = device_n_array(vec![
             ObjectVariant::Array(vec![name("Cyan"), name("Magenta"), name("Yellow")].into()),
             name("DeviceRGB"),
-            function_stream("pop pop 0.25 0.5", 2),
+            function_stream("{ pop pop 0.25 0.5 }", 2),
         ]);
 
         let parsed = read_device_n(arr).unwrap();
@@ -243,7 +243,7 @@ mod tests {
         let arr = device_n_array(vec![
             ObjectVariant::Array(vec![name("Cyan"), name("Magenta")].into()),
             name("DeviceRGB"),
-            function_stream("pop pop 0.1 0.2 0.3", 3),
+            function_stream("{ pop pop 0.1 0.2 0.3 }", 3),
         ]);
 
         let parsed = read_device_n(arr).unwrap();

crates/pdf-function/src/postscript_calculator.rs

@@ -86,12 +86,8 @@ impl FromPdfObject for PostScriptCalculatorFunction {
         let domain = dictionary.required_intervals(b"Domain")?;
         let range = dictionary.required_intervals(b"Range")?;
 
-        // Parse PostScript code: add spaces around braces for tokenization
-        let code_str = String::from_utf8_lossy(context.stream().raw_data());
-        let code_with_spaces = code_str.replace('{', " { ").replace('}', " } ");
-        let tokens: Vec<&str> = code_with_spaces.split_whitespace().collect();
-        let operators =
-            pdf_postscript::parser::parse_tokens(&tokens).map_err(FunctionReadError::from)?;
+        let operators = pdf_postscript::parser::parse(context.stream().raw_data())
+            .map_err(FunctionReadError::from)?;
 
         Ok(Self {
             operators,

crates/pdf-image/src/image_decoder.rs

@@ -301,7 +301,7 @@ mod tests {
                     ),
                 ),
             ])),
-            b"4 3 roll pop".to_vec(),
+            b"{ 4 3 roll pop }".to_vec(),
         );
 
         ObjectVariant::Array(
@@ -546,7 +546,7 @@ mod tests {
                     ),
                 ),
             ])),
-            b"pop 1 1 1".to_vec(),
+            b"{ pop 1 1 1 }".to_vec(),
         );
         let dictionary = Dictionary::new(BTreeMap::from([
             (Vec::from(b"BitsPerComponent"), ObjectVariant::Integer(8)),

crates/pdf-postscript/benches/calculator.rs

@@ -7,7 +7,7 @@
 use std::hint::black_box;
 
 use criterion::{Criterion, criterion_group, criterion_main};
-use pdf_postscript::{calculator::execute, operator::Operator, parser::parse_tokens, value::Value};
+use pdf_postscript::{calculator::execute, operator::Operator, parser, value::Value};
 
 const SAMPLES: usize = 256;
 
@@ -25,9 +25,7 @@ const STACK: &str = "{ 3 copy 2 index 1 index add 4 1 roll mul exch div 5 1 roll
     3 copy pop pop pop pop 4 2 roll add add exch pop }";
 
 fn parse(code: &str) -> Vec<Operator> {
-    let code = code.replace('{', " { ").replace('}', " } ");
-    parse_tokens(&code.split_whitespace().collect::<Vec<_>>())
-        .expect("benchmark program should parse")
+    parser::parse(code.as_bytes()).expect("benchmark program should parse")
 }
 
 fn benchmark_program(criterion: &mut Criterion, name: &str, code: &str, inputs: usize) {