perf: JIRA-17851 stream export rows instead of holding them in memory

phpstan-baseline.neon

@@ -120,11 +120,6 @@ parameters:
 			count: 1
 			path: src/Processor/EloquentProcessor.php
 
-		-
-			message: '#^Parameter \#2 \$value of method Illuminate\\Support\\Collection\<\(int\|string\),string\>\:\:put\(\) expects string, array given\.$#'
-			identifier: argument.type
-			count: 1
-			path: src/Processor/EloquentProcessor.php
 
 		-
 			message: '#^PHP files should declare strict types\.$#'

src/Generator/CsvDriver.php

@@ -69,7 +69,7 @@ public function saveToStorage(
             path: $filepath,
             size: Storage::size($filepath),
             url: Storage::url($filepath),
-            count: count($processorData->getData()),
+            count: $processorData->getCount(),
             mimeType: 'text/csv',
         );
     }
@@ -101,8 +101,7 @@ public function writeCsv(ProcessorData $processorData, $stream): int
             throw new ValueError('The stream must be a resource.');
         }
 
-        $rows = $processorData->getData();
-        $columns = $this->columns($rows);
+        $columns = $processorData->getColumns();
 
         if ($columns === []) {
             return 0;
@@ -112,7 +111,7 @@ public function writeCsv(ProcessorData $processorData, $stream): int
 
         $written = 0;
 
-        foreach ($rows as $row) {
+        foreach ($processorData->rows() as $row) {
             $fields = [];
 
             foreach ($columns as $column) {
@@ -126,27 +125,6 @@ public function writeCsv(ProcessorData $processorData, $stream): int
         return $written;
     }
 
-    /**
-     * The header names every column any row has, in the order first seen, so a
-     * row whose keys differ cannot line up against the wrong header.
-     *
-     * @param array<array-key, array<array-key, mixed>> $rows
-     *
-     * @return array<int, array-key>
-     */
-    private function columns(array $rows): array
-    {
-        $columns = [];
-
-        foreach ($rows as $row) {
-            foreach (array_keys($row) as $key) {
-                $columns[$key] = true;
-            }
-        }
-
-        return array_keys($columns);
-    }
-
     /**
      * @param resource                $stream
      * @param array<array-key, mixed> $fields

src/Processor/EloquentProcessor.php

@@ -87,16 +87,15 @@ protected function columns(): Collection
         });
     }
 
-    protected function filterQuery(Builder $query): array
+    protected function filterQuery(Builder $query): RowStream
     {
         $columns = $this->columns();
         $allowedColumns = $columns->keys();
 
-        $items = collect();
+        $rows = new RowStream();
 
-        $query->chunk(1000, function ($data) use ($columns, $allowedColumns, $items) {
-            // Combine columns plus additional and optional fields
-            $filtered = $data->map(function ($item) use ($columns, $allowedColumns) {
+        $query->chunk(1000, function ($data) use ($columns, $allowedColumns, $rows) {
+            foreach ($data as $item) {
                 $additional = $this->additional($item);
                 $optional = $this->optional($item);
                 $additionalKeys = array_keys($additional);
@@ -106,89 +105,29 @@ protected function filterQuery(Builder $query): array
                 $attributes = $allowedColumns->isEmpty() ? [] : $this->serialiseAttributes($item);
 
                 // Merge fieldsets
-                $item = collect(array_merge($attributes, $additional));
+                $row = collect(array_merge($attributes, $additional));
 
                 // Only let through the desired columns
-                $item = $item->only($allowedColumns->concat($additionalKeys));
+                $row = $row->only($allowedColumns->concat($additionalKeys));
 
                 // Ensure some values are properly casted
-                $item = $item->map(function ($value) {
+                $row = $row->map(function ($value) {
                     if (is_bool($value)) {
                         $value = (int) $value;
                     }
 
                     return strval($value);
                 });
 
-                // add optional into the item as array so we can later
-                // normalize the columns
-                $item->put('optional', $optional);
-
                 // update keys based on desired key names provided in the processor
-                $item = $item->keyBy(function ($value, $key) use ($columns) {
+                $row = $row->keyBy(function ($value, $key) use ($columns) {
                     return $columns->get($key, $key);
                 })->all();
 
-                // we return an array
-                return $item;
-            });
-
-            $items->push($filtered);
-        });
-
-        $formattedItems = $this->normalizeOptionalItems($items->flatten(1)->toArray());
-
-        return $formattedItems;
-    }
-
-    protected function getAllOptionalKeys(array $items): array
-    {
-        $optionalKeys = [];
-
-        foreach ($items as $item) {
-            $optional = $item['optional'];
-
-            if (count($optional)) {
-                foreach ($item['optional'] as $optional) {
-                    $key = key($optional);
-                    if (! in_array($key, $optionalKeys)) {
-                        $optionalKeys[] = $key;
-                    }
-                }
-            }
-        }
-
-        return $optionalKeys;
-    }
-
-    protected function normalizeOptionalItems(array $items): array
-    {
-        // get optional keys
-        $optionalKeys = $this->getAllOptionalKeys($items);
-
-        // go through each item
-        foreach ($items as $key => $item) {
-            $itemOptionals = call_user_func_array('array_merge', $items[$key]['optional']);
-
-            // for each item we wanna add each optional key
-            // and also populate value if we got one
-            foreach ($optionalKeys as $optionalArrKey => $optionalKey) {
-                $value = '';
-
-                // check if we have value for this item and populate it
-                if (isset($itemOptionals[$optionalKey])) {
-                    $value = $itemOptionals[$optionalKey];
-                }
-
-                // add the optional key and it's value
-                $items[$key][$optionalKey] = $value;
+                $rows->push($row, $optional);
             }
+        });
 
-            // unset our "temp" optional array as now we have them
-            // in the correct format
-            unset($items[$key]['optional']);
-        }
-
-        return $items;
+        return $rows;
     }
 }

src/Processor/ProcessorData.php

@@ -4,17 +4,55 @@
 
 class ProcessorData
 {
+    /**
+     * Rows either already built as an array, or held in a stream that can be read
+     * more than once. A one-shot iterable would be exhausted by the first reader.
+     *
+     * @param array<array-key, array<array-key, mixed>>|RowStream $data
+     */
     public function __construct(
-        private readonly array $data,
+        private readonly array|RowStream $data,
         private readonly string $type,
     ) {
     }
 
-    public function getData(): array
+    /**
+     * The rows, one at a time.
+     *
+     * @return iterable<array-key, array<array-key, mixed>>
+     */
+    public function rows(): iterable
     {
         return $this->data;
     }
 
+    public function getCount(): int
+    {
+        return count($this->data);
+    }
+
+    /**
+     * Every column any row has, in the order first seen.
+     *
+     * @return array<int, array-key>
+     */
+    public function getColumns(): array
+    {
+        if ($this->data instanceof RowStream) {
+            return $this->data->columns();
+        }
+
+        $columns = [];
+
+        foreach ($this->data as $row) {
+            foreach (array_keys($row) as $key) {
+                $columns[$key] = true;
+            }
+        }
+
+        return array_keys($columns);
+    }
+
     public function getType(): string
     {
         return $this->type;