perf: JIRA-17824 stream exports instead of building a spreadsheet

composer.json

@@ -5,14 +5,12 @@
     "license": "MIT",
     "require": {
         "php": "^8.4",
-        "guzzlehttp/guzzle": "^7.8",
         "illuminate/container": "^12.0 || ^13.0",
         "illuminate/contracts": "^12.0 || ^13.0",
         "illuminate/database": "^12.0 || ^13.0",
         "illuminate/support": "^12.0 || ^13.0",
         "nesbot/carbon": "^3.0",
         "nuwave/lighthouse": "^6.54",
-        "phpoffice/phpspreadsheet": "^1.30 || ^2.3.8 || ^5.4",
         "symfony/dependency-injection": "^7.0 || ^8.0",
         "worksome/graphql-helpers": "^0.1.18"
     },

src/Generator/CsvDriver.php

@@ -2,51 +2,68 @@
 
 namespace Worksome\DataExport\Generator;
 
-use GuzzleHttp\Psr7\Stream;
 use Illuminate\Support\Carbon;
-use Illuminate\Support\Collection;
 use Illuminate\Support\Facades\Storage;
 use Illuminate\Support\Str;
-use PhpOffice\PhpSpreadsheet\Spreadsheet;
-use PhpOffice\PhpSpreadsheet\Writer\Csv;
-use PhpOffice\PhpSpreadsheet\Writer\IWriter;
+use ValueError;
 use Worksome\DataExport\Generator\Contracts\GeneratorDriver;
 use Worksome\DataExport\Processor\ProcessorData;
 
 class CsvDriver implements GeneratorDriver
 {
+    private const string DELIMITER = ',';
+
+    private const string ENCLOSURE = '"';
+
+    private const string LINE_ENDING = "\r\n";
+
+    /** Rows are written to memory up to this size, and to a temp file beyond it. */
+    private const int BUFFER_BYTES = 8 * 1024 * 1024;
+
     public function generate(ProcessorData $processorData): GeneratorFile
     {
-        $csv = $this->exportToCsv($processorData);
-
         $filename = sprintf(
             'export-%s-%s-%s',
             $processorData->getType(),
             Carbon::now()->format('Y-m-d'),
             Str::random(40)
         );
 
-        return $this->saveToStorage($filename, $csv, $processorData);
+        return $this->saveToStorage($filename, $this->toStream($processorData), $processorData);
     }
 
-    public function exportToCsv(ProcessorData $processorData)
+    /**
+     * The whole file as a string. Streams via generate() instead for anything
+     * large enough that holding it in memory matters.
+     */
+    public function exportToCsv(ProcessorData $processorData): string
     {
-        $spreadsheet = $this->toSpreadsheet($processorData->getData());
-        $writer = $this->toCsvWriter($spreadsheet);
+        $stream = $this->toStream($processorData);
 
-        ob_start();
-        $writer->save('php://output');
-        $content = ob_get_contents();
-        ob_end_clean();
+        rewind($stream);
+        $contents = stream_get_contents($stream);
+        fclose($stream);
 
-        return $content;
+        return $contents === false ? '' : $contents;
     }
 
-    public function saveToStorage($filenameWithoutExtension, $content, ProcessorData $processorData): GeneratorFile
-    {
+    /**
+     * @param resource $stream
+     */
+    public function saveToStorage(
+        string $filenameWithoutExtension,
+        $stream,
+        ProcessorData $processorData,
+    ): GeneratorFile {
+        if (! is_resource($stream)) {
+            throw new ValueError('The stream must be a resource.');
+        }
+
         $filepath = sprintf('exports/%s.csv', $filenameWithoutExtension);
 
-        Storage::put($filepath, $content);
+        rewind($stream);
+        Storage::writeStream($filepath, $stream);
+        fclose($stream);
 
         return new GeneratorFile(
             path: $filepath,
@@ -57,44 +74,106 @@ public function saveToStorage($filenameWithoutExtension, $content, ProcessorData
         );
     }
 
-    public function toCsvWriter(Spreadsheet $spreadsheet): Csv
+    /**
+     * @return resource
+     */
+    public function toStream(ProcessorData $processorData)
     {
-        $writer = new Csv($spreadsheet);
-        $writer->setEnclosureRequired(false);
-        $writer->setLineEnding("\r\n");
-        $writer->setDelimiter(',');
+        $stream = fopen('php://temp/maxmemory:' . self::BUFFER_BYTES, 'w+b');
+
+        if ($stream === false) {
+            throw new \RuntimeException('Unable to open a stream to write the export to.');
+        }
 
-        return $writer;
+        $this->writeCsv($processorData, $stream);
+
+        return $stream;
     }
 
-    public function toSpreadsheet(array $entries): Spreadsheet
+    /**
+     * Writes the header and every row to $stream, and returns the rows written.
+     *
+     * @param resource $stream
+     */
+    public function writeCsv(ProcessorData $processorData, $stream): int
     {
-        $spreadsheet = new Spreadsheet();
+        if (! is_resource($stream)) {
+            throw new ValueError('The stream must be a resource.');
+        }
+
+        $rows = $processorData->getData();
+        $columns = $this->columns($rows);
 
-        $sheet = $spreadsheet->getActiveSheet();
+        if ($columns === []) {
+            return 0;
+        }
 
-        $data = Collection::make($entries);
-        $data = $data->map(fn ($item) => $item);
+        $this->writeRow($stream, $columns);
 
-        $spreadsheetData = $data
-            ->prepend(array_keys($data->first() ?? []))
-            ->toArray();
+        $written = 0;
 
-        $sheet->fromArray($spreadsheetData);
+        foreach ($rows as $row) {
+            $fields = [];
 
-        return $spreadsheet;
+            foreach ($columns as $column) {
+                $fields[] = $row[$column] ?? '';
+            }
+
+            $this->writeRow($stream, $fields);
+            $written++;
+        }
+
+        return $written;
     }
 
-    public function toStream(IWriter $writer): Stream
+    /**
+     * The header names every column any row has, in the order first seen, so a
+     * row whose keys differ cannot line up against the wrong header.
+     *
+     * @param array<array-key, array<array-key, mixed>> $rows
+     *
+     * @return array<int, array-key>
+     */
+    private function columns(array $rows): array
     {
-        // Write the spreadsheet to a resource.
-        $resource = fopen('php://temp', 'w+');
+        $columns = [];
 
-        $writer->save($resource);
+        foreach ($rows as $row) {
+            foreach (array_keys($row) as $key) {
+                $columns[$key] = true;
+            }
+        }
 
-        // Rewind the resource and convert to PSR stream
-        rewind($resource);
+        return array_keys($columns);
+    }
+
+    /**
+     * @param resource                $stream
+     * @param array<array-key, mixed> $fields
+     */
+    private function writeRow($stream, array $fields): void
+    {
+        $line = implode(self::DELIMITER, array_map(
+            fn (mixed $field): string => $this->field((string) $field),
+            $fields
+        ));
+
+        fwrite($stream, $line . self::LINE_ENDING);
+    }
+
+    /**
+     * A value is quoted only when leaving it bare would break the row, so the
+     * file keeps the shape it has always had. A bare carriage return counts,
+     * because rows are separated by one.
+     */
+    private function field(string $value): string
+    {
+        if (strpbrk($value, self::DELIMITER . self::ENCLOSURE . "\r\n") === false) {
+            return $value;
+        }
 
-        return new Stream($resource);
+        return self::ENCLOSURE
+            . str_replace(self::ENCLOSURE, self::ENCLOSURE . self::ENCLOSURE, $value)
+            . self::ENCLOSURE;
     }
 }

tests/Feature/Generator/CsvDriverTest.php

@@ -32,8 +32,7 @@
 
     $csvDriver = new CsvDriver();
 
-    $contents = $csvDriver->exportToCsv($processorData);
-    $savedFile = $csvDriver->saveToStorage('foo.csv', $contents, $processorData);
+    $savedFile = $csvDriver->saveToStorage('foo', $csvDriver->toStream($processorData), $processorData);
 
     Storage::assertExists($savedFile->getPath());
 
@@ -58,3 +57,69 @@
 
     expect($savedFile)->toBeInstanceOf(GeneratorFile::class);
 });
+
+it('writes values exactly as given', function () {
+    // A spreadsheet writer turns these into numbers and loses the formatting, and
+    // reads a leading = as a formula. A CSV holds text, so nothing is inferred.
+    $data = [[
+        'decimal' => '8.00',
+        'money' => '1234.50',
+        'small' => '0.10',
+        'formula' => '=SUM(A1)',
+        'signed' => '+4512',
+        'exponent' => '1e3',
+        'zeros' => '0000123',
+    ]];
+
+    $csv = (new CsvDriver())->exportToCsv(new ProcessorData($data, 'values'));
+
+    $lines = explode("\r\n", trim($csv));
+    $header = str_getcsv($lines[0], ',', '"', '');
+    $values = str_getcsv($lines[1], ',', '"', '');
+
+    expect(array_combine($header, $values))->toBe([
+        'decimal' => '8.00',
+        'money' => '1234.50',
+        'small' => '0.10',
+        'formula' => '=SUM(A1)',
+        'signed' => '+4512',
+        'exponent' => '1e3',
+        'zeros' => '0000123',
+    ]);
+});
+
+it('keeps every row aligned to the header when the keys differ', function () {
+    $data = [
+        ['a' => 'a1', 'b' => 'b1'],
+        ['b' => 'b2', 'a' => 'a2'],
+        ['a' => 'a3'],
+        ['a' => 'a4', 'b' => 'b4', 'c' => 'c4'],
+    ];
+
+    $csv = (new CsvDriver())->exportToCsv(new ProcessorData($data, 'shapes'));
+
+    expect($csv)->toBe("a,b,c\r\na1,b1,\r\na2,b2,\r\na3,,\r\na4,b4,c4\r\n");
+});
+
+it('quotes only the values that need it', function () {
+    $data = [[
+        'plain' => 'no quotes needed',
+        'comma' => 'a,b',
+        'quote' => 'say "hi"',
+        'newline' => "line1\nline2",
+        'backslash' => 'a\b',
+    ]];
+
+    $csv = (new CsvDriver())->exportToCsv(new ProcessorData($data, 'quoting'));
+
+    expect($csv)
+        ->toContain('no quotes needed')
+        ->toContain('"a,b"')
+        ->toContain('"say ""hi"""')
+        ->toContain("\"line1\nline2\"")
+        ->toContain('a\b');
+});
+
+it('returns an empty string when there is nothing to export', function () {
+    expect((new CsvDriver())->exportToCsv(new ProcessorData([], 'empty')))->toBe('');
+});