fix(plugin-grid,app-shell): a CSV file reaches the import door as CSV text

.changeset/13315-import-wizard-csv-door.md

@@ -0,0 +1,18 @@
+---
+'@object-ui/plugin-grid': patch
+'@object-ui/app-shell': patch
+---
+
+The Import Wizard sends a CSV file to the import door as CSV text (objectui#13315). When you import a `.csv`, `.tsv` or `.txt` file, the request is now `format: 'csv'` with the rows the preview shows written out as CSV. This holds on the hand-mapped path, and with a saved mapping that does not declare `sourceFormat: 'json'`. Before, those rows were sent as JSON, so the server never read the file's cells through its CSV import rule, the one its CSV export pairs with.
+
+- **The text is the preview's rows.** The wizard writes the CSV from the rows it shows, not from the file as uploaded, as it already did for a mapping that declares `sourceFormat: 'csv'`. A cell you corrected in the preview is sent corrected. The dry-run check sends the same request as the import.
+- **Unchanged:** an `.xlsx` sheet and a pasted table are sent as JSON rows. A saved mapping that declares `sourceFormat: 'json'` still gets JSON rows from any file, because the import door does not accept CSV text for it.
+- **User import** (`sys_user`) sends a CSV file as CSV text too, in batches of at most 500 rows. Each one-time password is still shown with the email or phone number of its row.
+
+Three shapes now import differently from what the preview shows:
+
+- **A last row whose mapped cells are all empty is not imported.** When the file's last row, or a run of last rows, has data only in columns you did not map, the server does not read it. The import counts one row fewer than the preview, and that row creates no record and reports no error. A row of that shape anywhere else in the file is imported as before, so row numbers keep matching the preview. User import sends the file in batches of 500 rows, so there the same holds for the last row of each batch.
+- **A cell that begins with an apostrophe and ends in a tab or a line break keeps the apostrophe** when the rest of it starts with `+` or `-` and holds only digits, spaces and `( ) + - . ,`. The preview trims the tab or line break before the cell is sent, and a server whose CSV import removes the apostrophe the CSV export adds leaves this trimmed cell as it is. The wizard imported such a cell with its apostrophe before this change too.
+- **A space before a leading apostrophe is trimmed first.** On a server whose CSV import removes the apostrophe the CSV export adds, a cell typed as a space, an apostrophe and a value the export would write with an apostrophe now imports without the apostrophe. Before this change it imported with it.
+
+No export, prop or type is added to either package entry.

packages/app-shell/src/views/__tests__/identityImport.csvBatches-13315.test.tsx

@@ -0,0 +1,218 @@
+/**
+ * ObjectUI
+ * Copyright (c) 2024-present ObjectStack Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/**
+ * The identity import sends a CSV file as CSV text, batch by batch
+ * (objectui#13315).
+ *
+ * The stock ImportWizard sends a CSV file's rows to the import door as CSV text
+ * (`format: 'csv'`, the preview's rows re-spelled), so the door reads them
+ * through its one CSV rule. The identity import goes through the same wizard
+ * and its adapter posts to `POST /api/v1/auth/admin/import-users` in batches of
+ * at most 500 rows, so the adapter takes the same change: each batch is the
+ * header record plus that batch's data records, as CSV text, and the rows each
+ * batch carries are still read to name every row's identity in the result.
+ *
+ *   - the pure half: how a CSV request is cut into batches (quoted commas, a
+ *     quoted line break, doubled quotes, a row with every cell empty);
+ *   - the wired half: the real wizard and the real adapter, a 1001-row .csv
+ *     file, three batches — each batch body is the wizard's own CSV text cut at
+ *     record boundaries, the results renumber onto the preview's rows, and a
+ *     one-time password is named by the identity of the row it belongs to;
+ *   - CONTROL: a paste still goes out as JSON rows.
+ *
+ * `ImportWizard` is imported at module scope rather than through `ObjectView`'s
+ * `React.lazy` boundary (AGENTS.md 测试纪律).
+ */
+import { describe, it, expect, beforeAll, vi } from 'vitest';
+import { render, screen, fireEvent, waitFor, act } from '@testing-library/react';
+import '@testing-library/jest-dom';
+import React from 'react';
+
+import { ImportWizard } from '@object-ui/plugin-grid';
+import type { ImportRequestOptions } from '@object-ui/types';
+import {
+  batchIdentityImportRequest,
+  createIdentityImportDataSource,
+  readCsvRequestRecords,
+  IDENTITY_IMPORT_OBJECT,
+} from '../identityImport';
+import { identityImportFields } from '../IdentityImportPanels';
+
+beforeAll(() => {
+  if (!Element.prototype.hasPointerCapture) Element.prototype.hasPointerCapture = vi.fn(() => false);
+  if (!Element.prototype.scrollIntoView) Element.prototype.scrollIntoView = vi.fn();
+});
+
+describe('batchIdentityImportRequest cuts a CSV request at record boundaries (objectui#13315)', () => {
+  const csv = [
+    'email,name',
+    'a@x.test,"Doe, Jane"',
+    'b@x.test,"Line one\nLine two"',
+    ',',
+    'c@x.test,"Say ""hi"""',
+  ].join('\r\n');
+
+  it('reads every record with its text as written and its cells', () => {
+    expect(readCsvRequestRecords(csv)).toEqual([
+      { text: 'email,name', cells: ['email', 'name'] },
+      { text: 'a@x.test,"Doe, Jane"', cells: ['a@x.test', 'Doe, Jane'] },
+      { text: 'b@x.test,"Line one\nLine two"', cells: ['b@x.test', 'Line one\nLine two'] },
+      { text: ',', cells: ['', ''] },
+      { text: 'c@x.test,"Say ""hi"""', cells: ['c@x.test', 'Say "hi"'] },
+    ]);
+  });
+
+  it('each batch is the header record plus its data records, and carries those rows in file order', () => {
+    expect(batchIdentityImportRequest({ format: 'csv', csv }, 2)).toEqual([
+      {
+        payload: { format: 'csv', csv: 'email,name\r\na@x.test,"Doe, Jane"\r\nb@x.test,"Line one\nLine two"' },
+        rows: [{ email: 'a@x.test', name: 'Doe, Jane' }, { email: 'b@x.test', name: 'Line one\nLine two' }],
+      },
+      {
+        payload: { format: 'csv', csv: 'email,name\r\n,\r\nc@x.test,"Say ""hi"""' },
+        rows: [{ email: '', name: '' }, { email: 'c@x.test', name: 'Say "hi"' }],
+      },
+    ]);
+  });
+
+  it('a header alone is no row, and the adapter refuses it before any request', async () => {
+    expect(batchIdentityImportRequest({ format: 'csv', csv: 'email,name' })).toEqual([]);
+    const authFetch = vi.fn();
+    const ds = createIdentityImportDataSource({
+      base: {}, authFetch, baseUrl: 'http://srv', getPasswordPolicy: () => 'auto',
+    }) as { importRecords: (o: string, r: ImportRequestOptions) => Promise<unknown> };
+    await expect(ds.importRecords(IDENTITY_IMPORT_OBJECT, { format: 'csv', csv: 'email,name' }))
+      .rejects.toThrow(/at least one mapped row/);
+    expect(authFetch).not.toHaveBeenCalled();
+  });
+
+  it('CONTROL: JSON rows are batched as JSON rows', () => {
+    const rows = [{ email: 'a@x.test' }, { email: 'b@x.test' }, { email: 'c@x.test' }];
+    expect(batchIdentityImportRequest({ format: 'json', rows }, 2)).toEqual([
+      { payload: { format: 'json', rows: rows.slice(0, 2) }, rows: rows.slice(0, 2) },
+      { payload: { format: 'json', rows: rows.slice(2) }, rows: rows.slice(2) },
+    ]);
+  });
+});
+
+/** The identity endpoint, faked: it records each body and answers one row
+ *  result per data record, with a one-time password on the rows asked for. */
+function fakeEndpoint(passwordFor: (batch: number, row: number) => string | undefined) {
+  const bodies: Array<Record<string, unknown>> = [];
+  const authFetch = vi.fn(async (_url: string, init?: RequestInit) => {
+    const body = JSON.parse(String(init?.body)) as Record<string, unknown>;
+    const batch = bodies.length;
+    bodies.push(body);
+    // This file's fixtures hold no quoted line break, so a CRLF ends a record.
+    const count = body.format === 'csv'
+      ? String(body.csv).split('\r\n').length - 1
+      : (body.rows as unknown[]).length;
+    const rows = Array.from({ length: count }, (_, i) => {
+      const temporaryPassword = passwordFor(batch, i + 1);
+      return { row: i + 1, ok: true, action: 'created', id: `u-${batch}-${i}`, ...(temporaryPassword ? { temporaryPassword } : {}) };
+    });
+    return new Response(JSON.stringify({
+      success: true,
+      data: { summary: { total: count, created: count, updated: 0, skipped: 0, errors: 0, dryRun: false }, rows },
+    }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+  });
+  return { bodies, authFetch };
+}
+
+function renderIdentityWizard(authFetch: (url: string, init?: RequestInit) => Promise<Response>, initialFile?: File) {
+  const ds = createIdentityImportDataSource({
+    base: {}, authFetch, baseUrl: 'http://srv', getPasswordPolicy: () => 'temporary',
+  }) as { importRecords: (o: string, r: ImportRequestOptions) => Promise<unknown> };
+  const seen: ImportRequestOptions[] = [];
+  const dataSource = {
+    ...ds,
+    importRecords: (object: string, request: ImportRequestOptions) => { seen.push(request); return ds.importRecords(object, request); },
+  };
+  const onComplete = vi.fn();
+  render(
+    <ImportWizard
+      objectName={IDENTITY_IMPORT_OBJECT}
+      fields={identityImportFields(undefined)}
+      dataSource={dataSource as never}
+      initialFile={initialFile}
+      open
+      onOpenChange={() => {}}
+      onComplete={onComplete}
+    />,
+  );
+  return { seen, onComplete };
+}
+
+async function runAsMapped(onComplete: ReturnType<typeof vi.fn>) {
+  const next = await screen.findByTestId('import-next-btn');
+  await waitFor(() => expect(next).toBeEnabled());
+  fireEvent.click(next);
+  const run = await screen.findByTestId('import-run-btn');
+  await waitFor(() => expect(run).toBeEnabled());
+  fireEvent.click(run);
+  await waitFor(() => expect(onComplete).toHaveBeenCalled());
+  return onComplete.mock.calls[0][0] as {
+    serverResult: { total: number; results: Array<{ row: number; identity?: string; temporaryPassword?: string }> };
+  };
+}
+
+describe('the identity import sends a CSV file as CSV text, batch by batch (objectui#13315)', () => {
+  it('a 1001-row .csv file: three CSV batches cut from the wizard\'s text, rows renumbered, identities named', async () => {
+    const people = Array.from({ length: 1001 }, (_, i) => [`user${i + 1}@example.test`, i === 2 ? '"Doe, Jane"' : `User ${i + 1}`]);
+    const file = new File([['Email,Name', ...people.map((p) => p.join(','))].join('\n')], 'users.csv', { type: 'text/csv' });
+    // A one-time password on row 277 of the second batch: row 777 of the file.
+    const { bodies, authFetch } = fakeEndpoint((batch, row) => (batch === 1 && row === 277 ? 'Pw-777' : undefined));
+    const { seen, onComplete } = renderIdentityWizard(authFetch, file);
+
+    const result = await runAsMapped(onComplete);
+
+    const sent = seen.filter((r) => !r.dryRun);
+    expect(sent).toHaveLength(1);
+    expect(sent[0]).toMatchObject({ format: 'csv' });
+    expect(sent[0]).not.toHaveProperty('rows');
+    const [header, ...records] = String(sent[0].csv).split('\r\n');
+    expect(header).toBe('email,name');
+    expect(records[2]).toBe('user3@example.test,"Doe, Jane"');
+
+    expect(bodies.map((b) => b.format)).toEqual(['csv', 'csv', 'csv']);
+    expect(bodies.map((b) => b.csv)).toEqual([
+      [header, ...records.slice(0, 500)].join('\r\n'),
+      [header, ...records.slice(500, 1000)].join('\r\n'),
+      [header, ...records.slice(1000)].join('\r\n'),
+    ]);
+    for (const body of bodies) {
+      expect(body).not.toHaveProperty('rows');
+      expect(body).toMatchObject({ mode: 'insert', passwordPolicy: 'temporary' });
+    }
+
+    expect(result.serverResult.total).toBe(1001);
+    expect(result.serverResult.results.map((r) => r.row)).toEqual(Array.from({ length: 1001 }, (_, i) => i + 1));
+    expect(result.serverResult.results.filter((r) => r.temporaryPassword)).toEqual([
+      expect.objectContaining({ row: 777, temporaryPassword: 'Pw-777', identity: 'user777@example.test' }),
+    ]);
+  });
+
+  it('CONTROL: a paste still goes out as JSON rows', async () => {
+    const { bodies, authFetch } = fakeEndpoint(() => undefined);
+    const { onComplete } = renderIdentityWizard(authFetch);
+    const evt = new Event('paste', { bubbles: true, cancelable: true }) as Event & {
+      clipboardData: { getData: (type: string) => string };
+    };
+    evt.clipboardData = { getData: (type: string) => (type === 'text/plain' ? 'Email\tName\na@x.test\tAda\nb@x.test\tGrace' : '') };
+    act(() => { window.dispatchEvent(evt); });
+
+    await runAsMapped(onComplete);
+    expect(bodies).toHaveLength(1);
+    expect(bodies[0]).toMatchObject({
+      format: 'json',
+      rows: [{ email: 'a@x.test', name: 'Ada' }, { email: 'b@x.test', name: 'Grace' }],
+    });
+    expect(bodies[0]).not.toHaveProperty('csv');
+  });
+});

packages/app-shell/src/views/identityImport.ts

@@ -12,7 +12,10 @@
 // into ≤500-row batches — the endpoint's hard cap, because password-related
 // work is CPU-bound server-side — and (2) inject the admin's chosen password
 // policy. The endpoint is idempotent on upsert (matched by email/phone), so
-// re-running a failed batch is safe.
+// re-running a failed batch is safe. A CSV file's request arrives as CSV text
+// (objectui#13315) and each batch is posted as CSV text too, so the endpoint
+// reads the file's cells through the same CSV rule as the generic import door;
+// JSON rows (a workbook, a paste) are batched as JSON rows.
 //
 // Password policies: `auto` (default, framework#3236 — per row: deliverable
 // rows get an invitation, unreachable rows fall back to a one-time password),
@@ -71,6 +74,90 @@ export function splitIntoBatches<T>(rows: T[], size = IDENTITY_IMPORT_BATCH_SIZE
   return batches;
 }
 
+/** One record of a CSV request's text: the record as written, without its
+ *  line break, and its cells. */
+export interface CsvRequestRecord {
+  text: string;
+  cells: string[];
+}
+
+/**
+ * Cut a CSV request's text into its records, header first (pure; exported for
+ * tests). The text is the import wizard's (`sourceRowsToCsv` in
+ * `@object-ui/plugin-grid`): records joined by CRLF, every cell holding a
+ * quote, a comma, a CR or an LF quoted (RFC 4180). The scan splits records the
+ * way the identity endpoint's CSV parse does — a quote opens a quoted run, a
+ * doubled quote inside one is a quote, an LF outside one ends the record, a CR
+ * outside one is dropped, and a last line holding nothing is no record — so a
+ * record cut here is a row the endpoint reads, in the same order.
+ *
+ * A record's text is kept as written, so a batch re-joined from these texts
+ * reaches the endpoint with the wizard's own bytes. The cells are read only to
+ * name each row's identity in the result (see mergeIdentityBatchResults); no
+ * imported value is decided here.
+ */
+export function readCsvRequestRecords(csv: string): CsvRequestRecord[] {
+  const text = csv.replace(/^\uFEFF/, '');
+  const records: CsvRequestRecord[] = [];
+  let cells: string[] = [];
+  let cell = '';
+  let start = 0;
+  let inQuotes = false;
+  const close = (end: number) => {
+    cells.push(cell);
+    const written = text.slice(start, end);
+    records.push({ text: written.endsWith('\r') ? written.slice(0, -1) : written, cells });
+    cells = [];
+    cell = '';
+    start = end + 1;
+  };
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i];
+    if (inQuotes) {
+      if (ch !== '"') cell += ch;
+      else if (text[i + 1] === '"') { cell += '"'; i++; }
+      else inQuotes = false;
+    } else if (ch === '"') inQuotes = true;
+    else if (ch === ',') { cells.push(cell); cell = ''; }
+    else if (ch === '\n') close(i);
+    else if (ch !== '\r') cell += ch;
+  }
+  if (cell.length > 0 || cells.length > 0) close(text.length);
+  return records;
+}
+
+/** One request to the identity endpoint: its payload, and the rows it carries
+ *  (read back to name each row's identity in the result). */
+export interface IdentityImportBatch {
+  payload: { format: 'csv'; csv: string } | { format: 'json'; rows: Array<Record<string, unknown>> };
+  rows: Array<Record<string, unknown>>;
+}
+
+/**
+ * Split the wizard's request into endpoint-sized batches (pure; exported for
+ * tests). CSV text (a CSV file, objectui#13315) is cut into records and every
+ * batch is the header record plus up to `size` data records, joined by CRLF —
+ * for the wizard's text, exactly `sourceRowsToCsv` of that batch's rows. JSON
+ * rows (a workbook, a paste) are batched as JSON rows. Each batch carries its
+ * rows in file order, so per-batch row numbers renumber onto the preview's.
+ */
+export function batchIdentityImportRequest(
+  request: ImportRequestOptions,
+  size = IDENTITY_IMPORT_BATCH_SIZE,
+): IdentityImportBatch[] {
+  if (request.format === 'csv' && typeof request.csv === 'string') {
+    const [header, ...records] = readCsvRequestRecords(request.csv);
+    if (!header) return [];
+    const columns = header.cells.map((c) => c.trim());
+    return splitIntoBatches(records, size).map((batch) => ({
+      payload: { format: 'csv' as const, csv: [header, ...batch].map((r) => r.text).join('\r\n') },
+      rows: batch.map((r) => Object.fromEntries(columns.map((column, i) => [column, r.cells[i] ?? '']))),
+    }));
+  }
+  const rows = Array.isArray(request.rows) ? request.rows : [];
+  return splitIntoBatches(rows, size).map((batch) => ({ payload: { format: 'json' as const, rows: batch }, rows: batch }));
+}
+
 /**
  * The write modes the identity import door accepts — stated once, here, beside
  * the refusal that enforces it (objectui#12697). The server serves no datum
@@ -206,20 +293,18 @@ export function createIdentityImportDataSource(opts: IdentityImportDataSourceOpt
   ): Promise<ImportRecordsResult> => {
     const { mode, matchBy } = resolveIdentityWriteOptions(request);
     const passwordPolicy = getPasswordPolicy();
-    const rows = Array.isArray(request.rows) ? request.rows : [];
-    if (rows.length === 0) {
-      throw new Error('Identity import needs mapped rows (rows[]) — file uploads are parsed client-side.');
+    const batches = batchIdentityImportRequest(request);
+    if (batches.length === 0) {
+      throw new Error('Identity import needs at least one mapped row (rows[] or csv).');
     }
 
-    const batches = splitIntoBatches(rows);
     const responses: Array<NonNullable<IdentityImportResponse['data']>> = [];
     for (const batch of batches) {
       const res = await authFetch(`${baseUrl}/api/v1/auth/admin/import-users`, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({
-          format: 'json',
-          rows: batch,
+          ...batch.payload,
           mode,
           ...(matchBy ? { matchBy } : {}),
           passwordPolicy,
@@ -241,7 +326,7 @@ export function createIdentityImportDataSource(opts: IdentityImportDataSourceOpt
 
     return mergeIdentityBatchResults(
       responses,
-      batches as Array<Array<Record<string, unknown>>>,
+      batches.map((batch) => batch.rows),
       { dryRun: request.dryRun === true, writeMode: mode },
     );
   };