diff --git a/.changeset/13315-import-wizard-csv-door.md b/.changeset/13315-import-wizard-csv-door.md new file mode 100644 index 0000000000..4a06cbd1fd --- /dev/null +++ b/.changeset/13315-import-wizard-csv-door.md @@ -0,0 +1,18 @@ +--- +'@object-ui/plugin-grid': patch +'@object-ui/app-shell': patch +--- + +The Import Wizard sends a CSV file to the import door as CSV text (objectui#13315). When you import a `.csv`, `.tsv` or `.txt` file, the request is now `format: 'csv'` with the rows the preview shows written out as CSV. This holds on the hand-mapped path, and with a saved mapping that does not declare `sourceFormat: 'json'`. Before, those rows were sent as JSON, so the server never read the file's cells through its CSV import rule, the one its CSV export pairs with. + +- **The text is the preview's rows.** The wizard writes the CSV from the rows it shows, not from the file as uploaded, as it already did for a mapping that declares `sourceFormat: 'csv'`. A cell you corrected in the preview is sent corrected. The dry-run check sends the same request as the import. +- **Unchanged:** an `.xlsx` sheet and a pasted table are sent as JSON rows. A saved mapping that declares `sourceFormat: 'json'` still gets JSON rows from any file, because the import door does not accept CSV text for it. +- **User import** (`sys_user`) sends a CSV file as CSV text too, in batches of at most 500 rows. Each one-time password is still shown with the email or phone number of its row. + +Three shapes now import differently from what the preview shows: + +- **A last row whose mapped cells are all empty is not imported.** When the file's last row, or a run of last rows, has data only in columns you did not map, the server does not read it. The import counts one row fewer than the preview, and that row creates no record and reports no error. A row of that shape anywhere else in the file is imported as before, so row numbers keep matching the preview. User import sends the file in batches of 500 rows, so there the same holds for the last row of each batch. +- **A cell that begins with an apostrophe and ends in a tab or a line break keeps the apostrophe** when the rest of it starts with `+` or `-` and holds only digits, spaces and `( ) + - . ,`. The preview trims the tab or line break before the cell is sent, and a server whose CSV import removes the apostrophe the CSV export adds leaves this trimmed cell as it is. The wizard imported such a cell with its apostrophe before this change too. +- **A space before a leading apostrophe is trimmed first.** On a server whose CSV import removes the apostrophe the CSV export adds, a cell typed as a space, an apostrophe and a value the export would write with an apostrophe now imports without the apostrophe. Before this change it imported with it. + +No export, prop or type is added to either package entry. diff --git a/packages/app-shell/src/views/__tests__/identityImport.csvBatches-13315.test.tsx b/packages/app-shell/src/views/__tests__/identityImport.csvBatches-13315.test.tsx new file mode 100644 index 0000000000..37cfa88ef3 --- /dev/null +++ b/packages/app-shell/src/views/__tests__/identityImport.csvBatches-13315.test.tsx @@ -0,0 +1,218 @@ +/** + * ObjectUI + * Copyright (c) 2024-present ObjectStack Inc. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +/** + * The identity import sends a CSV file as CSV text, batch by batch + * (objectui#13315). + * + * The stock ImportWizard sends a CSV file's rows to the import door as CSV text + * (`format: 'csv'`, the preview's rows re-spelled), so the door reads them + * through its one CSV rule. The identity import goes through the same wizard + * and its adapter posts to `POST /api/v1/auth/admin/import-users` in batches of + * at most 500 rows, so the adapter takes the same change: each batch is the + * header record plus that batch's data records, as CSV text, and the rows each + * batch carries are still read to name every row's identity in the result. + * + * - the pure half: how a CSV request is cut into batches (quoted commas, a + * quoted line break, doubled quotes, a row with every cell empty); + * - the wired half: the real wizard and the real adapter, a 1001-row .csv + * file, three batches — each batch body is the wizard's own CSV text cut at + * record boundaries, the results renumber onto the preview's rows, and a + * one-time password is named by the identity of the row it belongs to; + * - CONTROL: a paste still goes out as JSON rows. + * + * `ImportWizard` is imported at module scope rather than through `ObjectView`'s + * `React.lazy` boundary (AGENTS.md 测试纪律). + */ +import { describe, it, expect, beforeAll, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor, act } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import React from 'react'; + +import { ImportWizard } from '@object-ui/plugin-grid'; +import type { ImportRequestOptions } from '@object-ui/types'; +import { + batchIdentityImportRequest, + createIdentityImportDataSource, + readCsvRequestRecords, + IDENTITY_IMPORT_OBJECT, +} from '../identityImport'; +import { identityImportFields } from '../IdentityImportPanels'; + +beforeAll(() => { + if (!Element.prototype.hasPointerCapture) Element.prototype.hasPointerCapture = vi.fn(() => false); + if (!Element.prototype.scrollIntoView) Element.prototype.scrollIntoView = vi.fn(); +}); + +describe('batchIdentityImportRequest cuts a CSV request at record boundaries (objectui#13315)', () => { + const csv = [ + 'email,name', + 'a@x.test,"Doe, Jane"', + 'b@x.test,"Line one\nLine two"', + ',', + 'c@x.test,"Say ""hi"""', + ].join('\r\n'); + + it('reads every record with its text as written and its cells', () => { + expect(readCsvRequestRecords(csv)).toEqual([ + { text: 'email,name', cells: ['email', 'name'] }, + { text: 'a@x.test,"Doe, Jane"', cells: ['a@x.test', 'Doe, Jane'] }, + { text: 'b@x.test,"Line one\nLine two"', cells: ['b@x.test', 'Line one\nLine two'] }, + { text: ',', cells: ['', ''] }, + { text: 'c@x.test,"Say ""hi"""', cells: ['c@x.test', 'Say "hi"'] }, + ]); + }); + + it('each batch is the header record plus its data records, and carries those rows in file order', () => { + expect(batchIdentityImportRequest({ format: 'csv', csv }, 2)).toEqual([ + { + payload: { format: 'csv', csv: 'email,name\r\na@x.test,"Doe, Jane"\r\nb@x.test,"Line one\nLine two"' }, + rows: [{ email: 'a@x.test', name: 'Doe, Jane' }, { email: 'b@x.test', name: 'Line one\nLine two' }], + }, + { + payload: { format: 'csv', csv: 'email,name\r\n,\r\nc@x.test,"Say ""hi"""' }, + rows: [{ email: '', name: '' }, { email: 'c@x.test', name: 'Say "hi"' }], + }, + ]); + }); + + it('a header alone is no row, and the adapter refuses it before any request', async () => { + expect(batchIdentityImportRequest({ format: 'csv', csv: 'email,name' })).toEqual([]); + const authFetch = vi.fn(); + const ds = createIdentityImportDataSource({ + base: {}, authFetch, baseUrl: 'http://srv', getPasswordPolicy: () => 'auto', + }) as { importRecords: (o: string, r: ImportRequestOptions) => Promise }; + await expect(ds.importRecords(IDENTITY_IMPORT_OBJECT, { format: 'csv', csv: 'email,name' })) + .rejects.toThrow(/at least one mapped row/); + expect(authFetch).not.toHaveBeenCalled(); + }); + + it('CONTROL: JSON rows are batched as JSON rows', () => { + const rows = [{ email: 'a@x.test' }, { email: 'b@x.test' }, { email: 'c@x.test' }]; + expect(batchIdentityImportRequest({ format: 'json', rows }, 2)).toEqual([ + { payload: { format: 'json', rows: rows.slice(0, 2) }, rows: rows.slice(0, 2) }, + { payload: { format: 'json', rows: rows.slice(2) }, rows: rows.slice(2) }, + ]); + }); +}); + +/** The identity endpoint, faked: it records each body and answers one row + * result per data record, with a one-time password on the rows asked for. */ +function fakeEndpoint(passwordFor: (batch: number, row: number) => string | undefined) { + const bodies: Array> = []; + const authFetch = vi.fn(async (_url: string, init?: RequestInit) => { + const body = JSON.parse(String(init?.body)) as Record; + const batch = bodies.length; + bodies.push(body); + // This file's fixtures hold no quoted line break, so a CRLF ends a record. + const count = body.format === 'csv' + ? String(body.csv).split('\r\n').length - 1 + : (body.rows as unknown[]).length; + const rows = Array.from({ length: count }, (_, i) => { + const temporaryPassword = passwordFor(batch, i + 1); + return { row: i + 1, ok: true, action: 'created', id: `u-${batch}-${i}`, ...(temporaryPassword ? { temporaryPassword } : {}) }; + }); + return new Response(JSON.stringify({ + success: true, + data: { summary: { total: count, created: count, updated: 0, skipped: 0, errors: 0, dryRun: false }, rows }, + }), { status: 200, headers: { 'Content-Type': 'application/json' } }); + }); + return { bodies, authFetch }; +} + +function renderIdentityWizard(authFetch: (url: string, init?: RequestInit) => Promise, initialFile?: File) { + const ds = createIdentityImportDataSource({ + base: {}, authFetch, baseUrl: 'http://srv', getPasswordPolicy: () => 'temporary', + }) as { importRecords: (o: string, r: ImportRequestOptions) => Promise }; + const seen: ImportRequestOptions[] = []; + const dataSource = { + ...ds, + importRecords: (object: string, request: ImportRequestOptions) => { seen.push(request); return ds.importRecords(object, request); }, + }; + const onComplete = vi.fn(); + render( + {}} + onComplete={onComplete} + />, + ); + return { seen, onComplete }; +} + +async function runAsMapped(onComplete: ReturnType) { + const next = await screen.findByTestId('import-next-btn'); + await waitFor(() => expect(next).toBeEnabled()); + fireEvent.click(next); + const run = await screen.findByTestId('import-run-btn'); + await waitFor(() => expect(run).toBeEnabled()); + fireEvent.click(run); + await waitFor(() => expect(onComplete).toHaveBeenCalled()); + return onComplete.mock.calls[0][0] as { + serverResult: { total: number; results: Array<{ row: number; identity?: string; temporaryPassword?: string }> }; + }; +} + +describe('the identity import sends a CSV file as CSV text, batch by batch (objectui#13315)', () => { + it('a 1001-row .csv file: three CSV batches cut from the wizard\'s text, rows renumbered, identities named', async () => { + const people = Array.from({ length: 1001 }, (_, i) => [`user${i + 1}@example.test`, i === 2 ? '"Doe, Jane"' : `User ${i + 1}`]); + const file = new File([['Email,Name', ...people.map((p) => p.join(','))].join('\n')], 'users.csv', { type: 'text/csv' }); + // A one-time password on row 277 of the second batch: row 777 of the file. + const { bodies, authFetch } = fakeEndpoint((batch, row) => (batch === 1 && row === 277 ? 'Pw-777' : undefined)); + const { seen, onComplete } = renderIdentityWizard(authFetch, file); + + const result = await runAsMapped(onComplete); + + const sent = seen.filter((r) => !r.dryRun); + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ format: 'csv' }); + expect(sent[0]).not.toHaveProperty('rows'); + const [header, ...records] = String(sent[0].csv).split('\r\n'); + expect(header).toBe('email,name'); + expect(records[2]).toBe('user3@example.test,"Doe, Jane"'); + + expect(bodies.map((b) => b.format)).toEqual(['csv', 'csv', 'csv']); + expect(bodies.map((b) => b.csv)).toEqual([ + [header, ...records.slice(0, 500)].join('\r\n'), + [header, ...records.slice(500, 1000)].join('\r\n'), + [header, ...records.slice(1000)].join('\r\n'), + ]); + for (const body of bodies) { + expect(body).not.toHaveProperty('rows'); + expect(body).toMatchObject({ mode: 'insert', passwordPolicy: 'temporary' }); + } + + expect(result.serverResult.total).toBe(1001); + expect(result.serverResult.results.map((r) => r.row)).toEqual(Array.from({ length: 1001 }, (_, i) => i + 1)); + expect(result.serverResult.results.filter((r) => r.temporaryPassword)).toEqual([ + expect.objectContaining({ row: 777, temporaryPassword: 'Pw-777', identity: 'user777@example.test' }), + ]); + }); + + it('CONTROL: a paste still goes out as JSON rows', async () => { + const { bodies, authFetch } = fakeEndpoint(() => undefined); + const { onComplete } = renderIdentityWizard(authFetch); + const evt = new Event('paste', { bubbles: true, cancelable: true }) as Event & { + clipboardData: { getData: (type: string) => string }; + }; + evt.clipboardData = { getData: (type: string) => (type === 'text/plain' ? 'Email\tName\na@x.test\tAda\nb@x.test\tGrace' : '') }; + act(() => { window.dispatchEvent(evt); }); + + await runAsMapped(onComplete); + expect(bodies).toHaveLength(1); + expect(bodies[0]).toMatchObject({ + format: 'json', + rows: [{ email: 'a@x.test', name: 'Ada' }, { email: 'b@x.test', name: 'Grace' }], + }); + expect(bodies[0]).not.toHaveProperty('csv'); + }); +}); diff --git a/packages/app-shell/src/views/identityImport.ts b/packages/app-shell/src/views/identityImport.ts index c66ff6d757..7b2f0dc138 100644 --- a/packages/app-shell/src/views/identityImport.ts +++ b/packages/app-shell/src/views/identityImport.ts @@ -12,7 +12,10 @@ // into ≤500-row batches — the endpoint's hard cap, because password-related // work is CPU-bound server-side — and (2) inject the admin's chosen password // policy. The endpoint is idempotent on upsert (matched by email/phone), so -// re-running a failed batch is safe. +// re-running a failed batch is safe. A CSV file's request arrives as CSV text +// (objectui#13315) and each batch is posted as CSV text too, so the endpoint +// reads the file's cells through the same CSV rule as the generic import door; +// JSON rows (a workbook, a paste) are batched as JSON rows. // // Password policies: `auto` (default, framework#3236 — per row: deliverable // rows get an invitation, unreachable rows fall back to a one-time password), @@ -71,6 +74,90 @@ export function splitIntoBatches(rows: T[], size = IDENTITY_IMPORT_BATCH_SIZE return batches; } +/** One record of a CSV request's text: the record as written, without its + * line break, and its cells. */ +export interface CsvRequestRecord { + text: string; + cells: string[]; +} + +/** + * Cut a CSV request's text into its records, header first (pure; exported for + * tests). The text is the import wizard's (`sourceRowsToCsv` in + * `@object-ui/plugin-grid`): records joined by CRLF, every cell holding a + * quote, a comma, a CR or an LF quoted (RFC 4180). The scan splits records the + * way the identity endpoint's CSV parse does — a quote opens a quoted run, a + * doubled quote inside one is a quote, an LF outside one ends the record, a CR + * outside one is dropped, and a last line holding nothing is no record — so a + * record cut here is a row the endpoint reads, in the same order. + * + * A record's text is kept as written, so a batch re-joined from these texts + * reaches the endpoint with the wizard's own bytes. The cells are read only to + * name each row's identity in the result (see mergeIdentityBatchResults); no + * imported value is decided here. + */ +export function readCsvRequestRecords(csv: string): CsvRequestRecord[] { + const text = csv.replace(/^\uFEFF/, ''); + const records: CsvRequestRecord[] = []; + let cells: string[] = []; + let cell = ''; + let start = 0; + let inQuotes = false; + const close = (end: number) => { + cells.push(cell); + const written = text.slice(start, end); + records.push({ text: written.endsWith('\r') ? written.slice(0, -1) : written, cells }); + cells = []; + cell = ''; + start = end + 1; + }; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (inQuotes) { + if (ch !== '"') cell += ch; + else if (text[i + 1] === '"') { cell += '"'; i++; } + else inQuotes = false; + } else if (ch === '"') inQuotes = true; + else if (ch === ',') { cells.push(cell); cell = ''; } + else if (ch === '\n') close(i); + else if (ch !== '\r') cell += ch; + } + if (cell.length > 0 || cells.length > 0) close(text.length); + return records; +} + +/** One request to the identity endpoint: its payload, and the rows it carries + * (read back to name each row's identity in the result). */ +export interface IdentityImportBatch { + payload: { format: 'csv'; csv: string } | { format: 'json'; rows: Array> }; + rows: Array>; +} + +/** + * Split the wizard's request into endpoint-sized batches (pure; exported for + * tests). CSV text (a CSV file, objectui#13315) is cut into records and every + * batch is the header record plus up to `size` data records, joined by CRLF — + * for the wizard's text, exactly `sourceRowsToCsv` of that batch's rows. JSON + * rows (a workbook, a paste) are batched as JSON rows. Each batch carries its + * rows in file order, so per-batch row numbers renumber onto the preview's. + */ +export function batchIdentityImportRequest( + request: ImportRequestOptions, + size = IDENTITY_IMPORT_BATCH_SIZE, +): IdentityImportBatch[] { + if (request.format === 'csv' && typeof request.csv === 'string') { + const [header, ...records] = readCsvRequestRecords(request.csv); + if (!header) return []; + const columns = header.cells.map((c) => c.trim()); + return splitIntoBatches(records, size).map((batch) => ({ + payload: { format: 'csv' as const, csv: [header, ...batch].map((r) => r.text).join('\r\n') }, + rows: batch.map((r) => Object.fromEntries(columns.map((column, i) => [column, r.cells[i] ?? '']))), + })); + } + const rows = Array.isArray(request.rows) ? request.rows : []; + return splitIntoBatches(rows, size).map((batch) => ({ payload: { format: 'json' as const, rows: batch }, rows: batch })); +} + /** * The write modes the identity import door accepts — stated once, here, beside * the refusal that enforces it (objectui#12697). The server serves no datum @@ -206,20 +293,18 @@ export function createIdentityImportDataSource(opts: IdentityImportDataSourceOpt ): Promise => { const { mode, matchBy } = resolveIdentityWriteOptions(request); const passwordPolicy = getPasswordPolicy(); - const rows = Array.isArray(request.rows) ? request.rows : []; - if (rows.length === 0) { - throw new Error('Identity import needs mapped rows (rows[]) — file uploads are parsed client-side.'); + const batches = batchIdentityImportRequest(request); + if (batches.length === 0) { + throw new Error('Identity import needs at least one mapped row (rows[] or csv).'); } - const batches = splitIntoBatches(rows); const responses: Array> = []; for (const batch of batches) { const res = await authFetch(`${baseUrl}/api/v1/auth/admin/import-users`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - format: 'json', - rows: batch, + ...batch.payload, mode, ...(matchBy ? { matchBy } : {}), passwordPolicy, @@ -241,7 +326,7 @@ export function createIdentityImportDataSource(opts: IdentityImportDataSourceOpt return mergeIdentityBatchResults( responses, - batches as Array>>, + batches.map((batch) => batch.rows), { dryRun: request.dryRun === true, writeMode: mode }, ); }; diff --git a/packages/plugin-grid/src/ImportWizard.csvDoorParity-13315.test.tsx b/packages/plugin-grid/src/ImportWizard.csvDoorParity-13315.test.tsx new file mode 100644 index 0000000000..607b69c9d8 --- /dev/null +++ b/packages/plugin-grid/src/ImportWizard.csvDoorParity-13315.test.tsx @@ -0,0 +1,199 @@ +/** + * ObjectUI – Copyright (c) 2024-present ObjectStack Inc. + * Licensed under MIT. + */ + +/** + * Parity of the server's CSV import parse with the wizard's own parse, + * MEASURED before a CSV file source was sent to the server's CSV import door + * (objectui#13315, item 2). This file records the inputs on which the two + * differ in a way that changes an imported value. + * + * Two ways for the wizard to hand a CSV file to that door were measured: + * - FILE TEXT: the server parses the file's own decoded text. + * - RE-SPELLED: the server parses `sourceRowsToCsv(rows)`, the rows the + * preview shows written back out as CSV — what a csv-declared named + * mapping already sent (objectui#13156). + * + * RE-SPELLED is the route that shipped (the maintainer's ruling A on + * objectui#13315): a CSV file source sends `format: 'csv'` with + * `sourceRowsToCsv(rows)`. Its pins are in + * `ImportWizard.csvFileSource-13315.test.tsx`. The FILE TEXT cases below record + * why the file's own text was not sent. The RE-SPELLED cases record the three + * shapes that route was accepted with: a last row whose sent cells are all + * empty is dropped (B1, driven through the real wizard below), an exported + * +/- cell with trailing whitespace keeps its mark (B2), and a space before a + * mark is trimmed, so the mark is removed (B3). + * + * The browser half runs live here: `decodeSpreadsheetText`, `parseDelimited`, + * `sourceRowsToCsv`, `buildSourceRows`, and the real wizard's default-path + * request. The server half is a RECORDING, and nothing in this repository + * re-derives it (AGENTS.md #9): objectstack's `parseCsvToRows` + * (`packages/rest/src/import-prepare.ts`) was read out of the objectstack git + * objects at `main` abc25a3536 and at the head of objectstack-ai/objectstack#25130 + * (bf8244cb20), which adds the CSV text-cell rule to that parse, header row + * included. Each function was transpiled unchanged and run on the exact text a + * case names. To re-take a reading, run that function on the case's input at + * the ref you need. `imported` is the recorded server value after the door's + * default `trimWhitespace` (a blank cell leaves the field unset). + * + * Each case asserts the live browser side exactly, then asserts that it + * differs from the recorded server side, so a change to either browser half + * turns the case red and asks for a new reading. + */ +import { describe, it, expect, beforeAll, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import React from 'react'; + +import { ImportWizard, __testables } from './ImportWizard'; +import { parseDelimited, decodeSpreadsheetText } from './importParsers'; +import { buildSourceRows } from './savedMapping'; +import type { ImportRecordsResult, ImportRequestOptions } from '@object-ui/types'; + +const { sourceRowsToCsv } = __testables; + +/** The mark the server's CSV text-cell rule writes and reads (U+0027). */ +const MARK = String.fromCharCode(0x27); + +/** The wizard's file parse for delimited text (`parseSpreadsheetFile`'s text arm), on raw bytes. */ +function wizardGrid(text: string, delimiter: ',' | '\t' = ','): string[][] { + return parseDelimited(decodeSpreadsheetText(new TextEncoder().encode(text).buffer as ArrayBuffer), delimiter); +} + +/** The RE-SPELLED text for the named-mapping path: source-keyed rows, every column. */ +function respelled(grid: string[][]): string { + return sourceRowsToCsv(buildSourceRows(grid[0], grid.slice(1), {})); +} + +describe('FILE TEXT: the server parsing the file itself reads different cells from the wizard (objectui#13315)', () => { + it('a tab-delimited file is one column to the server', () => { + const text = 'name\tnote\nAda\thello, world\n'; + expect(wizardGrid(text, '\t')).toEqual([['name', 'note'], ['Ada', 'hello, world']]); + // RECORDED, main and PR head alike: the server splits on commas only. + const server = [{ 'name\tnote': 'Ada\thello' }]; + expect(server).not.toEqual([{ name: 'Ada', note: 'hello, world' }]); + }); + + it('a leading blank line becomes the header, and a blank line inside the file becomes a row', () => { + const text = '\nname,note\nAda,x\n\n,\nBob,y\n\n\n'; + expect(wizardGrid(text)).toEqual([['name', 'note'], ['Ada', 'x'], ['Bob', 'y']]); + // RECORDED, main and PR head alike: header [''], so every row is empty, and five rows, not two. + const server = [{}, {}, {}, {}, {}]; + expect(server).toHaveLength(5); + }); + + it('a bare CR inside an unquoted cell is dropped by the server and kept by the wizard', () => { + const text = 'name,note\nA\rB,x\n'; + expect(wizardGrid(text)).toEqual([['name', 'note'], ['A\rB', 'x']]); + // RECORDED, main and PR head alike. + const server = [{ name: 'AB', note: 'x' }]; + expect(server[0].name).not.toBe(wizardGrid(text)[1][0]); + }); + + it('two columns under one header: the wizard keeps both by position, the server keeps the last', () => { + const text = 'name,name\nA,B\n'; + expect(wizardGrid(text)).toEqual([['name', 'name'], ['A', 'B']]); + // RECORDED, main and PR head alike. + const server = [{ name: 'B' }]; + expect(Object.values(server[0])).toHaveLength(1); + }); +}); + +describe('RE-SPELLED (shipped): the wizard trims a cell before the server reads its mark (objectui#13315)', () => { + it('an exported +/- cell with trailing whitespace keeps its mark when re-spelled, and loses it when the server reads the file', () => { + // The CSV the server's own export writes for the stored values + // `+1 555 0100` + LF and `-42` + TAB (RECORDED: `formatCsvCell` at the PR head). + const text = `name,note\r\n"${MARK}+1 555 0100\n",${MARK}-42\t\r\n`; + const grid = wizardGrid(text); + expect(grid).toEqual([['name', 'note'], [`${MARK}+1 555 0100`, `${MARK}-42`]]); + expect(respelled(grid)).toBe(`name,note\r\n${MARK}+1 555 0100,${MARK}-42`); + // RECORDED at the PR head. + const fileText = { cells: [{ name: '+1 555 0100\n', note: '-42\t' }], imported: [{ name: '+1 555 0100', note: '-42' }] }; + const respelledRead = { cells: [{ name: `${MARK}+1 555 0100`, note: `${MARK}-42` }], imported: [{ name: `${MARK}+1 555 0100`, note: `${MARK}-42` }] }; + expect(respelledRead.imported).not.toEqual(fileText.imported); + // What the re-spelled read starts from is the preview, cell for cell. + expect(respelledRead.cells[0]).toEqual({ name: grid[1][0], note: grid[1][1] }); + }); + + it('a space before a mark is trimmed by the wizard, so the re-spelled read removes the mark the file read keeps', () => { + const text = `name,note\n ${MARK}=x,y\n`; + const grid = wizardGrid(text); + expect(grid).toEqual([['name', 'note'], [`${MARK}=x`, 'y']]); + expect(respelled(grid)).toBe(`name,note\r\n${MARK}=x,y`); + // RECORDED at the PR head; the JSON rows the wizard sent before it switched + // to CSV imported `'=x`, as the file read does. + const fileText = { cells: [{ name: ` ${MARK}=x`, note: 'y' }], imported: [{ name: `${MARK}=x`, note: 'y' }] }; + const respelledRead = { cells: [{ name: '=x', note: 'y' }], imported: [{ name: '=x', note: 'y' }] }; + expect(respelledRead.imported).not.toEqual(fileText.imported); + }); + + it('a header cell is read through the same rule at the PR head: the named path keys by it, the default path does not', () => { + const text = `${MARK}=Total,note\r\nv,w\r\n`; + const grid = wizardGrid(text); + expect(grid).toEqual([[`${MARK}=Total`, 'note'], ['v', 'w']]); + expect(respelled(grid)).toBe(`${MARK}=Total,note\r\nv,w`); + // RECORDED: main keys `'=Total`, the PR head keys `=Total` (the header as + // the export labelled it, before the mark). The default path's text is + // headed by field names, so its keys are untouched (sourceRowsToCsv over + // field-keyed rows: no header cell of the file reaches the server). + const server = { main: [{ [`${MARK}=Total`]: 'v', note: 'w' }], pr: [{ '=Total': 'v', note: 'w' }] }; + expect(Object.keys(server.pr[0])).not.toEqual(Object.keys(server.main[0])); + expect(sourceRowsToCsv([{ total: 'v', note: 'w' }])).toBe('total,note\r\nv,w'); + }); +}); + +describe('RE-SPELLED, default path (shipped): a last row with every mapped cell empty is dropped by the server (objectui#13315)', () => { + beforeAll(() => { + if (!Element.prototype.hasPointerCapture) Element.prototype.hasPointerCapture = vi.fn(() => false); + if (!Element.prototype.scrollIntoView) Element.prototype.scrollIntoView = vi.fn(); + }); + + it('the wizard sends that row as CSV text, and it is gone after the server reads it (B1)', async () => { + const sent: ImportRequestOptions[] = []; + const importRecords = vi.fn(async (_object: string, request: ImportRequestOptions): Promise => { + sent.push(request); + const rows = request.rows ?? []; + return { + object: 'contact', dryRun: request.dryRun === true, writeMode: 'insert', total: rows.length, + ok: rows.length, errors: 0, created: rows.length, updated: 0, skipped: 0, + results: rows.map((_, i) => ({ row: i + 1, ok: true, action: 'created' as const })), + } as ImportRecordsResult; + }); + const onComplete = vi.fn(); + // The `note` column has no field, so it stays unmapped; the last row has a + // value in that column only. + const file = new File(['name,note\nAda,x\n,only a note\n'], 'contacts.csv', { type: 'text/csv' }); + render( + {}} + onComplete={onComplete as never} + />, + ); + const next = await screen.findByTestId('import-next-btn'); + await waitFor(() => expect(next).toBeEnabled()); + fireEvent.click(next); + const run = await screen.findByTestId('import-run-btn'); + await waitFor(() => expect(run).toBeEnabled()); + fireEvent.click(run); + await waitFor(() => expect(onComplete).toHaveBeenCalled()); + + const real = sent.filter((r) => !r.dryRun); + expect(real).toHaveLength(1); + // The preview's two rows, the second with its one mapped cell empty, + // re-spelled: `sourceRowsToCsv([{ name: 'Ada' }, { name: '' }])`. + expect(real[0]).toEqual({ + format: 'csv', csv: 'name\r\nAda\r\n', writeMode: 'insert', createMissingOptions: false, skipBlankMatchKey: false, + }); + expect(sourceRowsToCsv([{ name: 'Ada' }, { name: '' }])).toBe(real[0].csv); + // RECORDED, main and PR head alike: the server drops trailing rows whose + // cells are all empty, so the preview's second row never reaches the runner. + const server = [{ name: 'Ada' }]; + expect(server).toHaveLength(2 - 1); + }); +}); diff --git a/packages/plugin-grid/src/ImportWizard.csvFileSource-13315.test.tsx b/packages/plugin-grid/src/ImportWizard.csvFileSource-13315.test.tsx new file mode 100644 index 0000000000..90c0c20176 --- /dev/null +++ b/packages/plugin-grid/src/ImportWizard.csvFileSource-13315.test.tsx @@ -0,0 +1,291 @@ +/** + * ObjectUI – Copyright (c) 2024-present ObjectStack Inc. + * Licensed under MIT. + */ + +/** + * A CSV file source reaches the server's CSV import door (objectui#13315). + * + * The import door reads a CSV payload's cells through one CSV rule, the one its + * CSV export pairs with. The wizard parsed every file in the browser and sent + * JSON rows on its default path, so a CSV file reached that rule only when a + * named mapping declared `sourceFormat: "csv"` (objectui#13156). + * + * These pins drive the real wizard to its real request (`importRecords`): + * - a CSV file — `.csv`, `.tsv` or `.txt`, the files `parseSpreadsheetFile` + * reads as delimited text — sends `format: 'csv'` with the preview's rows + * re-spelled as CSV (`sourceRowsToCsv`), never the file as uploaded: on the + * default path, where the dry-run pre-check sends the same payload, and on a + * named mapping that declares no format; + * - a header the CSV export marked still maps on the default path: the text + * is headed by field names, so no header cell of the file reaches the door; + * - a `json`-declared mapping, an .xlsx sheet and a paste send JSON rows, as + * before (CONTROLS). + */ +import { describe, it, expect, beforeAll, vi } from 'vitest'; +import { render, screen, fireEvent, waitFor, act, within } from '@testing-library/react'; +import '@testing-library/jest-dom'; +import React from 'react'; +// Imported at module scope so the wizard's lazy `import('exceljs')` resolves +// from the module cache instead of racing the findBy budget. +import ExcelJS from 'exceljs'; + +import { ImportWizard } from './ImportWizard'; +import type { SavedMapping } from './savedMapping'; +import type { ImportRecordsResult, ImportRequestOptions } from '@object-ui/types'; + +beforeAll(() => { + // Radix Select reads these on open; happy-dom lacks them. + if (!Element.prototype.hasPointerCapture) { + Element.prototype.hasPointerCapture = vi.fn(() => false); + } + if (!Element.prototype.scrollIntoView) { + Element.prototype.scrollIntoView = vi.fn(); + } +}); + +/** The mark the server's CSV text-cell rule writes on export (U+0027). */ +const MARK = String.fromCharCode(0x27); + +type WizardField = React.ComponentProps['fields'][number]; + +const PEOPLE: WizardField[] = [ + { name: 'name', label: 'Name', type: 'text', required: true }, + { name: 'email', label: 'Email', type: 'text' }, +]; + +const LEADS: WizardField[] = [ + { name: 'company', label: 'Company', type: 'text', required: true }, + { name: 'lead_source', label: 'Lead source', type: 'text' }, +]; + +/** A shipped mapping that declares no `sourceFormat`: the door takes any payload. */ +const UNDECLARED_MAPPING: SavedMapping = { + name: 'crm_lead_any', + label: 'Lead import (any format)', + targetObject: 'crm_lead', + fieldMapping: [ + { source: 'Company', target: 'company' }, + { source: 'Source', target: 'lead_source' }, + ], + mode: 'insert', +}; +const JSON_MAPPING: SavedMapping = { + ...UNDECLARED_MAPPING, + name: 'crm_lead_feed', + label: 'Lead feed (JSON)', + sourceFormat: 'json', +}; + +/** A data source whose import door records every request and accepts it. */ +function recordingDataSource() { + const sent: ImportRequestOptions[] = []; + const importRecords = vi.fn(async (_object: string, request: ImportRequestOptions): Promise => { + sent.push(request); + return { + object: 'x', dryRun: request.dryRun === true, writeMode: 'insert', total: 2, + ok: 2, errors: 0, created: 2, updated: 0, skipped: 0, + results: [{ row: 1, ok: true, action: 'created' }, { row: 2, ok: true, action: 'created' }], + } as ImportRecordsResult; + }); + return { sent, dataSource: { importRecords } }; +} + +function renderWizard(opts: { + fields: WizardField[]; + dataSource: object; + onComplete: (r: unknown) => void; + initialFile?: File; + savedMappings?: SavedMapping[]; +}) { + return render( + {}} + onComplete={opts.onComplete as never} + />, + ); +} + +/** Paste TSV into the upload step via the wizard's window-level handler. */ +function pasteRows(text: string) { + const evt = new Event('paste', { bubbles: true, cancelable: true }) as Event & { + clipboardData: { getData: (type: string) => string }; + }; + evt.clipboardData = { getData: (type: string) => (type === 'text/plain' ? text : '') }; + act(() => { window.dispatchEvent(evt); }); +} + +/** On the mapping step, with the hand mapping as auto-mapped: go to the preview. */ +async function previewAsMapped() { + const next = await screen.findByTestId('import-next-btn'); + await waitFor(() => expect(next).toBeEnabled()); + fireEvent.click(next); + return screen.findByTestId('import-run-btn'); +} + +/** On the mapping step: pick a saved mapping, then go to the preview. */ +async function chooseMappingAndPreview(name: string) { + await screen.findByTestId('import-saved-mapping-bar'); + fireEvent.pointerDown(screen.getByTestId('import-saved-mapping-select'), { button: 0, ctrlKey: false, pointerType: 'mouse' }); + const listbox = await screen.findByRole('listbox'); + fireEvent.click(within(listbox).getByTestId(`import-saved-mapping-option-${name}`)); + expect(await screen.findByTestId('import-saved-mapping-summary'), 'the mapping is active').toBeInTheDocument(); + return previewAsMapped(); +} + +/** Run the import and return the one real (non-dry-run) request it sent. */ +async function runImport(run: HTMLElement, sent: ImportRequestOptions[], onComplete: ReturnType) { + await waitFor(() => expect(run).toBeEnabled()); + fireEvent.click(run); + await waitFor(() => expect(onComplete).toHaveBeenCalled()); + const real = sent.filter((r) => !r.dryRun); + expect(real, 'exactly one import request').toHaveLength(1); + return real[0]; +} + +/** The default path's request around a payload: insert, no options set. */ +const defaultPath = (payload: Partial): ImportRequestOptions => ({ + ...payload, writeMode: 'insert', createMissingOptions: false, skipBlankMatchKey: false, +}); + +describe('ImportWizard sends a CSV file to the CSV import door (objectui#13315)', () => { + it('a .csv file on the default path sends format csv with the preview\'s cells, and the dry run sends the same payload', async () => { + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + const file = new File( + ['Name,Email\nAda Lovelace,ada@example.test\n"Hopper, Grace",grace@example.test\n'], + 'people.csv', + { type: 'text/csv' }, + ); + renderWizard({ fields: PEOPLE, dataSource, onComplete, initialFile: file }); + const run = await previewAsMapped(); + + // The user corrects a cell in the preview: the CSV carries the fix, so the + // text is the preview's rows re-spelled, not the file as uploaded. + fireEvent.change(screen.getByTestId('import-preview-cell-0-0'), { target: { value: 'Ada King' } }); + + fireEvent.click(screen.getByTestId('import-validate-btn')); + await screen.findByTestId('import-validate-result'); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual(defaultPath({ + format: 'csv', + csv: 'name,email\r\nAda King,ada@example.test\r\n"Hopper, Grace",grace@example.test', + })); + expect(request, 'no JSON rows ride along with the CSV text').not.toHaveProperty('rows'); + + const dry = sent.filter((r) => r.dryRun); + expect(dry).toHaveLength(1); + expect(dry[0]).toEqual({ ...request, dryRun: true }); + }); + + it.each([ + ['people.tsv', 'Name\tEmail\nAda\tada@example.test\nGrace\tgrace@example.test\n'], + ['people.txt', 'Name,Email\nAda,ada@example.test\nGrace,grace@example.test\n'], + ])('%s, picked in the upload step, is a CSV file too', async (fileName, text) => { + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + const { container } = renderWizard({ fields: PEOPLE, dataSource, onComplete }); + const input = container.ownerDocument.querySelector('input[type=file]') as HTMLInputElement; + fireEvent.change(input, { target: { files: [new File([text], fileName)] } }); + const run = await previewAsMapped(); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual(defaultPath({ + format: 'csv', + csv: 'name,email\r\nAda,ada@example.test\r\nGrace,grace@example.test', + })); + }); + + it('a header the CSV export marked still maps: the text is headed by the field name it maps to', async () => { + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + const fields: WizardField[] = [ + { name: 'region', label: 'Region', type: 'text' }, + { name: 'name', label: 'Name', type: 'text', required: true }, + ]; + const file = new File([`${MARK}=Region,Name\r\nNorth,Ada\r\nSouth,Grace\r\n`], 'export.csv', { type: 'text/csv' }); + renderWizard({ fields, dataSource, onComplete, initialFile: file }); + const run = await previewAsMapped(); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual(defaultPath({ format: 'csv', csv: 'region,name\r\nNorth,Ada\r\nSouth,Grace' })); + }); + + it('a named mapping that declares no format sends a CSV file as CSV text, with mappingName', async () => { + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + const file = new File(['Company,Source\nAcme,LinkedIn\n"Globex, Inc.",Google Ads\n'], 'leads.csv', { type: 'text/csv' }); + renderWizard({ fields: LEADS, dataSource, onComplete, initialFile: file, savedMappings: [UNDECLARED_MAPPING, JSON_MAPPING] }); + const run = await chooseMappingAndPreview('crm_lead_any'); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual({ + format: 'csv', + csv: 'Company,Source\r\nAcme,LinkedIn\r\n"Globex, Inc.",Google Ads', + mappingName: 'crm_lead_any', + }); + expect(request).not.toHaveProperty('rows'); + }); + + it('CONTROL: a json-declared mapping still sends a CSV file as JSON rows (the door refuses CSV for it)', async () => { + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + const file = new File(['Company,Source\nAcme,LinkedIn\n"Globex, Inc.",Google Ads\n'], 'leads.csv', { type: 'text/csv' }); + renderWizard({ fields: LEADS, dataSource, onComplete, initialFile: file, savedMappings: [UNDECLARED_MAPPING, JSON_MAPPING] }); + const run = await chooseMappingAndPreview('crm_lead_feed'); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual({ + format: 'json', + rows: [ + { Company: 'Acme', Source: 'LinkedIn' }, + { Company: 'Globex, Inc.', Source: 'Google Ads' }, + ], + mappingName: 'crm_lead_feed', + }); + expect(request).not.toHaveProperty('csv'); + }); + + it('CONTROL: an .xlsx sheet on the default path sends JSON rows, as before', async () => { + const wb = new ExcelJS.Workbook(); + const ws = wb.addWorksheet('People'); + ws.addRow(['Name', 'Email']); + ws.addRow(['Ada', 'ada@example.test']); + ws.addRow(['Grace', 'grace@example.test']); + const file = new File([await wb.xlsx.writeBuffer()], 'people.xlsx'); + + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + renderWizard({ fields: PEOPLE, dataSource, onComplete, initialFile: file }); + const run = await previewAsMapped(); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual(defaultPath({ + format: 'json', + rows: [{ name: 'Ada', email: 'ada@example.test' }, { name: 'Grace', email: 'grace@example.test' }], + })); + expect(request).not.toHaveProperty('csv'); + }); + + it('CONTROL: a paste on the default path sends JSON rows, as before', async () => { + const { sent, dataSource } = recordingDataSource(); + const onComplete = vi.fn(); + renderWizard({ fields: PEOPLE, dataSource, onComplete }); + pasteRows('Name\tEmail\nAda\tada@example.test\nGrace\tgrace@example.test'); + const run = await previewAsMapped(); + + const request = await runImport(run, sent, onComplete); + expect(request).toEqual(defaultPath({ + format: 'json', + rows: [{ name: 'Ada', email: 'ada@example.test' }, { name: 'Grace', email: 'grace@example.test' }], + })); + expect(request).not.toHaveProperty('csv'); + }); +}); diff --git a/packages/plugin-grid/src/ImportWizard.tsx b/packages/plugin-grid/src/ImportWizard.tsx index 03835ff85f..8a413a6fbb 100644 --- a/packages/plugin-grid/src/ImportWizard.tsx +++ b/packages/plugin-grid/src/ImportWizard.tsx @@ -36,7 +36,7 @@ import type { ImportJobUndoResult, } from '@object-ui/types'; import { - parseSpreadsheetFile, parseClipboardTable, inferColumnType, isTypeCompatible, + parseSpreadsheetFile, delimitedFileDelimiter, parseClipboardTable, inferColumnType, isTypeCompatible, suggestColumnMappings, ImportParseError, type InferredType, type ColumnSuggestion, type MappingConfidence, } from './importParsers'; @@ -952,7 +952,8 @@ function csvCell(value: string): string { return /[",\r\n]/.test(value) ? `"${value.replace(/"/g, '""')}"` : value; } -/** CSV text of SOURCE-keyed rows (see buildSourceRows): a header row naming +/** CSV text of the request's rows — SOURCE-keyed for a named mapping (see + * buildSourceRows), field-keyed on the hand-mapped path: a header row naming * every column the rows carry, then one line per row. The cells are the ones * the JSON rows would carry, so a corrected cell, a pasted table and an .xlsx * sheet travel exactly as the preview shows them (objectui#13156). The columns @@ -999,22 +1000,37 @@ function assembleImportRequest( * server serves and never inferred from the uploaded file's name. The * import door judges the declaration against the payload and refuses JSON * rows for a `csv` mapping (`400 MAPPING_FORMAT_MISMATCH`, objectui#13156), - * so a `csv` mapping's rows travel as CSV text; any other keeps JSON rows. */ + * so a `csv` mapping's rows travel as CSV text from any source, and a + * `json` mapping's as JSON rows from any source. An undeclared one follows + * the source, as the hand-mapped path does (`csvFile`). */ mappingSourceFormat?: SavedMapping['sourceFormat']; + /** The rows were read from a CSV file: a delimited-text file, by the name + * test `parseSpreadsheetFile` picks its text arm with + * (`delimitedFileDelimiter`). Its rows travel as CSV text, so the import + * door reads the file's cells through its own CSV rule — the one its CSV + * export pairs with (objectui#13315). The text is `sourceRowsToCsv(rows)`, + * the cells the preview parsed, never the file as uploaded. An .xlsx sheet + * and a paste keep JSON rows. */ + csvFile?: boolean; }, ): ImportRequestOptions { + // A `json`-declared mapping is the one CSV-file request that keeps JSON rows: + // the door refuses CSV text for it (`MAPPING_FORMAT_MISMATCH`). + const asCsv = opts.mappingName + ? opts.mappingSourceFormat === 'csv' || (opts.csvFile === true && opts.mappingSourceFormat !== 'json') + : opts.csvFile === true; + const payload = asCsv + ? { format: 'csv' as const, csv: sourceRowsToCsv(rows) } + : { format: 'json' as const, rows }; if (opts.mappingName) { return { - ...(opts.mappingSourceFormat === 'csv' - ? { format: 'csv' as const, csv: sourceRowsToCsv(rows) } - : { format: 'json' as const, rows }), + ...payload, mappingName: opts.mappingName, ...(opts.dryRun ? { dryRun: true } : {}), }; } return { - format: 'json', - rows, + ...payload, writeMode: opts.writeMode, ...(opts.writeMode !== 'insert' ? { matchFields: opts.matchFields } : {}), createMissingOptions: opts.createMissingOptions, @@ -1301,7 +1317,9 @@ function parseErrorMessage(err: unknown, t: (k: string, v?: Record void; + /** `csvFile`: the grid was read from a delimited-text file, not from a + * workbook or a paste (see `assembleImportRequest`). */ + onFileLoaded: (headers: string[], rows: string[][], csvFile: boolean) => void; /** Fetches the server's import template (objectui#9600). Absent ⇒ the * template is not offered: there is no client-built fallback. */ downloadTemplate?: () => Promise; @@ -1333,16 +1351,16 @@ const StepUpload: React.FC<{ }; /** Validate a freshly-parsed grid and hand it to the wizard, or report why not. */ - const acceptParsed = useCallback((parsed: string[][]) => { + const acceptParsed = useCallback((parsed: string[][], csvFile: boolean) => { if (parsed.length < 2) { setError(t('grid.import.fileNeedsHeader')); return false; } - onFileLoaded(parsed[0], parsed.slice(1)); + onFileLoaded(parsed[0], parsed.slice(1), csvFile); return true; }, [onFileLoaded, t]); const processFile = useCallback(async (file: File) => { setError(null); setBusy(true); try { - acceptParsed(await parseSpreadsheetFile(file)); + acceptParsed(await parseSpreadsheetFile(file), delimitedFileDelimiter(file.name) !== null); } catch (err) { setError(parseErrorMessage(err, t)); } finally { @@ -1362,7 +1380,7 @@ const StepUpload: React.FC<{ if (!parsed) return; e.preventDefault(); setError(null); - if (!acceptParsed(parsed)) { /* message already set */ } + if (!acceptParsed(parsed, false)) { /* message already set */ } }, [acceptParsed]); useEffect(() => { @@ -2207,6 +2225,9 @@ export const ImportWizard: React.FC = ({ const [step, setStep] = useState('upload'); const [headers, setHeaders] = useState([]); const [rows, setRows] = useState([]); + // The rows came from a CSV file (a delimited-text file), so the request + // travels as CSV text (assembleImportRequest, objectui#13315). + const [csvFile, setCsvFile] = useState(false); const [mapping, setMapping] = useState>({}); const [importing, setImporting] = useState(false); const [progress, setProgress] = useState(0); @@ -2345,7 +2366,8 @@ export const ImportWizard: React.FC = ({ // Select a registered mapping: the server owns rename + transforms, so we // set a read-only display mapping (for the preview grid) and reflect the // artifact's write semantics; submit sends source rows + mappingName, as CSV - // text when the mapping declares `sourceFormat: 'csv'`. + // text when the mapping declares `sourceFormat: 'csv'` or the source is a CSV + // file the mapping does not declare `json` for (assembleImportRequest). const handleSelectSavedMapping = useCallback((name: string) => { const m = availableMappings.find((sm) => sm.name === name); if (!m) return; @@ -2424,8 +2446,9 @@ export const ImportWizard: React.FC = ({ const missingRequiredHintId = React.useId(); const nextHeldByRequired = !mappingName && missingRequired.length > 0; - const handleFileLoaded = useCallback((h: string[], r: string[][]) => { - setHeaders(h); setRows(r); setMapping(autoMapColumns(h, fields, r)); setCorrections({}); setStep('mapping'); + const handleFileLoaded = useCallback((h: string[], r: string[][], fromCsvFile: boolean) => { + setHeaders(h); setRows(r); setCsvFile(fromCsvFile); + setMapping(autoMapColumns(h, fields, r)); setCorrections({}); setStep('mapping'); }, [fields]); // cloud#797 Excel→App: when opened with a pre-loaded file (from the AI build @@ -2444,7 +2467,7 @@ export const ImportWizard: React.FC = ({ const parsed = await parseSpreadsheetFile(initialFile); if (cancelled) return; if (parsed.length < 2) return; // needs a header + at least one row - handleFileLoaded(parsed[0], parsed.slice(1)); + handleFileLoaded(parsed[0], parsed.slice(1), delimitedFileDelimiter(initialFile.name) !== null); } catch { // Parse failed — leave the wizard on the upload step so the user can // retry / pick a different file; the upload UI shows its own errors. @@ -2707,16 +2730,16 @@ export const ImportWizard: React.FC = ({ const mappingSourceFormat = activeMapping?.sourceFormat; const buildImportRequest = useCallback((dryRun = false): ImportRequestOptions => assembleImportRequest( - // A named mapping is applied server-side over SOURCE-header rows, sent in - // the format it declares; the hand-mapped, field-keyed rows are for the - // manual path only. + // A named mapping is applied server-side over SOURCE-header rows; the + // hand-mapped, field-keyed rows are for the manual path only. Either + // travels as CSV text for a CSV file (assembleImportRequest). mappingName ? buildSourceRows(headers, rows, corrections) : buildRawRows(), { - writeMode, matchFields, createMissingOptions, skipBlankMatchKey, dryRun, + writeMode, matchFields, createMissingOptions, skipBlankMatchKey, dryRun, csvFile, ...(mappingName ? { mappingName, mappingSourceFormat } : {}), }, ), - [buildRawRows, mappingName, mappingSourceFormat, headers, rows, corrections, writeMode, matchFields, createMissingOptions, skipBlankMatchKey]); + [buildRawRows, mappingName, mappingSourceFormat, headers, rows, corrections, writeMode, matchFields, createMissingOptions, skipBlankMatchKey, csvFile]); const handleImport = useCallback(async () => { setImporting(true); setProgress(0); @@ -2915,7 +2938,7 @@ export const ImportWizard: React.FC = ({ const reset = useCallback(() => { cancelPollRef.current = false; setFinishedJob(null); setUndoOutcome(null); - setStep('upload'); setHeaders([]); setRows([]); setMapping({}); setProgress(0); setResult(null); + setStep('upload'); setHeaders([]); setRows([]); setCsvFile(false); setMapping({}); setProgress(0); setResult(null); setCorrections({}); setSelectedTemplateId(null); setMappingName(null); setWriteMode(defaultWriteMode); setMatchFields([]); setCreateMissingOptions(false); setSkipBlankMatchKey(false); diff --git a/packages/plugin-grid/src/importParsers.ts b/packages/plugin-grid/src/importParsers.ts index cc0bbf0d76..374a8b8e35 100644 --- a/packages/plugin-grid/src/importParsers.ts +++ b/packages/plugin-grid/src/importParsers.ts @@ -131,6 +131,21 @@ export function decodeSpreadsheetText(buffer: ArrayBuffer): string { } } +/** + * The delimiter a delimited-text import file is read with, picked off its + * name: `.tsv` is tab-delimited, `.csv` and `.txt` are comma-delimited, and + * every other file answers `null`. {@link parseSpreadsheetFile} picks its text + * arm by this answer, and the import wizard sends a file it answers for to the + * server's CSV import door (objectui#13315), so the two read "a CSV file" the + * same way. + */ +export function delimitedFileDelimiter(fileName: string): ',' | '\t' | null { + const name = fileName.toLowerCase(); + if (name.endsWith('.tsv')) return '\t'; + if (name.endsWith('.csv') || name.endsWith('.txt')) return ','; + return null; +} + /** * Parse any supported import file into `string[][]`. Delimited text is * decoded via {@link decodeSpreadsheetText} (UTF-8 with GB18030 fallback); @@ -138,11 +153,12 @@ export function decodeSpreadsheetText(buffer: ArrayBuffer): string { * {@link ImportParseError} code for unsupported / legacy formats. */ export async function parseSpreadsheetFile(file: File): Promise { - const name = file.name.toLowerCase(); - if (name.endsWith('.csv') || name.endsWith('.tsv') || name.endsWith('.txt')) { + const delimiter = delimitedFileDelimiter(file.name); + if (delimiter) { const text = decodeSpreadsheetText(await file.arrayBuffer()); - return parseDelimited(text, name.endsWith('.tsv') ? '\t' : ','); + return parseDelimited(text, delimiter); } + const name = file.name.toLowerCase(); if (name.endsWith('.xlsx') || name.endsWith('.xlsm')) { return parseExcelArrayBuffer(await file.arrayBuffer()); }